Northern-Thailand-Ping-River-Monitor/src/validators.py

#!/usr/bin/env python3
"""
Data validation utilities for water monitoring system
"""

import logging
from datetime import datetime
from typing import Any, Dict, List, Optional

from .exceptions import DataValidationError
from .models import StationInfo, WaterMeasurement

logger = logging.getLogger(__name__)


class DataValidator:
    """Validates water measurement data"""

    # Reasonable ranges for water measurements
    WATER_LEVEL_MIN = -10.0  # meters
    WATER_LEVEL_MAX = 50.0  # meters
    DISCHARGE_MIN = 0.0  # cms
    DISCHARGE_MAX = 10000.0  # cms
    DISCHARGE_PERCENT_MIN = 0.0
    # % of channel capacity. Major floods genuinely exceed 200% (Oct 2024 peaked
    # at 226%); values beyond 500% are treated as data errors.
    DISCHARGE_PERCENT_MAX = 500.0

    @classmethod
    def validate_measurement(cls, measurement: Dict[str, Any]) -> bool:
        """Validate a single measurement"""
        try:
            # Check required fields (discharge is now optional)
            required_fields = ["timestamp", "station_id", "water_level"]
            for field in required_fields:
                if field not in measurement:
                    logger.warning(f"Missing required field: {field}")
                    return False

            # Validate timestamp
            if not isinstance(measurement["timestamp"], datetime):
                logger.warning(
                    f"Invalid timestamp type: {type(measurement['timestamp'])}"
                )
                return False

            # Validate water level (required)
            if measurement["water_level"] is None:
                logger.warning("Water level cannot be None")
                return False
            water_level = float(measurement["water_level"])
            if not (cls.WATER_LEVEL_MIN <= water_level <= cls.WATER_LEVEL_MAX):
                logger.warning(f"Water level out of range: {water_level}")
                return False

            # Validate discharge (optional - can be None)
            discharge_value = measurement.get("discharge")
            if discharge_value is not None:
                discharge = float(discharge_value)
                if not (cls.DISCHARGE_MIN <= discharge <= cls.DISCHARGE_MAX):
                    logger.warning(f"Discharge out of range: {discharge}")
                    return False

            # Validate discharge percent if present. This is an auxiliary field:
            # an implausible value must not cost us the water level and discharge
            # (rejecting rows here silently deleted the Oct 2024 flood peaks), so
            # out-of-range percents are nulled and the measurement is kept.
            if measurement.get("discharge_percent") is not None:
                discharge_percent = float(measurement["discharge_percent"])
                if not (
                    cls.DISCHARGE_PERCENT_MIN
                    <= discharge_percent
                    <= cls.DISCHARGE_PERCENT_MAX
                ):
                    logger.warning(
                        f"Discharge percent out of range ({discharge_percent}); "
                        "keeping measurement with discharge_percent=None"
                    )
                    measurement["discharge_percent"] = None

            # Validate station ID
            station_id = measurement["station_id"]
            if not isinstance(station_id, int) or station_id < 1 or station_id > 16:
                logger.warning(f"Invalid station ID: {station_id}")
                return False

            return True

        except (ValueError, TypeError) as e:
            logger.warning(f"Data validation error: {e}")
            return False

    @classmethod
    def validate_measurements(
        cls, measurements: List[Dict[str, Any]]
    ) -> List[Dict[str, Any]]:
        """Validate and filter a list of measurements"""
        valid_measurements = []
        invalid_count = 0

        for measurement in measurements:
            if cls.validate_measurement(measurement):
                valid_measurements.append(measurement)
            else:
                invalid_count += 1

        if invalid_count > 0:
            logger.warning(f"Filtered out {invalid_count} invalid measurements")

        return valid_measurements

    @classmethod
    def validate_station_info(cls, station_info: Dict[str, Any]) -> bool:
        """Validate station information"""
        try:
            required_fields = [
                "station_id",
                "station_code",
                "thai_name",
                "english_name",
            ]
            for field in required_fields:
                if field not in station_info or not station_info[field]:
                    logger.warning(f"Missing or empty station field: {field}")
                    return False

            # Validate coordinates if present
            if station_info.get("latitude") is not None:
                lat = float(station_info["latitude"])
                if not (-90 <= lat <= 90):
                    logger.warning(f"Invalid latitude: {lat}")
                    return False

            if station_info.get("longitude") is not None:
                lon = float(station_info["longitude"])
                if not (-180 <= lon <= 180):
                    logger.warning(f"Invalid longitude: {lon}")
                    return False

            return True

        except (ValueError, TypeError) as e:
            logger.warning(f"Station validation error: {e}")
            return False