feat: --fill-gaps all repairs missing data across the whole DB range
Documentation / Generate API Documentation (push) Successful in 11s
Documentation / Build Sphinx Documentation (push) Successful in 17s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 1s
Documentation / Documentation Summary (push) Successful in 3s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 29s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 14s
Documentation / Validate Documentation (push) Failing after 9s
Documentation / Generate API Documentation (push) Successful in 11s
Documentation / Build Sphinx Documentation (push) Successful in 17s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 1s
Documentation / Documentation Summary (push) Successful in 3s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 29s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 14s
Documentation / Validate Documentation (push) Failing after 9s
Gap detection is now hour-granular: days with partial data (missing hourly slots) are re-fetched, not just days with no rows at all. The API's hour-24-is-next-midnight quirk is handled by re-fetching day D-1 when day D is missing its 00:00 slot. SQL adapters gain single-query range and recorded-hours lookups; non-SQL backends fall back to the old day-granular check.
This commit is contained in:
@@ -36,6 +36,26 @@ class DatabaseAdapter(ABC):
|
||||
def get_measurements_for_date(self, target_date: datetime.datetime) -> List[Dict]:
|
||||
pass
|
||||
|
||||
def get_measurement_date_range(
|
||||
self,
|
||||
) -> Optional[tuple]:
|
||||
"""Return (min_timestamp, max_timestamp) of stored measurements.
|
||||
|
||||
Returns None when the backend has no data or does not support the query.
|
||||
"""
|
||||
return None
|
||||
|
||||
def get_recorded_hours_by_day(
|
||||
self, start_date: datetime.date, end_date: datetime.date
|
||||
) -> Optional[Dict[datetime.date, set]]:
|
||||
"""Map each day in [start_date, end_date] to the set of hours (0-23)
|
||||
that have at least one measurement.
|
||||
|
||||
Returns None when the backend does not support hour-granular gap
|
||||
detection (callers should fall back to day-granular checks).
|
||||
"""
|
||||
return None
|
||||
|
||||
|
||||
# InfluxDB Adapter
|
||||
class InfluxDBAdapter(DatabaseAdapter):
|
||||
@@ -657,6 +677,93 @@ class SQLAdapter(DatabaseAdapter):
|
||||
)
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _coerce_date(value) -> Optional[datetime.date]:
|
||||
"""Normalize a DB-returned day value (str/date/datetime) to a date."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime.datetime):
|
||||
return value.date()
|
||||
if isinstance(value, datetime.date):
|
||||
return value
|
||||
# SQLite returns strings, e.g. '2024-09-15'
|
||||
return datetime.datetime.strptime(str(value)[:10], "%Y-%m-%d").date()
|
||||
|
||||
def get_measurement_date_range(self) -> Optional[tuple]:
|
||||
if not self.engine:
|
||||
return None
|
||||
|
||||
try:
|
||||
from sqlalchemy import text
|
||||
|
||||
query = "SELECT MIN(timestamp), MAX(timestamp) FROM water_measurements"
|
||||
with self.engine.connect() as conn:
|
||||
row = conn.execute(text(query)).fetchone()
|
||||
|
||||
if not row or row[0] is None:
|
||||
return None
|
||||
|
||||
def to_datetime(value):
|
||||
if isinstance(value, datetime.datetime):
|
||||
return value
|
||||
return datetime.datetime.fromisoformat(str(value)[:19])
|
||||
|
||||
return (to_datetime(row[0]), to_datetime(row[1]))
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error querying {self.db_type.upper()} date range: {e}")
|
||||
return None
|
||||
|
||||
def get_recorded_hours_by_day(
|
||||
self, start_date: datetime.date, end_date: datetime.date
|
||||
) -> Optional[Dict[datetime.date, set]]:
|
||||
if not self.engine:
|
||||
return None
|
||||
|
||||
try:
|
||||
from sqlalchemy import text
|
||||
|
||||
if self.db_type == "sqlite":
|
||||
day_expr = "DATE(timestamp)"
|
||||
hour_expr = "CAST(strftime('%H', timestamp) AS INTEGER)"
|
||||
elif self.db_type == "postgresql":
|
||||
day_expr = "CAST(timestamp AS DATE)"
|
||||
hour_expr = "CAST(EXTRACT(HOUR FROM timestamp) AS INTEGER)"
|
||||
else: # MySQL
|
||||
day_expr = "DATE(timestamp)"
|
||||
hour_expr = "HOUR(timestamp)"
|
||||
|
||||
query = f"""
|
||||
SELECT {day_expr} AS day, {hour_expr} AS hour
|
||||
FROM water_measurements
|
||||
WHERE timestamp >= :start_time AND timestamp < :end_time
|
||||
GROUP BY {day_expr}, {hour_expr}
|
||||
"""
|
||||
|
||||
start_time = datetime.datetime.combine(start_date, datetime.time.min)
|
||||
end_time = datetime.datetime.combine(
|
||||
end_date + datetime.timedelta(days=1), datetime.time.min
|
||||
)
|
||||
|
||||
hours_by_day: Dict[datetime.date, set] = {}
|
||||
with self.engine.connect() as conn:
|
||||
result = conn.execute(
|
||||
text(query), {"start_time": start_time, "end_time": end_time}
|
||||
)
|
||||
for row in result:
|
||||
day = self._coerce_date(row[0])
|
||||
if day is None:
|
||||
continue
|
||||
hours_by_day.setdefault(day, set()).add(int(row[1]))
|
||||
|
||||
return hours_by_day
|
||||
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
f"Error querying {self.db_type.upper()} recorded hours: {e}"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
# VictoriaMetrics Adapter (using Prometheus format)
|
||||
class VictoriaMetricsAdapter(DatabaseAdapter):
|
||||
|
||||
Reference in New Issue
Block a user