feat: --fill-gaps all repairs missing data across the whole DB range
Documentation / Generate API Documentation (push) Successful in 11s
Documentation / Build Sphinx Documentation (push) Successful in 17s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 1s
Documentation / Documentation Summary (push) Successful in 3s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 29s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 14s
Documentation / Validate Documentation (push) Failing after 9s

Gap detection is now hour-granular: days with partial data (missing
hourly slots) are re-fetched, not just days with no rows at all. The
API's hour-24-is-next-midnight quirk is handled by re-fetching day D-1
when day D is missing its 00:00 slot. SQL adapters gain single-query
range and recorded-hours lookups; non-SQL backends fall back to the
old day-granular check.
This commit is contained in:
2026-08-10 22:13:58 +07:00
parent 410faeddd5
commit 5e62ea529d
3 changed files with 236 additions and 37 deletions
+107
View File
@@ -36,6 +36,26 @@ class DatabaseAdapter(ABC):
def get_measurements_for_date(self, target_date: datetime.datetime) -> List[Dict]:
pass
def get_measurement_date_range(
self,
) -> Optional[tuple]:
"""Return (min_timestamp, max_timestamp) of stored measurements.
Returns None when the backend has no data or does not support the query.
"""
return None
def get_recorded_hours_by_day(
self, start_date: datetime.date, end_date: datetime.date
) -> Optional[Dict[datetime.date, set]]:
"""Map each day in [start_date, end_date] to the set of hours (0-23)
that have at least one measurement.
Returns None when the backend does not support hour-granular gap
detection (callers should fall back to day-granular checks).
"""
return None
# InfluxDB Adapter
class InfluxDBAdapter(DatabaseAdapter):
@@ -657,6 +677,93 @@ class SQLAdapter(DatabaseAdapter):
)
return []
@staticmethod
def _coerce_date(value) -> Optional[datetime.date]:
"""Normalize a DB-returned day value (str/date/datetime) to a date."""
if value is None:
return None
if isinstance(value, datetime.datetime):
return value.date()
if isinstance(value, datetime.date):
return value
# SQLite returns strings, e.g. '2024-09-15'
return datetime.datetime.strptime(str(value)[:10], "%Y-%m-%d").date()
def get_measurement_date_range(self) -> Optional[tuple]:
if not self.engine:
return None
try:
from sqlalchemy import text
query = "SELECT MIN(timestamp), MAX(timestamp) FROM water_measurements"
with self.engine.connect() as conn:
row = conn.execute(text(query)).fetchone()
if not row or row[0] is None:
return None
def to_datetime(value):
if isinstance(value, datetime.datetime):
return value
return datetime.datetime.fromisoformat(str(value)[:19])
return (to_datetime(row[0]), to_datetime(row[1]))
except Exception as e:
logging.error(f"Error querying {self.db_type.upper()} date range: {e}")
return None
def get_recorded_hours_by_day(
self, start_date: datetime.date, end_date: datetime.date
) -> Optional[Dict[datetime.date, set]]:
if not self.engine:
return None
try:
from sqlalchemy import text
if self.db_type == "sqlite":
day_expr = "DATE(timestamp)"
hour_expr = "CAST(strftime('%H', timestamp) AS INTEGER)"
elif self.db_type == "postgresql":
day_expr = "CAST(timestamp AS DATE)"
hour_expr = "CAST(EXTRACT(HOUR FROM timestamp) AS INTEGER)"
else: # MySQL
day_expr = "DATE(timestamp)"
hour_expr = "HOUR(timestamp)"
query = f"""
SELECT {day_expr} AS day, {hour_expr} AS hour
FROM water_measurements
WHERE timestamp >= :start_time AND timestamp < :end_time
GROUP BY {day_expr}, {hour_expr}
"""
start_time = datetime.datetime.combine(start_date, datetime.time.min)
end_time = datetime.datetime.combine(
end_date + datetime.timedelta(days=1), datetime.time.min
)
hours_by_day: Dict[datetime.date, set] = {}
with self.engine.connect() as conn:
result = conn.execute(
text(query), {"start_time": start_time, "end_time": end_time}
)
for row in result:
day = self._coerce_date(row[0])
if day is None:
continue
hours_by_day.setdefault(day, set()).add(int(row[1]))
return hours_by_day
except Exception as e:
logging.error(
f"Error querying {self.db_type.upper()} recorded hours: {e}"
)
return None
# VictoriaMetrics Adapter (using Prometheus format)
class VictoriaMetricsAdapter(DatabaseAdapter):