B4L/Northern-Thailand-Ping-River-Monitor
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

ci: green pipelines that check what exists; one formatting contract

Browse Source 
The Test Suite job failed on every push since the black check was added
because the tree had never been formatted, and pre-commit said 120
columns while CI ran black's default 88. pyproject.toml now carries
[tool.black] / [tool.isort] (88, black profile) as the single source;
pre-commit reads it; `make format` applied it (13 files, whitespace only,
146 insertions / 128 deletions, tests unchanged at 146 passed).

ci.yml: lint (black, isort, flake8 hard errors) + pytest. The Docker
registry push, VictoriaMetrics integration test, staging/production
deploy and Apache-Bench jobs were template scaffolding for hosts and
registries that do not exist; production is a systemd unit updated by
git pull. Removed rather than left permanently skipped.

docs.yml: the "Check markdown links" step curl'd every URL in every .md
and failed on localhost examples and the Tailscale IP, and the Sphinx
jobs built artifacts nobody read. Replaced by two checks that mean
something: relative links/images in README, CONTRIBUTING and docs/
resolve inside the repo, and the FastAPI OpenAPI schema exports with
the documented endpoints present (uploaded as an artifact).
This commit is contained in:
grabowski
2026-09-11 23:05:37 +02:00
parent 6f4a86edbb
commit 5ad8e4eac3
19 changed files with 290 additions and 807 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/__init__.py
+7 -3
View File
@@ -12,9 +12,13 @@ __description__ = "Northern Thailand Ping River Monitoring System"
from .config import Config
from .database_adapters import DatabaseAdapter, create_database_adapter
from .exceptions import (APIConnectionError, ConfigurationError,
DatabaseConnectionError, DataValidationError,
WaterMonitorException)
from .exceptions import (
APIConnectionError,
ConfigurationError,
DatabaseConnectionError,
DataValidationError,
WaterMonitorException,
)
from .models import DatabaseConfig, StationInfo, WaterMeasurement
from .water_scraper_v3 import EnhancedWaterMonitorScraper
src/database_adapters.py
+2 -6
View File
@@ -767,9 +767,7 @@ class SQLAdapter(DatabaseAdapter):
return hours_by_day
except Exception as e:
logging.error(
f"Error querying {self.db_type.upper()} recorded hours: {e}"
)
logging.error(f"Error querying {self.db_type.upper()} recorded hours: {e}")
return None
def get_database_stats(self) -> Optional[Dict]:
@@ -814,9 +812,7 @@ class SQLAdapter(DatabaseAdapter):
# the DISTINCT day-hour slots and coverage cannot exceed 100%
first_slot = first_ts.replace(minute=0, second=0, microsecond=0)
last_slot = last_ts.replace(minute=0, second=0, microsecond=0)
expected_hours = (
int((last_slot - first_slot).total_seconds() // 3600) + 1
)
expected_hours = int((last_slot - first_slot).total_seconds() // 3600) + 1
recorded_hours = int(row[4])
coverage_percent = round(100.0 * recorded_hours / expected_hours, 1)
src/hii_backfill.py
+2 -4
View File
@@ -17,9 +17,9 @@ import time
from typing import Dict, List, Optional
from .hii_collector import (
PING_BASIN_CODE,
HiiClient,
HiiStore,
PING_BASIN_CODE,
_parse_datetime,
_to_float,
)
@@ -145,9 +145,7 @@ def backfill(
station_rows += store.save_waterlevel_history(sid, rows)
except Exception as e:
totals["errors"] += 1
logger.warning(
f"{label}: {chunk_start}..{chunk_end} failed: {e}"
)
logger.warning(f"{label}: {chunk_start}..{chunk_end} failed: {e}")
time.sleep(sleep_seconds)
totals["rows"] += station_rows
logger.info(f"{label} (id {sid}): {station_rows} rows saved")
src/hii_collector.py
+3 -9
View File
@@ -66,9 +66,7 @@ def rid_code_from_oldcode(oldcode: Optional[str]) -> Optional[str]:
return match.group(1) if match else None
def parse_rain_records(
payload: Dict, basin_code: int = PING_BASIN_CODE
) -> List[Dict]:
def parse_rain_records(payload: Dict, basin_code: int = PING_BASIN_CODE) -> List[Dict]:
"""Extract per-station rainfall rows from a rain_24h payload."""
records = []
for row in payload.get("data") or []:
@@ -186,9 +184,7 @@ class HiiStore:
def __init__(self, connection_string: str, db_type: str):
self.db_type = db_type.lower()
if self.db_type not in ("sqlite", "postgresql", "mysql"):
raise ValueError(
f"HII collection requires a SQL database, got '{db_type}'"
)
raise ValueError(f"HII collection requires a SQL database, got '{db_type}'")
self.connection_string = connection_string
self.engine = None
@@ -400,9 +396,7 @@ class HiiStore:
from sqlalchemy import text
now = datetime.datetime.now()
station_sql = self._upsert(
station_table, ["id"], station_cols + ["updated_at"]
)
station_sql = self._upsert(station_table, ["id"], station_cols + ["updated_at"])
measurement_sql = self._upsert(
measurement_table, ["station_id", "timestamp"], measurement_cols
)
src/ml/dam.py
+1 -3
View File
@@ -61,9 +61,7 @@ def load_daily(
params["start"] = start
engine = create_engine(resolved, pool_pre_ping=True)
with engine.connect() as conn:
daily = pd.read_sql(
text(query + " ORDER BY date"), conn, params=params
)
daily = pd.read_sql(text(query + " ORDER BY date"), conn, params=params)
daily["date"] = pd.to_datetime(daily["date"])
daily = daily.set_index("date")
for col in DAM_COLUMNS:
src/ml/data.py
+1 -3
View File
@@ -213,9 +213,7 @@ def fill_from_hii(
}
)
fills.append(fill)
logger.info(
f"HII gap-fill {code}: +{len(fill)} hours (offset {offset:.3f} m)"
)
logger.info(f"HII gap-fill {code}: +{len(fill)} hours (offset {offset:.3f} m)")
if not fills:
return df
return _normalize_long(pd.concat([df] + fills, ignore_index=True))
src/ml/evaluate.py
+76 -51
View File
@@ -62,10 +62,17 @@ EXTRA_RAIN_FEATURES = ("rain_fc48",)
class Variant:
"""A trainable candidate producing (pred_abs, sigma_per_row) on test rows."""
def __init__(self, name: str, target: str, weighted: bool = False,
quantile: bool = False, use_rain: bool = False,
use_dam: bool = False, use_fc48: bool = False,
qsigma: bool = False):
def __init__(
self,
name: str,
target: str,
weighted: bool = False,
quantile: bool = False,
use_rain: bool = False,
use_dam: bool = False,
use_fc48: bool = False,
qsigma: bool = False,
):
self.name = name
self.target = target # 'abs' or 'rise'
self.weighted = weighted
@@ -78,9 +85,7 @@ class Variant:
# sigma-independent) and quantile heads ONLY for a per-row sigma.
self.qsigma = qsigma
def fit_predict(
self, X_tr, y_abs_tr, X_te
) -> Tuple[np.ndarray, np.ndarray]:
def fit_predict(self, X_tr, y_abs_tr, X_te) -> Tuple[np.ndarray, np.ndarray]:
if not self.use_rain:
drop = [c for c in features.RAIN_FEATURES if c in X_tr.columns]
X_tr = X_tr.drop(columns=drop)
@@ -135,24 +140,30 @@ VARIANTS: Dict[str, Variant] = {
"baseline_abs": Variant("baseline_abs", target="abs"),
"rise": Variant("rise", target="rise"),
"rise_weighted": Variant("rise_weighted", target="rise", weighted=True),
"rise_quantile": Variant("rise_quantile", target="rise", weighted=True,
quantile=True),
"rise_quantile": Variant(
"rise_quantile", target="rise", weighted=True, quantile=True
),
"rise_rain": Variant("rise_rain", target="rise", use_rain=True),
"rise_rain_dam": Variant("rise_rain_dam", target="rise", use_rain=True,
use_dam=True),
"rise_rain_dam": Variant(
"rise_rain_dam", target="rise", use_rain=True, use_dam=True
),
"rise_dam": Variant("rise_dam", target="rise", use_dam=True),
# 2026-09-12 experiments on top of the deployed rise_rain configuration:
# per-row sigma from quantile heads (the served sigma sits on the 0.15
# floor at every P.1 horizon, so stage probabilities are constant-
# calibrated), and a longer forecast-rain window for the 24 h horizon.
"rise_rain_quantile": Variant("rise_rain_quantile", target="rise",
weighted=True, quantile=True, use_rain=True),
"rise_rain_quantile_uw": Variant("rise_rain_quantile_uw", target="rise",
quantile=True, use_rain=True),
"rise_rain_fc48": Variant("rise_rain_fc48", target="rise", use_rain=True,
use_fc48=True),
"rise_rain_qsigma": Variant("rise_rain_qsigma", target="rise", use_rain=True,
qsigma=True),
"rise_rain_quantile": Variant(
"rise_rain_quantile", target="rise", weighted=True, quantile=True, use_rain=True
),
"rise_rain_quantile_uw": Variant(
"rise_rain_quantile_uw", target="rise", quantile=True, use_rain=True
),
"rise_rain_fc48": Variant(
"rise_rain_fc48", target="rise", use_rain=True, use_fc48=True
),
"rise_rain_qsigma": Variant(
"rise_rain_qsigma", target="rise", use_rain=True, qsigma=True
),
}
# Dam variants are opt-in by name: they require dam columns that only exist
@@ -160,8 +171,11 @@ VARIANTS: Dict[str, Variant] = {
# the 2026-08-13 ablation concluded them a negative result. The 2026-09-12
# experiments are opt-in too (see their results in docs/FLOOD_FORECASTING.md).
DEFAULT_VARIANTS = [
k for k, v in VARIANTS.items()
if not v.use_dam and not v.use_fc48 and not v.qsigma
k
for k, v in VARIANTS.items()
if not v.use_dam
and not v.use_fc48
and not v.qsigma
and not (v.quantile and v.use_rain)
]
@@ -209,12 +223,16 @@ def _first_alert_lead(
start = crossing - pd.Timedelta(hours=72)
if window_start_floor is not None and window_start_floor > start:
start = window_start_floor
window = p.loc[start: crossing + pd.Timedelta(hours=24)]
window = p.loc[start : crossing + pd.Timedelta(hours=24)]
if len(window) < 2:
return None
alert = (window >= ALERT_P) & (window.shift(-1) >= ALERT_P) & (
(window.index.to_series().shift(-1) - window.index.to_series())
<= pd.Timedelta(hours=2)
alert = (
(window >= ALERT_P)
& (window.shift(-1) >= ALERT_P)
& (
(window.index.to_series().shift(-1) - window.index.to_series())
<= pd.Timedelta(hours=2)
)
)
hits = window.index[alert.fillna(False)]
if len(hits) == 0:
@@ -222,9 +240,7 @@ def _first_alert_lead(
return float((crossing - hits[0]).total_seconds() / 3600.0)
def _false_alarm_episodes(
p: pd.Series, observed: pd.Series, thr: float
) -> int:
def _false_alarm_episodes(p: pd.Series, observed: pd.Series, thr: float) -> int:
"""Alert episodes with no observed >=thr within +/- FALSE_ALARM_GRACE_H."""
alert_hours = p[p >= ALERT_P].index
if len(alert_hours) == 0:
@@ -295,7 +311,9 @@ def evaluate_station(
tr = (X_all.index <= train_end) & y_abs.notna()
te = (X_all.index >= test_lo) & (X_all.index <= test_hi)
if tr.sum() < 5000 or te.sum() < 500:
logger.info(f"{station} {year}: skipped (train {tr.sum()}, test {te.sum()})")
logger.info(
f"{station} {year}: skipped (train {tr.sum()}, test {te.sum()})"
)
continue
X_tr, X_te = X_all.loc[tr], X_all.loc[te]
@@ -371,9 +389,7 @@ def evaluate_station(
)
fold["variants"][name] = {
"mae": float(errors.mean()) if len(errors) else None,
"mae_above_2p5": (
float(errors[high].mean()) if high.any() else None
),
"mae_above_2p5": (float(errors[high].mean()) if high.any() else None),
"brier_warn": brier,
"events": event_rows,
"false_alarm_episodes": _false_alarm_episodes(
@@ -394,17 +410,20 @@ def summarize(results: Dict) -> str:
lines.append(header)
for fold in results["folds"]:
for name, m in fold["variants"].items():
events = " ".join(
f"[{e['crossing'][:10]}: "
f"{'—' if e['lead_h'] is None else format(e['lead_h'], '+.0f')}h"
+ (
f" | {e['peak_pred_24h_before'] - e['peak_level']:+.2f}"
if e["peak_pred_24h_before"] is not None
else ""
events = (
" ".join(
f"[{e['crossing'][:10]}: "
f"{'—' if e['lead_h'] is None else format(e['lead_h'], '+.0f')}h"
+ (
f" | {e['peak_pred_24h_before'] - e['peak_level']:+.2f}"
if e["peak_pred_24h_before"] is not None
else ""
)
+ "]"
for e in m["events"]
)
+ "]"
for e in m["events"]
) or "no events"
or "no events"
)
lines.append(
f"{name:16} {fold['year']:>5} "
f"{m['mae'] if m['mae'] is not None else float('nan'):6.3f} "
@@ -421,16 +440,22 @@ def main(argv=None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--stations", default="P.1")
parser.add_argument("--db-url", default=None)
parser.add_argument("--variants", default=None,
help="comma list; default all")
parser.add_argument("--variants", default=None, help="comma list; default all")
parser.add_argument("--out", default="models/eval_variants.json")
parser.add_argument("--no-rain", action="store_true",
help="skip loading the Open-Meteo rain series")
parser.add_argument("--no-dam", action="store_true",
help="skip loading the Mae Ngat reservoir series")
parser.add_argument("--from-cache", action="store_true",
help="offline: read models/cache/ only (no DB, no API, "
"no Open-Meteo refresh) -- reproducible reruns")
parser.add_argument(
"--no-rain", action="store_true", help="skip loading the Open-Meteo rain series"
)
parser.add_argument(
"--no-dam",
action="store_true",
help="skip loading the Mae Ngat reservoir series",
)
parser.add_argument(
"--from-cache",
action="store_true",
help="offline: read models/cache/ only (no DB, no API, "
"no Open-Meteo refresh) -- reproducible reruns",
)
args = parser.parse_args(argv)
logging.basicConfig(
src/ml/hii_rain.py
+7 -4
View File
@@ -65,7 +65,12 @@ def load_gauge_mean(
"AND s.longitude BETWEEN :lon_lo AND :lon_hi "
"AND m.rain_1h IS NOT NULL"
)
params = {"lat_lo": lat_lo, "lat_hi": lat_hi, "lon_lo": lon_lo, "lon_hi": lon_hi}
params = {
"lat_lo": lat_lo,
"lat_hi": lat_hi,
"lon_lo": lon_lo,
"lon_hi": lon_hi,
}
if start is not None:
query += " AND m.timestamp >= :start"
params["start"] = pd.Timestamp(start).to_pydatetime()
@@ -98,9 +103,7 @@ def compare_with_openmeteo(
Both are summed over trailing `window_h` so single-hour timing offsets
(gauges report at :00, the model's hour is an interval) do not dominate.
"""
joined = pd.concat(
{"gauge": gauge, "openmeteo": openmeteo}, axis=1
).dropna()
joined = pd.concat({"gauge": gauge, "openmeteo": openmeteo}, axis=1).dropna()
if joined.empty:
return {"overlap_hours": 0}
g = joined["gauge"].rolling(window_h, min_periods=window_h).sum()
src/ml/predict.py
+7 -5
View File
@@ -182,14 +182,18 @@ def _model_forecast(
)
p_warning = _sigmoid_probability(predicted_max, warn_thr, sigma_h)
if warn_head is not None:
p_warning = max(p_warning, float(warn_head.predict_proba(feature_row)[0][1]))
p_warning = max(
p_warning, float(warn_head.predict_proba(feature_row)[0][1])
)
danger_head = (
None if thresholds_stale else bundle["heads"].get(f"danger_{horizon_h}")
)
p_danger = _sigmoid_probability(predicted_max, danger_thr, sigma_h)
if danger_head is not None:
p_danger = max(p_danger, float(danger_head.predict_proba(feature_row)[0][1]))
p_danger = max(
p_danger, float(danger_head.predict_proba(feature_row)[0][1])
)
p_warning = _clip_probability(p_warning)
p_danger = min(_clip_probability(p_danger), p_warning)
@@ -400,6 +404,4 @@ def get_latest_forecasts(
logger.warning("dam state unavailable; dam features will be NaN")
dam = pd.DataFrame()
return get_forecasts(
readings_by_station, models_dir=models_dir, rain=rain, dam=dam
)
return get_forecasts(readings_by_station, models_dir=models_dir, rain=rain, dam=dam)
src/ml/rain.py
+4 -9
View File
@@ -121,12 +121,8 @@ def load_history(
cursor = fetch_from.date()
try:
while cursor <= end:
chunk_end = min(
datetime.date(cursor.year, 12, 31), end
)
chunks.append(
fetch_history(cursor.isoformat(), chunk_end.isoformat())
)
chunk_end = min(datetime.date(cursor.year, 12, 31), end)
chunks.append(fetch_history(cursor.isoformat(), chunk_end.isoformat()))
cursor = datetime.date(cursor.year + 1, 1, 1)
except Exception as error:
logger.warning(f"Open-Meteo history fetch failed: {error}")
@@ -174,7 +170,7 @@ def backfill_db(engine, db_type: str, chunk_rows: int = 5000) -> int:
return 0
total = 0
for start in range(0, len(history), chunk_rows):
part = history.iloc[start: start + chunk_rows]
part = history.iloc[start : start + chunk_rows]
total += save_to_db(part, engine, db_type)
logger.info(f"openmeteo_rain backfill: {total}/{len(history)} rows")
return total
@@ -204,8 +200,7 @@ def save_to_db(df: pd.DataFrame, engine, db_type: str) -> int:
cols = ["timestamp"] + point_cols + ["catchment_mean"]
placeholders = ", ".join(f":{c}" for c in cols)
updates = ", ".join(
f"{c} = "
+ (f"VALUES({c})" if db_type == "mysql" else f"EXCLUDED.{c}")
f"{c} = " + (f"VALUES({c})" if db_type == "mysql" else f"EXCLUDED.{c}")
for c in cols[1:]
)
if db_type == "mysql":
src/ml/train.py
+2 -3
View File
@@ -53,6 +53,7 @@ class RainUnavailableError(RuntimeError):
overwrite the deployed v3 artifacts without anyone noticing.
"""
HGB_PARAMS = {
"max_iter": 300,
"learning_rate": 0.06,
@@ -518,9 +519,7 @@ def train_all(
"training v3-style bundles WITHOUT dam features"
)
if dam_frame is not None:
logger.info(
f"dam series: {dam_frame.index.min()} .. {dam_frame.index.max()}"
)
logger.info(f"dam series: {dam_frame.index.min()} .. {dam_frame.index.max()}")
# Run-level version: v4 only if some requested station actually receives
# dam columns (they are gated to DAM_STATIONS; per-bundle versions are
src/rid_reservoir.py
+2 -6
View File
@@ -402,9 +402,7 @@ class RidReservoirStore:
measure_row = {
c: _bounded(record.get(c), _MEASURE_BOUNDS[c]) for c in measure_cols
}
measure_row.update(
{"dam_id": record["dam_id"], "date": record["date"]}
)
measure_row.update({"dam_id": record["dam_id"], "date": record["date"]})
measurements.append(measure_row)
try:
with self.engine.begin() as conn:
@@ -495,9 +493,7 @@ def backfill(
if not store.engine and not store.connect():
logger.error("backfill aborted: database connection failed")
return 0
span = [
start + datetime.timedelta(days=i) for i in range((end - start).days + 1)
]
span = [start + datetime.timedelta(days=i) for i in range((end - start).days + 1)]
present = store.present_dates(start, end)
targets = [d for d in span if d not in present]
logger.info(
src/water_scraper_v3.py
+1 -3
View File
@@ -632,9 +632,7 @@ class EnhancedWaterMonitorScraper:
if data:
if self.save_to_database(data):
filled_count += len(data)
logger.info(
f"Filled {len(data)} measurements for {fetch_date}"
)
logger.info(f"Filled {len(data)} measurements for {fetch_date}")
else:
logger.warning(f"Failed to save data for {fetch_date}")
else:
src/web_api.py
+17 -16
View File
@@ -374,9 +374,7 @@ async def background_scraping_task():
hii_counts = await asyncio.get_event_loop().run_in_executor(
None, hii_collector.run_cycle
)
set_gauge(
"hii_rainfall_rows_saved", hii_counts["rainfall"]
)
set_gauge("hii_rainfall_rows_saved", hii_counts["rainfall"])
set_gauge(
"hii_waterlevel_rows_saved", hii_counts["waterlevel"]
)
@@ -521,7 +519,9 @@ _STATIC_DIR = os.path.dirname(_DASHBOARD_HTML_PATH)
@app.get("/robots.txt", include_in_schema=False)
async def robots_txt():
return FileResponse(os.path.join(_STATIC_DIR, "robots.txt"), media_type="text/plain")
return FileResponse(
os.path.join(_STATIC_DIR, "robots.txt"), media_type="text/plain"
)
@app.get("/llms.txt", include_in_schema=False)
@@ -1022,8 +1022,12 @@ async def get_hii_rainfall_catchment(
engine = _hii_engine()
if engine is None:
return {"box": hii_rain.CATCHMENT_BOX, "gauge": [], "openmeteo": [],
"comparison_24h_sums": {"overlap_hours": 0}}
return {
"box": hii_rain.CATCHMENT_BOX,
"gauge": [],
"openmeteo": [],
"comparison_24h_sums": {"overlap_hours": 0},
}
gauge = hii_rain.load_gauge_mean(start=pd.Timestamp(start), engine=engine)
openmeteo = None
try:
@@ -1050,7 +1054,10 @@ async def get_hii_rainfall_catchment(
if s is None:
return []
return [
{"timestamp": ts.isoformat(), "rain_mm": None if pd.isna(v) else round(float(v), 2)}
{
"timestamp": ts.isoformat(),
"rain_mm": None if pd.isna(v) else round(float(v), 2),
}
for ts, v in s.items()
]
@@ -1121,9 +1128,7 @@ async def get_postgres_history(
return cached[1]
try:
db_config = Config.get_database_config()
end_time = (
datetime.combine(end, datetime.max.time()) if end else datetime.now()
)
end_time = datetime.combine(end, datetime.max.time()) if end else datetime.now()
start_time = (
datetime.combine(start, datetime.min.time())
if start
@@ -1216,17 +1221,13 @@ async def get_forecast_history(
store = app_state.get("forecast_store")
if not store:
return []
end_dt = (
datetime.combine(end, datetime.max.time()) if end else datetime.now()
)
end_dt = datetime.combine(end, datetime.max.time()) if end else datetime.now()
start_dt = (
datetime.combine(start, datetime.min.time())
if start
else end_dt - timedelta(hours=hours)
)
return await asyncio.to_thread(
store.fetch, station_code, start_dt, end_dt, horizon
)
return await asyncio.to_thread(store.fetch, station_code, start_dt, end_dt, horizon)
@app.get("/measurements/latest", response_model=List[MeasurementResponse])