grabowski
5c4f464383
CI / Test suite (push) Successful in 1m2s
CI / Format & lint (push) Successful in 22s
Rust / fmt, clippy, test, build (push) Successful in 9m53s
Security / Dependency vulnerabilities (push) Successful in 1m52s
Security / Static analysis (push) Successful in 20s
Security / License report (push) Successful in 1m41s
Phase 2a of docs/RUST_REWRITE.md. rust/forecast (ping-forecast) ports the serving half of src/ml: the HII mirror gap fill, the hourly grid, every feature (pandas' Kahan rolling sums included), the Mae Ngat dam frame, HistGradientBoosting tree inference, the heuristic fallback, stale-threshold handling and the P.1 flood-stage probabilities. Hand-over from Python (src/ml/export.py): - models/flood_<station>.json.gz next to each joblib bundle; train_all() writes it, retrain.sh promotes it, `python -m src.ml.export` exports existing ones. - rust/forecast/flood_config.json: thresholds, stages, leads and notify constants, compiled in. tests/test_model_export.py fails while it differs from the Python source. Parity (rust/forecast/tests/parity.rs, fixtures from testdata/gen_forecast_fixtures.py on the production code path): four scenarios incl. the 2024 flood with RID lagging behind HII. Merged readings, all features and all levels identical; with all 15 local models 377/384 probabilities bit-identical, the rest within 2 ulp (exp). Python refactor without behaviour change: predict.forecasts_from_long, and get_latest_forecasts/load_latest/serving_frame take now/rain for reproducible comparisons; the HII query orders its rows.
92 lines
3.6 KiB
Bash
92 lines
3.6 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# Retrain the flood forecast models safely. Run by water-monitor-retrain.timer
|
|
# (monthly) or by hand: sudo systemctl start water-monitor-retrain.service
|
|
#
|
|
# Why a script rather than ExecStart=train_flood_model.py:
|
|
# * train.py writes each station's bundle straight into models/ over ~12 min,
|
|
# and the API's hourly precompute reloads bundles by mtime. Training into
|
|
# a staging dir and mv-ing (atomic on one filesystem) means the API never
|
|
# sees a half-written joblib file or a mixed old/new set.
|
|
# * A run that produced gauge-only (v2) bundles, or trained too few stations,
|
|
# must NOT replace the deployed models. train.py already aborts on a
|
|
# missing rain series; this script re-checks the written metrics anyway.
|
|
# * No restart is needed: predict.py (Python) and ping-collector (Rust,
|
|
# FORECAST_IN_COLLECTOR=true) reload changed bundles on their next
|
|
# forecast run, so the new models are live within an hour.
|
|
#
|
|
# Exit codes: 0 ok, 2 training refused (see log), 3 verification failed.
|
|
set -euo pipefail
|
|
|
|
APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
|
|
PYTHON="${PYTHON:-${APP_DIR}/.venv/bin/python}"
|
|
MODELS_DIR="${APP_DIR}/models"
|
|
STAGE_DIR="${MODELS_DIR}/.staging"
|
|
# P.4A is NOT_TRAINABLE by design (17% fill); 15 of 16 is the normal outcome.
|
|
MIN_TRAINED="${MIN_TRAINED:-14}"
|
|
EXPECT_VERSION_PREFIX="${EXPECT_VERSION_PREFIX:-hgb-v3+}"
|
|
|
|
log() { printf '%s retrain: %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; }
|
|
|
|
cd "${APP_DIR}"
|
|
[ -x "${PYTHON}" ] || { log "no interpreter at ${PYTHON} (run uv sync)"; exit 3; }
|
|
|
|
rm -rf "${STAGE_DIR}"
|
|
mkdir -p "${STAGE_DIR}"
|
|
log "training into ${STAGE_DIR} (python=${PYTHON}, OMP_NUM_THREADS=${OMP_NUM_THREADS:-unset})"
|
|
|
|
# train_flood_model.py exits 2 on a missing rain series (RainUnavailableError)
|
|
# instead of silently writing v2 bundles -- propagate that unchanged.
|
|
set +e
|
|
"${PYTHON}" scripts/train_flood_model.py --stations all --models-dir "${STAGE_DIR}" "$@"
|
|
rc=$?
|
|
set -e
|
|
if [ "${rc}" -ne 0 ]; then
|
|
log "training failed (exit ${rc}); deployed models untouched"
|
|
rm -rf "${STAGE_DIR}"
|
|
exit "${rc}"
|
|
fi
|
|
|
|
# Verify before promoting. Reads metrics.json from the stage dir.
|
|
VERSION="$("${PYTHON}" - "${STAGE_DIR}/metrics.json" <<'PY'
|
|
import json, sys
|
|
m = json.load(open(sys.argv[1]))
|
|
print(m["model_version"])
|
|
PY
|
|
)"
|
|
TRAINED="$("${PYTHON}" - "${STAGE_DIR}/metrics.json" <<'PY'
|
|
import json, sys
|
|
m = json.load(open(sys.argv[1]))
|
|
print(sum(1 for s in m["stations"].values() if s.get("status") == "trained"))
|
|
PY
|
|
)"
|
|
log "staged model_version=${VERSION} trained_stations=${TRAINED}"
|
|
|
|
case "${VERSION}" in
|
|
"${EXPECT_VERSION_PREFIX}"*) ;;
|
|
*)
|
|
log "REFUSING to deploy: version '${VERSION}' does not start with '${EXPECT_VERSION_PREFIX}'"
|
|
rm -rf "${STAGE_DIR}"
|
|
exit 3
|
|
;;
|
|
esac
|
|
if [ "${TRAINED}" -lt "${MIN_TRAINED}" ]; then
|
|
log "REFUSING to deploy: only ${TRAINED} stations trained (< ${MIN_TRAINED})"
|
|
rm -rf "${STAGE_DIR}"
|
|
exit 3
|
|
fi
|
|
|
|
# Promote: per-file rename is atomic; readers see either the old or the new
|
|
# bundle, never a partial one. Keep one previous generation for rollback.
|
|
mkdir -p "${MODELS_DIR}/.previous"
|
|
# flood_*.json.gz is the Rust forecaster's export of the same bundle.
|
|
for f in "${STAGE_DIR}"/flood_*.joblib "${STAGE_DIR}"/flood_*.json.gz "${STAGE_DIR}/metrics.json"; do
|
|
name="$(basename "${f}")"
|
|
if [ -f "${MODELS_DIR}/${name}" ]; then
|
|
mv -f "${MODELS_DIR}/${name}" "${MODELS_DIR}/.previous/${name}"
|
|
fi
|
|
mv -f "${f}" "${MODELS_DIR}/${name}"
|
|
done
|
|
rm -rf "${STAGE_DIR}"
|
|
log "deployed ${VERSION} (${TRAINED} stations); previous generation in models/.previous. The API picks it up on its next hourly precompute."
|