feat: refuse silent v3->v2 downgrade; monthly retrain timer with staged promote
train_all() now raises RainUnavailableError when use_rain=True and the
Open-Meteo history cannot be loaded, instead of logging a warning and
writing gauge-only (v2) bundles over the deployed v3 set -- which is what
the 2026-09-01 server retrain did unnoticed. --no-rain remains the explicit
way to get v2. CLI exits 2 with a one-line error. Three tests cover the
guard, the opt-out, and the v3 happy path.
scripts/retrain.sh trains into models/.staging, refuses to promote unless
metrics.json shows hgb-v3+ and >=14 trained stations, then renames bundles
into place (previous generation kept in models/.previous). No API restart:
predict.py reloads by mtime on the hourly precompute.
water-monitor-retrain.{service,timer}: 1st of each month 03:30, Persistent,
OMP_NUM_THREADS=4, Nice=15, same sandbox as the API unit. install.sh now
does `uv sync` into .venv (one env rule; removes a stale venv/) and enables
the timer. water-monitor.service in the repo matched neither the deployed
unit nor the uv env; it now does (run.py --web-api, .venv, EnvironmentFile).
This commit is contained in:
+19
-6
@@ -18,6 +18,7 @@ APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
|
||||
SERVICE_USER="${SERVICE_USER:-water-monitor}"
|
||||
SERVICE_GROUP="${SERVICE_GROUP:-${SERVICE_USER}}"
|
||||
SERVICE_NAME="water-monitor.service"
|
||||
RETRAIN_NAME="water-monitor-retrain"
|
||||
|
||||
# Resolve the repo root (parent of this scripts/ directory).
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -72,11 +73,18 @@ if ! command -v uv >/dev/null 2>&1; then
|
||||
fi
|
||||
UV="$(command -v uv)"
|
||||
|
||||
log "Creating virtualenv at ${APP_DIR}/venv"
|
||||
log "Syncing uv-managed virtualenv at ${APP_DIR}/.venv"
|
||||
cd "${APP_DIR}"
|
||||
# Named 'venv' (not uv's default .venv) to match the systemd unit's ExecStart.
|
||||
"${UV}" venv venv
|
||||
"${UV}" pip install --python venv/bin/python -r requirements.txt
|
||||
# ONE environment: uv sync owns .venv/ (from pyproject.toml + uv.lock, so the
|
||||
# ML extras such as scikit-learn/joblib are present) and both systemd units
|
||||
# run its interpreter directly. Never create a second env by another name --
|
||||
# a stale 'venv/' once coexisted here and broke manual retrains with
|
||||
# ModuleNotFoundError while the service itself ran fine.
|
||||
"${UV}" sync --python 3.11 --frozen
|
||||
if [ -d "${APP_DIR}/venv" ]; then
|
||||
warn "Removing stale ${APP_DIR}/venv (superseded by .venv)"
|
||||
rm -rf "${APP_DIR}/venv"
|
||||
fi
|
||||
|
||||
# 4. Environment file ----------------------------------------------------------
|
||||
if [ ! -f "${APP_DIR}/.env" ]; then
|
||||
@@ -100,11 +108,14 @@ if [ -f "${APP_DIR}/.env" ]; then
|
||||
chmod 0600 "${APP_DIR}/.env"
|
||||
fi
|
||||
|
||||
# 6. Install and enable the systemd unit --------------------------------------
|
||||
log "Installing systemd unit"
|
||||
# 6. Install and enable the systemd units -------------------------------------
|
||||
log "Installing systemd units"
|
||||
install -m 0644 "${SCRIPT_DIR}/${SERVICE_NAME}" "/etc/systemd/system/${SERVICE_NAME}"
|
||||
install -m 0644 "${SCRIPT_DIR}/${RETRAIN_NAME}.service" "/etc/systemd/system/${RETRAIN_NAME}.service"
|
||||
install -m 0644 "${SCRIPT_DIR}/${RETRAIN_NAME}.timer" "/etc/systemd/system/${RETRAIN_NAME}.timer"
|
||||
systemctl daemon-reload
|
||||
systemctl enable "${SERVICE_NAME}"
|
||||
systemctl enable --now "${RETRAIN_NAME}.timer"
|
||||
|
||||
log "Done."
|
||||
echo
|
||||
@@ -112,3 +123,5 @@ echo "Next steps:"
|
||||
echo " sudo systemctl start ${SERVICE_NAME}"
|
||||
echo " systemctl status ${SERVICE_NAME}"
|
||||
echo " sudo journalctl -u ${SERVICE_NAME} -f"
|
||||
echo " systemctl list-timers ${RETRAIN_NAME}.timer # monthly flood-model retrain"
|
||||
echo " sudo systemctl start ${RETRAIN_NAME}.service # retrain now"
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Retrain the flood forecast models safely. Run by water-monitor-retrain.timer
|
||||
# (monthly) or by hand: sudo systemctl start water-monitor-retrain.service
|
||||
#
|
||||
# Why a script rather than ExecStart=train_flood_model.py:
|
||||
# * train.py writes each station's bundle straight into models/ over ~12 min,
|
||||
# and the API's hourly precompute reloads bundles by mtime. Training into
|
||||
# a staging dir and mv-ing (atomic on one filesystem) means the API never
|
||||
# sees a half-written joblib file or a mixed old/new set.
|
||||
# * A run that produced gauge-only (v2) bundles, or trained too few stations,
|
||||
# must NOT replace the deployed models. train.py already aborts on a
|
||||
# missing rain series; this script re-checks the written metrics anyway.
|
||||
# * No API restart is needed: predict.py reloads changed bundles on the next
|
||||
# precompute (every scrape cycle, hourly), so the new models are live
|
||||
# within an hour. Restart manually if you want them live immediately.
|
||||
#
|
||||
# Exit codes: 0 ok, 2 training refused (see log), 3 verification failed.
|
||||
set -euo pipefail
|
||||
|
||||
APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
|
||||
PYTHON="${PYTHON:-${APP_DIR}/.venv/bin/python}"
|
||||
MODELS_DIR="${APP_DIR}/models"
|
||||
STAGE_DIR="${MODELS_DIR}/.staging"
|
||||
# P.4A is NOT_TRAINABLE by design (17% fill); 15 of 16 is the normal outcome.
|
||||
MIN_TRAINED="${MIN_TRAINED:-14}"
|
||||
EXPECT_VERSION_PREFIX="${EXPECT_VERSION_PREFIX:-hgb-v3+}"
|
||||
|
||||
log() { printf '%s retrain: %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; }
|
||||
|
||||
cd "${APP_DIR}"
|
||||
[ -x "${PYTHON}" ] || { log "no interpreter at ${PYTHON} (run uv sync)"; exit 3; }
|
||||
|
||||
rm -rf "${STAGE_DIR}"
|
||||
mkdir -p "${STAGE_DIR}"
|
||||
log "training into ${STAGE_DIR} (python=${PYTHON}, OMP_NUM_THREADS=${OMP_NUM_THREADS:-unset})"
|
||||
|
||||
# train_flood_model.py exits 2 on a missing rain series (RainUnavailableError)
|
||||
# instead of silently writing v2 bundles -- propagate that unchanged.
|
||||
set +e
|
||||
"${PYTHON}" scripts/train_flood_model.py --stations all --models-dir "${STAGE_DIR}" "$@"
|
||||
rc=$?
|
||||
set -e
|
||||
if [ "${rc}" -ne 0 ]; then
|
||||
log "training failed (exit ${rc}); deployed models untouched"
|
||||
rm -rf "${STAGE_DIR}"
|
||||
exit "${rc}"
|
||||
fi
|
||||
|
||||
# Verify before promoting. Reads metrics.json from the stage dir.
|
||||
VERSION="$("${PYTHON}" - "${STAGE_DIR}/metrics.json" <<'PY'
|
||||
import json, sys
|
||||
m = json.load(open(sys.argv[1]))
|
||||
print(m["model_version"])
|
||||
PY
|
||||
)"
|
||||
TRAINED="$("${PYTHON}" - "${STAGE_DIR}/metrics.json" <<'PY'
|
||||
import json, sys
|
||||
m = json.load(open(sys.argv[1]))
|
||||
print(sum(1 for s in m["stations"].values() if s.get("status") == "trained"))
|
||||
PY
|
||||
)"
|
||||
log "staged model_version=${VERSION} trained_stations=${TRAINED}"
|
||||
|
||||
case "${VERSION}" in
|
||||
"${EXPECT_VERSION_PREFIX}"*) ;;
|
||||
*)
|
||||
log "REFUSING to deploy: version '${VERSION}' does not start with '${EXPECT_VERSION_PREFIX}'"
|
||||
rm -rf "${STAGE_DIR}"
|
||||
exit 3
|
||||
;;
|
||||
esac
|
||||
if [ "${TRAINED}" -lt "${MIN_TRAINED}" ]; then
|
||||
log "REFUSING to deploy: only ${TRAINED} stations trained (< ${MIN_TRAINED})"
|
||||
rm -rf "${STAGE_DIR}"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# Promote: per-file rename is atomic; readers see either the old or the new
|
||||
# bundle, never a partial one. Keep one previous generation for rollback.
|
||||
mkdir -p "${MODELS_DIR}/.previous"
|
||||
for f in "${STAGE_DIR}"/flood_*.joblib "${STAGE_DIR}/metrics.json"; do
|
||||
name="$(basename "${f}")"
|
||||
if [ -f "${MODELS_DIR}/${name}" ]; then
|
||||
mv -f "${MODELS_DIR}/${name}" "${MODELS_DIR}/.previous/${name}"
|
||||
fi
|
||||
mv -f "${f}" "${MODELS_DIR}/${name}"
|
||||
done
|
||||
rm -rf "${STAGE_DIR}"
|
||||
log "deployed ${VERSION} (${TRAINED} stations); previous generation in models/.previous. The API picks it up on its next hourly precompute."
|
||||
@@ -11,7 +11,7 @@ import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
from src.ml.train import main
|
||||
from src.ml.train import cli
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
raise SystemExit(cli())
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
[Unit]
|
||||
Description=Retrain the Ping River flood forecast models
|
||||
Documentation=https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/-/blob/master/docs/FLOOD_FORECASTING.md
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=water-monitor
|
||||
Group=water-monitor
|
||||
WorkingDirectory=/opt/thailand-water-monitor
|
||||
EnvironmentFile=/opt/thailand-water-monitor/.env
|
||||
# Same interpreter as water-monitor.service -- the uv-managed .venv.
|
||||
# scripts/retrain.sh trains into models/.staging, refuses to promote anything
|
||||
# that is not a rain-enabled (hgb-v3) set covering the expected stations, then
|
||||
# renames the bundles into place. The API reloads them on its next hourly
|
||||
# precompute; no restart, so a failed run leaves the old models serving.
|
||||
ExecStart=/bin/bash /opt/thailand-water-monitor/scripts/retrain.sh
|
||||
# HistGradientBoosting is CPU-bound; cap threads so training cannot starve
|
||||
# the API (docs/FLOOD_FORECASTING.md section 6 measured 4 as the sweet spot).
|
||||
Environment=OMP_NUM_THREADS=4
|
||||
Environment=PYTHONPATH=/opt/thailand-water-monitor
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
Nice=15
|
||||
IOSchedulingClass=idle
|
||||
# 15 stations at ~50 s each plus data load: 12 min observed on 2026-09-12.
|
||||
TimeoutStartSec=45min
|
||||
|
||||
# Same sandbox as the API unit.
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/opt/thailand-water-monitor
|
||||
CapabilityBoundingSet=
|
||||
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=water-monitor-retrain
|
||||
@@ -0,0 +1,17 @@
|
||||
[Unit]
|
||||
Description=Monthly flood-model retrain (docs/FLOOD_FORECASTING.md section 7)
|
||||
|
||||
[Timer]
|
||||
# Policy: at minimum once pre-monsoon (May-June), monthly through the season
|
||||
# (July-November), and after any major flood. A retrain costs ~12 min and RAM
|
||||
# peaks ~300 MB, so running it every month all year is cheaper than remembering
|
||||
# which months matter. 1st of the month, 03:30 server-local -- between the
|
||||
# hourly scrapes and outside Thai daytime traffic.
|
||||
OnCalendar=*-*-01 03:30:00
|
||||
# Catch up if the box was off at the scheduled time.
|
||||
Persistent=true
|
||||
RandomizedDelaySec=20min
|
||||
Unit=water-monitor-retrain.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -9,17 +9,19 @@ Type=simple
|
||||
User=water-monitor
|
||||
Group=water-monitor
|
||||
WorkingDirectory=/opt/thailand-water-monitor
|
||||
ExecStart=/opt/thailand-water-monitor/venv/bin/python src/water_scraper_v3.py
|
||||
# The uv-managed env (uv sync -> .venv). Same interpreter for water-monitor-retrain.service.
|
||||
ExecStart=/opt/thailand-water-monitor/.venv/bin/python run.py --web-api
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
Restart=always
|
||||
RestartSec=60
|
||||
TimeoutStopSec=30
|
||||
|
||||
# Environment variables
|
||||
Environment=DB_TYPE=victoriametrics
|
||||
Environment=VM_HOST=localhost
|
||||
Environment=VM_PORT=8428
|
||||
# DB_TYPE / POSTGRES_CONNECTION_STRING / MATRIX_* come from the .env file.
|
||||
EnvironmentFile=/opt/thailand-water-monitor/.env
|
||||
Environment=PYTHONPATH=/opt/thailand-water-monitor
|
||||
# Serving path is latency-bound; single-threaded BLAS is 2.6x faster per call
|
||||
# (docs/FLOOD_FORECASTING.md section 6). Training sets its own value.
|
||||
Environment=OMP_NUM_THREADS=1
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
# Security settings
|
||||
|
||||
Reference in New Issue
Block a user