feat: refuse silent v3->v2 downgrade; monthly retrain timer with staged promote

train_all() now raises RainUnavailableError when use_rain=True and the
Open-Meteo history cannot be loaded, instead of logging a warning and
writing gauge-only (v2) bundles over the deployed v3 set -- which is what
the 2026-09-01 server retrain did unnoticed. --no-rain remains the explicit
way to get v2. CLI exits 2 with a one-line error. Three tests cover the
guard, the opt-out, and the v3 happy path.

scripts/retrain.sh trains into models/.staging, refuses to promote unless
metrics.json shows hgb-v3+ and >=14 trained stations, then renames bundles
into place (previous generation kept in models/.previous). No API restart:
predict.py reloads by mtime on the hourly precompute.

water-monitor-retrain.{service,timer}: 1st of each month 03:30, Persistent,
OMP_NUM_THREADS=4, Nice=15, same sandbox as the API unit. install.sh now
does `uv sync` into .venv (one env rule; removes a stale venv/) and enables
the timer. water-monitor.service in the repo matched neither the deployed
unit nor the uv env; it now does (run.py --web-api, .venv, EnvironmentFile).

scripts/install.sh

@@ -18,6 +18,7 @@ APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
 SERVICE_USER="${SERVICE_USER:-water-monitor}"
 SERVICE_GROUP="${SERVICE_GROUP:-${SERVICE_USER}}"
 SERVICE_NAME="water-monitor.service"
+RETRAIN_NAME="water-monitor-retrain"
 
 # Resolve the repo root (parent of this scripts/ directory).
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -72,11 +73,18 @@ if ! command -v uv >/dev/null 2>&1; then
 fi
 UV="$(command -v uv)"
 
-log "Creating virtualenv at ${APP_DIR}/venv"
+log "Syncing uv-managed virtualenv at ${APP_DIR}/.venv"
 cd "${APP_DIR}"
-# Named 'venv' (not uv's default .venv) to match the systemd unit's ExecStart.
-"${UV}" venv venv
-"${UV}" pip install --python venv/bin/python -r requirements.txt
+# ONE environment: uv sync owns .venv/ (from pyproject.toml + uv.lock, so the
+# ML extras such as scikit-learn/joblib are present) and both systemd units
+# run its interpreter directly. Never create a second env by another name --
+# a stale 'venv/' once coexisted here and broke manual retrains with
+# ModuleNotFoundError while the service itself ran fine.
+"${UV}" sync --python 3.11 --frozen
+if [ -d "${APP_DIR}/venv" ]; then
+    warn "Removing stale ${APP_DIR}/venv (superseded by .venv)"
+    rm -rf "${APP_DIR}/venv"
+fi
 
 # 4. Environment file ----------------------------------------------------------
 if [ ! -f "${APP_DIR}/.env" ]; then
@@ -100,11 +108,14 @@ if [ -f "${APP_DIR}/.env" ]; then
     chmod 0600 "${APP_DIR}/.env"
 fi
 
-# 6. Install and enable the systemd unit --------------------------------------
-log "Installing systemd unit"
+# 6. Install and enable the systemd units -------------------------------------
+log "Installing systemd units"
 install -m 0644 "${SCRIPT_DIR}/${SERVICE_NAME}" "/etc/systemd/system/${SERVICE_NAME}"
+install -m 0644 "${SCRIPT_DIR}/${RETRAIN_NAME}.service" "/etc/systemd/system/${RETRAIN_NAME}.service"
+install -m 0644 "${SCRIPT_DIR}/${RETRAIN_NAME}.timer" "/etc/systemd/system/${RETRAIN_NAME}.timer"
 systemctl daemon-reload
 systemctl enable "${SERVICE_NAME}"
+systemctl enable --now "${RETRAIN_NAME}.timer"
 
 log "Done."
 echo
@@ -112,3 +123,5 @@ echo "Next steps:"
 echo "  sudo systemctl start ${SERVICE_NAME}"
 echo "  systemctl status ${SERVICE_NAME}"
 echo "  sudo journalctl -u ${SERVICE_NAME} -f"
+echo "  systemctl list-timers ${RETRAIN_NAME}.timer   # monthly flood-model retrain"
+echo "  sudo systemctl start ${RETRAIN_NAME}.service   # retrain now"