grabowski
764764e07e
train_all() now raises RainUnavailableError when use_rain=True and the
Open-Meteo history cannot be loaded, instead of logging a warning and
writing gauge-only (v2) bundles over the deployed v3 set -- which is what
the 2026-09-01 server retrain did unnoticed. --no-rain remains the explicit
way to get v2. CLI exits 2 with a one-line error. Three tests cover the
guard, the opt-out, and the v3 happy path.
scripts/retrain.sh trains into models/.staging, refuses to promote unless
metrics.json shows hgb-v3+ and >=14 trained stations, then renames bundles
into place (previous generation kept in models/.previous). No API restart:
predict.py reloads by mtime on the hourly precompute.
water-monitor-retrain.{service,timer}: 1st of each month 03:30, Persistent,
OMP_NUM_THREADS=4, Nice=15, same sandbox as the API unit. install.sh now
does `uv sync` into .venv (one env rule; removes a stale venv/) and enables
the timer. water-monitor.service in the repo matched neither the deployed
unit nor the uv env; it now does (run.py --web-api, .venv, EnvironmentFile).
128 lines
5.3 KiB
Bash
Executable File
128 lines
5.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Install the Thailand Water Level Monitor as a hardened systemd service.
|
|
#
|
|
# Creates a dedicated system user, deploys the code to /opt, builds a uv-managed
|
|
# virtualenv, installs the systemd unit, and enables the service. Idempotent:
|
|
# safe to re-run to update an existing install.
|
|
#
|
|
# Usage (as root, from a checkout of the repo):
|
|
# sudo bash scripts/install.sh
|
|
#
|
|
# Override defaults via environment variables:
|
|
# APP_DIR=/opt/thailand-water-monitor SERVICE_USER=water-monitor sudo -E bash scripts/install.sh
|
|
#
|
|
set -euo pipefail
|
|
|
|
APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
|
|
SERVICE_USER="${SERVICE_USER:-water-monitor}"
|
|
SERVICE_GROUP="${SERVICE_GROUP:-${SERVICE_USER}}"
|
|
SERVICE_NAME="water-monitor.service"
|
|
RETRAIN_NAME="water-monitor-retrain"
|
|
|
|
# Resolve the repo root (parent of this scripts/ directory).
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
|
|
|
log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; }
|
|
warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*"; }
|
|
die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }
|
|
|
|
[ "$(id -u)" -eq 0 ] || die "This script must be run as root (use sudo)."
|
|
|
|
# 1. Dedicated system user/group (no login, no home) --------------------------
|
|
if ! getent group "${SERVICE_GROUP}" >/dev/null; then
|
|
log "Creating group ${SERVICE_GROUP}"
|
|
groupadd --system "${SERVICE_GROUP}"
|
|
fi
|
|
if ! id "${SERVICE_USER}" >/dev/null 2>&1; then
|
|
log "Creating system user ${SERVICE_USER}"
|
|
useradd --system --no-create-home --shell /usr/sbin/nologin \
|
|
--gid "${SERVICE_GROUP}" "${SERVICE_USER}"
|
|
fi
|
|
|
|
# 2. Deploy code to APP_DIR ----------------------------------------------------
|
|
log "Deploying code to ${APP_DIR}"
|
|
mkdir -p "${APP_DIR}"
|
|
if command -v rsync >/dev/null 2>&1; then
|
|
rsync -a --delete \
|
|
--exclude '.git' --exclude '.venv' --exclude 'venv' \
|
|
--exclude '__pycache__' --exclude '*.pyc' \
|
|
--exclude '*.db' --exclude '.env' --exclude 'stations.json' \
|
|
"${REPO_DIR}/" "${APP_DIR}/"
|
|
else
|
|
warn "rsync not found; falling back to cp (will not prune deleted files)"
|
|
cp -r "${REPO_DIR}/." "${APP_DIR}/"
|
|
fi
|
|
|
|
# 3. Build the uv-managed virtualenv ------------------------------------------
|
|
# Prefer an already-installed uv. For stricter supply-chain control install uv
|
|
# ahead of time via your distro / package manager; this script only fetches the
|
|
# upstream installer (piped to a root shell) when AUTO_INSTALL_UV=1 is set, and
|
|
# pins the version so the fetched script is reproducible.
|
|
UV_VERSION="${UV_VERSION:-0.5.11}"
|
|
if ! command -v uv >/dev/null 2>&1; then
|
|
if [ "${AUTO_INSTALL_UV:-0}" = "1" ]; then
|
|
warn "uv not found; installing pinned uv ${UV_VERSION} from astral.sh (runs as root)"
|
|
curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" \
|
|
| env UV_INSTALL_DIR=/usr/local/bin sh
|
|
else
|
|
die "uv not found. Install it (e.g. your package manager, or 'pipx install uv'),
|
|
or re-run with AUTO_INSTALL_UV=1 to fetch the pinned upstream installer."
|
|
fi
|
|
fi
|
|
UV="$(command -v uv)"
|
|
|
|
log "Syncing uv-managed virtualenv at ${APP_DIR}/.venv"
|
|
cd "${APP_DIR}"
|
|
# ONE environment: uv sync owns .venv/ (from pyproject.toml + uv.lock, so the
|
|
# ML extras such as scikit-learn/joblib are present) and both systemd units
|
|
# run its interpreter directly. Never create a second env by another name --
|
|
# a stale 'venv/' once coexisted here and broke manual retrains with
|
|
# ModuleNotFoundError while the service itself ran fine.
|
|
"${UV}" sync --python 3.11 --frozen
|
|
if [ -d "${APP_DIR}/venv" ]; then
|
|
warn "Removing stale ${APP_DIR}/venv (superseded by .venv)"
|
|
rm -rf "${APP_DIR}/venv"
|
|
fi
|
|
|
|
# 4. Environment file ----------------------------------------------------------
|
|
if [ ! -f "${APP_DIR}/.env" ]; then
|
|
if [ -f "${REPO_DIR}/.env" ]; then
|
|
log "Copying .env from checkout"
|
|
cp "${REPO_DIR}/.env" "${APP_DIR}/.env"
|
|
else
|
|
warn "No .env found. Copy .env.example to ${APP_DIR}/.env and fill in"
|
|
warn "MATRIX_ACCESS_TOKEN / MATRIX_ROOM_ID and DB settings before starting."
|
|
fi
|
|
fi
|
|
|
|
# 5. Ownership and permissions -------------------------------------------------
|
|
# Service user needs write access for logs / stations.json.
|
|
log "Setting ownership to ${SERVICE_USER}:${SERVICE_GROUP}"
|
|
chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "${APP_DIR}"
|
|
# Restrict traversal to root + the service user, and lock down the secrets file
|
|
# (contains the Matrix token and DB credentials).
|
|
chmod 0750 "${APP_DIR}"
|
|
if [ -f "${APP_DIR}/.env" ]; then
|
|
chmod 0600 "${APP_DIR}/.env"
|
|
fi
|
|
|
|
# 6. Install and enable the systemd units -------------------------------------
|
|
log "Installing systemd units"
|
|
install -m 0644 "${SCRIPT_DIR}/${SERVICE_NAME}" "/etc/systemd/system/${SERVICE_NAME}"
|
|
install -m 0644 "${SCRIPT_DIR}/${RETRAIN_NAME}.service" "/etc/systemd/system/${RETRAIN_NAME}.service"
|
|
install -m 0644 "${SCRIPT_DIR}/${RETRAIN_NAME}.timer" "/etc/systemd/system/${RETRAIN_NAME}.timer"
|
|
systemctl daemon-reload
|
|
systemctl enable "${SERVICE_NAME}"
|
|
systemctl enable --now "${RETRAIN_NAME}.timer"
|
|
|
|
log "Done."
|
|
echo
|
|
echo "Next steps:"
|
|
echo " sudo systemctl start ${SERVICE_NAME}"
|
|
echo " systemctl status ${SERVICE_NAME}"
|
|
echo " sudo journalctl -u ${SERVICE_NAME} -f"
|
|
echo " systemctl list-timers ${RETRAIN_NAME}.timer # monthly flood-model retrain"
|
|
echo " sudo systemctl start ${RETRAIN_NAME}.service # retrain now"
|