grabowski
7f29b1831e
Run as root, SUDO_USER is unset and uv is not on root's PATH, so the training environment was not synced. Now the sync runs as the owner of .venv, with uv from PATH, that user's ~/.local/bin or ~/.cargo/bin, /usr/local/bin, or UV=/path/to/uv.
221 lines
10 KiB
Bash
Executable File
221 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Install or update the Ping River Monitor on a systemd host.
|
|
#
|
|
# sudo bash scripts/install.sh
|
|
#
|
|
# What runs:
|
|
# ping-collector.service the service (rust/collector): RID/HII/dam/rain
|
|
# collection, forecast, ntfy + Matrix
|
|
# notifications, website + API on WEB_LISTEN
|
|
# water-monitor-retrain.timer monthly model retrain (Python, scripts/retrain.sh)
|
|
#
|
|
# Re-run after every update (git pull + build). Steps, all idempotent:
|
|
# 1. system user/group (water-monitor)
|
|
# 2. code: run from a checkout other than APP_DIR, it is rsynced there (the
|
|
# trained models, .env, stations.json and bin/ are kept) and handed to
|
|
# the service user. Production runs in place (APP_DIR is the checkout):
|
|
# then nothing is copied and no ownership changes.
|
|
# 3. the binary: BIN=/path/to/ping-collector (e.g. the CI artifact
|
|
# "ping-collector-linux-x86_64") or rust/target/release/ping-collector,
|
|
# built beforehand as a normal user:
|
|
# cargo build --release --locked --manifest-path rust/Cargo.toml
|
|
# (Rust >= 1.88 from rustup, plus pkg-config and libssl-dev)
|
|
# 4. the Python training environment (.venv, `uv sync --frozen`), as the
|
|
# owner of .venv (in place; else the user running sudo) or as root for a
|
|
# copy. uv is looked up on PATH and in that user's ~/.local/bin and
|
|
# ~/.cargo/bin (sudo and root shells often lack them); UV=/path/to/uv
|
|
# overrides, SKIP_UV=1 skips the step
|
|
# 5. .env: created from .env.example on a fresh install (edit, then re-run);
|
|
# WEB_LISTEN and FORECAST_IN_COLLECTOR default to the production values
|
|
# when absent
|
|
# 6. systemd units; the retired Python web app (water-monitor.service) and
|
|
# the old `run.py --alert-check` Matrix cron job are removed
|
|
# 7. checks (SKIP_CHECKS=1 skips them): a dry run of every source against
|
|
# the database, and the forecast compared with the Python reference on
|
|
# the live data (scripts/forecast_parity.py)
|
|
# 8. (re)start, then wait for the website to answer
|
|
set -euo pipefail
|
|
|
|
APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
|
|
SERVICE_USER="${SERVICE_USER:-water-monitor}" # as in the units
|
|
SERVICE_GROUP="${SERVICE_GROUP:-${SERVICE_USER}}"
|
|
ENV_FILE="${APP_DIR}/.env"
|
|
PYTHON="${APP_DIR}/.venv/bin/python"
|
|
REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
RETRAIN="water-monitor-retrain"
|
|
|
|
log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; }
|
|
warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*" >&2; }
|
|
die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }
|
|
|
|
[ "$(id -u)" -eq 0 ] || die "run as root (sudo)"
|
|
|
|
set_default() { # key value: append when the key is not set at all
|
|
grep -q "^$1=" "$ENV_FILE" || printf '%s=%s\n' "$1" "$2" >>"$ENV_FILE"
|
|
}
|
|
|
|
# A command the way the services run it: same user, working directory and
|
|
# EnvironmentFile parsing (systemd's).
|
|
as_service() {
|
|
if systemd-run --version >/dev/null 2>&1; then
|
|
systemd-run --quiet --pipe --wait --collect \
|
|
-p User="$SERVICE_USER" -p Group="$SERVICE_GROUP" \
|
|
-p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" "$@"
|
|
else
|
|
(cd "$APP_DIR" && runuser -u "$SERVICE_USER" -- "$@")
|
|
fi
|
|
}
|
|
|
|
wait_http() { # url seconds
|
|
local i
|
|
for i in $(seq "$2"); do
|
|
curl -fs -o /dev/null --max-time 5 "$1" 2>/dev/null && return 0
|
|
sleep 1
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# 1. user ---------------------------------------------------------------------
|
|
getent group "$SERVICE_GROUP" >/dev/null || groupadd --system "$SERVICE_GROUP"
|
|
if ! id "$SERVICE_USER" >/dev/null 2>&1; then
|
|
log "Creating system user $SERVICE_USER"
|
|
useradd --system --no-create-home --shell /usr/sbin/nologin \
|
|
--gid "$SERVICE_GROUP" "$SERVICE_USER"
|
|
fi
|
|
|
|
# 2. code ---------------------------------------------------------------------
|
|
in_place=0
|
|
[ -d "$APP_DIR" ] && [ "$(cd "$APP_DIR" && pwd -P)" = "$(cd "$REPO_DIR" && pwd -P)" ] && in_place=1
|
|
if [ "$in_place" = 0 ]; then
|
|
command -v rsync >/dev/null || die "rsync is needed to copy $REPO_DIR to $APP_DIR"
|
|
log "Copying $REPO_DIR to $APP_DIR"
|
|
mkdir -p "$APP_DIR"
|
|
rsync -a --delete \
|
|
--exclude '.git' --exclude '.venv' --exclude '__pycache__' --exclude '/rust/target' \
|
|
--exclude '/.env' --exclude '/stations.json' --exclude '/bin' \
|
|
--exclude '/collector-status.json' --exclude '/crontab-*.bak' \
|
|
--exclude '/models/flood_*' --exclude '/models/metrics.json' \
|
|
--exclude '/models/cache' --exclude '/models/.staging' --exclude '/models/.previous' \
|
|
"$REPO_DIR/" "$APP_DIR/"
|
|
fi
|
|
|
|
# 3. binary -------------------------------------------------------------------
|
|
BIN="${BIN:-${REPO_DIR}/rust/target/release/ping-collector}"
|
|
[ -x "$BIN" ] || die "no binary at $BIN: build it (see the header) or pass BIN=/path/to/ping-collector"
|
|
log "Installing $BIN -> ${APP_DIR}/bin/ping-collector ($("$BIN" --version))"
|
|
install -D -m 0755 -o root -g root "$BIN" "${APP_DIR}/bin/ping-collector"
|
|
|
|
# 4. Python training environment ------------------------------------------------
|
|
find_uv() { # user -> path of a uv binary that user can run
|
|
local home c
|
|
home="$(getent passwd "$1" | cut -d: -f6)"
|
|
for c in "${UV:-}" "$(command -v uv 2>/dev/null || true)" "${home}/.local/bin/uv" \
|
|
"${home}/.cargo/bin/uv" /root/.local/bin/uv /usr/local/bin/uv /usr/bin/uv; do
|
|
[ -n "$c" ] && [ -x "$c" ] && runuser -u "$1" -- test -x "$c" && {
|
|
echo "$c"
|
|
return 0
|
|
}
|
|
done
|
|
return 1
|
|
}
|
|
|
|
if [ "${SKIP_UV:-0}" != 1 ]; then
|
|
uv_user=root # copy mode: handed to the service user below
|
|
if [ "$in_place" = 1 ]; then
|
|
uv_user="${SUDO_USER:-root}"
|
|
[ -d "${APP_DIR}/.venv" ] && uv_user="$(stat -c %U "${APP_DIR}/.venv")"
|
|
fi
|
|
if uv_bin="$(find_uv "$uv_user")"; then
|
|
log "Syncing the Python training environment (${APP_DIR}/.venv) as $uv_user with $uv_bin"
|
|
(cd "$APP_DIR" && runuser -u "$uv_user" -- env HOME="$(getent passwd "$uv_user" | cut -d: -f6)" \
|
|
"$uv_bin" sync --python 3.11 --frozen) ||
|
|
warn "uv sync failed: the monthly retrain needs ${APP_DIR}/.venv (uv sync --python 3.11 --frozen as $uv_user)"
|
|
else
|
|
warn "uv not found for $uv_user (pass UV=/path/to/uv): ${APP_DIR}/.venv not synced"
|
|
fi
|
|
fi
|
|
|
|
# 5. .env ---------------------------------------------------------------------
|
|
fresh_env=0
|
|
if [ ! -f "$ENV_FILE" ]; then
|
|
cp "${APP_DIR}/.env.example" "$ENV_FILE"
|
|
chmod 0600 "$ENV_FILE"
|
|
fresh_env=1
|
|
fi
|
|
set_default WEB_LISTEN 0.0.0.0:8000
|
|
set_default FORECAST_IN_COLLECTOR true
|
|
|
|
if [ "$in_place" = 0 ]; then
|
|
# a copy belongs to the service (it writes models/, stations.json and the
|
|
# status file); an in-place checkout keeps its owner (git pull, builds)
|
|
chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$APP_DIR"
|
|
chown root:root "${APP_DIR}/bin/ping-collector"
|
|
chmod 0750 "$APP_DIR"
|
|
fi
|
|
|
|
# 6. units --------------------------------------------------------------------
|
|
log "Installing systemd units"
|
|
install -m 0644 "${APP_DIR}/scripts/ping-collector.service" /etc/systemd/system/ping-collector.service
|
|
install -m 0644 "${APP_DIR}/scripts/${RETRAIN}.service" "/etc/systemd/system/${RETRAIN}.service"
|
|
install -m 0644 "${APP_DIR}/scripts/${RETRAIN}.timer" "/etc/systemd/system/${RETRAIN}.timer"
|
|
if [ -f /etc/systemd/system/water-monitor.service ]; then
|
|
log "Removing the retired Python web app (water-monitor.service)"
|
|
systemctl disable --now water-monitor.service 2>/dev/null || true
|
|
rm -f /etc/systemd/system/water-monitor.service
|
|
fi
|
|
systemctl daemon-reload
|
|
systemctl enable ping-collector.service
|
|
systemctl enable --now "${RETRAIN}.timer"
|
|
|
|
# The old Matrix cron job (`run.py --alert-check`, repeating an alert every
|
|
# 15 minutes while a level stayed high): ping-collector sends Matrix now.
|
|
for user in root "$SERVICE_USER"; do
|
|
crontab -l -u "$user" 2>/dev/null | grep -v '^\s*#' | grep -q -- '--alert-check' || continue
|
|
backup="${APP_DIR}/crontab-${user}.$(date +%Y%m%d%H%M%S).bak"
|
|
crontab -l -u "$user" >"$backup"
|
|
sed '/--alert-check/ s|^\([^#]\)|# retired: Matrix is sent by ping-collector # \1|' "$backup" |
|
|
crontab -u "$user" -
|
|
log "Commented out the old --alert-check cron job of $user (backup: $backup)"
|
|
done
|
|
if grep -rlqs -- '--alert-check' /etc/cron.d /etc/crontab 2>/dev/null; then
|
|
warn "an --alert-check job is still in $(grep -rls -- '--alert-check' /etc/cron.d /etc/crontab | tr '\n' ' '): remove it"
|
|
fi
|
|
|
|
if [ "$fresh_env" = 1 ]; then
|
|
warn "created $ENV_FILE from .env.example: set the database (POSTGRES_*), NTFY_* and MATRIX_*, then re-run"
|
|
exit 0
|
|
fi
|
|
|
|
# 7. checks -------------------------------------------------------------------
|
|
if [ "${SKIP_CHECKS:-0}" != 1 ]; then
|
|
log "Dry run of every source against the configured database (writes nothing)"
|
|
as_service "${APP_DIR}/bin/ping-collector" once --dry-run
|
|
if compgen -G "${APP_DIR}/models/flood_*.joblib" >/dev/null && [ -x "$PYTHON" ]; then
|
|
log "Exporting trained models for the forecaster (models/flood_*.json.gz)"
|
|
as_service "$PYTHON" -m src.ml.export
|
|
log "Forecast check: ping-collector against the Python reference, same data, moment and rain"
|
|
as_service "$PYTHON" scripts/forecast_parity.py ||
|
|
warn "the forecast check found differences (see above); the service runs regardless"
|
|
else
|
|
warn "no trained models or no ${PYTHON}: forecast check skipped (the forecast falls back to the heuristic)"
|
|
fi
|
|
fi
|
|
|
|
# 8. (re)start ----------------------------------------------------------------
|
|
systemctl restart ping-collector.service
|
|
listen="$(grep '^WEB_LISTEN=' "$ENV_FILE" | tail -1 | cut -d= -f2- | tr -d '"'"'"' ')"
|
|
if [ -n "$listen" ]; then
|
|
port="${listen##*:}"
|
|
# readiness: a route that answers 200 whatever the data (/health may be 503)
|
|
if wait_http "http://127.0.0.1:${port}/api/notifications" 60; then
|
|
log "Website and API answer on :${port} ($(curl -s --max-time 5 "http://127.0.0.1:${port}/scraping/status" | grep -o '"collector":"[^"]*"'))"
|
|
else
|
|
warn "no answer on port ${port} yet: journalctl -u ping-collector"
|
|
fi
|
|
fi
|
|
sleep 3
|
|
systemctl --no-pager --lines=0 status ping-collector.service || true
|
|
journalctl -u ping-collector -n 25 --no-pager || true
|
|
echo
|
|
log "Done. Logs: journalctl -u ping-collector -f; retrain: systemctl list-timers ${RETRAIN}.timer"
|