Northern-Thailand-Ping-River-Monitor/scripts/install.sh

#!/usr/bin/env bash
# Install or update the Ping River Monitor on a systemd host.
#
#   sudo bash scripts/install.sh
#
# What runs:
#   ping-collector.service        the service (rust/collector): RID/HII/dam/rain
#                                 collection, forecast, ntfy + Matrix
#                                 notifications, website + API on WEB_LISTEN
#   water-monitor-retrain.timer   monthly model retrain (Python, scripts/retrain.sh)
#
# Re-run after every update (git pull + build). Steps, all idempotent:
#   1. system user/group (water-monitor)
#   2. code: run from a checkout other than APP_DIR, it is rsynced there (the
#      trained models, .env, stations.json and bin/ are kept) and handed to
#      the service user. Production runs in place (APP_DIR is the checkout):
#      then nothing is copied and no ownership changes.
#   3. the binary: BIN=/path/to/ping-collector (e.g. the CI artifact
#      "ping-collector-linux-x86_64") or rust/target/release/ping-collector,
#      built beforehand as a normal user:
#        cargo build --release --locked --manifest-path rust/Cargo.toml
#      (Rust >= 1.88 from rustup, plus pkg-config and libssl-dev)
#   4. the Python training environment (.venv, `uv sync --frozen`): in place
#      as the user running sudo (who pulled and built), otherwise as root;
#      SKIP_UV=1 skips it
#   5. .env: created from .env.example on a fresh install (edit, then re-run);
#      WEB_LISTEN and FORECAST_IN_COLLECTOR default to the production values
#      when absent
#   6. systemd units; the retired Python web app (water-monitor.service) and
#      the old `run.py --alert-check` Matrix cron job are removed
#   7. checks (SKIP_CHECKS=1 skips them): a dry run of every source against
#      the database, and the forecast compared with the Python reference on
#      the live data (scripts/forecast_parity.py)
#   8. (re)start, then wait for the website to answer
set -euo pipefail

APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
SERVICE_USER="${SERVICE_USER:-water-monitor}" # as in the units
SERVICE_GROUP="${SERVICE_GROUP:-${SERVICE_USER}}"
ENV_FILE="${APP_DIR}/.env"
PYTHON="${APP_DIR}/.venv/bin/python"
REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RETRAIN="water-monitor-retrain"

log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; }
warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*" >&2; }
die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }

[ "$(id -u)" -eq 0 ] || die "run as root (sudo)"

set_default() { # key value: append when the key is not set at all
    grep -q "^$1=" "$ENV_FILE" || printf '%s=%s\n' "$1" "$2" >>"$ENV_FILE"
}

# A command the way the services run it: same user, working directory and
# EnvironmentFile parsing (systemd's).
as_service() {
    if systemd-run --version >/dev/null 2>&1; then
        systemd-run --quiet --pipe --wait --collect \
            -p User="$SERVICE_USER" -p Group="$SERVICE_GROUP" \
            -p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" "$@"
    else
        (cd "$APP_DIR" && runuser -u "$SERVICE_USER" -- "$@")
    fi
}

wait_http() { # url seconds
    local i
    for i in $(seq "$2"); do
        curl -fs -o /dev/null --max-time 5 "$1" 2>/dev/null && return 0
        sleep 1
    done
    return 1
}

# 1. user ---------------------------------------------------------------------
getent group "$SERVICE_GROUP" >/dev/null || groupadd --system "$SERVICE_GROUP"
if ! id "$SERVICE_USER" >/dev/null 2>&1; then
    log "Creating system user $SERVICE_USER"
    useradd --system --no-create-home --shell /usr/sbin/nologin \
        --gid "$SERVICE_GROUP" "$SERVICE_USER"
fi

# 2. code ---------------------------------------------------------------------
in_place=0
[ -d "$APP_DIR" ] && [ "$(cd "$APP_DIR" && pwd -P)" = "$(cd "$REPO_DIR" && pwd -P)" ] && in_place=1
if [ "$in_place" = 0 ]; then
    command -v rsync >/dev/null || die "rsync is needed to copy $REPO_DIR to $APP_DIR"
    log "Copying $REPO_DIR to $APP_DIR"
    mkdir -p "$APP_DIR"
    rsync -a --delete \
        --exclude '.git' --exclude '.venv' --exclude '__pycache__' --exclude '/rust/target' \
        --exclude '/.env' --exclude '/stations.json' --exclude '/bin' \
        --exclude '/collector-status.json' --exclude '/crontab-*.bak' \
        --exclude '/models/flood_*' --exclude '/models/metrics.json' \
        --exclude '/models/cache' --exclude '/models/.staging' --exclude '/models/.previous' \
        "$REPO_DIR/" "$APP_DIR/"
fi

# 3. binary -------------------------------------------------------------------
BIN="${BIN:-${REPO_DIR}/rust/target/release/ping-collector}"
[ -x "$BIN" ] || die "no binary at $BIN: build it (see the header) or pass BIN=/path/to/ping-collector"
log "Installing $BIN -> ${APP_DIR}/bin/ping-collector ($("$BIN" --version))"
install -D -m 0755 -o root -g root "$BIN" "${APP_DIR}/bin/ping-collector"

# 4. Python training environment ------------------------------------------------
if [ "${SKIP_UV:-0}" != 1 ]; then
    uv_user=root # copy mode: handed to the service user below
    [ "$in_place" = 1 ] && uv_user="${SUDO_USER:-root}"
    log "Syncing the Python training environment (${APP_DIR}/.venv) as $uv_user"
    if [ "$uv_user" = root ]; then
        (cd "$APP_DIR" && uv sync --python 3.11 --frozen) ||
            warn "uv sync failed: the monthly retrain needs ${APP_DIR}/.venv (uv sync --python 3.11 --frozen)"
    else
        runuser -u "$uv_user" -- bash -lc "cd '$APP_DIR' && uv sync --python 3.11 --frozen" ||
            warn "uv sync as $uv_user failed: run 'uv sync --python 3.11 --frozen' in $APP_DIR as the owner of .venv"
    fi
fi

# 5. .env ---------------------------------------------------------------------
fresh_env=0
if [ ! -f "$ENV_FILE" ]; then
    cp "${APP_DIR}/.env.example" "$ENV_FILE"
    chmod 0600 "$ENV_FILE"
    fresh_env=1
fi
set_default WEB_LISTEN 0.0.0.0:8000
set_default FORECAST_IN_COLLECTOR true

if [ "$in_place" = 0 ]; then
    # a copy belongs to the service (it writes models/, stations.json and the
    # status file); an in-place checkout keeps its owner (git pull, builds)
    chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$APP_DIR"
    chown root:root "${APP_DIR}/bin/ping-collector"
    chmod 0750 "$APP_DIR"
fi

# 6. units --------------------------------------------------------------------
log "Installing systemd units"
install -m 0644 "${APP_DIR}/scripts/ping-collector.service" /etc/systemd/system/ping-collector.service
install -m 0644 "${APP_DIR}/scripts/${RETRAIN}.service" "/etc/systemd/system/${RETRAIN}.service"
install -m 0644 "${APP_DIR}/scripts/${RETRAIN}.timer" "/etc/systemd/system/${RETRAIN}.timer"
if [ -f /etc/systemd/system/water-monitor.service ]; then
    log "Removing the retired Python web app (water-monitor.service)"
    systemctl disable --now water-monitor.service 2>/dev/null || true
    rm -f /etc/systemd/system/water-monitor.service
fi
systemctl daemon-reload
systemctl enable ping-collector.service
systemctl enable --now "${RETRAIN}.timer"

# The old Matrix cron job (`run.py --alert-check`, repeating an alert every
# 15 minutes while a level stayed high): ping-collector sends Matrix now.
for user in root "$SERVICE_USER"; do
    crontab -l -u "$user" 2>/dev/null | grep -v '^\s*#' | grep -q -- '--alert-check' || continue
    backup="${APP_DIR}/crontab-${user}.$(date +%Y%m%d%H%M%S).bak"
    crontab -l -u "$user" >"$backup"
    sed '/--alert-check/ s|^\([^#]\)|# retired: Matrix is sent by ping-collector # \1|' "$backup" |
        crontab -u "$user" -
    log "Commented out the old --alert-check cron job of $user (backup: $backup)"
done
if grep -rlqs -- '--alert-check' /etc/cron.d /etc/crontab 2>/dev/null; then
    warn "an --alert-check job is still in $(grep -rls -- '--alert-check' /etc/cron.d /etc/crontab | tr '\n' ' '): remove it"
fi

if [ "$fresh_env" = 1 ]; then
    warn "created $ENV_FILE from .env.example: set the database (POSTGRES_*), NTFY_* and MATRIX_*, then re-run"
    exit 0
fi

# 7. checks -------------------------------------------------------------------
if [ "${SKIP_CHECKS:-0}" != 1 ]; then
    log "Dry run of every source against the configured database (writes nothing)"
    as_service "${APP_DIR}/bin/ping-collector" once --dry-run
    if compgen -G "${APP_DIR}/models/flood_*.joblib" >/dev/null && [ -x "$PYTHON" ]; then
        log "Exporting trained models for the forecaster (models/flood_*.json.gz)"
        as_service "$PYTHON" -m src.ml.export
        log "Forecast check: ping-collector against the Python reference, same data, moment and rain"
        as_service "$PYTHON" scripts/forecast_parity.py ||
            warn "the forecast check found differences (see above); the service runs regardless"
    else
        warn "no trained models or no ${PYTHON}: forecast check skipped (the forecast falls back to the heuristic)"
    fi
fi

# 8. (re)start ----------------------------------------------------------------
systemctl restart ping-collector.service
listen="$(grep '^WEB_LISTEN=' "$ENV_FILE" | tail -1 | cut -d= -f2- | tr -d '"'"'"' ')"
if [ -n "$listen" ]; then
    port="${listen##*:}"
    # readiness: a route that answers 200 whatever the data (/health may be 503)
    if wait_http "http://127.0.0.1:${port}/api/notifications" 60; then
        log "Website and API answer on :${port} ($(curl -s --max-time 5 "http://127.0.0.1:${port}/scraping/status" | grep -o '"collector":"[^"]*"'))"
    else
        warn "no answer on port ${port} yet: journalctl -u ping-collector"
    fi
fi
sleep 3
systemctl --no-pager --lines=0 status ping-collector.service || true
journalctl -u ping-collector -n 25 --no-pager || true
echo
log "Done. Logs: journalctl -u ping-collector -f; retrain: systemctl list-timers ${RETRAIN}.timer"