287 lines
13 KiB
Bash
Executable File
287 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Install the Rust service (rust/collector, `ping-collector`) and move work
|
|
# from the Python app to it, or back.
|
|
#
|
|
# sudo bash scripts/install_collector.sh # install binary + unit; checks only
|
|
# sudo bash scripts/install_collector.sh --cutover # Rust collects (COLLECTION_ENABLED=false)
|
|
# sudo bash scripts/install_collector.sh --cutover-forecast # Rust also forecasts and notifies:
|
|
# # ntfy + Matrix (FORECAST_IN_COLLECTOR=true)
|
|
# sudo bash scripts/install_collector.sh --check-api # compare the Rust website/API with the
|
|
# # Python app's, change nothing
|
|
# sudo bash scripts/install_collector.sh --cutover-api # Rust serves the website and API on
|
|
# # port 8000 (WEB_LISTEN); the Python app stops
|
|
# sudo bash scripts/install_collector.sh --rollback-api # the Python app serves the website again
|
|
# sudo bash scripts/install_collector.sh --rollback-forecast # forecasts + notifications back to Python
|
|
# sudo bash scripts/install_collector.sh --rollback # Python does everything again
|
|
#
|
|
# Every install also exports the trained models for the Rust forecaster and
|
|
# compares its forecast with Python's on the live data
|
|
# (scripts/forecast_parity.py); --cutover-forecast refuses to switch when they
|
|
# differ. --cutover-api first runs `ping-collector serve` next to the Python
|
|
# app and compares every route (scripts/api_parity.py); it refuses to switch
|
|
# when they differ. FORCE=1 overrides either check.
|
|
#
|
|
# Once ping-collector sends the Matrix messages (FORECAST_IN_COLLECTOR=true
|
|
# and MATRIX_* set), the old `run.py --alert-check` cron job is commented out
|
|
# (backup next to the app); it repeated an alert on every run.
|
|
#
|
|
# The binary comes from BIN=/path/to/ping-collector (e.g. the "ping-collector-
|
|
# linux-x86_64" CI artifact) or from rust/target/release/ping-collector, built
|
|
# beforehand as a normal user:
|
|
# cargo build --release --locked --manifest-path rust/Cargo.toml
|
|
# (needs Rust >= 1.88 from rustup -- distro cargo is too old -- plus pkg-config
|
|
# and libssl-dev; see rust/README.md).
|
|
set -euo pipefail
|
|
|
|
APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
|
|
SERVICE_USER="${SERVICE_USER:-water-monitor}" # as in scripts/ping-collector.service
|
|
ENV_FILE="${APP_DIR}/.env"
|
|
STATUS_FILE="${APP_DIR}/collector-status.json"
|
|
PYTHON="${APP_DIR}/.venv/bin/python"
|
|
REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
WEB_ADDR="${WEB_ADDR:-0.0.0.0:8000}" # where the Python app listens today
|
|
CHECK_PORT="${CHECK_PORT:-8001}" # the Rust side during --check-api
|
|
MODE="${1:-install}"
|
|
|
|
log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; }
|
|
warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*" >&2; }
|
|
die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }
|
|
|
|
[ "$(id -u)" -eq 0 ] || die "run as root (sudo)"
|
|
[ -f "$ENV_FILE" ] || die "$ENV_FILE not found"
|
|
|
|
set_env() {
|
|
local key="$1" value="$2"
|
|
if grep -q "^${key}=" "$ENV_FILE"; then
|
|
sed -i "s|^${key}=.*|${key}=${value}|" "$ENV_FILE"
|
|
else
|
|
printf '%s=%s\n' "$key" "$value" >>"$ENV_FILE"
|
|
fi
|
|
}
|
|
|
|
# Run a command the way the services run: same user, working directory and
|
|
# EnvironmentFile parsing (systemd's, not a .env library's).
|
|
as_service() {
|
|
if systemd-run --version >/dev/null 2>&1; then
|
|
systemd-run --quiet --pipe --wait --collect \
|
|
-p User="$SERVICE_USER" -p Group="$(id -gn "$SERVICE_USER")" \
|
|
-p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" "$@"
|
|
else
|
|
(cd "$APP_DIR" && runuser -u "$SERVICE_USER" -- "$@")
|
|
fi
|
|
}
|
|
|
|
show() {
|
|
sleep "$1"
|
|
systemctl --no-pager --lines=0 status ping-collector.service || true
|
|
journalctl -u ping-collector -n 25 --no-pager || true
|
|
}
|
|
|
|
env_is() { grep -q "^$1=$2\s*$" "$ENV_FILE"; }
|
|
|
|
wait_http() { # url seconds
|
|
local i
|
|
for i in $(seq "$2"); do
|
|
curl -fs -o /dev/null --max-time 5 "$1" 2>/dev/null && return 0
|
|
sleep 1
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# The Matrix messages come from ping-collector once it notifies; the old
|
|
# cron job (`run.py --alert-check`, every 15 min, repeating while a level
|
|
# stays high) is commented out, with a backup of the crontab.
|
|
retire_matrix_cron() {
|
|
env_is FORECAST_IN_COLLECTOR true || return 0
|
|
grep -q '^MATRIX_ACCESS_TOKEN=..*' "$ENV_FILE" || return 0
|
|
local user backup
|
|
for user in root "$SERVICE_USER"; do
|
|
crontab -l -u "$user" 2>/dev/null | grep -v '^\s*#' | grep -q -- '--alert-check' || continue
|
|
backup="${APP_DIR}/crontab-${user}.$(date +%Y%m%d%H%M%S).bak"
|
|
crontab -l -u "$user" >"$backup"
|
|
sed '/--alert-check/ s|^\([^#]\)|# retired: Matrix is sent by ping-collector # \1|' "$backup" |
|
|
crontab -u "$user" -
|
|
log "Matrix: commented out the old --alert-check cron job of $user (backup: $backup)"
|
|
done
|
|
if grep -rlqs -- '--alert-check' /etc/cron.d /etc/crontab 2>/dev/null; then
|
|
warn "an --alert-check job is still in $(grep -rls -- '--alert-check' /etc/cron.d /etc/crontab | tr '\n' ' '): remove it, ping-collector sends Matrix now"
|
|
fi
|
|
if systemctl list-timers --all --no-pager 2>/dev/null | grep -qi alert; then
|
|
warn "a systemd timer mentions 'alert' (systemctl list-timers --all): if it runs --alert-check, disable it"
|
|
fi
|
|
}
|
|
|
|
# ping-collector serve next to the Python app; every route compared.
|
|
api_check() {
|
|
local unit=ping-api-check ok=0
|
|
systemctl stop "$unit" 2>/dev/null || true
|
|
systemd-run --quiet --unit="$unit" --collect \
|
|
-p User="$SERVICE_USER" -p Group="$(id -gn "$SERVICE_USER")" \
|
|
-p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" \
|
|
"${APP_DIR}/bin/ping-collector" serve --listen "127.0.0.1:${CHECK_PORT}" --no-analytics
|
|
if wait_http "http://127.0.0.1:${CHECK_PORT}/api/notifications" 30; then
|
|
as_service "$PYTHON" scripts/api_parity.py \
|
|
--python "http://127.0.0.1:${WEB_ADDR##*:}" --rust "http://127.0.0.1:${CHECK_PORT}" && ok=1
|
|
else
|
|
warn "ping-collector serve did not answer on port ${CHECK_PORT}:"
|
|
journalctl -u "$unit" -n 20 --no-pager || true
|
|
fi
|
|
systemctl stop "$unit" 2>/dev/null || true
|
|
[ "$ok" = 1 ]
|
|
}
|
|
|
|
rollback_api() {
|
|
if env_is WEB_LISTEN '..*'; then
|
|
log "The Python app serves the website and API again"
|
|
set_env WEB_LISTEN ""
|
|
systemctl restart ping-collector.service
|
|
fi
|
|
systemctl enable water-monitor.service
|
|
systemctl restart water-monitor.service
|
|
}
|
|
|
|
matrix_backups() {
|
|
ls "${APP_DIR}"/crontab-*.bak 2>/dev/null | sed 's/^/ /' || true
|
|
}
|
|
|
|
case "$MODE" in
|
|
--rollback)
|
|
log "Stopping ping-collector; the Python app collects, serves, forecasts and notifies again"
|
|
systemctl disable --now ping-collector.service 2>/dev/null || true
|
|
set_env WEB_LISTEN ""
|
|
set_env FORECAST_IN_COLLECTOR false
|
|
set_env COLLECTION_ENABLED true
|
|
systemctl enable water-monitor.service
|
|
systemctl restart water-monitor.service
|
|
if [ -n "$(matrix_backups)" ]; then
|
|
warn "Matrix alerts came from ping-collector; to restore the old cron job: crontab -u USER BACKUP"
|
|
matrix_backups
|
|
fi
|
|
log "Done. Check: journalctl -u water-monitor -n 50"
|
|
exit 0
|
|
;;
|
|
--rollback-api)
|
|
rollback_api
|
|
log "Done. Check: journalctl -u water-monitor -n 50; curl -s http://127.0.0.1:${WEB_ADDR##*:}/health"
|
|
exit 0
|
|
;;
|
|
--rollback-forecast)
|
|
# the Python app forecasts in its own process: it must serve again
|
|
rollback_api
|
|
log "Forecasts and notifications back to the Python app; ping-collector keeps collecting"
|
|
set_env FORECAST_IN_COLLECTOR false
|
|
systemctl restart ping-collector.service water-monitor.service
|
|
if [ -n "$(matrix_backups)" ]; then
|
|
warn "Matrix alerts stop with this (the Python app sends ntfy only); old cron job: crontab -u USER BACKUP"
|
|
matrix_backups
|
|
fi
|
|
log "Done. Check: journalctl -u water-monitor -n 50"
|
|
exit 0
|
|
;;
|
|
--check-api)
|
|
[ -x "${APP_DIR}/bin/ping-collector" ] || die "install first"
|
|
if env_is WEB_LISTEN '..*'; then
|
|
die "ping-collector already serves the website (WEB_LISTEN is set)"
|
|
fi
|
|
api_check && log "The Rust API answers as the Python app" || die "the API check found differences"
|
|
exit 0
|
|
;;
|
|
install | --cutover | --cutover-forecast | --cutover-api) ;;
|
|
*) die "unknown option: $MODE" ;;
|
|
esac
|
|
|
|
BIN="${BIN:-${REPO_DIR}/rust/target/release/ping-collector}"
|
|
[ -x "$BIN" ] || die "no binary at $BIN: build it (see header) or pass BIN=/path/to/ping-collector"
|
|
|
|
log "Installing $BIN -> ${APP_DIR}/bin/ping-collector ($("$BIN" --version))"
|
|
install -D -m 0755 -o root -g root "$BIN" "${APP_DIR}/bin/ping-collector"
|
|
install -m 0644 "${REPO_DIR}/scripts/ping-collector.service" /etc/systemd/system/ping-collector.service
|
|
systemctl daemon-reload
|
|
|
|
# Both services read and write /scraping/status through this file (inside
|
|
# the app dir: the units use PrivateTmp).
|
|
set_env SCRAPING_STATUS_FILE "$STATUS_FILE"
|
|
touch "$STATUS_FILE"
|
|
chown "${SERVICE_USER}:$(id -gn "$SERVICE_USER")" "$STATUS_FILE"
|
|
|
|
log "Dry run against the configured database (writes nothing)"
|
|
as_service "${APP_DIR}/bin/ping-collector" once --dry-run
|
|
|
|
forecast_ok=1
|
|
if compgen -G "${APP_DIR}/models/flood_*.joblib" >/dev/null; then
|
|
log "Exporting the trained models for the Rust forecaster (models/flood_*.json.gz)"
|
|
as_service "$PYTHON" -m src.ml.export
|
|
log "Forecast check: Python and ping-collector on the same data, moment and rain"
|
|
as_service "$PYTHON" scripts/forecast_parity.py || forecast_ok=0
|
|
else
|
|
warn "no trained models in ${APP_DIR}/models: forecast check skipped"
|
|
forecast_ok=0
|
|
fi
|
|
|
|
case "$MODE" in
|
|
--cutover)
|
|
log "Cutover: ping-collector collects; the Python app forecasts and notifies"
|
|
set_env COLLECTION_ENABLED false
|
|
systemctl enable ping-collector.service
|
|
systemctl restart ping-collector.service water-monitor.service
|
|
show 20
|
|
log "Next step: --cutover-forecast. Rollback: sudo bash scripts/install_collector.sh --rollback"
|
|
;;
|
|
--cutover-forecast)
|
|
grep -q '^COLLECTION_ENABLED=false' "$ENV_FILE" ||
|
|
die "ping-collector must collect first: run --cutover"
|
|
if [ "$forecast_ok" != 1 ] && [ "${FORCE:-0}" != 1 ]; then
|
|
die "the forecast check did not pass; not switching (FORCE=1 overrides)"
|
|
fi
|
|
log "Cutover: ping-collector also forecasts and notifies; the Python app serves its forecast"
|
|
set_env FORECAST_IN_COLLECTOR true
|
|
retire_matrix_cron
|
|
systemctl enable ping-collector.service
|
|
systemctl restart ping-collector.service water-monitor.service
|
|
show 30
|
|
log "Next step: --cutover-api. Rollback: sudo bash scripts/install_collector.sh --rollback-forecast"
|
|
;;
|
|
--cutover-api)
|
|
env_is FORECAST_IN_COLLECTOR true || die "ping-collector must forecast first: run --cutover-forecast"
|
|
if env_is WEB_LISTEN '..*'; then
|
|
log "ping-collector already serves the website; restarting it on the new binary"
|
|
retire_matrix_cron
|
|
systemctl restart ping-collector.service
|
|
show 10
|
|
exit 0
|
|
fi
|
|
api_ok=1
|
|
if [ "$(systemctl is-active water-monitor.service 2>/dev/null)" = active ]; then
|
|
log "API check: ping-collector serve on :${CHECK_PORT} next to the Python app, every route compared"
|
|
api_check || api_ok=0
|
|
else
|
|
warn "the Python app is not running: nothing to compare the API with"
|
|
api_ok=0
|
|
fi
|
|
if [ "$api_ok" != 1 ] && [ "${FORCE:-0}" != 1 ]; then
|
|
die "the API check did not pass; not switching (FORCE=1 overrides)"
|
|
fi
|
|
log "Cutover: ping-collector serves the website and API on ${WEB_ADDR}; the Python app stops"
|
|
set_env WEB_LISTEN "$WEB_ADDR"
|
|
retire_matrix_cron
|
|
systemctl disable --now water-monitor.service
|
|
systemctl restart ping-collector.service
|
|
if wait_http "http://127.0.0.1:${WEB_ADDR##*:}/health" 60; then
|
|
log "The website answers from ping-collector: $(curl -s --max-time 5 "http://127.0.0.1:${WEB_ADDR##*:}/scraping/status" | grep -o '"collector":"[^"]*"')"
|
|
else
|
|
warn "no answer on port ${WEB_ADDR##*:} yet; check journalctl -u ping-collector"
|
|
fi
|
|
show 5
|
|
log "Rollback if needed: sudo bash scripts/install_collector.sh --rollback-api"
|
|
;;
|
|
*)
|
|
retire_matrix_cron
|
|
if [ "$(systemctl is-active ping-collector.service 2>/dev/null)" = active ]; then
|
|
systemctl restart ping-collector.service
|
|
log "Installed; ping-collector restarted on the new binary"
|
|
else
|
|
log "Installed, not started. Switch over with: sudo bash scripts/install_collector.sh --cutover"
|
|
fi
|
|
;;
|
|
esac
|