feat(web): the website and API in ping-collector; Matrix gets the ntfy transitions
CI / Format & lint (push) Successful in 26s
CI / Test suite (push) Successful in 1m9s
Docs / Validate documentation (push) Successful in 25s
Rust / fmt, clippy, test, build (push) Successful in 11m46s

Phase 3 of docs/RUST_REWRITE.md: rust/collector/src/web (axum) serves what
the FastAPI app (src/web_api.py) served, so one process collects, forecasts,
notifies and serves. With WEB_LISTEN=0.0.0.0:8000 it runs inside
`ping-collector run`; `ping-collector serve` runs it alone.

Same routes, parameters, JSON fields and field order, status codes,
{"detail": ...} errors and pydantic-style 422 lists, Cache-Control, gzip
(>500 B), CORS, Umami API tracking, the dashboard and static files, and the
stale-while-revalidate response caches (now with byte budgets: a history
response can be 10 MB). /docs is the same Swagger UI (+ /redoc); its OpenAPI
document is built from the route table, whose per-route parameter
declarations also validate the requests, so docs and behaviour cannot drift.
Python output conventions are kept: isoformat(), round(), numpy's pairwise
sums (catchment means), Python's summation order (skill metrics).

scripts/api_parity.py compares the two servers route by route on one
database (53 cases incl. errors, admin and static files; query values vary
per run so FastAPI's caches answer fresh). Locally: 53 identical; the
station CRUD writes byte-identical stations.json files; the skill/trend
logic matches on 9 station/horizon pairs with synthetic model versions.
Intended differences: /health counts the ThaiWater telemetry of the same
gauges while RID lags (as the notifier); /scraping/status adds the collector
version and last forecast run; an old stored forecast is served marked
X-Data-Stale instead of recomputed; /sensors/thaiwater is cached a minute;
.geojson is served as application/geo+json. Load (local, 16 clients): the
dashboard's routes 1375 vs 552 req/s, uncached history 714 vs 53 req/s;
25 MB resident when serving only.

Matrix (MATRIX_*): the notifier sends the room the transitions ntfy gets
(per station, outlook, feed; not ntfy's per-topic summaries), each once,
with its own matrix: keys in notification_state, seeded from ntfy's on first
use so nothing already announced repeats. Checked against a fake homeserver.
It replaces the `run.py --alert-check` cron job (repeated alerts every 15
min), which install_collector.sh now comments out (crontab backup kept).

install_collector.sh: --check-api (ping-collector serve on :8001 next to
FastAPI, api_parity.py), --cutover-api (refuses on a difference; sets
WEB_LISTEN, stops and disables water-monitor.service), --rollback-api;
--rollback-forecast and --rollback bring the Python app back as server.
ping-collector.service: MemoryMax 512M. Config::from_vars for tests; route
tests against a throwaway schema.
This commit is contained in:
grabowski committed 2026-10-18 16:08:06 +07:00
1 parent 1d1c000922
commit 779728e38b
32 files changed
+5692 -160

No files matched your search

+144 -44
View File
@@ -48,28 +48,74 @@ pub struct Config {
pub public_url: String,
/// Report RID lagging this long while ThaiWater is current (0 = never).
pub rid_stale_alert_hours: f64,
/// Public ntfy URL (what subscribers use), shown by /api/notifications.
pub ntfy_server: Option<String>,
/// MATRIX_*: the notifications also go to this room (off when unset).
pub matrix: Option<MatrixConfig>,
pub web: WebConfig,
}
fn var(name: &str) -> Option<String> {
env::var(name)
.ok()
.map(|v| v.trim().to_string())
.filter(|v| !v.is_empty())
#[derive(Clone, Debug)]
pub struct MatrixConfig {
pub homeserver: String,
pub access_token: String,
pub room_id: String,
}
fn flag(name: &str, default: bool) -> bool {
var(name).map_or(default, |v| {
matches!(v.to_lowercase().as_str(), "1" | "true" | "yes")
})
/// The website and read API (src/web_api.py's settings, same names).
#[derive(Clone, Debug)]
pub struct WebConfig {
/// WEB_LISTEN: `run` also serves the website on this address
/// (e.g. 0.0.0.0:8000); unset = collection only.
pub listen: Option<String>,
/// Dashboard, robots/llms/sitemap and /static files.
pub static_dir: PathBuf,
/// X-API-Key for the station editing, /config and /scrape/trigger;
/// unset = those endpoints are disabled.
pub admin_api_key: Option<String>,
pub cors_allow_origins: Vec<String>,
pub umami_api_url: String,
/// Empty = no server-side analytics.
pub umami_website_id: String,
pub umami_track_api: bool,
pub hii_cache_ttl_s: u64,
pub latest_cache_ttl_s: u64,
pub health_cache_ttl_s: u64,
/// A stored forecast older than this is served marked `X-Data-Stale`.
pub forecast_max_age_s: u64,
pub thaiwater_api_key: Option<String>,
pub request_timeout_s: u64,
}
fn number<T: std::str::FromStr>(name: &str, default: T) -> Result<T>
where
T::Err: std::fmt::Display,
{
match var(name) {
None => Ok(default),
Some(v) => v.parse().map_err(|e| anyhow::anyhow!("{name}={v:?}: {e}")),
/// Variable lookup (the process environment, or a map in tests).
struct Vars<'a>(&'a dyn Fn(&str) -> Option<String>);
impl Vars<'_> {
fn var(&self, name: &str) -> Option<String> {
(self.0)(name)
.map(|v| v.trim().to_string())
.filter(|v| !v.is_empty())
}
/// `os.getenv(name, default)`: a variable set to "" stays "".
fn raw_var(&self, name: &str, default: &str) -> String {
(self.0)(name).map_or_else(|| default.to_string(), |v| v.trim().to_string())
}
fn flag(&self, name: &str, default: bool) -> bool {
self.var(name).map_or(default, |v| {
matches!(v.to_lowercase().as_str(), "1" | "true" | "yes")
})
}
fn number<T: std::str::FromStr>(&self, name: &str, default: T) -> Result<T>
where
T::Err: std::fmt::Display,
{
match self.var(name) {
None => Ok(default),
Some(v) => v.parse().map_err(|e| anyhow::anyhow!("{name}={v:?}: {e}")),
}
}
}
@@ -101,52 +147,106 @@ pub fn normalize_database_url(url: &str) -> String {
impl Config {
pub fn from_env() -> Result<Self> {
let database_url = match var("POSTGRES_CONNECTION_STRING") {
Self::from_vars(&|name| env::var(name).ok())
}
pub fn from_vars(get: &dyn Fn(&str) -> Option<String>) -> Result<Self> {
let v = Vars(get);
let database_url = match v.var("POSTGRES_CONNECTION_STRING") {
Some(url) => normalize_database_url(&url),
None => {
let password = var("POSTGRES_PASSWORD")
let password = v
.var("POSTGRES_PASSWORD")
.context("POSTGRES_CONNECTION_STRING or POSTGRES_PASSWORD must be set")?;
format!(
"postgres://{}:{}@{}:{}/{}",
encode_userinfo(&var("POSTGRES_USER").unwrap_or_else(|| "postgres".into())),
encode_userinfo(&v.var("POSTGRES_USER").unwrap_or_else(|| "postgres".into())),
encode_userinfo(&password),
var("POSTGRES_HOST").unwrap_or_else(|| "localhost".into()),
var("POSTGRES_PORT").unwrap_or_else(|| "5432".into()),
var("POSTGRES_DB").unwrap_or_else(|| "water_monitoring".into()),
v.var("POSTGRES_HOST").unwrap_or_else(|| "localhost".into()),
v.var("POSTGRES_PORT").unwrap_or_else(|| "5432".into()),
v.var("POSTGRES_DB")
.unwrap_or_else(|| "water_monitoring".into()),
)
}
};
if let Some(db_type) = var("DB_TYPE")
if let Some(db_type) = v.var("DB_TYPE")
&& db_type != "postgresql"
{
bail!("ping-collector supports PostgreSQL only (DB_TYPE={db_type})");
}
let cfg = Config {
database_url,
rid_api_url: var("RID_API_URL").unwrap_or_else(|| RID_API_URL.into()),
hii_api_base: var("HII_API_BASE").unwrap_or_else(|| HII_API_BASE.into()),
dams_url: var("RID_DAMS_URL").unwrap_or_else(|| RID_DAMS_URL.into()),
openmeteo_url: var("OPENMETEO_URL").unwrap_or_else(|| OPENMETEO_URL.into()),
hii_enabled: flag("ENABLE_HII_COLLECTION", true),
dams_enabled: flag("ENABLE_RESERVOIR_COLLECTION", true),
rain_enabled: flag("ENABLE_RAIN_COLLECTION", true),
hii_basin_code: number("HII_BASIN_CODE", 6)?,
poll_offset_minutes: number("RID_POLL_OFFSET_MINUTES", 5)?,
retry_minutes: number("RID_RETRY_MINUTES", 10)?,
gap_fill_days_on_start: number("GAP_FILL_DAYS_ON_START", 14)?,
station_config_path: var("STATION_CONFIG_PATH")
rid_api_url: v.var("RID_API_URL").unwrap_or_else(|| RID_API_URL.into()),
hii_api_base: v.var("HII_API_BASE").unwrap_or_else(|| HII_API_BASE.into()),
dams_url: v.var("RID_DAMS_URL").unwrap_or_else(|| RID_DAMS_URL.into()),
openmeteo_url: v
.var("OPENMETEO_URL")
.unwrap_or_else(|| OPENMETEO_URL.into()),
hii_enabled: v.flag("ENABLE_HII_COLLECTION", true),
dams_enabled: v.flag("ENABLE_RESERVOIR_COLLECTION", true),
rain_enabled: v.flag("ENABLE_RAIN_COLLECTION", true),
hii_basin_code: v.number("HII_BASIN_CODE", 6)?,
poll_offset_minutes: v.number("RID_POLL_OFFSET_MINUTES", 5)?,
retry_minutes: v.number("RID_RETRY_MINUTES", 10)?,
gap_fill_days_on_start: v.number("GAP_FILL_DAYS_ON_START", 14)?,
station_config_path: v
.var("STATION_CONFIG_PATH")
.unwrap_or_else(|| "stations.json".into())
.into(),
status_file: var("SCRAPING_STATUS_FILE").map(PathBuf::from),
forecast_enabled: flag("FORECAST_IN_COLLECTOR", false),
models_dir: var("MODELS_DIR").unwrap_or_else(|| "models".into()).into(),
status_file: v.var("SCRAPING_STATUS_FILE").map(PathBuf::from),
forecast_enabled: v.flag("FORECAST_IN_COLLECTOR", false),
models_dir: v
.var("MODELS_DIR")
.unwrap_or_else(|| "models".into())
.into(),
// as src/config.py: off unless NTFY_SERVER is set
ntfy_publish_url: var("NTFY_SERVER")
.map(|server| var("NTFY_PUBLISH_URL").unwrap_or(server)),
ntfy_token: var("NTFY_TOKEN"),
ntfy_topic_prefix: var("NTFY_TOPIC_PREFIX").unwrap_or_else(|| "ping".into()),
public_url: var("PUBLIC_URL").unwrap_or_else(|| "https://water.buildfor.life/".into()),
rid_stale_alert_hours: number("RID_STALE_ALERT_HOURS", 18.0)?,
ntfy_publish_url: v
.var("NTFY_SERVER")
.map(|server| v.var("NTFY_PUBLISH_URL").unwrap_or(server)),
ntfy_token: v.var("NTFY_TOKEN"),
ntfy_topic_prefix: v.var("NTFY_TOPIC_PREFIX").unwrap_or_else(|| "ping".into()),
public_url: v
.var("PUBLIC_URL")
.unwrap_or_else(|| "https://water.buildfor.life/".into()),
rid_stale_alert_hours: v.number("RID_STALE_ALERT_HOURS", 18.0)?,
ntfy_server: v.var("NTFY_SERVER"),
matrix: match (v.var("MATRIX_ACCESS_TOKEN"), v.var("MATRIX_ROOM_ID")) {
(Some(access_token), Some(room_id)) => Some(MatrixConfig {
homeserver: v
.var("MATRIX_HOMESERVER")
.unwrap_or_else(|| "https://matrix.org".into()),
access_token,
room_id,
}),
_ => None,
},
web: WebConfig {
listen: v.var("WEB_LISTEN"),
static_dir: v
.var("STATIC_DIR")
.unwrap_or_else(|| "src/static".into())
.into(),
admin_api_key: v.var("ADMIN_API_KEY"),
cors_allow_origins: v
.raw_var("CORS_ALLOW_ORIGINS", "")
.split(',')
.map(str::trim)
.filter(|o| !o.is_empty())
.map(String::from)
.collect(),
umami_api_url: v
.var("UMAMI_API_URL")
.unwrap_or_else(|| "https://stats.buildfor.life/api/send".into()),
umami_website_id: v
.raw_var("UMAMI_WEBSITE_ID", "00b2be73-8f5f-4400-9029-3be852eb08f7"),
umami_track_api: v.flag("UMAMI_TRACK_API", true),
hii_cache_ttl_s: v.number("HII_CACHE_TTL_SECONDS", 120)?,
latest_cache_ttl_s: v.number("LATEST_CACHE_TTL_SECONDS", 45)?,
health_cache_ttl_s: v.number("HEALTH_CACHE_TTL_SECONDS", 30)?,
forecast_max_age_s: v.number("FORECAST_MAX_AGE_SECONDS", 7200)?,
thaiwater_api_key: v.var("THAIWATER_API_KEY"),
request_timeout_s: v.number("REQUEST_TIMEOUT", 30)?,
},
};
if cfg.poll_offset_minutes >= 60 {
bail!("RID_POLL_OFFSET_MINUTES must be between 0 and 59");