feat(web): the website and API in ping-collector; Matrix gets the ntfy transitions
Phase 3 of docs/RUST_REWRITE.md: rust/collector/src/web (axum) serves what
the FastAPI app (src/web_api.py) served, so one process collects, forecasts,
notifies and serves. With WEB_LISTEN=0.0.0.0:8000 it runs inside
`ping-collector run`; `ping-collector serve` runs it alone.
Same routes, parameters, JSON fields and field order, status codes,
{"detail": ...} errors and pydantic-style 422 lists, Cache-Control, gzip
(>500 B), CORS, Umami API tracking, the dashboard and static files, and the
stale-while-revalidate response caches (now with byte budgets: a history
response can be 10 MB). /docs is the same Swagger UI (+ /redoc); its OpenAPI
document is built from the route table, whose per-route parameter
declarations also validate the requests, so docs and behaviour cannot drift.
Python output conventions are kept: isoformat(), round(), numpy's pairwise
sums (catchment means), Python's summation order (skill metrics).
scripts/api_parity.py compares the two servers route by route on one
database (53 cases incl. errors, admin and static files; query values vary
per run so FastAPI's caches answer fresh). Locally: 53 identical; the
station CRUD writes byte-identical stations.json files; the skill/trend
logic matches on 9 station/horizon pairs with synthetic model versions.
Intended differences: /health counts the ThaiWater telemetry of the same
gauges while RID lags (as the notifier); /scraping/status adds the collector
version and last forecast run; an old stored forecast is served marked
X-Data-Stale instead of recomputed; /sensors/thaiwater is cached a minute;
.geojson is served as application/geo+json. Load (local, 16 clients): the
dashboard's routes 1375 vs 552 req/s, uncached history 714 vs 53 req/s;
25 MB resident when serving only.
Matrix (MATRIX_*): the notifier sends the room the transitions ntfy gets
(per station, outlook, feed; not ntfy's per-topic summaries), each once,
with its own matrix: keys in notification_state, seeded from ntfy's on first
use so nothing already announced repeats. Checked against a fake homeserver.
It replaces the `run.py --alert-check` cron job (repeated alerts every 15
min), which install_collector.sh now comments out (crontab backup kept).
install_collector.sh: --check-api (ping-collector serve on :8001 next to
FastAPI, api_parity.py), --cutover-api (refuses on a difference; sets
WEB_LISTEN, stops and disables water-monitor.service), --rollback-api;
--rollback-forecast and --rollback bring the Python app back as server.
ping-collector.service: MemoryMax 512M. Config::from_vars for tests; route
tests against a throwaway schema.
This commit is contained in:
1 parent
1d1c000922
commit
779728e38b
32 files changed
+5692
-160
No files matched your search
+144
-44
@@ -48,28 +48,74 @@ pub struct Config {
|
||||
pub public_url: String,
|
||||
/// Report RID lagging this long while ThaiWater is current (0 = never).
|
||||
pub rid_stale_alert_hours: f64,
|
||||
/// Public ntfy URL (what subscribers use), shown by /api/notifications.
|
||||
pub ntfy_server: Option<String>,
|
||||
/// MATRIX_*: the notifications also go to this room (off when unset).
|
||||
pub matrix: Option<MatrixConfig>,
|
||||
pub web: WebConfig,
|
||||
}
|
||||
|
||||
fn var(name: &str) -> Option<String> {
|
||||
env::var(name)
|
||||
.ok()
|
||||
.map(|v| v.trim().to_string())
|
||||
.filter(|v| !v.is_empty())
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MatrixConfig {
|
||||
pub homeserver: String,
|
||||
pub access_token: String,
|
||||
pub room_id: String,
|
||||
}
|
||||
|
||||
fn flag(name: &str, default: bool) -> bool {
|
||||
var(name).map_or(default, |v| {
|
||||
matches!(v.to_lowercase().as_str(), "1" | "true" | "yes")
|
||||
})
|
||||
/// The website and read API (src/web_api.py's settings, same names).
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WebConfig {
|
||||
/// WEB_LISTEN: `run` also serves the website on this address
|
||||
/// (e.g. 0.0.0.0:8000); unset = collection only.
|
||||
pub listen: Option<String>,
|
||||
/// Dashboard, robots/llms/sitemap and /static files.
|
||||
pub static_dir: PathBuf,
|
||||
/// X-API-Key for the station editing, /config and /scrape/trigger;
|
||||
/// unset = those endpoints are disabled.
|
||||
pub admin_api_key: Option<String>,
|
||||
pub cors_allow_origins: Vec<String>,
|
||||
pub umami_api_url: String,
|
||||
/// Empty = no server-side analytics.
|
||||
pub umami_website_id: String,
|
||||
pub umami_track_api: bool,
|
||||
pub hii_cache_ttl_s: u64,
|
||||
pub latest_cache_ttl_s: u64,
|
||||
pub health_cache_ttl_s: u64,
|
||||
/// A stored forecast older than this is served marked `X-Data-Stale`.
|
||||
pub forecast_max_age_s: u64,
|
||||
pub thaiwater_api_key: Option<String>,
|
||||
pub request_timeout_s: u64,
|
||||
}
|
||||
|
||||
fn number<T: std::str::FromStr>(name: &str, default: T) -> Result<T>
|
||||
where
|
||||
T::Err: std::fmt::Display,
|
||||
{
|
||||
match var(name) {
|
||||
None => Ok(default),
|
||||
Some(v) => v.parse().map_err(|e| anyhow::anyhow!("{name}={v:?}: {e}")),
|
||||
/// Variable lookup (the process environment, or a map in tests).
|
||||
struct Vars<'a>(&'a dyn Fn(&str) -> Option<String>);
|
||||
|
||||
impl Vars<'_> {
|
||||
fn var(&self, name: &str) -> Option<String> {
|
||||
(self.0)(name)
|
||||
.map(|v| v.trim().to_string())
|
||||
.filter(|v| !v.is_empty())
|
||||
}
|
||||
|
||||
/// `os.getenv(name, default)`: a variable set to "" stays "".
|
||||
fn raw_var(&self, name: &str, default: &str) -> String {
|
||||
(self.0)(name).map_or_else(|| default.to_string(), |v| v.trim().to_string())
|
||||
}
|
||||
|
||||
fn flag(&self, name: &str, default: bool) -> bool {
|
||||
self.var(name).map_or(default, |v| {
|
||||
matches!(v.to_lowercase().as_str(), "1" | "true" | "yes")
|
||||
})
|
||||
}
|
||||
|
||||
fn number<T: std::str::FromStr>(&self, name: &str, default: T) -> Result<T>
|
||||
where
|
||||
T::Err: std::fmt::Display,
|
||||
{
|
||||
match self.var(name) {
|
||||
None => Ok(default),
|
||||
Some(v) => v.parse().map_err(|e| anyhow::anyhow!("{name}={v:?}: {e}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,52 +147,106 @@ pub fn normalize_database_url(url: &str) -> String {
|
||||
|
||||
impl Config {
|
||||
pub fn from_env() -> Result<Self> {
|
||||
let database_url = match var("POSTGRES_CONNECTION_STRING") {
|
||||
Self::from_vars(&|name| env::var(name).ok())
|
||||
}
|
||||
|
||||
pub fn from_vars(get: &dyn Fn(&str) -> Option<String>) -> Result<Self> {
|
||||
let v = Vars(get);
|
||||
let database_url = match v.var("POSTGRES_CONNECTION_STRING") {
|
||||
Some(url) => normalize_database_url(&url),
|
||||
None => {
|
||||
let password = var("POSTGRES_PASSWORD")
|
||||
let password = v
|
||||
.var("POSTGRES_PASSWORD")
|
||||
.context("POSTGRES_CONNECTION_STRING or POSTGRES_PASSWORD must be set")?;
|
||||
format!(
|
||||
"postgres://{}:{}@{}:{}/{}",
|
||||
encode_userinfo(&var("POSTGRES_USER").unwrap_or_else(|| "postgres".into())),
|
||||
encode_userinfo(&v.var("POSTGRES_USER").unwrap_or_else(|| "postgres".into())),
|
||||
encode_userinfo(&password),
|
||||
var("POSTGRES_HOST").unwrap_or_else(|| "localhost".into()),
|
||||
var("POSTGRES_PORT").unwrap_or_else(|| "5432".into()),
|
||||
var("POSTGRES_DB").unwrap_or_else(|| "water_monitoring".into()),
|
||||
v.var("POSTGRES_HOST").unwrap_or_else(|| "localhost".into()),
|
||||
v.var("POSTGRES_PORT").unwrap_or_else(|| "5432".into()),
|
||||
v.var("POSTGRES_DB")
|
||||
.unwrap_or_else(|| "water_monitoring".into()),
|
||||
)
|
||||
}
|
||||
};
|
||||
if let Some(db_type) = var("DB_TYPE")
|
||||
if let Some(db_type) = v.var("DB_TYPE")
|
||||
&& db_type != "postgresql"
|
||||
{
|
||||
bail!("ping-collector supports PostgreSQL only (DB_TYPE={db_type})");
|
||||
}
|
||||
let cfg = Config {
|
||||
database_url,
|
||||
rid_api_url: var("RID_API_URL").unwrap_or_else(|| RID_API_URL.into()),
|
||||
hii_api_base: var("HII_API_BASE").unwrap_or_else(|| HII_API_BASE.into()),
|
||||
dams_url: var("RID_DAMS_URL").unwrap_or_else(|| RID_DAMS_URL.into()),
|
||||
openmeteo_url: var("OPENMETEO_URL").unwrap_or_else(|| OPENMETEO_URL.into()),
|
||||
hii_enabled: flag("ENABLE_HII_COLLECTION", true),
|
||||
dams_enabled: flag("ENABLE_RESERVOIR_COLLECTION", true),
|
||||
rain_enabled: flag("ENABLE_RAIN_COLLECTION", true),
|
||||
hii_basin_code: number("HII_BASIN_CODE", 6)?,
|
||||
poll_offset_minutes: number("RID_POLL_OFFSET_MINUTES", 5)?,
|
||||
retry_minutes: number("RID_RETRY_MINUTES", 10)?,
|
||||
gap_fill_days_on_start: number("GAP_FILL_DAYS_ON_START", 14)?,
|
||||
station_config_path: var("STATION_CONFIG_PATH")
|
||||
rid_api_url: v.var("RID_API_URL").unwrap_or_else(|| RID_API_URL.into()),
|
||||
hii_api_base: v.var("HII_API_BASE").unwrap_or_else(|| HII_API_BASE.into()),
|
||||
dams_url: v.var("RID_DAMS_URL").unwrap_or_else(|| RID_DAMS_URL.into()),
|
||||
openmeteo_url: v
|
||||
.var("OPENMETEO_URL")
|
||||
.unwrap_or_else(|| OPENMETEO_URL.into()),
|
||||
hii_enabled: v.flag("ENABLE_HII_COLLECTION", true),
|
||||
dams_enabled: v.flag("ENABLE_RESERVOIR_COLLECTION", true),
|
||||
rain_enabled: v.flag("ENABLE_RAIN_COLLECTION", true),
|
||||
hii_basin_code: v.number("HII_BASIN_CODE", 6)?,
|
||||
poll_offset_minutes: v.number("RID_POLL_OFFSET_MINUTES", 5)?,
|
||||
retry_minutes: v.number("RID_RETRY_MINUTES", 10)?,
|
||||
gap_fill_days_on_start: v.number("GAP_FILL_DAYS_ON_START", 14)?,
|
||||
station_config_path: v
|
||||
.var("STATION_CONFIG_PATH")
|
||||
.unwrap_or_else(|| "stations.json".into())
|
||||
.into(),
|
||||
status_file: var("SCRAPING_STATUS_FILE").map(PathBuf::from),
|
||||
forecast_enabled: flag("FORECAST_IN_COLLECTOR", false),
|
||||
models_dir: var("MODELS_DIR").unwrap_or_else(|| "models".into()).into(),
|
||||
status_file: v.var("SCRAPING_STATUS_FILE").map(PathBuf::from),
|
||||
forecast_enabled: v.flag("FORECAST_IN_COLLECTOR", false),
|
||||
models_dir: v
|
||||
.var("MODELS_DIR")
|
||||
.unwrap_or_else(|| "models".into())
|
||||
.into(),
|
||||
// as src/config.py: off unless NTFY_SERVER is set
|
||||
ntfy_publish_url: var("NTFY_SERVER")
|
||||
.map(|server| var("NTFY_PUBLISH_URL").unwrap_or(server)),
|
||||
ntfy_token: var("NTFY_TOKEN"),
|
||||
ntfy_topic_prefix: var("NTFY_TOPIC_PREFIX").unwrap_or_else(|| "ping".into()),
|
||||
public_url: var("PUBLIC_URL").unwrap_or_else(|| "https://water.buildfor.life/".into()),
|
||||
rid_stale_alert_hours: number("RID_STALE_ALERT_HOURS", 18.0)?,
|
||||
ntfy_publish_url: v
|
||||
.var("NTFY_SERVER")
|
||||
.map(|server| v.var("NTFY_PUBLISH_URL").unwrap_or(server)),
|
||||
ntfy_token: v.var("NTFY_TOKEN"),
|
||||
ntfy_topic_prefix: v.var("NTFY_TOPIC_PREFIX").unwrap_or_else(|| "ping".into()),
|
||||
public_url: v
|
||||
.var("PUBLIC_URL")
|
||||
.unwrap_or_else(|| "https://water.buildfor.life/".into()),
|
||||
rid_stale_alert_hours: v.number("RID_STALE_ALERT_HOURS", 18.0)?,
|
||||
ntfy_server: v.var("NTFY_SERVER"),
|
||||
matrix: match (v.var("MATRIX_ACCESS_TOKEN"), v.var("MATRIX_ROOM_ID")) {
|
||||
(Some(access_token), Some(room_id)) => Some(MatrixConfig {
|
||||
homeserver: v
|
||||
.var("MATRIX_HOMESERVER")
|
||||
.unwrap_or_else(|| "https://matrix.org".into()),
|
||||
access_token,
|
||||
room_id,
|
||||
}),
|
||||
_ => None,
|
||||
},
|
||||
web: WebConfig {
|
||||
listen: v.var("WEB_LISTEN"),
|
||||
static_dir: v
|
||||
.var("STATIC_DIR")
|
||||
.unwrap_or_else(|| "src/static".into())
|
||||
.into(),
|
||||
admin_api_key: v.var("ADMIN_API_KEY"),
|
||||
cors_allow_origins: v
|
||||
.raw_var("CORS_ALLOW_ORIGINS", "")
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|o| !o.is_empty())
|
||||
.map(String::from)
|
||||
.collect(),
|
||||
umami_api_url: v
|
||||
.var("UMAMI_API_URL")
|
||||
.unwrap_or_else(|| "https://stats.buildfor.life/api/send".into()),
|
||||
umami_website_id: v
|
||||
.raw_var("UMAMI_WEBSITE_ID", "00b2be73-8f5f-4400-9029-3be852eb08f7"),
|
||||
umami_track_api: v.flag("UMAMI_TRACK_API", true),
|
||||
hii_cache_ttl_s: v.number("HII_CACHE_TTL_SECONDS", 120)?,
|
||||
latest_cache_ttl_s: v.number("LATEST_CACHE_TTL_SECONDS", 45)?,
|
||||
health_cache_ttl_s: v.number("HEALTH_CACHE_TTL_SECONDS", 30)?,
|
||||
forecast_max_age_s: v.number("FORECAST_MAX_AGE_SECONDS", 7200)?,
|
||||
thaiwater_api_key: v.var("THAIWATER_API_KEY"),
|
||||
request_timeout_s: v.number("REQUEST_TIMEOUT", 30)?,
|
||||
},
|
||||
};
|
||||
if cfg.poll_offset_minutes >= 60 {
|
||||
bail!("RID_POLL_OFFSET_MINUTES must be between 0 and 59");
|
||||
|
||||
Reference in new issue
Block a user