feat(web): the website and API in ping-collector; Matrix gets the ntfy transitions
Phase 3 of docs/RUST_REWRITE.md: rust/collector/src/web (axum) serves what
the FastAPI app (src/web_api.py) served, so one process collects, forecasts,
notifies and serves. With WEB_LISTEN=0.0.0.0:8000 it runs inside
`ping-collector run`; `ping-collector serve` runs it alone.
Same routes, parameters, JSON fields and field order, status codes,
{"detail": ...} errors and pydantic-style 422 lists, Cache-Control, gzip
(>500 B), CORS, Umami API tracking, the dashboard and static files, and the
stale-while-revalidate response caches (now with byte budgets: a history
response can be 10 MB). /docs is the same Swagger UI (+ /redoc); its OpenAPI
document is built from the route table, whose per-route parameter
declarations also validate the requests, so docs and behaviour cannot drift.
Python output conventions are kept: isoformat(), round(), numpy's pairwise
sums (catchment means), Python's summation order (skill metrics).
scripts/api_parity.py compares the two servers route by route on one
database (53 cases incl. errors, admin and static files; query values vary
per run so FastAPI's caches answer fresh). Locally: 53 identical; the
station CRUD writes byte-identical stations.json files; the skill/trend
logic matches on 9 station/horizon pairs with synthetic model versions.
Intended differences: /health counts the ThaiWater telemetry of the same
gauges while RID lags (as the notifier); /scraping/status adds the collector
version and last forecast run; an old stored forecast is served marked
X-Data-Stale instead of recomputed; /sensors/thaiwater is cached a minute;
.geojson is served as application/geo+json. Load (local, 16 clients): the
dashboard's routes 1375 vs 552 req/s, uncached history 714 vs 53 req/s;
25 MB resident when serving only.
Matrix (MATRIX_*): the notifier sends the room the transitions ntfy gets
(per station, outlook, feed; not ntfy's per-topic summaries), each once,
with its own matrix: keys in notification_state, seeded from ntfy's on first
use so nothing already announced repeats. Checked against a fake homeserver.
It replaces the `run.py --alert-check` cron job (repeated alerts every 15
min), which install_collector.sh now comments out (crontab backup kept).
install_collector.sh: --check-api (ping-collector serve on :8001 next to
FastAPI, api_parity.py), --cutover-api (refuses on a difference; sets
WEB_LISTEN, stops and disables water-monitor.service), --rollback-api;
--rollback-forecast and --rollback bring the Python app back as server.
ping-collector.service: MemoryMax 512M. Config::from_vars for tests; route
tests against a throwaway schema.
This commit is contained in:
grabowski committed
1 parent
1d1c000922
commit
779728e38b
32 files changed
+5692
-160
No files matched your search
@@ -0,0 +1,293 @@
|
||||
//! Route tests: the router as the server runs it, against a throwaway
|
||||
//! schema (TEST_DATABASE_URL; see db_tests.rs) or, for the routes that do
|
||||
//! not touch the database, a pool that never connects. Output formats are
|
||||
//! the FastAPI app's (scripts/api_parity.py compares the two live).
|
||||
|
||||
use super::{AppState, router};
|
||||
use crate::config::Config;
|
||||
use crate::db_tests::TestDb;
|
||||
use crate::stations::Stations;
|
||||
use axum::Router;
|
||||
use axum::body::Body;
|
||||
use axum::http::{HeaderMap, Request, StatusCode};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tower::ServiceExt;
|
||||
|
||||
fn config(extra: &[(&str, &str)]) -> Config {
|
||||
let static_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../../src/static");
|
||||
let mut vars: HashMap<String, String> = [
|
||||
(
|
||||
"POSTGRES_CONNECTION_STRING",
|
||||
"postgres://test@localhost:1/none",
|
||||
),
|
||||
("UMAMI_TRACK_API", "false"),
|
||||
("STATIC_DIR", static_dir),
|
||||
("NTFY_SERVER", "https://ntfy.example"),
|
||||
("STATION_CONFIG_PATH", "/nonexistent/stations.json"),
|
||||
]
|
||||
.iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||
.collect();
|
||||
for (k, v) in extra {
|
||||
vars.insert(k.to_string(), v.to_string());
|
||||
}
|
||||
Config::from_vars(&|k| vars.get(k).cloned()).unwrap()
|
||||
}
|
||||
|
||||
fn app(cfg: Config, pool: sqlx::PgPool) -> Router {
|
||||
let state = AppState::new(
|
||||
cfg,
|
||||
pool,
|
||||
reqwest::Client::new(),
|
||||
Arc::new(RwLock::new(Stations::bundled())),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
router(state)
|
||||
}
|
||||
|
||||
async fn call(
|
||||
app: &Router,
|
||||
method: &str,
|
||||
uri: &str,
|
||||
headers: &[(&str, &str)],
|
||||
) -> (StatusCode, HeaderMap, Vec<u8>) {
|
||||
let mut req = Request::builder().method(method).uri(uri);
|
||||
for (k, v) in headers {
|
||||
req = req.header(*k, *v);
|
||||
}
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(req.body(Body::empty()).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let (parts, body) = resp.into_parts();
|
||||
let bytes = axum::body::to_bytes(body, usize::MAX).await.unwrap();
|
||||
(parts.status, parts.headers, bytes.to_vec())
|
||||
}
|
||||
|
||||
async fn get_json(app: &Router, uri: &str) -> (StatusCode, HeaderMap, Value) {
|
||||
let (s, h, b) = call(app, "GET", uri, &[]).await;
|
||||
let v = serde_json::from_slice(&b)
|
||||
.unwrap_or_else(|e| panic!("{uri}: {e}: {}", String::from_utf8_lossy(&b)));
|
||||
(s, h, v)
|
||||
}
|
||||
|
||||
fn header<'a>(h: &'a HeaderMap, name: &str) -> Option<&'a str> {
|
||||
h.get(name).and_then(|v| v.to_str().ok())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn routes_without_database() {
|
||||
let pool = sqlx::PgPool::connect_lazy("postgres://test@localhost:1/none").unwrap();
|
||||
let app = app(config(&[]), pool);
|
||||
|
||||
let (s, h, stations) = get_json(&app, "/stations").await;
|
||||
assert_eq!(s, StatusCode::OK);
|
||||
assert_eq!(header(&h, "cache-control"), Some("public, max-age=300"));
|
||||
assert_eq!(header(&h, "content-type"), Some("application/json"));
|
||||
assert_eq!(stations.as_array().unwrap().len(), 16);
|
||||
// the list leaves geohash out, the single station has it (as FastAPI)
|
||||
assert_eq!(stations[7]["station_code"], "P.1");
|
||||
assert_eq!(stations[7]["geohash"], Value::Null);
|
||||
let (_, _, p1) = get_json(&app, "/stations/8").await;
|
||||
assert_eq!(
|
||||
p1.as_object().unwrap().keys().collect::<Vec<_>>(),
|
||||
[
|
||||
"station_id",
|
||||
"station_code",
|
||||
"thai_name",
|
||||
"english_name",
|
||||
"latitude",
|
||||
"longitude",
|
||||
"geohash",
|
||||
"status"
|
||||
]
|
||||
);
|
||||
|
||||
let (s, _, e) = get_json(&app, "/stations/abc").await;
|
||||
assert_eq!(s, StatusCode::UNPROCESSABLE_ENTITY);
|
||||
assert_eq!(e["detail"][0]["type"], "int_parsing");
|
||||
assert_eq!(e["detail"][0]["loc"], json!(["path", "station_id"]));
|
||||
assert_eq!(
|
||||
get_json(&app, "/stations/99").await.2,
|
||||
json!({"detail": "Station not found"})
|
||||
);
|
||||
assert_eq!(
|
||||
get_json(&app, "/nope").await.2,
|
||||
json!({"detail": "Not Found"})
|
||||
);
|
||||
let (s, _, b) = call(&app, "DELETE", "/forecast", &[]).await;
|
||||
assert_eq!(s, StatusCode::METHOD_NOT_ALLOWED);
|
||||
assert_eq!(b, br#"{"detail":"Method Not Allowed"}"#);
|
||||
|
||||
let (s, _, e) = get_json(&app, "/config").await;
|
||||
assert_eq!(s, StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(
|
||||
e["detail"],
|
||||
"Admin API disabled: ADMIN_API_KEY is not configured on the server"
|
||||
);
|
||||
let (s, _, e) = get_json(&app, "/measurements/history/P.1?hours=0").await;
|
||||
assert_eq!(s, StatusCode::UNPROCESSABLE_ENTITY);
|
||||
assert_eq!(e["detail"][0]["ctx"], json!({"ge": 1}));
|
||||
|
||||
let (_, _, n) = get_json(&app, "/api/notifications").await;
|
||||
assert_eq!(n["topics"]["warning"], "ping-warning");
|
||||
assert_eq!(n["server"], "https://ntfy.example");
|
||||
|
||||
let (s, h, spec) = get_json(&app, "/openapi.json").await;
|
||||
assert_eq!(s, StatusCode::OK);
|
||||
assert_eq!(header(&h, "content-encoding"), None);
|
||||
assert!(spec["paths"]["/forecast"]["get"].is_object());
|
||||
|
||||
let (s, h, html) = call(&app, "GET", "/", &[("accept-encoding", "gzip")]).await;
|
||||
assert_eq!(s, StatusCode::OK);
|
||||
assert_eq!(header(&h, "cache-control"), Some("public, max-age=120"));
|
||||
assert_eq!(header(&h, "content-encoding"), Some("gzip"));
|
||||
assert!(html.len() > 100);
|
||||
let (s, h, _) = call(&app, "GET", "/docs", &[]).await;
|
||||
assert_eq!(s, StatusCode::OK);
|
||||
assert_eq!(header(&h, "content-type"), Some("text/html; charset=utf-8"));
|
||||
let (s, h, _) = call(&app, "GET", "/static/custom-markers.json", &[]).await;
|
||||
assert_eq!(s, StatusCode::OK);
|
||||
assert_eq!(header(&h, "cache-control"), Some("public, max-age=3600"));
|
||||
let (s, h, _) = call(&app, "GET", "/robots.txt", &[]).await;
|
||||
assert_eq!(s, StatusCode::OK);
|
||||
assert_eq!(
|
||||
header(&h, "content-type"),
|
||||
Some("text/plain; charset=utf-8")
|
||||
);
|
||||
assert_eq!(
|
||||
call(&app, "GET", "/static/nope.json", &[]).await.0,
|
||||
StatusCode::NOT_FOUND
|
||||
);
|
||||
|
||||
// admin key
|
||||
let app = super::router(AppState::new(
|
||||
config(&[("ADMIN_API_KEY", "k3y")]),
|
||||
sqlx::PgPool::connect_lazy("postgres://test@localhost:1/none").unwrap(),
|
||||
reqwest::Client::new(),
|
||||
Arc::new(RwLock::new(Stations::bundled())),
|
||||
None,
|
||||
None,
|
||||
));
|
||||
assert_eq!(
|
||||
call(&app, "GET", "/config", &[("x-api-key", "nope")])
|
||||
.await
|
||||
.0,
|
||||
StatusCode::UNAUTHORIZED
|
||||
);
|
||||
let (s, _, b) = call(&app, "GET", "/config", &[("x-api-key", "k3y")]).await;
|
||||
assert_eq!(s, StatusCode::OK, "{}", String::from_utf8_lossy(&b));
|
||||
assert!(String::from_utf8_lossy(&b).contains("postgres://test:***@localhost:1/none"));
|
||||
let (s, _, b) = call(&app, "POST", "/scrape/trigger", &[("x-api-key", "k3y")]).await;
|
||||
assert_eq!(s, StatusCode::CONFLICT, "{}", String::from_utf8_lossy(&b));
|
||||
}
|
||||
|
||||
async fn seed(pool: &sqlx::PgPool) {
|
||||
for sql in [
|
||||
"INSERT INTO stations (id, station_code, thai_name, english_name) VALUES
|
||||
(1, 'P.20', 'บ้านเชียงดาว', 'Ban Chiang Dao'), (8, 'P.1', 'สะพานนวรัฐ', 'Nawarat Bridge')",
|
||||
// P.1 hourly 2026-10-01 00:00 .. 10-03 23:00, one missing discharge, a hole at 10-02 05:00
|
||||
"INSERT INTO water_measurements (timestamp, station_id, water_level, discharge, discharge_percent, status)
|
||||
SELECT t, 8, round((1.2 + extract(hour from t) / 100.0)::numeric, 3),
|
||||
CASE WHEN t = timestamp '2026-10-02 03:00' THEN NULL ELSE 55.5 END, 12.94, 'active'
|
||||
FROM generate_series(timestamp '2026-10-01 00:00', timestamp '2026-10-03 23:00', interval '1 hour') t
|
||||
WHERE t <> timestamp '2026-10-02 05:00'",
|
||||
"INSERT INTO water_measurements (timestamp, station_id, water_level, discharge, discharge_percent, status)
|
||||
VALUES (timestamp '2026-10-03 22:00', 1, 0.95, NULL, NULL, 'active')",
|
||||
"INSERT INTO forecast_history (as_of, station_code, horizon_hours, predicted_max_level, p_warning,
|
||||
p_danger, current_level, model_version, source)
|
||||
SELECT t, 'P.1', 6, 1.5, 0.01, 0.0, 1.2, CASE WHEN t < timestamp '2026-10-02 00:00' THEN 'v1' ELSE 'v2' END, 'model'
|
||||
FROM generate_series(timestamp '2026-10-01 00:00', timestamp '2026-10-03 12:00', interval '1 hour') t",
|
||||
"INSERT INTO forecast_current (id, computed_at, model_versions, payload)
|
||||
VALUES ('all', timestamp '2026-10-03 23:10', 'v2', '[{\"station_code\":\"P.1\",\"horizon_hours\":6}]')",
|
||||
] {
|
||||
sqlx::query(sql).execute(pool).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn routes_against_the_database() {
|
||||
let Some(db) = TestDb::new().await else {
|
||||
return;
|
||||
};
|
||||
seed(&db.pool).await;
|
||||
let app = app(
|
||||
config(&[("FORECAST_MAX_AGE_SECONDS", "60")]),
|
||||
db.pool.clone(),
|
||||
);
|
||||
|
||||
let (s, h, latest) = get_json(&app, "/measurements/latest?limit=5").await;
|
||||
assert_eq!(s, StatusCode::OK);
|
||||
assert_eq!(header(&h, "cache-control"), Some("public, max-age=30"));
|
||||
// ORDER BY station_code: P.1 before P.20
|
||||
assert_eq!(
|
||||
latest,
|
||||
json!([
|
||||
{"timestamp": "2026-10-03T23:00:00", "station_code": "P.1", "station_name_en": "Nawarat Bridge",
|
||||
"station_name_th": "สะพานนวรัฐ", "water_level": 1.43, "discharge": 55.5,
|
||||
"discharge_percent": 12.94, "status": "active"},
|
||||
{"timestamp": "2026-10-03T22:00:00", "station_code": "P.20", "station_name_en": "Ban Chiang Dao",
|
||||
"station_name_th": "บ้านเชียงดาว", "water_level": 0.95, "discharge": null,
|
||||
"discharge_percent": null, "status": "active"}
|
||||
])
|
||||
);
|
||||
|
||||
let (_, _, h) = get_json(
|
||||
&app,
|
||||
"/measurements/history/P.1?start=2026-10-02&end=2026-10-02&limit=6",
|
||||
)
|
||||
.await;
|
||||
let rows = h.as_array().unwrap();
|
||||
assert_eq!(rows.len(), 6);
|
||||
assert_eq!(rows[0]["timestamp"], "2026-10-02T00:00:00");
|
||||
// discharge missing in the database: 20 * H, rounded
|
||||
assert_eq!(rows[3]["discharge"], json!(24.6));
|
||||
assert_eq!(rows[5]["timestamp"], "2026-10-02T06:00:00"); // the hole at 05:00
|
||||
let (s, _, e) = get_json(
|
||||
&app,
|
||||
"/measurements/history/P.1?start=2026-10-03&end=2026-10-02",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(s, StatusCode::BAD_GATEWAY);
|
||||
assert_eq!(e["detail"], "Measurement history unavailable");
|
||||
|
||||
let (_, _, stats) = get_json(&app, "/api/stats").await;
|
||||
assert_eq!(stats["rid_measurements"], 72);
|
||||
assert_eq!(stats["rid_station_count"], 2);
|
||||
assert_eq!(stats["first_timestamp"], "2026-10-01T00:00:00");
|
||||
assert_eq!(stats["days_spanned"], 3);
|
||||
assert_eq!(stats["coverage_percent"], json!(98.6)); // 71 of 72 hours
|
||||
|
||||
let (_, _, fh) = get_json(&app, "/api/forecast/history/P.1?start=2026-10-03&horizon=6").await;
|
||||
assert_eq!(fh.as_array().unwrap().len(), 13);
|
||||
assert_eq!(fh[0]["predicted_max_level"], json!(1.5));
|
||||
|
||||
let (s, h, f) = get_json(&app, "/forecast").await;
|
||||
assert_eq!(s, StatusCode::OK);
|
||||
assert_eq!(f[0]["station_code"], "P.1");
|
||||
// computed long ago: served, marked
|
||||
assert_eq!(header(&h, "x-data-stale"), Some("true"));
|
||||
|
||||
let (_, _, sk) = get_json(&app, "/api/forecast/skill?station_code=P.1&horizon=6").await;
|
||||
let versions = sk["versions"].as_array().unwrap();
|
||||
assert_eq!(versions.len(), 2);
|
||||
assert_eq!(versions[0]["model_version"], "v1");
|
||||
assert_eq!(versions[0]["n"], 24);
|
||||
assert_eq!(versions[0]["enough_data"], true);
|
||||
assert_eq!(sk["current"]["model_version"], "v2");
|
||||
assert_eq!(sk["trend"]["previous_version"], "v1");
|
||||
|
||||
let (_, _, health) = get_json(&app, "/health").await;
|
||||
assert_eq!(health["checks"]["database"]["status"], "degraded"); // data from 2026-10-03
|
||||
assert!(
|
||||
health["checks"]["database"]["message"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.starts_with("Latest data is old: 2026-10-03 23:00:00")
|
||||
);
|
||||
db.drop().await;
|
||||
}
|
||||
Reference in new issue
Block a user