grabowski
779728e38b
Phase 3 of docs/RUST_REWRITE.md: rust/collector/src/web (axum) serves what
the FastAPI app (src/web_api.py) served, so one process collects, forecasts,
notifies and serves. With WEB_LISTEN=0.0.0.0:8000 it runs inside
`ping-collector run`; `ping-collector serve` runs it alone.
Same routes, parameters, JSON fields and field order, status codes,
{"detail": ...} errors and pydantic-style 422 lists, Cache-Control, gzip
(>500 B), CORS, Umami API tracking, the dashboard and static files, and the
stale-while-revalidate response caches (now with byte budgets: a history
response can be 10 MB). /docs is the same Swagger UI (+ /redoc); its OpenAPI
document is built from the route table, whose per-route parameter
declarations also validate the requests, so docs and behaviour cannot drift.
Python output conventions are kept: isoformat(), round(), numpy's pairwise
sums (catchment means), Python's summation order (skill metrics).
scripts/api_parity.py compares the two servers route by route on one
database (53 cases incl. errors, admin and static files; query values vary
per run so FastAPI's caches answer fresh). Locally: 53 identical; the
station CRUD writes byte-identical stations.json files; the skill/trend
logic matches on 9 station/horizon pairs with synthetic model versions.
Intended differences: /health counts the ThaiWater telemetry of the same
gauges while RID lags (as the notifier); /scraping/status adds the collector
version and last forecast run; an old stored forecast is served marked
X-Data-Stale instead of recomputed; /sensors/thaiwater is cached a minute;
.geojson is served as application/geo+json. Load (local, 16 clients): the
dashboard's routes 1375 vs 552 req/s, uncached history 714 vs 53 req/s;
25 MB resident when serving only.
Matrix (MATRIX_*): the notifier sends the room the transitions ntfy gets
(per station, outlook, feed; not ntfy's per-topic summaries), each once,
with its own matrix: keys in notification_state, seeded from ntfy's on first
use so nothing already announced repeats. Checked against a fake homeserver.
It replaces the `run.py --alert-check` cron job (repeated alerts every 15
min), which install_collector.sh now comments out (crontab backup kept).
install_collector.sh: --check-api (ping-collector serve on :8001 next to
FastAPI, api_parity.py), --cutover-api (refuses on a difference; sets
WEB_LISTEN, stops and disables water-monitor.service), --rollback-api;
--rollback-forecast and --rollback bring the Python app back as server.
ping-collector.service: MemoryMax 512M. Config::from_vars for tests; route
tests against a throwaway schema.
390 lines
15 KiB
Python
390 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare the Rust website/API with the FastAPI app, route by route.
|
|
|
|
Phase 3 of docs/RUST_REWRITE.md: before the Rust service takes over port
|
|
8000, both answer the same requests against the same database and every
|
|
response is compared: status, content type, Cache-Control and the JSON body
|
|
(floats exactly; lists of database rows without ORDER BY as sets). Static
|
|
files and the dashboard byte for byte.
|
|
|
|
# the FastAPI app on :8000, `ping-collector serve --listen 127.0.0.1:8001`
|
|
uv run python scripts/api_parity.py \\
|
|
--python http://127.0.0.1:8000 --rust http://127.0.0.1:8001
|
|
|
|
Query parameters vary per run (limits, hours) so the FastAPI app's response
|
|
caches (up to 15 min) answer with fresh data, not an older copy. Data the
|
|
collector writes during the run can still make a route differ once: a
|
|
mismatch is retried after a pause before it counts.
|
|
|
|
Intended differences (the Rust side may answer differently):
|
|
/health a lagging RID report with current ThaiWater telemetry for
|
|
the same gauges is healthy (as the notifier, phase 2)
|
|
/scraping/status extra fields (collector version, forecast run)
|
|
/metrics process counters: compared by shape only
|
|
/sensors/thaiwater live upstream: compared, a difference only warns
|
|
/api/hii/rainfall/catchment correlation within 1e-9 (numpy uses BLAS)
|
|
/api/stats newer counts on the Rust side are accepted (the FastAPI app
|
|
caches this one for 5 minutes)
|
|
*.geojson content type application/geo+json (FastAPI: octet-stream)
|
|
|
|
Exit status 0 when every hard check passed.
|
|
"""
|
|
|
|
import argparse
|
|
import random
|
|
import sys
|
|
import time
|
|
from typing import Any, Callable, List, Optional, Tuple
|
|
|
|
import requests
|
|
|
|
SOFT = "soft" # a difference warns, does not fail
|
|
|
|
|
|
def get(base: str, path: str, method: str = "GET", **kw) -> requests.Response:
|
|
last = None
|
|
for _ in range(3):
|
|
try:
|
|
return requests.request(method, base + path, timeout=120, **kw)
|
|
except requests.RequestException as e: # connection reset etc.
|
|
last = e
|
|
time.sleep(1)
|
|
raise last
|
|
|
|
|
|
def first_diff(a: Any, b: Any, path: str = "$", rel: float = 0.0) -> Optional[str]:
|
|
"""Where two decoded JSON values differ (None if equal)."""
|
|
if isinstance(a, bool) or isinstance(b, bool):
|
|
return None if a is b else f"{path}: {a!r} != {b!r}"
|
|
if isinstance(a, (int, float)) and isinstance(b, (int, float)):
|
|
if a == b:
|
|
return None
|
|
if rel and abs(a - b) <= rel * max(abs(a), abs(b)):
|
|
return None
|
|
return f"{path}: {a!r} != {b!r}"
|
|
if type(a) is not type(b):
|
|
return f"{path}: {type(a).__name__} {a!r} != {type(b).__name__} {b!r}"
|
|
if isinstance(a, dict):
|
|
if list(a.keys()) != list(b.keys()):
|
|
if set(a.keys()) != set(b.keys()):
|
|
return f"{path}: keys {sorted(set(a) ^ set(b))} only on one side"
|
|
for k in a:
|
|
d = first_diff(a[k], b[k], f"{path}.{k}", rel)
|
|
if d:
|
|
return d
|
|
return None
|
|
if isinstance(a, list):
|
|
if len(a) != len(b):
|
|
return f"{path}: {len(a)} items != {len(b)} items"
|
|
for i, (x, y) in enumerate(zip(a, b)):
|
|
d = first_diff(x, y, f"{path}[{i}]", rel)
|
|
if d:
|
|
return d
|
|
return None
|
|
return None if a == b else f"{path}: {a!r} != {b!r}"
|
|
|
|
|
|
def media_type(r: requests.Response) -> str:
|
|
return r.headers.get("content-type", "").split(";")[0].strip()
|
|
|
|
|
|
class Case:
|
|
def __init__(
|
|
self,
|
|
path: str,
|
|
method: str = "GET",
|
|
compare: Optional[Callable] = None,
|
|
sort_key: Optional[str] = None,
|
|
soft: bool = False,
|
|
body: Any = None,
|
|
headers: Optional[dict] = None,
|
|
):
|
|
self.path, self.method, self.compare = path, method, compare
|
|
self.sort_key, self.soft, self.body = sort_key, soft, body
|
|
self.headers = headers or {}
|
|
|
|
def run(self, py: str, rs: str) -> Tuple[Optional[str], Optional[str]]:
|
|
"""(hard difference, warning)"""
|
|
kw = {"headers": self.headers}
|
|
if self.body is not None:
|
|
kw["json"] = self.body
|
|
a = get(py, self.path, self.method, **kw)
|
|
b = get(rs, self.path, self.method, **kw)
|
|
if a.status_code != b.status_code:
|
|
return self._fail(
|
|
f"status {a.status_code} != {b.status_code} "
|
|
f"({a.text[:120]!r} / {b.text[:120]!r})"
|
|
)
|
|
if media_type(a) != media_type(b):
|
|
warn = f"content-type {media_type(a)!r} != {media_type(b)!r}"
|
|
if media_type(a).startswith("application/json") or media_type(b).startswith(
|
|
"application/json"
|
|
):
|
|
return self._fail(warn)
|
|
return None, warn
|
|
if self.method == "GET" and a.status_code == 200:
|
|
ca, cb = a.headers.get("cache-control"), b.headers.get("cache-control")
|
|
if ca != cb:
|
|
return self._fail(f"Cache-Control {ca!r} != {cb!r}")
|
|
if self.compare is not None:
|
|
d = self.compare(a, b)
|
|
elif media_type(a) == "application/json":
|
|
ja, jb = a.json(), b.json()
|
|
if a.status_code == 422:
|
|
# pydantic's messages are approximated; type and location must agree
|
|
ja = [(e["type"], e["loc"]) for e in ja["detail"]]
|
|
jb = [(e["type"], e["loc"]) for e in jb["detail"]]
|
|
if any(t.startswith("date") for t, _ in ja):
|
|
ja, jb = [loc for _, loc in ja], [loc for _, loc in jb]
|
|
elif self.sort_key and isinstance(ja, list):
|
|
ja = sorted(ja, key=lambda r: str(r.get(self.sort_key)))
|
|
jb = sorted(jb, key=lambda r: str(r.get(self.sort_key)))
|
|
d = first_diff(ja, jb)
|
|
else:
|
|
d = (
|
|
None
|
|
if a.content == b.content
|
|
else f"bodies differ ({len(a.content)} / {len(b.content)} bytes)"
|
|
)
|
|
return self._fail(d) if d else (None, None)
|
|
|
|
def _fail(self, d: str) -> Tuple[Optional[str], Optional[str]]:
|
|
return (None, d) if self.soft else (d, None)
|
|
|
|
|
|
def compare_health(a, b) -> Optional[str]:
|
|
ja, jb = a.json(), b.json()
|
|
if set(ja["checks"]) != set(jb["checks"]):
|
|
return f"checks {sorted(ja['checks'])} != {sorted(jb['checks'])}"
|
|
for name, ca in ja["checks"].items():
|
|
cb = jb["checks"][name]
|
|
if ca["status"] == cb["status"]:
|
|
continue
|
|
# the intended change: RID lagging, ThaiWater current
|
|
if (
|
|
name == "database"
|
|
and ca["status"] == "degraded"
|
|
and cb["status"] == "healthy"
|
|
and "ThaiWater telemetry current" in cb["message"]
|
|
):
|
|
continue
|
|
if name in ("api", "memory"):
|
|
continue # probes of the RID API / own process, compared loosely
|
|
return f"{name}: {ca} != {cb}"
|
|
return None
|
|
|
|
|
|
def compare_metrics(a, b) -> Optional[str]:
|
|
ja, jb = a.json(), b.json()
|
|
return None if set(ja) == set(jb) else f"keys {sorted(ja)} != {sorted(jb)}"
|
|
|
|
|
|
def compare_status(a, b) -> Optional[str]:
|
|
ja, jb = a.json(), b.json()
|
|
for k, v in ja.items():
|
|
if jb.get(k) != v:
|
|
return f"{k}: {v!r} != {jb.get(k)!r}"
|
|
return None
|
|
|
|
|
|
def compare_skill(a, b) -> Optional[str]:
|
|
ja, jb = a.json(), b.json()
|
|
va, vb = ja.pop("verified_until"), jb.pop("verified_until")
|
|
gap = abs(
|
|
time.mktime(time.strptime(va[:19], "%Y-%m-%dT%H:%M:%S"))
|
|
- time.mktime(time.strptime(vb[:19], "%Y-%m-%dT%H:%M:%S"))
|
|
)
|
|
if gap > 300:
|
|
return f"verified_until {va} != {vb}"
|
|
return first_diff(ja, jb)
|
|
|
|
|
|
def compare_stats(a, b) -> Optional[str]:
|
|
"""Exact, except that the FastAPI app may answer from its 5-minute cache
|
|
while the collector has written newer rows since."""
|
|
ja, jb = a.json(), b.json()
|
|
if ja == jb:
|
|
return None
|
|
for k, v in ja.items():
|
|
w = jb.get(k)
|
|
if k.endswith("_measurements") or k == "days_spanned":
|
|
if not isinstance(w, int) or w < v:
|
|
return f"{k}: {v!r} > {w!r}"
|
|
elif k == "last_timestamp":
|
|
if not isinstance(w, str) or w < v:
|
|
return f"{k}: {v!r} > {w!r}"
|
|
elif k == "coverage_percent":
|
|
if not isinstance(w, (int, float)) or abs(w - v) > 0.2:
|
|
return f"{k}: {v!r} != {w!r}"
|
|
elif v != w:
|
|
return f"{k}: {v!r} != {w!r}"
|
|
return None
|
|
|
|
|
|
def compare_catchment(a, b) -> Optional[str]:
|
|
return first_diff(a.json(), b.json(), rel=1e-9)
|
|
|
|
|
|
def compare_openapi(a, b) -> Optional[str]:
|
|
ja, jb = a.json(), b.json()
|
|
if ja["info"]["title"] != jb["info"]["title"]:
|
|
return "title differs"
|
|
ops = lambda spec: { # noqa: E731
|
|
(m, p): op for p, item in spec["paths"].items() for m, op in item.items()
|
|
}
|
|
oa, ob = ops(ja), ops(jb)
|
|
if set(oa) != set(ob):
|
|
return f"operations only on one side: {sorted(set(oa) ^ set(ob))}"
|
|
for k, op in oa.items():
|
|
other = ob[k]
|
|
for field in ("operationId", "summary", "parameters", "requestBody"):
|
|
d = first_diff(op.get(field), other.get(field), f"{k}.{field}")
|
|
if d:
|
|
return d
|
|
if set(op["responses"]) != set(other["responses"]):
|
|
return f"{k}: responses {sorted(op['responses'])} != {sorted(other['responses'])}"
|
|
for name in ja["components"]["schemas"]:
|
|
if name not in jb["components"]["schemas"]:
|
|
return f"schema {name} missing"
|
|
return None
|
|
|
|
|
|
def compare_html_docs(a, b) -> Optional[str]:
|
|
return (
|
|
None
|
|
if "swagger-ui" in b.text and "/openapi.json" in b.text
|
|
else "no Swagger UI"
|
|
)
|
|
|
|
|
|
def cases(admin_key: Optional[str]) -> List[Case]:
|
|
r = random.Random(time.time())
|
|
hours = r.randint(150, 190) # fresh keys for the Python app's caches
|
|
limit = r.randint(40000, 49999)
|
|
feed_hours = r.randint(20, 30)
|
|
skill_station = r.choice(
|
|
["P.67", "P.103", "P.20", "P.75", "P.92", "P.21", "P.5", "P.82"]
|
|
)
|
|
day = time.strftime("%Y-%m-%d", time.localtime(time.time() - 86400 * 3))
|
|
day2 = time.strftime("%Y-%m-%d", time.localtime(time.time() - 86400))
|
|
out = [
|
|
Case("/"),
|
|
Case("/robots.txt"),
|
|
Case("/llms.txt"),
|
|
Case("/sitemap.xml"),
|
|
Case("/static/ping-river-network.geojson", soft=True),
|
|
Case("/static/custom-markers.json"),
|
|
Case("/static/flood-zones-p1.geojson", soft=True),
|
|
Case("/static/flood-2024-all.json"),
|
|
Case("/static/nope.json"),
|
|
Case("/openapi.json", compare=compare_openapi),
|
|
Case("/docs", compare=compare_html_docs),
|
|
Case("/health", compare=compare_health),
|
|
Case("/metrics", compare=compare_metrics),
|
|
Case("/stations"),
|
|
Case("/stations/8"),
|
|
Case("/stations/1"),
|
|
Case("/stations/999"),
|
|
Case("/stations/abc"),
|
|
Case(f"/measurements/latest?limit={r.randint(200, 499)}"),
|
|
Case(f"/measurements/latest?limit={r.randint(1, 15)}"),
|
|
Case(f"/measurements/history/P.1?hours={hours}&limit={limit}"),
|
|
Case(f"/measurements/history/P.67?hours={hours + 1}"),
|
|
Case(f"/measurements/history/P.1?start={day}&end={day2}&limit={limit - 1}"),
|
|
Case(f"/measurements/history/P.1?start={day}&limit={r.randint(5, 50)}"),
|
|
Case(f"/measurements/history/P.1?start={day2}&end={day}"),
|
|
Case(f"/measurements/history/NOPE?hours={hours}"),
|
|
Case("/measurements/history/P.1?hours=0"),
|
|
Case("/measurements/history/P.1?limit=100001"),
|
|
Case("/measurements/history/P.1?start=yesterday"),
|
|
Case(f"/measurements/station/P.1?hours={r.randint(20, 48)}"),
|
|
Case("/measurements/station/P.103?hours=6&limit=2"),
|
|
Case(f"/api/hii/rainfall/latest?hours={feed_hours}", sort_key="station_id"),
|
|
Case(
|
|
f"/api/hii/waterlevel/latest?hours={feed_hours + 1}", sort_key="station_id"
|
|
),
|
|
Case("/api/hii/waterlevel/latest?hours=500"),
|
|
Case(
|
|
f"/api/hii/rainfall/catchment?days={r.randint(15, 30)}",
|
|
compare=compare_catchment,
|
|
),
|
|
Case("/sensors/thaiwater", sort_key="id", soft=True),
|
|
Case("/forecast"),
|
|
Case(f"/api/forecast/history/P.1?hours={hours}&horizon=24"),
|
|
Case(f"/api/forecast/history/P.103?hours={feed_hours}"),
|
|
Case(f"/api/forecast/history/P.1?start={day}&end={day2}&horizon=6"),
|
|
Case("/api/forecast/history/P.1?horizon=99"),
|
|
Case("/api/notifications"),
|
|
# a station/horizon pair the dashboard does not ask for: not in the
|
|
# FastAPI app's 15-minute cache; the dashboard's own one may be
|
|
Case(
|
|
f"/api/forecast/skill?station_code={skill_station}"
|
|
f"&horizon={r.randint(1, 48)}",
|
|
compare=compare_skill,
|
|
),
|
|
Case(
|
|
"/api/forecast/skill?station_code=P.1&horizon=24",
|
|
compare=compare_skill,
|
|
soft=True,
|
|
),
|
|
Case("/api/forecast/skill?horizon=0"),
|
|
Case("/api/stats", compare=compare_stats),
|
|
Case("/scraping/status", compare=compare_status),
|
|
Case("/nope"),
|
|
Case("/forecast", method="DELETE"),
|
|
Case("/config"),
|
|
Case("/scrape/trigger", method="POST"),
|
|
Case("/stations", method="POST", body={"station_code": "X"}),
|
|
Case("/stations/8", method="DELETE"),
|
|
]
|
|
if admin_key:
|
|
key = {"X-API-Key": admin_key}
|
|
out += [
|
|
Case("/config", headers={"X-API-Key": "wrong"}),
|
|
Case("/stations", method="POST", body={"thai_name": 1}, headers=key),
|
|
Case("/stations/999", method="PUT", body={}, headers=key),
|
|
]
|
|
return out
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
|
ap.add_argument("--python", default="http://127.0.0.1:8000")
|
|
ap.add_argument("--rust", default="http://127.0.0.1:8001")
|
|
ap.add_argument(
|
|
"--admin-key", help="ADMIN_API_KEY, to compare the admin errors too"
|
|
)
|
|
ap.add_argument("--retries", type=int, default=2)
|
|
ap.add_argument("--pause", type=float, default=30.0, help="seconds before a retry")
|
|
args = ap.parse_args()
|
|
|
|
failed, warned, passed = [], [], 0
|
|
for case in cases(args.admin_key):
|
|
label = f"{case.method} {case.path}"
|
|
for attempt in range(args.retries + 1):
|
|
try:
|
|
hard, warn = case.run(args.python, args.rust)
|
|
except Exception as e: # noqa: BLE001 - report and go on
|
|
hard, warn = f"{type(e).__name__}: {e}", None
|
|
if not hard or attempt == args.retries:
|
|
break
|
|
print(f" retry {label}: {hard}", flush=True)
|
|
time.sleep(args.pause)
|
|
if hard:
|
|
failed.append((label, hard))
|
|
print(f"FAIL {label}: {hard}", flush=True)
|
|
elif warn:
|
|
warned.append((label, warn))
|
|
print(f"warn {label}: {warn}", flush=True)
|
|
else:
|
|
passed += 1
|
|
print(
|
|
f"{passed} identical, {len(warned)} warnings, {len(failed)} different "
|
|
f"({args.python} vs {args.rust})"
|
|
)
|
|
return 1 if failed else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|