grabowski
7e642555b4
Phase 4 of docs/RUST_REWRITE.md. Since 2026-10-18 ping-collector (rust/) collects, forecasts, notifies (ntfy + Matrix) and serves the website/API; nothing in production ran the Python service code any more. Removed (~17k lines): the FastAPI app (web_api, schemas, health_check, metrics, postgres_history, forecast_history, thaiwater), the Python RID collector and CLI (water_scraper_v3, main, run.py), the notifier and Matrix alerting (notify, alerting), the multi-backend adapters (SQLite, MySQL, InfluxDB, VictoriaMetrics) with demo databases, models/validators/exceptions/ logging/rate limiter, ml/skill.py (served by Rust now), the migration tooling (api_parity, collector_parity, e2e_notify, install_collector.sh, gen_notify_cases.py, spikes/rust-forecast), Docker/compose/Grafana files, sql/, the release workflow and the docs for all of that. Their tests went with them. Kept in Python: src/ml (training, evaluation harness, export; predict.py as the reference the Rust forecaster is tested against), the HII and RID-dam clients/stores for the archive backfills (their live collectors removed), the backfill scripts, retrain.sh + timer, forecast_parity.py. - PostgreSQL only: src/config.py is database_url() from POSTGRES_* / .env; the stores lost their sqlite/mysql branches (and the 2026-08 column migration). The database tests run against a throwaway schema from TEST_DATABASE_URL (tests/conftest.py pg_url, skipped without it), as the Rust tests do; CI gets a PostgreSQL service. 100 tests (66 without a DB). - The notifier settings moved from src/notify.py into src/ml/export.py; flood_config.json regenerates byte-identical. - tests/test_dashboard.py checks every path the dashboard fetches against ping-collector's router. - Dependencies: fastapi, uvicorn, pydantic, influxdb, pymysql, psutil, schedule, sphinx, pyinstaller, mypy, pytest-asyncio dropped; python-dotenv 1.1.1 explicit (it came in through uvicorn[standard]). - Formatting contract extended to scripts/ and tests/ (reformatted once). - scripts/install.sh is the one installer (fresh host or update, in place in a checkout without touching its ownership): binary, units, uv sync, .env defaults, removes the retired water-monitor.service and any old --alert-check cron job, dry run + forecast check, restart, readiness wait. - Docs, CLAUDE.md, CONTRIBUTING.md, .env.example, Makefile, unit comments and Rust doc comments updated to the Rust service.
233 lines
8.6 KiB
Python
233 lines
8.6 KiB
Python
import re
|
|
from pathlib import Path
|
|
|
|
DASHBOARD_PATH = Path(__file__).parents[1] / "src" / "static" / "dashboard.html"
|
|
ROUTER_PATH = (
|
|
Path(__file__).parents[1] / "rust" / "collector" / "src" / "web" / "mod.rs"
|
|
)
|
|
|
|
|
|
def _rust_routes() -> set:
|
|
"""Paths the website serves (ping-collector's axum router)."""
|
|
source = ROUTER_PATH.read_text(encoding="utf-8")
|
|
return set(re.findall(r'\.route\(\s*"([^"]+)"', source))
|
|
|
|
|
|
def test_every_api_path_the_dashboard_fetches_is_served():
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
routes = _rust_routes()
|
|
patterns = [
|
|
re.compile("^" + re.sub(r"\\\{[^}]+\\\}", "[^/]+", re.escape(r)) + "$")
|
|
for r in routes
|
|
]
|
|
fetched = set(re.findall(r"fetch\(\s*[`'\"](/[^`'\"?$]*)", html))
|
|
assert fetched, "no fetch() calls found"
|
|
missing = [
|
|
path
|
|
for path in fetched
|
|
if not path.startswith("/static/")
|
|
and not any(p.match(path.rstrip("/") or "/") for p in patterns)
|
|
and not any(r.startswith(path) for r in routes)
|
|
]
|
|
assert not missing, f"dashboard fetches paths the server does not route: {missing}"
|
|
|
|
|
|
def test_dashboard_contains_live_map_and_flow_visualization():
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
|
|
assert 'id="station-map"' in html
|
|
assert 'id="river-flow"' in html
|
|
assert "fetch('/stations')" in html or 'fetch("/stations")' in html
|
|
assert (
|
|
"fetch('/measurements/latest" in html or 'fetch("/measurements/latest' in html
|
|
)
|
|
assert "leaflet" in html.lower()
|
|
|
|
|
|
def _body(html: str) -> str:
|
|
"""Markup only. The STRINGS table repeats every English phrase, so a
|
|
whole-file search would pass even if an element were deleted."""
|
|
return html[html.index("<body>") : html.index("<script src=")]
|
|
|
|
|
|
def test_dashboard_explains_flow_legend_and_refresh():
|
|
body = _body(DASHBOARD_PATH.read_text(encoding="utf-8"))
|
|
|
|
assert 'data-i18n="legend.flow"' in body
|
|
assert 'data-i18n="stat.updated"' in body
|
|
assert 'data-i18n="action.refresh"' in body
|
|
|
|
|
|
def test_dashboard_uses_mapped_river_network_instead_of_station_connections():
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
river_network = DASHBOARD_PATH.with_name("ping-river-network.geojson")
|
|
|
|
assert river_network.exists()
|
|
assert "fetch('/static/ping-river-network.geojson')" in html
|
|
assert "mainBasin.map" not in html
|
|
|
|
|
|
def test_dashboard_loads_additional_thaiwater_sensors():
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
|
|
assert "fetch('/sensors/thaiwater')" in html
|
|
assert 'data-i18n="sensors.title"' in _body(html)
|
|
assert 'id="station-search"' in html
|
|
assert "applyStationSearch" in html
|
|
|
|
|
|
def test_dashboard_has_seo_and_indexing_files():
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
static_dir = DASHBOARD_PATH.parent
|
|
|
|
assert '<meta name="description"' in html
|
|
assert '<link rel="canonical" href="https://water.buildfor.life/">' in html
|
|
assert 'property="og:title"' in html
|
|
assert 'rel="icon"' in html
|
|
|
|
robots = (static_dir / "robots.txt").read_text(encoding="utf-8")
|
|
assert "Sitemap: https://water.buildfor.life/sitemap.xml" in robots
|
|
assert "<loc>https://water.buildfor.life/</loc>" in (
|
|
static_dir / "sitemap.xml"
|
|
).read_text(encoding="utf-8")
|
|
assert "Ping River Live Monitor" in (static_dir / "llms.txt").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
assert {"/robots.txt", "/llms.txt", "/sitemap.xml"} <= _rust_routes()
|
|
|
|
|
|
def test_dashboard_shows_hii_rainfall_layer():
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
|
|
assert "fetch('/api/hii/rainfall/latest')" in html
|
|
assert "fetch('/api/hii/waterlevel/latest')" in html
|
|
assert "renderRainLayer" in html
|
|
assert "rain-toggle" in html
|
|
body = _body(html)
|
|
assert 'data-i18n="legend.rain"' in body
|
|
# TMD rain classes on the legend
|
|
assert 'data-i18n="legend.rain.heavy"' in body
|
|
assert 'data-i18n="legend.rain.extreme"' in body
|
|
|
|
|
|
def test_dashboard_loads_station_history_chart():
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
|
|
assert 'id="history-card"' in html
|
|
assert "/api/forecast/history/" in html
|
|
assert "'chart.model'" in html
|
|
assert "PostgreSQL" not in html
|
|
assert "/measurements/history/" in html
|
|
assert "history-chart" in html
|
|
# Date-range picker alongside the quick-range dropdown
|
|
assert 'id="history-start"' in html
|
|
assert 'id="history-end"' in html
|
|
assert 'data-i18n="range.7d"' in _body(html)
|
|
assert 'data-i18n="range.90d"' in _body(html)
|
|
|
|
|
|
def _extract_lang_tables(html: str) -> dict:
|
|
"""Pull the `en:` / `th:` key sets out of the STRINGS literal.
|
|
|
|
Parsing the JS with a regex is crude, but it is enough to catch the failure
|
|
that matters: a key added to one language and forgotten in the other, which
|
|
silently falls back to English for Thai readers.
|
|
"""
|
|
import re
|
|
|
|
start = html.index("const STRINGS = {")
|
|
end = html.index("// Thai unless the visitor's browser", start)
|
|
block = html[start:end]
|
|
tables = {}
|
|
for lang in ("en", "th"):
|
|
section = re.search(rf"\n {lang}: {{\n(.*?)\n }},\n", block, re.S)
|
|
assert section, f"{lang} table not found in STRINGS"
|
|
tables[lang] = set(re.findall(r"^\s{12}'([^']+)':", section.group(1), re.M))
|
|
return tables
|
|
|
|
|
|
def test_dashboard_translations_cover_both_languages():
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
tables = _extract_lang_tables(html)
|
|
|
|
assert len(tables["en"]) > 100, "expected the full English string table"
|
|
missing_th = tables["en"] - tables["th"]
|
|
missing_en = tables["th"] - tables["en"]
|
|
assert not missing_th, f"keys missing a Thai translation: {sorted(missing_th)}"
|
|
assert (
|
|
not missing_en
|
|
), f"Thai-only keys with no English fallback: {sorted(missing_en)}"
|
|
|
|
|
|
def test_dashboard_i18n_markup_keys_exist():
|
|
"""Every data-i18n attribute must resolve to a real string key."""
|
|
import re
|
|
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
keys = _extract_lang_tables(html)["en"]
|
|
used = set(re.findall(r'data-i18n(?:-placeholder|-title|-aria)?="([^"]+)"', html))
|
|
unknown = used - keys
|
|
assert not unknown, f"markup references undefined string keys: {sorted(unknown)}"
|
|
|
|
|
|
def test_dashboard_is_mobile_portrait_safe():
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
|
|
# The header row overflowed a 412 px Android viewport by 64 px until it wrapped
|
|
assert "flex-wrap: wrap" in html
|
|
assert "overflow-x: hidden" in html
|
|
assert 'id="lang-toggle"' in html
|
|
assert "ping-monitor-lang" in html # remembered language choice
|
|
|
|
|
|
def test_dashboard_translates_user_visible_aria_labels():
|
|
"""A Thai page must not hand screen-reader users English landmarks."""
|
|
import re
|
|
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
body = _body(html)
|
|
for match in re.finditer(r'<[^>]*\saria-label="[^"]+"[^>]*>', body):
|
|
tag = match.group(0)
|
|
if "data-i18n-aria" in tag or 'id="lang-toggle"' in tag:
|
|
continue # the toggle sets its own label per language in JS
|
|
raise AssertionError(f"aria-label without a translation key: {tag[:120]}")
|
|
|
|
|
|
def test_dashboard_keeps_simulation_and_replay_labels_on_language_switch():
|
|
"""Relabelling a pinned simulation as LIVE would present fake flood data as real."""
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
|
|
assert "setLiveIndicator(state.liveMode || 'live', state.liveLabelKey)" in html
|
|
assert "if (!state.replayTimer) setLiveIndicator('live');" not in html
|
|
|
|
|
|
def test_dashboard_default_language_respects_browser_order():
|
|
"""navigator.languages = ['th-TH','en-US'] must resolve to Thai, not English."""
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
|
|
assert "langs.some" not in html # the old any-English-wins test
|
|
assert "if (code.startsWith('th')) return 'th';" in html
|
|
|
|
|
|
def test_dashboard_shows_one_current_river_level():
|
|
"""The verdict banner and the P.1 outlook must not disagree about "now".
|
|
|
|
Production served 1.66 m in the banner and 1.52 m in the outlook at the
|
|
same moment: the banner used the latest measurement, the outlook used the
|
|
forecast payload's current_level from an older as_of.
|
|
"""
|
|
html = DASHBOARD_PATH.read_text(encoding="utf-8")
|
|
|
|
# One arbitrated writer, freshest-wins
|
|
assert "function setP1Level(" in html
|
|
assert "state.p1NowAt" in html
|
|
|
|
# Neither feed may assign the level directly any more
|
|
assert "state.p1Now = Number(p1.water_level)" not in html
|
|
assert "state.p1Now = row.current_level" not in html
|
|
|
|
# The outlook renders the arbitrated level, and never a peak below it
|
|
assert "const shownNow = state.p1Now != null" in html
|
|
assert "Math.max(Number(row.predicted_max_level), shownNow)" in html
|