feat(forecast): Rust forecast crate, bit-for-bit with the Python serving path

Phase 2a of docs/RUST_REWRITE.md. rust/forecast (ping-forecast) ports the
serving half of src/ml: the HII mirror gap fill, the hourly grid, every
feature (pandas' Kahan rolling sums included), the Mae Ngat dam frame,
HistGradientBoosting tree inference, the heuristic fallback, stale-threshold
handling and the P.1 flood-stage probabilities.

Hand-over from Python (src/ml/export.py):
- models/flood_<station>.json.gz next to each joblib bundle; train_all() writes
  it, retrain.sh promotes it, `python -m src.ml.export` exports existing ones.
- rust/forecast/flood_config.json: thresholds, stages, leads and notify
  constants, compiled in. tests/test_model_export.py fails while it differs
  from the Python source.

Parity (rust/forecast/tests/parity.rs, fixtures from
testdata/gen_forecast_fixtures.py on the production code path): four
scenarios incl. the 2024 flood with RID lagging behind HII. Merged readings,
all features and all levels identical; with all 15 local models 377/384
probabilities bit-identical, the rest within 2 ulp (exp).

Python refactor without behaviour change: predict.forecasts_from_long, and
get_latest_forecasts/load_latest/serving_frame take now/rain for
reproducible comparisons; the HII query orders its rows.

rust/forecast/testdata/gen_forecast_fixtures.py

@@ -0,0 +1,310 @@
+"""Parity fixtures for rust/forecast: inputs plus what Python computes from them.
+
+Run from the repo root (needs the trained bundles in models/ and the
+models/cache/ snapshot, i.e. a machine that has trained or pulled models):
+
+    uv run python rust/forecast/testdata/gen_forecast_fixtures.py
+
+Writes, next to this file:
+  models/flood_P.1.json.gz       the P.1 bundle as src/ml/export.py writes it
+  fixture_<scenario>.json.gz     inputs (RID rows, HII mirror rows, rain points,
+                                 dam days, now) and Python's outputs: merged
+                                 rows, the feature row at each station's as_of,
+                                 forecasts with only P.1's model (checked in CI)
+                                 and with every local model (checked when
+                                 ../../models has the exports).
+
+The Python side is the production code path: data._normalize_long and
+data.fill_from_hii (with the HII query replaced by the fixture rows),
+rain.catchment_mean, dam.hourly_frame and predict.forecasts_from_long.
+"""
+
+import datetime
+import gzip
+import json
+import math
+import shutil
+import sys
+import tempfile
+from pathlib import Path
+from unittest import mock
+
+import joblib
+import numpy as np
+import pandas as pd
+
+HERE = Path(__file__).resolve().parent
+REPO = HERE.parents[2]
+sys.path.insert(0, str(REPO))
+
+from src.ml import dam as dam_mod  # noqa: E402
+from src.ml import data, export, features, predict  # noqa: E402
+from src.ml import rain as rain_mod  # noqa: E402
+
+MODELS = REPO / "models"
+CACHE = MODELS / "cache"
+FMT = "%Y-%m-%dT%H:%M:%S"
+
+# Synthetic MSL offsets of the HII twins (P.81 also gets a noisy bias).
+HII_OFFSETS = {
+    "P.1": 300.0,
+    "P.103": 296.473,
+    "P.20": 410.31,
+    "P.4A": 352.09,
+    "P.67": 318.552,
+    "P.75": 389.11,
+    "P.82": 520.6,
+    "P.84": 288.125,
+    "P.92": 371.0,
+    "P.81": 281.402,
+    "P.77": 402.5,  # a different sensor: must never be merged
+}
+
+
+def num(v):
+    if v is None:
+        return None
+    v = float(v)
+    return None if math.isnan(v) else v
+
+
+def scenario_inputs(long, rain_points, dam_daily, now, rid_lag, null_levels):
+    end = now
+    start = now - datetime.timedelta(hours=336)
+    window = long[(long.timestamp >= start) & (long.timestamp <= end)].copy()
+
+    # HII twins see every hour the gauges saw, including those RID "lags" on
+    rng = np.random.default_rng(42)
+    hii = []
+    for code, offset in HII_OFFSETS.items():
+        rows = window[(window.station_code == code) & window.water_level.notna()]
+        for ts, level, q in zip(rows.timestamp, rows.water_level, rows.discharge):
+            wl = level + offset
+            if code == "P.81":
+                wl += rng.normal(0, 0.05) + 0.19
+            hii.append(
+                [ts, code, round(wl, 3), num(None if pd.isna(q) else round(q + 1.5, 2))]
+            )
+            if code == "P.1" and ts.hour % 5 == 0:
+                # a second, later reading in the same hour wins; an earlier
+                # one and a level-less one must not
+                hii.append(
+                    [ts + pd.Timedelta(minutes=40), code, round(wl + 0.004, 3), None]
+                )
+                hii.append(
+                    [ts + pd.Timedelta(minutes=10), code, round(wl - 0.25, 3), None]
+                )
+                hii.append([ts + pd.Timedelta(minutes=50), code, None, None])
+    hii = pd.DataFrame(
+        hii, columns=["timestamp", "station_code", "wl_msl", "discharge"]
+    )
+    hii = hii[(hii.timestamp >= start) & (hii.timestamp <= end)]
+    # the query's ORDER BY m.timestamp, m.station_id (code stands in for the id)
+    hii = hii.sort_values(["timestamp", "station_code"], kind="stable").reset_index(
+        drop=True
+    )
+
+    # RID behind on some stations (the night case), NULL levels on others
+    newest = window.timestamp.max()
+    for code, hours in rid_lag.items():
+        cut = newest - pd.Timedelta(hours=hours)
+        window = window[~((window.station_code == code) & (window.timestamp > cut))]
+    for code, every in null_levels.items():
+        sel = (window.station_code == code) & (window.timestamp.dt.hour % every == 0)
+        window.loc[sel, "water_level"] = np.nan
+    window = window.sort_values(["timestamp", "station_code"], kind="stable")
+
+    day0 = pd.Timestamp(now.date())
+    rain_rows = rain_points[
+        (rain_points.index >= day0 - pd.Timedelta(days=5))
+        & (rain_points.index < day0 + pd.Timedelta(days=2))
+    ]
+    dam_days = dam_daily[
+        dam_daily.index >= pd.Timestamp(now.date() - datetime.timedelta(days=21))
+    ]
+    return start, end, window, hii, rain_rows, dam_days
+
+
+def python_side(window, hii, rain_rows, dam_days, start, end, now, models_dirs):
+    df = data._normalize_long(window)
+
+    def fake_fetch(db_url, codes, s, e):
+        assert (s, e) == (start, end)
+        return data._prepare_hii_levels(hii[hii.station_code.isin(codes)])
+
+    with mock.patch.object(data, "_fetch_hii_levels", fake_fetch):
+        merged = data.fill_from_hii(df, "fixture://", start=start, end=end)
+
+    rain = rain_mod.catchment_mean(rain_rows)
+    if rain is None:
+        rain = pd.Series(dtype=float)
+    dam_frame = dam_mod.hourly_frame(dam_days)
+    if dam_frame is None:
+        dam_frame = pd.DataFrame()
+
+    grid = features.make_hourly_grid(merged)
+    feature_rows = {}
+    for code in sorted(merged.station_code.unique()):
+        observed = grid.observed[(code, "water_level")].dropna()
+        if observed.empty:
+            continue
+        as_of = observed.index.max()
+        X = features.build_features(grid, code, rain=rain, dam=dam_frame)
+        feature_rows[code] = {
+            "as_of": as_of.strftime(FMT),
+            "features": {k: num(v) for k, v in X.loc[as_of].items()},
+        }
+
+    forecasts = {
+        name: json.loads(
+            json.dumps(
+                predict.forecasts_from_long(
+                    merged, models_dir=d, now=now, rain=rain, dam=dam_frame
+                ),
+                default=float,
+            )
+        )
+        for name, d in models_dirs.items()
+    }
+    merged_rows = [
+        [t.strftime(FMT), c, num(l), num(q)]
+        for t, c, l, q in zip(
+            merged.timestamp, merged.station_code, merged.water_level, merged.discharge
+        )
+    ]
+    merged_rows.sort(key=lambda r: (r[1], r[0]))
+    return merged_rows, feature_rows, forecasts
+
+
+def main():
+    long = data._read_cache(CACHE, None)
+    rain_points = pd.read_csv(
+        CACHE / "rain_openmeteo.csv.gz", index_col=0, parse_dates=True
+    )
+    rain_points = rain_points[[p[0] for p in rain_mod.CATCHMENT_POINTS]]
+    dam_daily = pd.read_csv(CACHE / "dam_200103.csv.gz", index_col=0, parse_dates=True)
+
+    out_models = HERE / "models"
+    out_models.mkdir(exist_ok=True)
+    p1 = joblib.load(MODELS / "flood_P.1.joblib")
+    export.write_bundle_json(p1, out_models / "flood_P.1.json.gz")
+
+    tmp = Path(tempfile.mkdtemp())
+    try:
+        p1_only = tmp / "p1"
+        p1_only.mkdir()
+        shutil.copy(MODELS / "flood_P.1.joblib", p1_only)
+        # the configured thresholds moved since training -> regression path
+        p1_moved = tmp / "p1_moved"
+        p1_moved.mkdir()
+        moved = dict(p1)
+        moved["thresholds"] = {"warning": 3.5, "danger": 4.0}
+        joblib.dump(moved, p1_moved / "flood_P.1.joblib")
+
+        scenarios = {
+            # rising limb of the October 2024 record flood, RID 3 h behind
+            # on three HII-mirrored stations
+            "flood2024": dict(
+                now=datetime.datetime(2024, 10, 3, 1, 17, 31, 500000),
+                rid_lag={"P.1": 3, "P.103": 3, "P.67": 2},
+                null_levels={"P.21": 7},
+            ),
+            "peak2024": dict(
+                now=datetime.datetime(2024, 10, 5, 3, 5, 0),
+                rid_lag={"P.1": 1},
+                null_levels={},
+            ),
+            # quiet season, the newest cached data, with dam state
+            "recent2026": dict(
+                now=datetime.datetime(2026, 8, 13, 19, 40, 12),
+                rid_lag={},
+                null_levels={"P.85": 5},
+            ),
+            # every station > 6 h old: heuristic only
+            "stale2026": dict(
+                now=datetime.datetime(2026, 8, 14, 3, 45, 0),
+                rid_lag={"P.20": 30},
+                null_levels={},
+            ),
+        }
+        for name, sc in scenarios.items():
+            start, end, window, hii, rain_rows, dam_days = scenario_inputs(
+                long,
+                rain_points,
+                dam_daily,
+                sc["now"],
+                sc["rid_lag"],
+                sc["null_levels"],
+            )
+            dirs = {"p1_only": p1_only, "all_models": MODELS}
+            if name == "recent2026":
+                dirs["p1_moved_thresholds"] = p1_moved
+            merged, feats, fcs = python_side(
+                window, hii, rain_rows, dam_days, start, end, sc["now"], dirs
+            )
+            fixture = {
+                "now": sc["now"].strftime(FMT + ".%f"),
+                "window": [start.strftime(FMT + ".%f"), end.strftime(FMT + ".%f")],
+                "rid_rows": [
+                    [t.strftime(FMT), c, num(l), num(q)]
+                    for t, c, l, q in zip(
+                        window.timestamp,
+                        window.station_code,
+                        window.water_level,
+                        window.discharge,
+                    )
+                ],
+                "hii_rows": [
+                    [t.strftime(FMT), c, num(w), num(q)]
+                    for t, c, w, q in zip(
+                        hii.timestamp, hii.station_code, hii.wl_msl, hii.discharge
+                    )
+                ],
+                "rain_points": [
+                    [t.strftime(FMT), [num(v) for v in row]]
+                    for t, row in zip(rain_rows.index, rain_rows.to_numpy().tolist())
+                ],
+                "dam_days": [
+                    [d.strftime("%Y-%m-%d"), num(s), num(i), num(o)]
+                    for d, s, i, o in zip(
+                        dam_days.index,
+                        dam_days.storage_pct,
+                        dam_days.inflow_mcm,
+                        dam_days.outflow_mcm,
+                    )
+                ],
+                "expected": {
+                    "merged_rows": merged,
+                    "features": feats,
+                    "forecasts": fcs,
+                    "all_models_versions": sorted(
+                        {
+                            joblib.load(p)["model_version"]
+                            for p in MODELS.glob("flood_*.joblib")
+                        }
+                    ),
+                },
+            }
+            path = HERE / f"fixture_{name}.json.gz"
+            with (
+                open(path, "wb") as raw,
+                gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as gz,
+            ):
+                gz.write(json.dumps(fixture, separators=(",", ":")).encode())
+            n_model = sum(r["source"] == "model" for r in fcs["p1_only"])
+            p1_24 = [
+                r
+                for r in fcs["all_models"]
+                if r["station_code"] == "P.1" and r["horizon_hours"] == 24
+            ]
+            print(
+                f"{name}: {len(fixture['rid_rows'])} RID rows, {len(fixture['hii_rows'])} HII rows, "
+                f"{len(merged)} merged, {len(feats)} stations, model rows (P.1 only) {n_model}, "
+                f"P.1 24h {[(round(r['p_warning'], 3), round(r['predicted_max_level'], 2)) for r in p1_24]}"
+            )
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    main()