fix: RID collector keeps late readings instead of losing every evening

RID's hourly report often lags 1-2 h in the evening. The cycle discarded the
whole report unless it already held the current hour, the web-API loop then
slept a full hour, and after 01:00 only today's report was fetched. Result:
the 19:00-24:00 readings were dropped most nights (P.1 lost 00:00 on 23 of
the last 60 days, 13 evening hours this week), the feed went stale from
~21:00 until morning and ntfy ping-status sent a stale/recovered pair daily.

- run_scraping_cycle() upserts every reading the report has; "current hour
  present" now only decides the retry cadence (returned as before).
- Yesterday's report is re-fetched while the store has holes in it (hours
  22-24 are only reachable through that day's report).
- Leader loop polls at :05 (RID_POLL_OFFSET_MINUTES) and retries every
  RID_RETRY_MINUTES (10) while RID is behind; HII/dams/rain once per clock
  hour; forecast precompute + ntfy when newer readings arrive or hourly.
- Startup re-fetches holes from the last GAP_FILL_DAYS_ON_START (14) days.
- /scraping/status: the leader shares its stats via a JSON file, so any
  worker reports them (was always 0 runs); adds feed_current,
  latest_reading, readings_saved.

tests/test_scraping_cycle.py

@@ -0,0 +1,307 @@
+"""The RID scraping cycle must keep every reading of a late report.
+
+RID's hourly report often lags 1-2 h in the evening. The cycle used to discard
+a whole report unless it already held the current hour, and after 01:00 it
+only fetched today's report, so the evening hours (and the "hour 24" =
+00:00 reading) were lost for good. These tests pin the fixed behaviour with a
+mocked HTTP session and DB adapter: no network, no database.
+"""
+
+import datetime
+from unittest.mock import MagicMock
+
+import pytest
+
+import src.water_scraper_v3 as scraper_mod
+from src.water_scraper_v3 import EnhancedWaterMonitorScraper as Scraper
+
+TODAY = datetime.date(2026, 10, 17)
+YESTERDAY = TODAY - datetime.timedelta(days=1)
+
+
+def _thai(d: datetime.date) -> str:
+    return f"{d.day:02d}/{d.month:02d}/{d.year + 543}"
+
+
+def _report(last_hour: int):
+    """RID rows 1.00..24.00 with readings up to ``last_hour`` (rest null)."""
+    return [
+        {
+            "hourlytime": f"{h}.00",
+            "wlvalues8": 1.0 + h / 100 if h <= last_hour else None,
+        }
+        for h in range(1, 25)
+    ]
+
+
+@pytest.fixture
+def make_scraper(monkeypatch):
+    monkeypatch.setattr(
+        scraper_mod.DataValidator,
+        "validate_measurements",
+        staticmethod(lambda m: m),
+    )
+
+    def _factory(reports, hours_by_day):
+        """reports: {date: last published hour}; hours_by_day: DB contents."""
+        scraper = Scraper.__new__(Scraper)  # bypass __init__ (no DB/network)
+        scraper.api_url = "https://example.invalid/api"
+        scraper.rate_limiter = MagicMock()
+        scraper.request_tracker = MagicMock()
+        scraper.station_config_path = "/nonexistent/stations.json"
+        scraper.station_mapping = scraper._load_station_mapping()
+        scraper.last_cycle = None
+
+        by_thai = {_thai(d): last for d, last in reports.items()}
+
+        def post(url, data, timeout):
+            response = MagicMock()
+            response.raise_for_status.return_value = None
+            last = by_thai.get(data["DW[TimeCurrent]"])
+            response.json.return_value = {
+                "rows": _report(last) if last is not None else []
+            }
+            return response
+
+        scraper.session = MagicMock()
+        scraper.session.post.side_effect = post
+
+        scraper.db_adapter = MagicMock()
+        scraper.db_adapter.get_recorded_hours_by_day.return_value = hours_by_day
+        scraper.db_adapter.save_measurements.return_value = True
+        return scraper
+
+    return _factory
+
+
+def _fetched_dates(scraper):
+    return [
+        c.kwargs["data"]["DW[TimeCurrent]"] for c in scraper.session.post.call_args_list
+    ]
+
+
+def _saved_times(scraper):
+    saved = []
+    for c in scraper.db_adapter.save_measurements.call_args_list:
+        saved.extend(m["timestamp"] for m in c.args[0])
+    return sorted(set(saved))
+
+
+def test_late_report_is_saved_but_not_current(make_scraper):
+    # 21:30, RID has only published up to 20:00 (the case that lost data)
+    scraper = make_scraper(
+        {TODAY: 20},
+        {YESTERDAY: set(range(24)), TODAY: set(range(0, 19))},
+    )
+    now = datetime.datetime(2026, 10, 17, 21, 30)
+
+    assert scraper.run_scraping_cycle(now=now) is False
+
+    saved = _saved_times(scraper)
+    assert datetime.datetime(2026, 10, 17, 19) in saved
+    assert datetime.datetime(2026, 10, 17, 20) in saved
+    assert scraper.last_cycle["latest_reading"] == datetime.datetime(2026, 10, 17, 20)
+    assert scraper.last_cycle["current"] is False
+
+
+def test_current_hour_published_is_current(make_scraper):
+    scraper = make_scraper({TODAY: 21}, {YESTERDAY: set(range(24))})
+    now = datetime.datetime(2026, 10, 17, 21, 5)
+
+    assert scraper.run_scraping_cycle(now=now) is True
+    assert scraper.last_cycle["latest_reading"] == datetime.datetime(2026, 10, 17, 21)
+
+
+def test_holes_in_yesterday_refetch_yesterdays_report(make_scraper):
+    # 02:10: yesterday's 22:00-23:00 and today's 00:00 (yesterday's hour 24)
+    # were published late; only yesterday's report still holds them
+    scraper = make_scraper(
+        {YESTERDAY: 24, TODAY: 2},
+        {YESTERDAY: set(range(22)), TODAY: {1}},
+    )
+    now = datetime.datetime(2026, 10, 17, 2, 10)
+
+    assert scraper.run_scraping_cycle(now=now) is True
+
+    assert _fetched_dates(scraper) == [_thai(YESTERDAY), _thai(TODAY)]
+    saved = _saved_times(scraper)
+    for hour in (
+        datetime.datetime(2026, 10, 16, 22),
+        datetime.datetime(2026, 10, 16, 23),
+        datetime.datetime(2026, 10, 17, 0),
+    ):
+        assert hour in saved
+
+
+def test_complete_yesterday_is_not_refetched(make_scraper):
+    scraper = make_scraper(
+        {TODAY: 15},
+        {YESTERDAY: set(range(24)), TODAY: set(range(15))},
+    )
+    scraper.run_scraping_cycle(now=datetime.datetime(2026, 10, 17, 15, 5))
+
+    assert _fetched_dates(scraper) == [_thai(TODAY)]
+
+
+def test_before_one_am_uses_yesterdays_report(make_scraper):
+    # 00:20: today's report is empty; midnight is yesterday's "hour 24"
+    scraper = make_scraper({YESTERDAY: 24}, {YESTERDAY: set(range(24))})
+
+    assert scraper.run_scraping_cycle(now=datetime.datetime(2026, 10, 17, 0, 20))
+    assert _fetched_dates(scraper) == [_thai(YESTERDAY)]
+    assert scraper.last_cycle["latest_reading"] == datetime.datetime(2026, 10, 17, 0)
+
+
+def test_nothing_published_returns_false_without_saving(make_scraper):
+    scraper = make_scraper({}, {YESTERDAY: set(range(24))})
+
+    assert (
+        scraper.run_scraping_cycle(now=datetime.datetime(2026, 10, 17, 9, 5)) is False
+    )
+    scraper.db_adapter.save_measurements.assert_not_called()
+    assert scraper.last_cycle["error"] == "no data fetched"
+
+
+def test_gap_detection_failure_still_fetches_current_report(make_scraper):
+    scraper = make_scraper({TODAY: 9}, {})
+    scraper.db_adapter.get_recorded_hours_by_day.side_effect = RuntimeError("db down")
+
+    assert scraper.run_scraping_cycle(now=datetime.datetime(2026, 10, 17, 9, 5))
+    assert _fetched_dates(scraper) == [_thai(TODAY)]
+
+
+class TestNextPollTime:
+    @pytest.fixture(autouse=True)
+    def _config(self, monkeypatch):
+        from src.config import Config
+
+        monkeypatch.setattr(Config, "SCRAPING_INTERVAL_HOURS", 1)
+        monkeypatch.setattr(Config, "RID_POLL_OFFSET_MINUTES", 5)
+        monkeypatch.setattr(Config, "RID_RETRY_MINUTES", 10)
+
+    def test_current_feed_waits_for_next_hour_plus_offset(self):
+        from src.web_api import _next_poll_time
+
+        now = datetime.datetime(2026, 10, 17, 21, 6)
+        assert _next_poll_time(now, True) == datetime.datetime(2026, 10, 17, 22, 5)
+
+    def test_behind_feed_retries_soon(self):
+        from src.web_api import _next_poll_time
+
+        now = datetime.datetime(2026, 10, 17, 21, 6)
+        assert _next_poll_time(now, False) == datetime.datetime(2026, 10, 17, 21, 16)
+
+    def test_retry_never_skips_the_hourly_poll(self):
+        from src.web_api import _next_poll_time
+
+        now = datetime.datetime(2026, 10, 17, 21, 58)
+        assert _next_poll_time(now, False) == datetime.datetime(2026, 10, 17, 22, 5)
+
+
+def test_scraping_status_shared_through_file(tmp_path, monkeypatch):
+    from src import web_api
+    from src.config import Config
+
+    monkeypatch.setattr(Config, "SCRAPING_STATUS_FILE", str(tmp_path / "s.json"))
+    monkeypatch.setitem(web_api.app_state, "leader_lock", object())
+    monkeypatch.setitem(web_api.app_state, "is_scraping", False)
+    stats = dict(web_api.app_state["scraping_stats"])
+    stats.update(
+        total_runs=3,
+        successful_runs=2,
+        failed_runs=1,
+        feed_current=True,
+        latest_reading=datetime.datetime(2026, 10, 17, 21),
+    )
+    monkeypatch.setitem(web_api.app_state, "scraping_stats", stats)
+
+    web_api._publish_scraping_status()
+    shared = web_api._read_scraping_status()
+
+    assert shared["total_runs"] == 3
+    assert shared["latest_reading"] == "2026-10-17T21:00:00"
+    resp = web_api.ScrapingStatusResponse(
+        **{
+            k: v
+            for k, v in shared.items()
+            if k in web_api.ScrapingStatusResponse.model_fields
+        }
+    )
+    assert resp.latest_reading == datetime.datetime(2026, 10, 17, 21)
+    assert resp.feed_current is True
+
+
+def test_loop_retries_while_behind_and_runs_hourly_work_once(monkeypatch):
+    """Leader loop: 10-min retries while RID is behind; HII/dams/rain once per
+    clock hour; forecasts + ntfy only when newer readings arrive (or hourly)."""
+    import asyncio
+
+    from src import web_api
+    from src.config import Config
+
+    clock = datetime.datetime(2026, 10, 17, 21, 30)
+
+    class FixedClock(datetime.datetime):
+        @classmethod
+        def now(cls, tz=None):
+            return clock
+
+    class FakeScraper:
+        last_cycle = None
+        results = [
+            (False, datetime.datetime(2026, 10, 17, 20)),
+            (False, datetime.datetime(2026, 10, 17, 20)),
+            (True, datetime.datetime(2026, 10, 17, 21)),
+        ]
+
+        def fill_data_gaps(self, days):
+            calls["gap_fill"] = days
+            return 0
+
+        def run_scraping_cycle(self):
+            current, latest = self.results.pop(0)
+            self.last_cycle = {"latest_reading": latest, "saved": 16}
+            return current
+
+    calls = {"hourly": 0, "precompute": 0, "notify": 0, "gap_fill": None}
+    sleeps = []
+
+    def counter(name):
+        async def _fn():
+            calls[name] += 1
+
+        return _fn
+
+    async def fake_sleep(seconds):
+        sleeps.append(seconds)
+        if len(sleeps) == 3:
+            raise asyncio.CancelledError
+
+    monkeypatch.setattr(web_api, "datetime", FixedClock)
+    monkeypatch.setattr(web_api.asyncio, "sleep", fake_sleep)
+    monkeypatch.setattr(web_api, "_collect_hourly_sources", counter("hourly"))
+    monkeypatch.setattr(web_api, "_precompute_forecasts", counter("precompute"))
+    monkeypatch.setattr(web_api, "_notify_transitions", counter("notify"))
+    monkeypatch.setattr(Config, "SCRAPING_INTERVAL_HOURS", 1)
+    monkeypatch.setattr(Config, "RID_POLL_OFFSET_MINUTES", 5)
+    monkeypatch.setattr(Config, "RID_RETRY_MINUTES", 10)
+    monkeypatch.setattr(Config, "GAP_FILL_DAYS_ON_START", 14)
+    monkeypatch.setitem(web_api.app_state, "scraper", FakeScraper())
+    monkeypatch.setitem(web_api.app_state, "leader_lock", None)
+    monkeypatch.setitem(web_api.app_state, "is_scraping", False)
+    monkeypatch.setitem(
+        web_api.app_state, "scraping_stats", dict(web_api.app_state["scraping_stats"])
+    )
+
+    asyncio.run(web_api.background_scraping_task())
+
+    assert calls == {"hourly": 1, "precompute": 2, "notify": 2, "gap_fill": 14}
+    assert sleeps == [600.0, 600.0, 2100.0]  # retry, retry, then 22:05
+    stats = web_api.app_state["scraping_stats"]
+    assert (stats["total_runs"], stats["successful_runs"], stats["failed_runs"]) == (
+        3,
+        1,
+        2,
+    )
+    assert stats["latest_reading"] == datetime.datetime(2026, 10, 17, 21)
+    assert web_api.app_state["is_scraping"] is False