Northern-Thailand-Ping-River-Monitor/tests/test_scraping_cycle.py

"""The RID scraping cycle must keep every reading of a late report.

RID's hourly report often lags 1-2 h in the evening. The cycle used to discard
a whole report unless it already held the current hour, and after 01:00 it
only fetched today's report, so the evening hours (and the "hour 24" =
00:00 reading) were lost for good. These tests pin the fixed behaviour with a
mocked HTTP session and DB adapter: no network, no database.
"""

import datetime
from unittest.mock import MagicMock

import pytest

import src.water_scraper_v3 as scraper_mod
from src.water_scraper_v3 import EnhancedWaterMonitorScraper as Scraper

TODAY = datetime.date(2026, 10, 17)
YESTERDAY = TODAY - datetime.timedelta(days=1)


def _thai(d: datetime.date) -> str:
    return f"{d.day:02d}/{d.month:02d}/{d.year + 543}"


def _report(last_hour: int):
    """RID rows 1.00..24.00 with readings up to ``last_hour`` (rest null)."""
    return [
        {
            "hourlytime": f"{h}.00",
            "wlvalues8": 1.0 + h / 100 if h <= last_hour else None,
        }
        for h in range(1, 25)
    ]


@pytest.fixture
def make_scraper(monkeypatch):
    monkeypatch.setattr(
        scraper_mod.DataValidator,
        "validate_measurements",
        staticmethod(lambda m: m),
    )

    def _factory(reports, hours_by_day):
        """reports: {date: last published hour}; hours_by_day: DB contents."""
        scraper = Scraper.__new__(Scraper)  # bypass __init__ (no DB/network)
        scraper.api_url = "https://example.invalid/api"
        scraper.rate_limiter = MagicMock()
        scraper.request_tracker = MagicMock()
        scraper.station_config_path = "/nonexistent/stations.json"
        scraper.station_mapping = scraper._load_station_mapping()
        scraper.last_cycle = None

        by_thai = {_thai(d): last for d, last in reports.items()}

        def post(url, data, timeout):
            response = MagicMock()
            response.raise_for_status.return_value = None
            last = by_thai.get(data["DW[TimeCurrent]"])
            response.json.return_value = {
                "rows": _report(last) if last is not None else []
            }
            return response

        scraper.session = MagicMock()
        scraper.session.post.side_effect = post

        scraper.db_adapter = MagicMock()
        scraper.db_adapter.get_recorded_hours_by_day.return_value = hours_by_day
        scraper.db_adapter.save_measurements.return_value = True
        return scraper

    return _factory


def _fetched_dates(scraper):
    return [
        c.kwargs["data"]["DW[TimeCurrent]"] for c in scraper.session.post.call_args_list
    ]


def _saved_times(scraper):
    saved = []
    for c in scraper.db_adapter.save_measurements.call_args_list:
        saved.extend(m["timestamp"] for m in c.args[0])
    return sorted(set(saved))


def test_late_report_is_saved_but_not_current(make_scraper):
    # 21:30, RID has only published up to 20:00 (the case that lost data)
    scraper = make_scraper(
        {TODAY: 20},
        {YESTERDAY: set(range(24)), TODAY: set(range(0, 19))},
    )
    now = datetime.datetime(2026, 10, 17, 21, 30)

    assert scraper.run_scraping_cycle(now=now) is False

    saved = _saved_times(scraper)
    assert datetime.datetime(2026, 10, 17, 19) in saved
    assert datetime.datetime(2026, 10, 17, 20) in saved
    assert scraper.last_cycle["latest_reading"] == datetime.datetime(2026, 10, 17, 20)
    assert scraper.last_cycle["current"] is False


def test_current_hour_published_is_current(make_scraper):
    scraper = make_scraper({TODAY: 21}, {YESTERDAY: set(range(24))})
    now = datetime.datetime(2026, 10, 17, 21, 5)

    assert scraper.run_scraping_cycle(now=now) is True
    assert scraper.last_cycle["latest_reading"] == datetime.datetime(2026, 10, 17, 21)


def test_holes_in_yesterday_refetch_yesterdays_report(make_scraper):
    # 02:10: yesterday's 22:00-23:00 and today's 00:00 (yesterday's hour 24)
    # were published late; only yesterday's report still holds them
    scraper = make_scraper(
        {YESTERDAY: 24, TODAY: 2},
        {YESTERDAY: set(range(22)), TODAY: {1}},
    )
    now = datetime.datetime(2026, 10, 17, 2, 10)

    assert scraper.run_scraping_cycle(now=now) is True

    assert _fetched_dates(scraper) == [_thai(YESTERDAY), _thai(TODAY)]
    saved = _saved_times(scraper)
    for hour in (
        datetime.datetime(2026, 10, 16, 22),
        datetime.datetime(2026, 10, 16, 23),
        datetime.datetime(2026, 10, 17, 0),
    ):
        assert hour in saved


def test_complete_yesterday_is_not_refetched(make_scraper):
    scraper = make_scraper(
        {TODAY: 15},
        {YESTERDAY: set(range(24)), TODAY: set(range(15))},
    )
    scraper.run_scraping_cycle(now=datetime.datetime(2026, 10, 17, 15, 5))

    assert _fetched_dates(scraper) == [_thai(TODAY)]


def test_before_one_am_uses_yesterdays_report(make_scraper):
    # 00:20: today's report is empty; midnight is yesterday's "hour 24"
    scraper = make_scraper({YESTERDAY: 24}, {YESTERDAY: set(range(24))})

    assert scraper.run_scraping_cycle(now=datetime.datetime(2026, 10, 17, 0, 20))
    assert _fetched_dates(scraper) == [_thai(YESTERDAY)]
    assert scraper.last_cycle["latest_reading"] == datetime.datetime(2026, 10, 17, 0)


def test_nothing_published_returns_false_without_saving(make_scraper):
    scraper = make_scraper({}, {YESTERDAY: set(range(24))})

    assert (
        scraper.run_scraping_cycle(now=datetime.datetime(2026, 10, 17, 9, 5)) is False
    )
    scraper.db_adapter.save_measurements.assert_not_called()
    assert scraper.last_cycle["error"] == "no data fetched"


def test_gap_detection_failure_still_fetches_current_report(make_scraper):
    scraper = make_scraper({TODAY: 9}, {})
    scraper.db_adapter.get_recorded_hours_by_day.side_effect = RuntimeError("db down")

    assert scraper.run_scraping_cycle(now=datetime.datetime(2026, 10, 17, 9, 5))
    assert _fetched_dates(scraper) == [_thai(TODAY)]


class TestNextPollTime:
    @pytest.fixture(autouse=True)
    def _config(self, monkeypatch):
        from src.config import Config

        monkeypatch.setattr(Config, "SCRAPING_INTERVAL_HOURS", 1)
        monkeypatch.setattr(Config, "RID_POLL_OFFSET_MINUTES", 5)
        monkeypatch.setattr(Config, "RID_RETRY_MINUTES", 10)

    def test_current_feed_waits_for_next_hour_plus_offset(self):
        from src.web_api import _next_poll_time

        now = datetime.datetime(2026, 10, 17, 21, 6)
        assert _next_poll_time(now, True) == datetime.datetime(2026, 10, 17, 22, 5)

    def test_behind_feed_retries_soon(self):
        from src.web_api import _next_poll_time

        now = datetime.datetime(2026, 10, 17, 21, 6)
        assert _next_poll_time(now, False) == datetime.datetime(2026, 10, 17, 21, 16)

    def test_retry_never_skips_the_hourly_poll(self):
        from src.web_api import _next_poll_time

        now = datetime.datetime(2026, 10, 17, 21, 58)
        assert _next_poll_time(now, False) == datetime.datetime(2026, 10, 17, 22, 5)


def test_scraping_status_shared_through_file(tmp_path, monkeypatch):
    from src import web_api
    from src.config import Config

    monkeypatch.setattr(Config, "SCRAPING_STATUS_FILE", str(tmp_path / "s.json"))
    monkeypatch.setitem(web_api.app_state, "leader_lock", object())
    monkeypatch.setitem(web_api.app_state, "is_scraping", False)
    stats = dict(web_api.app_state["scraping_stats"])
    stats.update(
        total_runs=3,
        successful_runs=2,
        failed_runs=1,
        feed_current=True,
        latest_reading=datetime.datetime(2026, 10, 17, 21),
    )
    monkeypatch.setitem(web_api.app_state, "scraping_stats", stats)

    web_api._publish_scraping_status()
    shared = web_api._read_scraping_status()

    assert shared["total_runs"] == 3
    assert shared["latest_reading"] == "2026-10-17T21:00:00"
    resp = web_api.ScrapingStatusResponse(
        **{
            k: v
            for k, v in shared.items()
            if k in web_api.ScrapingStatusResponse.model_fields
        }
    )
    assert resp.latest_reading == datetime.datetime(2026, 10, 17, 21)
    assert resp.feed_current is True


def test_loop_retries_while_behind_and_runs_hourly_work_once(monkeypatch):
    """Leader loop: 10-min retries while RID is behind; HII/dams/rain once per
    clock hour; forecasts + ntfy only when newer readings arrive (or hourly)."""
    import asyncio

    from src import web_api
    from src.config import Config

    clock = datetime.datetime(2026, 10, 17, 21, 30)

    class FixedClock(datetime.datetime):
        @classmethod
        def now(cls, tz=None):
            return clock

    class FakeScraper:
        last_cycle = None
        results = [
            (False, datetime.datetime(2026, 10, 17, 20)),
            (False, datetime.datetime(2026, 10, 17, 20)),
            (True, datetime.datetime(2026, 10, 17, 21)),
        ]

        def fill_data_gaps(self, days):
            calls["gap_fill"] = days
            return 0

        def run_scraping_cycle(self):
            current, latest = self.results.pop(0)
            self.last_cycle = {"latest_reading": latest, "saved": 16}
            return current

    calls = {"hourly": 0, "precompute": 0, "notify": 0, "gap_fill": None}
    sleeps = []

    def counter(name):
        async def _fn():
            calls[name] += 1

        return _fn

    async def fake_sleep(seconds):
        sleeps.append(seconds)
        if len(sleeps) == 3:
            raise asyncio.CancelledError

    monkeypatch.setattr(web_api, "datetime", FixedClock)
    monkeypatch.setattr(web_api.asyncio, "sleep", fake_sleep)
    monkeypatch.setattr(web_api, "_collect_hourly_sources", counter("hourly"))
    monkeypatch.setattr(web_api, "_precompute_forecasts", counter("precompute"))
    monkeypatch.setattr(web_api, "_notify_transitions", counter("notify"))
    monkeypatch.setattr(Config, "SCRAPING_INTERVAL_HOURS", 1)
    monkeypatch.setattr(Config, "RID_POLL_OFFSET_MINUTES", 5)
    monkeypatch.setattr(Config, "RID_RETRY_MINUTES", 10)
    monkeypatch.setattr(Config, "GAP_FILL_DAYS_ON_START", 14)
    monkeypatch.setitem(web_api.app_state, "scraper", FakeScraper())
    monkeypatch.setitem(web_api.app_state, "leader_lock", None)
    monkeypatch.setitem(web_api.app_state, "is_scraping", False)
    monkeypatch.setitem(
        web_api.app_state, "scraping_stats", dict(web_api.app_state["scraping_stats"])
    )

    asyncio.run(web_api.background_scraping_task())

    assert calls == {"hourly": 1, "precompute": 2, "notify": 2, "gap_fill": 14}
    assert sleeps == [600.0, 600.0, 2100.0]  # retry, retry, then 22:05
    stats = web_api.app_state["scraping_stats"]
    assert (stats["total_runs"], stats["successful_runs"], stats["failed_runs"]) == (
        3,
        1,
        2,
    )
    assert stats["latest_reading"] == datetime.datetime(2026, 10, 17, 21)
    assert web_api.app_state["is_scraping"] is False