Add unit tests for RID API response parsing
Cover the previously-untested, highest-risk parsing in fetch_water_data_for_date by mocking the HTTP call: - hour 1-23 map to the same day; hour 24 rolls to next-day midnight - qvalues "***" / None yield discharge None (no crash) - None water level is skipped - out-of-range (0, 25) and empty hourlytime rows are skipped - missing "rows" key returns an empty list This gives the scraper a safety net before its module is split.
This commit is contained in:
@@ -0,0 +1,125 @@
|
|||||||
|
"""Assert-based tests for the RID API response parsing.
|
||||||
|
|
||||||
|
The parsing in ``fetch_water_data_for_date`` is the riskiest, previously
|
||||||
|
untested code: it maps the API's 1..24 "hourlytime" onto real timestamps
|
||||||
|
(hour 24 rolls to next-day midnight) and treats ``"***"``/``None`` discharge as
|
||||||
|
missing. These tests mock the HTTP call so no network is touched and stub the
|
||||||
|
validator so we assert on the parser's output directly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import src.water_scraper_v3 as scraper_mod
|
||||||
|
from src.water_scraper_v3 import EnhancedWaterMonitorScraper as Scraper
|
||||||
|
|
||||||
|
TARGET = datetime.datetime(2026, 7, 22)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def make_scraper(monkeypatch):
|
||||||
|
"""Return a factory that builds a bare scraper returning the given API rows."""
|
||||||
|
|
||||||
|
def _factory(rows):
|
||||||
|
scraper = Scraper.__new__(Scraper) # bypass __init__ (no DB/network)
|
||||||
|
scraper.api_url = "https://example.invalid/api"
|
||||||
|
scraper.rate_limiter = MagicMock()
|
||||||
|
scraper.request_tracker = MagicMock()
|
||||||
|
scraper.station_config_path = "/nonexistent/stations.json"
|
||||||
|
scraper.station_mapping = scraper._load_station_mapping() # bundled defaults
|
||||||
|
|
||||||
|
response = MagicMock()
|
||||||
|
response.json.return_value = {"rows": rows}
|
||||||
|
response.raise_for_status.return_value = None
|
||||||
|
scraper.session = MagicMock()
|
||||||
|
scraper.session.post.return_value = response
|
||||||
|
|
||||||
|
# Isolate parsing from validation.
|
||||||
|
monkeypatch.setattr(
|
||||||
|
scraper_mod.DataValidator,
|
||||||
|
"validate_measurements",
|
||||||
|
staticmethod(lambda m: m),
|
||||||
|
)
|
||||||
|
return scraper
|
||||||
|
|
||||||
|
return _factory
|
||||||
|
|
||||||
|
|
||||||
|
def test_parses_water_level_and_discharge(make_scraper):
|
||||||
|
rows = [
|
||||||
|
{
|
||||||
|
"hourlytime": "9.00",
|
||||||
|
"wlvalues1": "3.50",
|
||||||
|
"qvalues1": "120.5",
|
||||||
|
"QPercent1": "45.2",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
data = make_scraper(rows).fetch_water_data_for_date(TARGET)
|
||||||
|
|
||||||
|
p20 = [d for d in data if d["station_code"] == "P.20"]
|
||||||
|
assert len(p20) == 1
|
||||||
|
m = p20[0]
|
||||||
|
assert m["water_level"] == 3.5
|
||||||
|
assert m["discharge"] == 120.5
|
||||||
|
assert m["discharge_percent"] == 45.2
|
||||||
|
assert m["timestamp"] == datetime.datetime(2026, 7, 22, 9, 0)
|
||||||
|
assert m["station_name_en"] == "Ban Chiang Dao"
|
||||||
|
|
||||||
|
|
||||||
|
def test_discharge_asterisks_becomes_none(make_scraper):
|
||||||
|
rows = [{"hourlytime": "10.00", "wlvalues8": "4.20", "qvalues8": "***"}]
|
||||||
|
data = make_scraper(rows).fetch_water_data_for_date(TARGET)
|
||||||
|
|
||||||
|
p1 = [d for d in data if d["station_code"] == "P.1"][0]
|
||||||
|
assert p1["water_level"] == 4.2
|
||||||
|
assert p1["discharge"] is None
|
||||||
|
assert p1["discharge_percent"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_hour_24_rolls_to_next_day_midnight(make_scraper):
|
||||||
|
rows = [{"hourlytime": "24.00", "wlvalues1": "3.00"}]
|
||||||
|
data = make_scraper(rows).fetch_water_data_for_date(TARGET)
|
||||||
|
|
||||||
|
assert data[0]["timestamp"] == datetime.datetime(2026, 7, 23, 0, 0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_hours_1_to_23_stay_same_day(make_scraper):
|
||||||
|
rows = [
|
||||||
|
{"hourlytime": "1.00", "wlvalues1": "3.00"},
|
||||||
|
{"hourlytime": "23.00", "wlvalues1": "3.10"},
|
||||||
|
]
|
||||||
|
data = make_scraper(rows).fetch_water_data_for_date(TARGET)
|
||||||
|
|
||||||
|
times = sorted(d["timestamp"] for d in data)
|
||||||
|
assert times == [
|
||||||
|
datetime.datetime(2026, 7, 22, 1, 0),
|
||||||
|
datetime.datetime(2026, 7, 22, 23, 0),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_none_water_level_is_skipped(make_scraper):
|
||||||
|
rows = [{"hourlytime": "9.00", "wlvalues1": None, "wlvalues2": "2.5"}]
|
||||||
|
data = make_scraper(rows).fetch_water_data_for_date(TARGET)
|
||||||
|
|
||||||
|
codes = {d["station_code"] for d in data}
|
||||||
|
assert "P.20" not in codes # station 1 skipped (None water level)
|
||||||
|
assert "P.75" in codes # station 2 present
|
||||||
|
|
||||||
|
|
||||||
|
def test_out_of_range_and_empty_hours_skipped(make_scraper):
|
||||||
|
rows = [
|
||||||
|
{"hourlytime": "25.00", "wlvalues1": "3.0"},
|
||||||
|
{"hourlytime": "0.00", "wlvalues1": "3.0"},
|
||||||
|
{"hourlytime": "", "wlvalues1": "3.0"},
|
||||||
|
]
|
||||||
|
data = make_scraper(rows).fetch_water_data_for_date(TARGET)
|
||||||
|
|
||||||
|
assert data == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_rows_key_returns_empty(make_scraper):
|
||||||
|
scraper = make_scraper([])
|
||||||
|
scraper.session.post.return_value.json.return_value = {"unexpected": True}
|
||||||
|
assert scraper.fetch_water_data_for_date(TARGET) == []
|
||||||
Reference in New Issue
Block a user