fix(api): /health never serves a cached result older than 4 TTLs
CI / Format & lint (push) Successful in 27s
CI / Test suite (push) Successful in 56s
Docs / Validate documentation (push) Successful in 26s
Security / Dependency vulnerabilities (push) Successful in 2m2s
Security / Static analysis (push) Successful in 21s
Security / License report (push) Successful in 1m50s
CI / Format & lint (push) Successful in 27s
CI / Test suite (push) Successful in 56s
Docs / Validate documentation (push) Successful in 26s
Security / Dependency vulnerabilities (push) Successful in 2m2s
Security / Static analysis (push) Successful in 21s
Security / License report (push) Successful in 1m50s
Stale-while-revalidate answered the first /health after a quiet night with the previous evening's state (seen after the collector cutover: 'Latest data is old: 21:00' at 13:47 while data was current). _cached_swr gains max_stale; /health uses 4 x HEALTH_CACHE_TTL_SECONDS.
This commit is contained in:
1 parent
9b910c3cbf
commit
64c5b02e06
2 files changed
+39
-2
No files matched your search
+12
-2
@@ -784,6 +784,8 @@ async def get_health():
|
|||||||
"health",
|
"health",
|
||||||
Config.HEALTH_CACHE_TTL_SECONDS,
|
Config.HEALTH_CACHE_TTL_SECONDS,
|
||||||
compute,
|
compute,
|
||||||
|
# never report a state older than a few TTLs: monitors poll /health
|
||||||
|
max_stale=4 * Config.HEALTH_CACHE_TTL_SECONDS,
|
||||||
)
|
)
|
||||||
|
|
||||||
return HealthResponse(**summary)
|
return HealthResponse(**summary)
|
||||||
@@ -1102,7 +1104,9 @@ _REFRESH_IN_FLIGHT: set = set()
|
|||||||
_REFRESH_FLAG_LOCK = Lock()
|
_REFRESH_FLAG_LOCK = Lock()
|
||||||
|
|
||||||
|
|
||||||
async def _cached_swr(cache, cache_lock, compute_lock, key, ttl, compute):
|
async def _cached_swr(
|
||||||
|
cache, cache_lock, compute_lock, key, ttl, compute, max_stale=None
|
||||||
|
):
|
||||||
"""Stale-while-revalidate. Returns (value, served_stale).
|
"""Stale-while-revalidate. Returns (value, served_stale).
|
||||||
|
|
||||||
fresh -> return inline; expired-but-present -> return the stale value
|
fresh -> return inline; expired-but-present -> return the stale value
|
||||||
@@ -1110,13 +1114,19 @@ async def _cached_swr(cache, cache_lock, compute_lock, key, ttl, compute):
|
|||||||
thread is parked on the single-flight lock — under stampede that parking
|
thread is parked on the single-flight lock — under stampede that parking
|
||||||
exhausted the pool and timed out unrelated endpoints); absent -> cold
|
exhausted the pool and timed out unrelated endpoints); absent -> cold
|
||||||
single-flight compute (first request per key since startup).
|
single-flight compute (first request per key since startup).
|
||||||
|
|
||||||
|
``max_stale`` (seconds) bounds how old a served stale copy may be; older
|
||||||
|
entries are recomputed inline. A worker that idles overnight would
|
||||||
|
otherwise answer its first request with a result from the evening before.
|
||||||
"""
|
"""
|
||||||
fresh = _cache_fresh(cache, cache_lock, key, ttl)
|
fresh = _cache_fresh(cache, cache_lock, key, ttl)
|
||||||
if fresh is not None:
|
if fresh is not None:
|
||||||
return fresh, False
|
return fresh, False
|
||||||
with cache_lock:
|
with cache_lock:
|
||||||
entry = cache.get(key)
|
entry = cache.get(key)
|
||||||
if entry is not None:
|
if entry is not None and (
|
||||||
|
max_stale is None or time.monotonic() - entry[0] <= max_stale
|
||||||
|
):
|
||||||
with _REFRESH_FLAG_LOCK:
|
with _REFRESH_FLAG_LOCK:
|
||||||
should_start = key not in _REFRESH_IN_FLIGHT
|
should_start = key not in _REFRESH_IN_FLIGHT
|
||||||
if should_start:
|
if should_start:
|
||||||
|
|||||||
@@ -94,3 +94,30 @@ def test_manual_trigger_is_refused_when_collection_runs_elsewhere(monkeypatch):
|
|||||||
with pytest.raises(HTTPException) as err:
|
with pytest.raises(HTTPException) as err:
|
||||||
asyncio.run(web_api.trigger_scraping(BackgroundTasks()))
|
asyncio.run(web_api.trigger_scraping(BackgroundTasks()))
|
||||||
assert err.value.status_code == 409
|
assert err.value.status_code == 409
|
||||||
|
|
||||||
|
|
||||||
|
def test_stale_cache_older_than_max_stale_is_recomputed(monkeypatch):
|
||||||
|
"""/health must not answer with a result from hours ago."""
|
||||||
|
import asyncio
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
from src import web_api
|
||||||
|
|
||||||
|
cache, lock, compute_lock = {}, threading.Lock(), threading.Lock()
|
||||||
|
cache["k"] = (time.monotonic() - 3600, "an hour old")
|
||||||
|
|
||||||
|
value, stale = asyncio.run(
|
||||||
|
web_api._cached_swr(
|
||||||
|
cache, lock, compute_lock, "k", 30, lambda: "fresh", max_stale=120
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert (value, stale) == ("fresh", False)
|
||||||
|
|
||||||
|
cache["k"] = (time.monotonic() - 60, "a minute old")
|
||||||
|
value, stale = asyncio.run(
|
||||||
|
web_api._cached_swr(
|
||||||
|
cache, lock, compute_lock, "k", 30, lambda: "fresh", max_stale=120
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert (value, stale) == ("a minute old", True) # SWR within the bound
|
||||||
Reference in new issue
Block a user