fix(gateway): the fire startup wait stays under the dashboard forwarder timeout and retries after 60s
(cherry picked from commit e5d54c20ec719dafaac88c7469f34f02c5eec462)
This commit is contained in:
@@ -14,10 +14,10 @@ from typing import Any, Optional
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
# How long a fire that lands during startup waits before it is refused as retryable. A cold boot
|
||||
# publishes its adapters within seconds; the bound keeps a gateway that never finishes starting from
|
||||
# holding the request open (NAS's callback timeout is 30s).
|
||||
FIRE_STARTUP_WAIT_SECONDS = 10.0
|
||||
# How long a fire that lands during startup waits before it is refused as retryable. Kept under the
|
||||
# dashboard forwarder's 10s timeout: past it NAS sees a 503 while this handler still runs the job, and
|
||||
# the retry would run it twice.
|
||||
FIRE_STARTUP_WAIT_SECONDS = 7.0
|
||||
_POLL_SECONDS = 0.1
|
||||
|
||||
|
||||
@@ -25,12 +25,12 @@ async def refuse_until_started(runner: Any, job_id: str) -> Optional["web.Respon
|
||||
"""None once ``runner`` has finished starting (``_running``: every platform connect attempted and its
|
||||
adapters published), or when there is no runner to ask (a self-hosted api_server, a test double).
|
||||
Otherwise the retryable 503, worded like the dashboard's own "gateway unreachable" so NAS classifies
|
||||
it the same."""
|
||||
it the same, with the dashboard's 60s Retry-After (``_CRON_FIRE_RETRY_AFTER_SECONDS``)."""
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + FIRE_STARTUP_WAIT_SECONDS
|
||||
while runner is not None and not getattr(runner, "_running", True):
|
||||
if loop.time() >= deadline:
|
||||
return web.json_response(
|
||||
{"error": "gateway unreachable; retry", "job_id": job_id}, status=503, headers={"Retry-After": "2"})
|
||||
{"error": "gateway unreachable; retry", "job_id": job_id}, status=503, headers={"Retry-After": "60"})
|
||||
await asyncio.sleep(_POLL_SECONDS)
|
||||
return None
|
||||
|
||||
@@ -98,7 +98,9 @@ async def test_fire_is_retryable_and_never_claimed_when_startup_does_not_finish(
|
||||
|
||||
resp = await _post_fire(adapter, runner)
|
||||
|
||||
assert resp.status == 503
|
||||
# NAS classifies this refusal by the exact body string and honours Retry-After.
|
||||
assert resp.status == 503 and resp.headers["Retry-After"] == "60"
|
||||
assert (await resp.json())["error"] == "gateway unreachable; retry"
|
||||
assert provider.claimed == [] and provider.fired == [] # nothing durably claimed for a fire we refused
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user