diff --git a/apps/api/tests/test_awooop_source_correlation_apply_smoke.py b/apps/api/tests/test_awooop_source_correlation_apply_smoke.py index 322c0ee2..7146bc81 100644 --- a/apps/api/tests/test_awooop_source_correlation_apply_smoke.py +++ b/apps/api/tests/test_awooop_source_correlation_apply_smoke.py @@ -1,7 +1,10 @@ from __future__ import annotations import importlib.util +import io +import json import sys +import urllib.error from types import SimpleNamespace from pathlib import Path @@ -22,6 +25,20 @@ sys.modules[SPEC.name] = awooop_source_correlation_apply_smoke SPEC.loader.exec_module(awooop_source_correlation_apply_smoke) +class _JsonResponse: + def __init__(self, payload: dict[str, object]) -> None: + self._payload = payload + + def __enter__(self) -> "_JsonResponse": + return self + + def __exit__(self, *_: object) -> None: + return None + + def read(self) -> bytes: + return json.dumps(self._payload).encode("utf-8") + + def test_failed_check_summary_lists_preflight_failures() -> None: payload = { "checks": [ @@ -169,3 +186,63 @@ def test_wait_for_review_readback_retries_until_accepted(monkeypatch) -> None: "review_id": "review-2", } assert len(calls) == 2 + + +def test_http_json_retries_transient_get_502(monkeypatch) -> None: + calls: list[str] = [] + + def fake_urlopen(request: object, *, timeout: int) -> _JsonResponse: + calls.append(str(getattr(request, "full_url", ""))) + if len(calls) == 1: + raise urllib.error.HTTPError( + url="https://awoooi.wooo.work/api", + code=502, + msg="Bad Gateway", + hdrs=None, + fp=io.BytesIO(b"bad gateway"), + ) + return _JsonResponse({"ok": True}) + + monkeypatch.setattr( + awooop_source_correlation_apply_smoke.urllib.request, + "urlopen", + fake_urlopen, + ) + monkeypatch.setattr(awooop_source_correlation_apply_smoke.time, "sleep", lambda _: None) + + assert awooop_source_correlation_apply_smoke._http_json( + "https://awoooi.wooo.work/api", + ) == {"ok": True} + assert len(calls) == 2 + + +def test_http_json_does_not_retry_post_502(monkeypatch) -> None: + calls: list[str] = [] + + def fake_urlopen(request: object, *, timeout: int) -> _JsonResponse: + calls.append(str(getattr(request, "full_url", ""))) + raise urllib.error.HTTPError( + url="https://awoooi.wooo.work/api", + code=502, + msg="Bad Gateway", + hdrs=None, + fp=io.BytesIO(b"bad gateway"), + ) + + monkeypatch.setattr( + awooop_source_correlation_apply_smoke.urllib.request, + "urlopen", + fake_urlopen, + ) + + try: + awooop_source_correlation_apply_smoke._http_json( + "https://awoooi.wooo.work/api", + method="POST", + payload={"write": True}, + ) + except awooop_source_correlation_apply_smoke.SmokeError as exc: + assert "HTTP 502" in str(exc) + else: + raise AssertionError("POST 502 should fail without retry") + assert len(calls) == 1 diff --git a/scripts/awooop_source_correlation_apply_smoke.py b/scripts/awooop_source_correlation_apply_smoke.py index cd01681d..25684909 100644 --- a/scripts/awooop_source_correlation_apply_smoke.py +++ b/scripts/awooop_source_correlation_apply_smoke.py @@ -20,6 +20,9 @@ from typing import Any SAFE_WORK_ITEM_TERMS = ("canary", "smoke", "codex") +TRANSIENT_GET_HTTP_CODES = {502, 503, 504} +GET_READBACK_ATTEMPTS = 4 +GET_READBACK_RETRY_DELAY_SECONDS = 2.0 class SmokeError(RuntimeError): @@ -72,14 +75,37 @@ def _http_json( data = json.dumps(payload).encode("utf-8") headers["Content-Type"] = "application/json" request = urllib.request.Request(url, data=data, headers=headers, method=method) - try: - with urllib.request.urlopen(request, timeout=timeout) as response: - return json.loads(response.read().decode("utf-8")) - except urllib.error.HTTPError as exc: - body = exc.read().decode("utf-8", errors="replace")[:500] - raise SmokeError(f"HTTP {exc.code} from {url}: {body}") from exc - except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: - raise SmokeError(f"request failed for {url}: {exc}") from exc + attempts = GET_READBACK_ATTEMPTS if method.upper() == "GET" else 1 + last_error: Exception | None = None + for attempt in range(max(attempts, 1)): + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + body = ( + exc.read() + .decode("utf-8", errors="replace") + .encode("utf-8", errors="replace") + .decode("utf-8") + )[:500] + last_error = SmokeError(f"HTTP {exc.code} from {url}: {body}") + should_retry = ( + method.upper() == "GET" + and exc.code in TRANSIENT_GET_HTTP_CODES + and attempt + 1 < attempts + ) + if should_retry: + time.sleep(GET_READBACK_RETRY_DELAY_SECONDS) + continue + raise last_error from exc + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: + last_error = exc + should_retry = method.upper() == "GET" and attempt + 1 < attempts + if should_retry: + time.sleep(GET_READBACK_RETRY_DELAY_SECONDS) + continue + raise SmokeError(f"request failed for {url}: {exc}") from exc + raise SmokeError(f"request failed for {url}: {last_error}") def _find_work_item(