fix(awooop): 重試 source correlation 讀取瞬斷

2026-06-15 12:50:46 +08:00
parent 5389e9dbc5
commit c2c55a0d72
2 changed files with 111 additions and 8 deletions
--- a/apps/api/tests/test_awooop_source_correlation_apply_smoke.py
+++ b/apps/api/tests/test_awooop_source_correlation_apply_smoke.py
@@ -1,7 +1,10 @@
 from __future__ import annotations

 import importlib.util
+import io
+import json
 import sys
+import urllib.error
 from types import SimpleNamespace
 from pathlib import Path

@@ -22,6 +25,20 @@ sys.modules[SPEC.name] = awooop_source_correlation_apply_smoke
 SPEC.loader.exec_module(awooop_source_correlation_apply_smoke)


+class _JsonResponse:
+    def __init__(self, payload: dict[str, object]) -> None:
+        self._payload = payload
+
+    def __enter__(self) -> "_JsonResponse":
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        return None
+
+    def read(self) -> bytes:
+        return json.dumps(self._payload).encode("utf-8")
+
+
 def test_failed_check_summary_lists_preflight_failures() -> None:
    payload = {
        "checks": [
@@ -169,3 +186,63 @@ def test_wait_for_review_readback_retries_until_accepted(monkeypatch) -> None:
        "review_id": "review-2",
    }
    assert len(calls) == 2
+
+
+def test_http_json_retries_transient_get_502(monkeypatch) -> None:
+    calls: list[str] = []
+
+    def fake_urlopen(request: object, *, timeout: int) -> _JsonResponse:
+        calls.append(str(getattr(request, "full_url", "")))
+        if len(calls) == 1:
+            raise urllib.error.HTTPError(
+                url="https://awoooi.wooo.work/api",
+                code=502,
+                msg="Bad Gateway",
+                hdrs=None,
+                fp=io.BytesIO(b"<html>bad gateway</html>"),
+            )
+        return _JsonResponse({"ok": True})
+
+    monkeypatch.setattr(
+        awooop_source_correlation_apply_smoke.urllib.request,
+        "urlopen",
+        fake_urlopen,
+    )
+    monkeypatch.setattr(awooop_source_correlation_apply_smoke.time, "sleep", lambda _: None)
+
+    assert awooop_source_correlation_apply_smoke._http_json(
+        "https://awoooi.wooo.work/api",
+    ) == {"ok": True}
+    assert len(calls) == 2
+
+
+def test_http_json_does_not_retry_post_502(monkeypatch) -> None:
+    calls: list[str] = []
+
+    def fake_urlopen(request: object, *, timeout: int) -> _JsonResponse:
+        calls.append(str(getattr(request, "full_url", "")))
+        raise urllib.error.HTTPError(
+            url="https://awoooi.wooo.work/api",
+            code=502,
+            msg="Bad Gateway",
+            hdrs=None,
+            fp=io.BytesIO(b"<html>bad gateway</html>"),
+        )
+
+    monkeypatch.setattr(
+        awooop_source_correlation_apply_smoke.urllib.request,
+        "urlopen",
+        fake_urlopen,
+    )
+
+    try:
+        awooop_source_correlation_apply_smoke._http_json(
+            "https://awoooi.wooo.work/api",
+            method="POST",
+            payload={"write": True},
+        )
+    except awooop_source_correlation_apply_smoke.SmokeError as exc:
+        assert "HTTP 502" in str(exc)
+    else:
+        raise AssertionError("POST 502 should fail without retry")
+    assert len(calls) == 1
--- a/scripts/awooop_source_correlation_apply_smoke.py
+++ b/scripts/awooop_source_correlation_apply_smoke.py
@@ -20,6 +20,9 @@ from typing import Any


 SAFE_WORK_ITEM_TERMS = ("canary", "smoke", "codex")
+TRANSIENT_GET_HTTP_CODES = {502, 503, 504}
+GET_READBACK_ATTEMPTS = 4
+GET_READBACK_RETRY_DELAY_SECONDS = 2.0


 class SmokeError(RuntimeError):
@@ -72,14 +75,37 @@ def _http_json(
        data = json.dumps(payload).encode("utf-8")
        headers["Content-Type"] = "application/json"
    request = urllib.request.Request(url, data=data, headers=headers, method=method)
-    try:
-        with urllib.request.urlopen(request, timeout=timeout) as response:
-            return json.loads(response.read().decode("utf-8"))
-    except urllib.error.HTTPError as exc:
-        body = exc.read().decode("utf-8", errors="replace")[:500]
-        raise SmokeError(f"HTTP {exc.code} from {url}: {body}") from exc
-    except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
-        raise SmokeError(f"request failed for {url}: {exc}") from exc
+    attempts = GET_READBACK_ATTEMPTS if method.upper() == "GET" else 1
+    last_error: Exception | None = None
+    for attempt in range(max(attempts, 1)):
+        try:
+            with urllib.request.urlopen(request, timeout=timeout) as response:
+                return json.loads(response.read().decode("utf-8"))
+        except urllib.error.HTTPError as exc:
+            body = (
+                exc.read()
+                .decode("utf-8", errors="replace")
+                .encode("utf-8", errors="replace")
+                .decode("utf-8")
+            )[:500]
+            last_error = SmokeError(f"HTTP {exc.code} from {url}: {body}")
+            should_retry = (
+                method.upper() == "GET"
+                and exc.code in TRANSIENT_GET_HTTP_CODES
+                and attempt + 1 < attempts
+            )
+            if should_retry:
+                time.sleep(GET_READBACK_RETRY_DELAY_SECONDS)
+                continue
+            raise last_error from exc
+        except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
+            last_error = exc
+            should_retry = method.upper() == "GET" and attempt + 1 < attempts
+            if should_retry:
+                time.sleep(GET_READBACK_RETRY_DELAY_SECONDS)
+                continue
+            raise SmokeError(f"request failed for {url}: {exc}") from exc
+    raise SmokeError(f"request failed for {url}: {last_error}")


 def _find_work_item(