feat(recovery): classify non110 harbor repair lane blocker

apps/api/src/services/harbor_registry_controlled_recovery_receipt.py

@@ -241,6 +241,17 @@ def validate_harbor_registry_controlled_recovery_receipt(
             "gitea_queue_current_cd_harbor_retrying_unavailable": gitea_queue[
                 "current_cd_harbor_retrying_unavailable"
             ],
+            "gitea_queue_current_cd_harbor_repair_requires_110_controlled_lane": (
+                gitea_queue["current_cd_harbor_repair_requires_110_controlled_lane"]
+            ),
+            "gitea_queue_current_cd_harbor_repair_blocked_by_awoooi_host": (
+                gitea_queue[
+                    "current_cd_harbor_repair_blocked_by_no_matching_awoooi_host"
+                ]
+            ),
+            "gitea_queue_current_cd_harbor_repair_lane_classifier": gitea_queue[
+                "current_cd_harbor_repair_lane_classifier"
+            ],
             "gitea_queue_cd_jobs_stale_or_mismatched": gitea_queue[
                 "cd_run_jobs_stale_or_mismatched"
             ],
@@ -332,6 +343,12 @@ def _control_path_readiness(
     queue_jobs_cross_workflow = bool(
         gitea_queue["harbor_110_repair_jobs_cross_workflow_mismatch"]
     )
+    cd_harbor_repair_requires_110_lane = bool(
+        gitea_queue["current_cd_harbor_repair_requires_110_controlled_lane"]
+    )
+    cd_harbor_repair_blocked_by_awoooi_host = bool(
+        gitea_queue["current_cd_harbor_repair_blocked_by_no_matching_awoooi_host"]
+    )
     cd_jobs_stale = bool(gitea_queue["cd_run_jobs_stale_or_mismatched"])
     cd_jobs_head_sha_mismatch = bool(gitea_queue["cd_run_jobs_head_sha_mismatch"])
     runner_timeout = bool(ssh_diagnosis["runner_systemctl_show_timeout_seen"])
@@ -350,6 +367,10 @@ def _control_path_readiness(
         queue_no_matching_runner=queue_no_matching_runner,
         queue_jobs_stale=queue_jobs_stale,
         queue_jobs_cross_workflow=queue_jobs_cross_workflow,
+        cd_harbor_repair_requires_110_lane=cd_harbor_repair_requires_110_lane,
+        cd_harbor_repair_blocked_by_awoooi_host=(
+            cd_harbor_repair_blocked_by_awoooi_host
+        ),
         cd_jobs_head_sha_mismatch=cd_jobs_head_sha_mismatch,
         cd_jobs_stale=cd_jobs_stale,
         public_registry_ready=public_registry_ready,
@@ -385,6 +406,15 @@ def _control_path_readiness(
         ],
         "harbor_110_repair_jobs_stale_or_mismatched": queue_jobs_stale,
         "harbor_110_repair_jobs_cross_workflow_mismatch": queue_jobs_cross_workflow,
+        "cd_harbor_repair_requires_110_controlled_lane": (
+            cd_harbor_repair_requires_110_lane
+        ),
+        "cd_harbor_repair_blocked_by_no_matching_awoooi_host": (
+            cd_harbor_repair_blocked_by_awoooi_host
+        ),
+        "cd_harbor_repair_lane_classifier": gitea_queue[
+            "current_cd_harbor_repair_lane_classifier"
+        ],
         "cd_jobs_stale_or_mismatched": cd_jobs_stale,
         "cd_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch,
         "registry_v2_public_http_status": verifier[
@@ -410,6 +440,8 @@ def _control_path_signal_ids(
     queue_no_matching_runner: bool,
     queue_jobs_stale: bool,
     queue_jobs_cross_workflow: bool,
+    cd_harbor_repair_requires_110_lane: bool,
+    cd_harbor_repair_blocked_by_awoooi_host: bool,
     cd_jobs_head_sha_mismatch: bool,
     cd_jobs_stale: bool,
     public_registry_ready: bool,
@@ -430,6 +462,12 @@ def _control_path_signal_ids(
         signal_ids.append("gitea_queue_harbor_110_repair_jobs_cross_workflow_mismatch")
     if queue_jobs_stale:
         signal_ids.append("gitea_queue_harbor_110_repair_jobs_stale_or_mismatched")
+    if cd_harbor_repair_blocked_by_awoooi_host:
+        signal_ids.append(
+            "gitea_queue_cd_harbor_repair_blocked_by_no_matching_awoooi_host"
+        )
+    if cd_harbor_repair_requires_110_lane:
+        signal_ids.append("gitea_queue_cd_harbor_repair_requires_110_controlled_lane")
     if cd_jobs_head_sha_mismatch:
         signal_ids.append("gitea_queue_cd_jobs_head_sha_mismatch")
     if cd_jobs_stale:
@@ -858,6 +896,9 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
             "current_cd_harbor_retrying_unavailable": False,
             "controlled_profile_no_matching_runner_labels": {},
             "controlled_profile_no_matching_runner_label_count": 0,
+            "current_cd_harbor_repair_requires_110_controlled_lane": False,
+            "current_cd_harbor_repair_blocked_by_no_matching_awoooi_host": False,
+            "current_cd_harbor_repair_lane_classifier": "",
             "cd_run_jobs_stale_or_mismatched": False,
             "cd_run_jobs_payload_classifier": "",
             "cd_run_jobs_head_sha_mismatch": False,
@@ -949,6 +990,25 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
         or current_cd_inflight_classifier
         == "harbor_registry_public_route_unavailable_pending_retry"
     )
+    current_cd_harbor_repair_requires_110_lane = bool(
+        rollups.get("current_main_cd_harbor_repair_requires_110_controlled_lane")
+        is True
+        or readback.get("latest_visible_cd_harbor_repair_requires_110_controlled_lane")
+        is True
+    )
+    current_cd_harbor_repair_blocked_by_awoooi_host = bool(
+        rollups.get("current_main_cd_harbor_repair_blocked_by_no_matching_awoooi_host")
+        is True
+        or readback.get(
+            "latest_visible_cd_harbor_repair_blocked_by_no_matching_awoooi_host"
+        )
+        is True
+    )
+    current_cd_harbor_repair_lane_classifier = str(
+        rollups.get("current_main_cd_harbor_repair_lane_classifier")
+        or readback.get("latest_visible_cd_harbor_repair_lane_classifier")
+        or ""
+    )
     cd_jobs_stale = bool(
         rollups.get("cd_run_jobs_stale_or_mismatched") is True
         or readback.get("cd_run_jobs_stale_or_mismatched") is True
@@ -989,6 +1049,12 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
         current_cd_harbor_retrying=current_cd_harbor_retrying,
         current_cd_waiting=current_cd_waiting,
         current_cd_no_matching_runner=bool(current_cd_no_matching_label),
+        current_cd_harbor_repair_requires_110_lane=(
+            current_cd_harbor_repair_requires_110_lane
+        ),
+        current_cd_harbor_repair_blocked_by_awoooi_host=(
+            current_cd_harbor_repair_blocked_by_awoooi_host
+        ),
         cd_jobs_stale=cd_jobs_stale,
         cd_jobs_head_sha_mismatch=cd_jobs_head_sha_mismatch,
         cd_jobs_run_id_mismatch=cd_jobs_run_id_mismatch,
@@ -1028,6 +1094,15 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
         "controlled_profile_no_matching_runner_label_count": len(
             controlled_profile_no_matching_labels
         ),
+        "current_cd_harbor_repair_requires_110_controlled_lane": (
+            current_cd_harbor_repair_requires_110_lane
+        ),
+        "current_cd_harbor_repair_blocked_by_no_matching_awoooi_host": (
+            current_cd_harbor_repair_blocked_by_awoooi_host
+        ),
+        "current_cd_harbor_repair_lane_classifier": (
+            current_cd_harbor_repair_lane_classifier
+        ),
         "cd_run_jobs_stale_or_mismatched": cd_jobs_stale,
         "cd_run_jobs_payload_classifier": cd_jobs_payload_classifier,
         "cd_run_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch,
@@ -1147,6 +1222,8 @@ def _gitea_queue_blockers(
     current_cd_harbor_retrying: bool,
     current_cd_waiting: bool,
     current_cd_no_matching_runner: bool,
+    current_cd_harbor_repair_requires_110_lane: bool,
+    current_cd_harbor_repair_blocked_by_awoooi_host: bool,
     cd_jobs_stale: bool,
     cd_jobs_head_sha_mismatch: bool,
     cd_jobs_run_id_mismatch: bool,
@@ -1154,6 +1231,12 @@ def _gitea_queue_blockers(
     boundary_violation: bool,
 ) -> list[str]:
     blockers: list[str] = []
+    if current_cd_harbor_repair_blocked_by_awoooi_host:
+        blockers.append(
+            "gitea_queue_cd_harbor_repair_blocked_by_no_matching_awoooi_host"
+        )
+    if current_cd_harbor_repair_requires_110_lane:
+        blockers.append("gitea_queue_cd_harbor_repair_requires_110_controlled_lane")
     if current_cd_harbor_retrying:
         blockers.append("gitea_queue_current_cd_harbor_retrying_unavailable")
     if current_cd_no_matching_runner:

apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py

@@ -689,6 +689,50 @@ def test_harbor_recovery_receipt_surfaces_inflight_cd_harbor_retry() -> None:
     assert payload["rollups"]["gitea_queue_blocker_count"] == 3
 
 
+def test_harbor_recovery_receipt_surfaces_non110_cd_repair_lane_blocker() -> None:
+    payload = validate_harbor_registry_controlled_recovery_receipt(
+        {
+            "gitea_actions_queue_readback": (
+                _gitea_queue_cd_harbor_repair_requires_110_lane()
+            ),
+        }
+    )
+
+    assert (
+        "gitea_queue_cd_harbor_repair_blocked_by_no_matching_awoooi_host"
+        in payload["active_blockers"]
+    )
+    assert (
+        "gitea_queue_cd_harbor_repair_requires_110_controlled_lane"
+        in payload["active_blockers"]
+    )
+    queue = payload["readback"]["gitea_actions_queue"]
+    assert queue["current_cd_harbor_repair_requires_110_controlled_lane"] is True
+    assert (
+        queue["current_cd_harbor_repair_blocked_by_no_matching_awoooi_host"]
+        is True
+    )
+    assert queue["current_cd_harbor_repair_lane_classifier"] == (
+        "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host"
+    )
+    readiness = payload["readback"]["control_path_readiness"]
+    assert (
+        "gitea_queue_cd_harbor_repair_blocked_by_no_matching_awoooi_host"
+        in readiness["signal_ids"]
+    )
+    assert (
+        "gitea_queue_cd_harbor_repair_requires_110_controlled_lane"
+        in readiness["signal_ids"]
+    )
+    assert payload["rollups"][
+        "gitea_queue_current_cd_harbor_repair_requires_110_controlled_lane"
+    ] is True
+    assert (
+        payload["rollups"]["gitea_queue_current_cd_harbor_repair_lane_classifier"]
+        == "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host"
+    )
+
+
 def test_harbor_recovery_receipt_endpoint_redacts_raw_output() -> None:
     app = FastAPI()
     app.include_router(router, prefix="/api/v1")
@@ -1034,3 +1078,26 @@ def _gitea_queue_no_matching_runner_with_cd_retry() -> dict:
         }
     )
     return payload
+
+
+def _gitea_queue_cd_harbor_repair_requires_110_lane() -> dict:
+    payload = _gitea_queue_no_matching_runner()
+    payload["readback"].update(
+        {
+            "latest_visible_cd_harbor_repair_requires_110_controlled_lane": True,
+            "latest_visible_cd_harbor_repair_blocked_by_no_matching_awoooi_host": True,
+            "latest_visible_cd_harbor_repair_lane_classifier": (
+                "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host"
+            ),
+        }
+    )
+    payload["rollups"].update(
+        {
+            "current_main_cd_harbor_repair_requires_110_controlled_lane": True,
+            "current_main_cd_harbor_repair_blocked_by_no_matching_awoooi_host": True,
+            "current_main_cd_harbor_repair_lane_classifier": (
+                "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host"
+            ),
+        }
+    )
+    return payload

ops/runner/read-public-gitea-actions-queue.py

@@ -421,6 +421,25 @@ def build_readback(
     harbor_110_repair_visible_waiting_stale = (
         harbor_110_repair_waiting and harbor_110_repair_jobs_all_success
     )
+    cd_harbor_repair_requires_110_controlled_lane = (
+        build_log_classifier["harbor_public_route_blocked_or_retrying"]
+        and build_log_classifier["harbor_controlled_repair_skip_reason"]
+        == "not_110_host"
+    )
+    cd_harbor_repair_blocked_by_no_matching_awoooi_host = (
+        cd_harbor_repair_requires_110_controlled_lane
+        and harbor_110_repair_no_matching_runner_label == "awoooi-host"
+    )
+    cd_harbor_repair_lane_classifier = (
+        "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host"
+        if cd_harbor_repair_blocked_by_no_matching_awoooi_host
+        else "cd_harbor_repair_requires_110_controlled_lane_waiting"
+        if cd_harbor_repair_requires_110_controlled_lane
+        and harbor_110_repair_waiting
+        else "cd_harbor_repair_requires_110_controlled_lane"
+        if cd_harbor_repair_requires_110_controlled_lane
+        else ""
+    )
     harbor_110_repair_waiting_after_cd_harbor_blocker = (
         build_log_classifier["harbor_public_route_blocked_or_retrying"]
         and harbor_110_repair_waiting
@@ -504,6 +523,15 @@ def build_readback(
                 "harbor_controlled_repair_public_registry_v2_status"
             ]
         ),
+        "latest_visible_cd_harbor_repair_requires_110_controlled_lane": (
+            cd_harbor_repair_requires_110_controlled_lane
+        ),
+        "latest_visible_cd_harbor_repair_blocked_by_no_matching_awoooi_host": (
+            cd_harbor_repair_blocked_by_no_matching_awoooi_host
+        ),
+        "latest_visible_cd_harbor_repair_lane_classifier": (
+            cd_harbor_repair_lane_classifier
+        ),
         "latest_visible_cd_harbor_public_route_blocked": build_log_classifier[
             "harbor_public_route_blocked"
         ],
@@ -689,6 +717,15 @@ def build_readback(
             "current_main_cd_harbor_controlled_repair_skip_reason": (
                 build_log_classifier["harbor_controlled_repair_skip_reason"]
             ),
+            "current_main_cd_harbor_repair_requires_110_controlled_lane": (
+                cd_harbor_repair_requires_110_controlled_lane
+            ),
+            "current_main_cd_harbor_repair_blocked_by_no_matching_awoooi_host": (
+                cd_harbor_repair_blocked_by_no_matching_awoooi_host
+            ),
+            "current_main_cd_harbor_repair_lane_classifier": (
+                cd_harbor_repair_lane_classifier
+            ),
             "current_main_cd_host_pressure_classifier": tests_log_classifier[
                 "host_pressure_classifier"
             ],

ops/runner/test_read_public_gitea_actions_queue.py

@@ -699,6 +699,22 @@ def test_build_readback_classifies_harbor_public_route_blocker() -> None:
         ]
         == "not_110_host"
     )
+    assert (
+        payload["readback"][
+            "latest_visible_cd_harbor_repair_requires_110_controlled_lane"
+        ]
+        is True
+    )
+    assert (
+        payload["readback"][
+            "latest_visible_cd_harbor_repair_blocked_by_no_matching_awoooi_host"
+        ]
+        is False
+    )
+    assert (
+        payload["readback"]["latest_visible_cd_harbor_repair_lane_classifier"]
+        == "cd_harbor_repair_requires_110_controlled_lane"
+    )
     assert payload["readback"]["latest_visible_cd_harbor_public_route_blocked"] is True
     assert payload["rollups"]["current_main_cd_harbor_public_route_blocked"] is True
     assert (
@@ -707,6 +723,55 @@ def test_build_readback_classifies_harbor_public_route_blocker() -> None:
         ]
         == "not_110_host"
     )
+    assert (
+        payload["rollups"][
+            "current_main_cd_harbor_repair_requires_110_controlled_lane"
+        ]
+        is True
+    )
+
+
+def test_harbor_repair_no_matching_explains_non110_cd_skip_reason() -> None:
+    module = _load_module()
+    payload = module.build_readback(
+        actions_html=_actions_html_harbor_repair_waiting_with_workflow_no_matching(),
+        actions_list_http_status=401,
+        actions_list_payload={"message": "token is required"},
+        cd_jobs_http_status=200,
+        cd_jobs_payload={"jobs": [], "total_count": 0},
+        latest_cd_build_log_http_status=200,
+        latest_cd_build_log_text=_harbor_blocked_log(),
+    )
+
+    assert payload["status"] == "blocked_harbor_110_repair_no_matching_runner"
+    assert (
+        payload["readback"][
+            "latest_visible_cd_harbor_controlled_repair_skip_reason"
+        ]
+        == "not_110_host"
+    )
+    assert (
+        payload["readback"][
+            "latest_visible_cd_harbor_repair_requires_110_controlled_lane"
+        ]
+        is True
+    )
+    assert (
+        payload["readback"][
+            "latest_visible_cd_harbor_repair_blocked_by_no_matching_awoooi_host"
+        ]
+        is True
+    )
+    assert (
+        payload["readback"]["latest_visible_cd_harbor_repair_lane_classifier"]
+        == "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host"
+    )
+    assert (
+        payload["rollups"][
+            "current_main_cd_harbor_repair_lane_classifier"
+        ]
+        == "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host"
+    )
 
 
 def test_build_readback_prefers_visible_blocked_over_stale_jobs() -> None: