From 77e0b58ee4236f03c2e2a2fa18ab953a5d622f2d Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 1 Jul 2026 08:25:35 +0800 Subject: [PATCH] feat(recovery): classify non110 harbor repair lane blocker --- ...or_registry_controlled_recovery_receipt.py | 83 +++++++++++++++++++ ...or_registry_controlled_recovery_receipt.py | 67 +++++++++++++++ ops/runner/read-public-gitea-actions-queue.py | 37 +++++++++ .../test_read_public_gitea_actions_queue.py | 65 +++++++++++++++ 4 files changed, 252 insertions(+) diff --git a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py index 035d9a63..f1e143ef 100644 --- a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py @@ -241,6 +241,17 @@ def validate_harbor_registry_controlled_recovery_receipt( "gitea_queue_current_cd_harbor_retrying_unavailable": gitea_queue[ "current_cd_harbor_retrying_unavailable" ], + "gitea_queue_current_cd_harbor_repair_requires_110_controlled_lane": ( + gitea_queue["current_cd_harbor_repair_requires_110_controlled_lane"] + ), + "gitea_queue_current_cd_harbor_repair_blocked_by_awoooi_host": ( + gitea_queue[ + "current_cd_harbor_repair_blocked_by_no_matching_awoooi_host" + ] + ), + "gitea_queue_current_cd_harbor_repair_lane_classifier": gitea_queue[ + "current_cd_harbor_repair_lane_classifier" + ], "gitea_queue_cd_jobs_stale_or_mismatched": gitea_queue[ "cd_run_jobs_stale_or_mismatched" ], @@ -332,6 +343,12 @@ def _control_path_readiness( queue_jobs_cross_workflow = bool( gitea_queue["harbor_110_repair_jobs_cross_workflow_mismatch"] ) + cd_harbor_repair_requires_110_lane = bool( + gitea_queue["current_cd_harbor_repair_requires_110_controlled_lane"] + ) + cd_harbor_repair_blocked_by_awoooi_host = bool( + gitea_queue["current_cd_harbor_repair_blocked_by_no_matching_awoooi_host"] + ) cd_jobs_stale = bool(gitea_queue["cd_run_jobs_stale_or_mismatched"]) cd_jobs_head_sha_mismatch = bool(gitea_queue["cd_run_jobs_head_sha_mismatch"]) runner_timeout = bool(ssh_diagnosis["runner_systemctl_show_timeout_seen"]) @@ -350,6 +367,10 @@ def _control_path_readiness( queue_no_matching_runner=queue_no_matching_runner, queue_jobs_stale=queue_jobs_stale, queue_jobs_cross_workflow=queue_jobs_cross_workflow, + cd_harbor_repair_requires_110_lane=cd_harbor_repair_requires_110_lane, + cd_harbor_repair_blocked_by_awoooi_host=( + cd_harbor_repair_blocked_by_awoooi_host + ), cd_jobs_head_sha_mismatch=cd_jobs_head_sha_mismatch, cd_jobs_stale=cd_jobs_stale, public_registry_ready=public_registry_ready, @@ -385,6 +406,15 @@ def _control_path_readiness( ], "harbor_110_repair_jobs_stale_or_mismatched": queue_jobs_stale, "harbor_110_repair_jobs_cross_workflow_mismatch": queue_jobs_cross_workflow, + "cd_harbor_repair_requires_110_controlled_lane": ( + cd_harbor_repair_requires_110_lane + ), + "cd_harbor_repair_blocked_by_no_matching_awoooi_host": ( + cd_harbor_repair_blocked_by_awoooi_host + ), + "cd_harbor_repair_lane_classifier": gitea_queue[ + "current_cd_harbor_repair_lane_classifier" + ], "cd_jobs_stale_or_mismatched": cd_jobs_stale, "cd_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch, "registry_v2_public_http_status": verifier[ @@ -410,6 +440,8 @@ def _control_path_signal_ids( queue_no_matching_runner: bool, queue_jobs_stale: bool, queue_jobs_cross_workflow: bool, + cd_harbor_repair_requires_110_lane: bool, + cd_harbor_repair_blocked_by_awoooi_host: bool, cd_jobs_head_sha_mismatch: bool, cd_jobs_stale: bool, public_registry_ready: bool, @@ -430,6 +462,12 @@ def _control_path_signal_ids( signal_ids.append("gitea_queue_harbor_110_repair_jobs_cross_workflow_mismatch") if queue_jobs_stale: signal_ids.append("gitea_queue_harbor_110_repair_jobs_stale_or_mismatched") + if cd_harbor_repair_blocked_by_awoooi_host: + signal_ids.append( + "gitea_queue_cd_harbor_repair_blocked_by_no_matching_awoooi_host" + ) + if cd_harbor_repair_requires_110_lane: + signal_ids.append("gitea_queue_cd_harbor_repair_requires_110_controlled_lane") if cd_jobs_head_sha_mismatch: signal_ids.append("gitea_queue_cd_jobs_head_sha_mismatch") if cd_jobs_stale: @@ -858,6 +896,9 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: "current_cd_harbor_retrying_unavailable": False, "controlled_profile_no_matching_runner_labels": {}, "controlled_profile_no_matching_runner_label_count": 0, + "current_cd_harbor_repair_requires_110_controlled_lane": False, + "current_cd_harbor_repair_blocked_by_no_matching_awoooi_host": False, + "current_cd_harbor_repair_lane_classifier": "", "cd_run_jobs_stale_or_mismatched": False, "cd_run_jobs_payload_classifier": "", "cd_run_jobs_head_sha_mismatch": False, @@ -949,6 +990,25 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: or current_cd_inflight_classifier == "harbor_registry_public_route_unavailable_pending_retry" ) + current_cd_harbor_repair_requires_110_lane = bool( + rollups.get("current_main_cd_harbor_repair_requires_110_controlled_lane") + is True + or readback.get("latest_visible_cd_harbor_repair_requires_110_controlled_lane") + is True + ) + current_cd_harbor_repair_blocked_by_awoooi_host = bool( + rollups.get("current_main_cd_harbor_repair_blocked_by_no_matching_awoooi_host") + is True + or readback.get( + "latest_visible_cd_harbor_repair_blocked_by_no_matching_awoooi_host" + ) + is True + ) + current_cd_harbor_repair_lane_classifier = str( + rollups.get("current_main_cd_harbor_repair_lane_classifier") + or readback.get("latest_visible_cd_harbor_repair_lane_classifier") + or "" + ) cd_jobs_stale = bool( rollups.get("cd_run_jobs_stale_or_mismatched") is True or readback.get("cd_run_jobs_stale_or_mismatched") is True @@ -989,6 +1049,12 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: current_cd_harbor_retrying=current_cd_harbor_retrying, current_cd_waiting=current_cd_waiting, current_cd_no_matching_runner=bool(current_cd_no_matching_label), + current_cd_harbor_repair_requires_110_lane=( + current_cd_harbor_repair_requires_110_lane + ), + current_cd_harbor_repair_blocked_by_awoooi_host=( + current_cd_harbor_repair_blocked_by_awoooi_host + ), cd_jobs_stale=cd_jobs_stale, cd_jobs_head_sha_mismatch=cd_jobs_head_sha_mismatch, cd_jobs_run_id_mismatch=cd_jobs_run_id_mismatch, @@ -1028,6 +1094,15 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: "controlled_profile_no_matching_runner_label_count": len( controlled_profile_no_matching_labels ), + "current_cd_harbor_repair_requires_110_controlled_lane": ( + current_cd_harbor_repair_requires_110_lane + ), + "current_cd_harbor_repair_blocked_by_no_matching_awoooi_host": ( + current_cd_harbor_repair_blocked_by_awoooi_host + ), + "current_cd_harbor_repair_lane_classifier": ( + current_cd_harbor_repair_lane_classifier + ), "cd_run_jobs_stale_or_mismatched": cd_jobs_stale, "cd_run_jobs_payload_classifier": cd_jobs_payload_classifier, "cd_run_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch, @@ -1147,6 +1222,8 @@ def _gitea_queue_blockers( current_cd_harbor_retrying: bool, current_cd_waiting: bool, current_cd_no_matching_runner: bool, + current_cd_harbor_repair_requires_110_lane: bool, + current_cd_harbor_repair_blocked_by_awoooi_host: bool, cd_jobs_stale: bool, cd_jobs_head_sha_mismatch: bool, cd_jobs_run_id_mismatch: bool, @@ -1154,6 +1231,12 @@ def _gitea_queue_blockers( boundary_violation: bool, ) -> list[str]: blockers: list[str] = [] + if current_cd_harbor_repair_blocked_by_awoooi_host: + blockers.append( + "gitea_queue_cd_harbor_repair_blocked_by_no_matching_awoooi_host" + ) + if current_cd_harbor_repair_requires_110_lane: + blockers.append("gitea_queue_cd_harbor_repair_requires_110_controlled_lane") if current_cd_harbor_retrying: blockers.append("gitea_queue_current_cd_harbor_retrying_unavailable") if current_cd_no_matching_runner: diff --git a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py index 7a93968d..9670fd3b 100644 --- a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py @@ -689,6 +689,50 @@ def test_harbor_recovery_receipt_surfaces_inflight_cd_harbor_retry() -> None: assert payload["rollups"]["gitea_queue_blocker_count"] == 3 +def test_harbor_recovery_receipt_surfaces_non110_cd_repair_lane_blocker() -> None: + payload = validate_harbor_registry_controlled_recovery_receipt( + { + "gitea_actions_queue_readback": ( + _gitea_queue_cd_harbor_repair_requires_110_lane() + ), + } + ) + + assert ( + "gitea_queue_cd_harbor_repair_blocked_by_no_matching_awoooi_host" + in payload["active_blockers"] + ) + assert ( + "gitea_queue_cd_harbor_repair_requires_110_controlled_lane" + in payload["active_blockers"] + ) + queue = payload["readback"]["gitea_actions_queue"] + assert queue["current_cd_harbor_repair_requires_110_controlled_lane"] is True + assert ( + queue["current_cd_harbor_repair_blocked_by_no_matching_awoooi_host"] + is True + ) + assert queue["current_cd_harbor_repair_lane_classifier"] == ( + "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host" + ) + readiness = payload["readback"]["control_path_readiness"] + assert ( + "gitea_queue_cd_harbor_repair_blocked_by_no_matching_awoooi_host" + in readiness["signal_ids"] + ) + assert ( + "gitea_queue_cd_harbor_repair_requires_110_controlled_lane" + in readiness["signal_ids"] + ) + assert payload["rollups"][ + "gitea_queue_current_cd_harbor_repair_requires_110_controlled_lane" + ] is True + assert ( + payload["rollups"]["gitea_queue_current_cd_harbor_repair_lane_classifier"] + == "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host" + ) + + def test_harbor_recovery_receipt_endpoint_redacts_raw_output() -> None: app = FastAPI() app.include_router(router, prefix="/api/v1") @@ -1034,3 +1078,26 @@ def _gitea_queue_no_matching_runner_with_cd_retry() -> dict: } ) return payload + + +def _gitea_queue_cd_harbor_repair_requires_110_lane() -> dict: + payload = _gitea_queue_no_matching_runner() + payload["readback"].update( + { + "latest_visible_cd_harbor_repair_requires_110_controlled_lane": True, + "latest_visible_cd_harbor_repair_blocked_by_no_matching_awoooi_host": True, + "latest_visible_cd_harbor_repair_lane_classifier": ( + "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host" + ), + } + ) + payload["rollups"].update( + { + "current_main_cd_harbor_repair_requires_110_controlled_lane": True, + "current_main_cd_harbor_repair_blocked_by_no_matching_awoooi_host": True, + "current_main_cd_harbor_repair_lane_classifier": ( + "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host" + ), + } + ) + return payload diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index 77b45568..eaa4fb87 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -421,6 +421,25 @@ def build_readback( harbor_110_repair_visible_waiting_stale = ( harbor_110_repair_waiting and harbor_110_repair_jobs_all_success ) + cd_harbor_repair_requires_110_controlled_lane = ( + build_log_classifier["harbor_public_route_blocked_or_retrying"] + and build_log_classifier["harbor_controlled_repair_skip_reason"] + == "not_110_host" + ) + cd_harbor_repair_blocked_by_no_matching_awoooi_host = ( + cd_harbor_repair_requires_110_controlled_lane + and harbor_110_repair_no_matching_runner_label == "awoooi-host" + ) + cd_harbor_repair_lane_classifier = ( + "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host" + if cd_harbor_repair_blocked_by_no_matching_awoooi_host + else "cd_harbor_repair_requires_110_controlled_lane_waiting" + if cd_harbor_repair_requires_110_controlled_lane + and harbor_110_repair_waiting + else "cd_harbor_repair_requires_110_controlled_lane" + if cd_harbor_repair_requires_110_controlled_lane + else "" + ) harbor_110_repair_waiting_after_cd_harbor_blocker = ( build_log_classifier["harbor_public_route_blocked_or_retrying"] and harbor_110_repair_waiting @@ -504,6 +523,15 @@ def build_readback( "harbor_controlled_repair_public_registry_v2_status" ] ), + "latest_visible_cd_harbor_repair_requires_110_controlled_lane": ( + cd_harbor_repair_requires_110_controlled_lane + ), + "latest_visible_cd_harbor_repair_blocked_by_no_matching_awoooi_host": ( + cd_harbor_repair_blocked_by_no_matching_awoooi_host + ), + "latest_visible_cd_harbor_repair_lane_classifier": ( + cd_harbor_repair_lane_classifier + ), "latest_visible_cd_harbor_public_route_blocked": build_log_classifier[ "harbor_public_route_blocked" ], @@ -689,6 +717,15 @@ def build_readback( "current_main_cd_harbor_controlled_repair_skip_reason": ( build_log_classifier["harbor_controlled_repair_skip_reason"] ), + "current_main_cd_harbor_repair_requires_110_controlled_lane": ( + cd_harbor_repair_requires_110_controlled_lane + ), + "current_main_cd_harbor_repair_blocked_by_no_matching_awoooi_host": ( + cd_harbor_repair_blocked_by_no_matching_awoooi_host + ), + "current_main_cd_harbor_repair_lane_classifier": ( + cd_harbor_repair_lane_classifier + ), "current_main_cd_host_pressure_classifier": tests_log_classifier[ "host_pressure_classifier" ], diff --git a/ops/runner/test_read_public_gitea_actions_queue.py b/ops/runner/test_read_public_gitea_actions_queue.py index 81ea7830..0dbbe581 100644 --- a/ops/runner/test_read_public_gitea_actions_queue.py +++ b/ops/runner/test_read_public_gitea_actions_queue.py @@ -699,6 +699,22 @@ def test_build_readback_classifies_harbor_public_route_blocker() -> None: ] == "not_110_host" ) + assert ( + payload["readback"][ + "latest_visible_cd_harbor_repair_requires_110_controlled_lane" + ] + is True + ) + assert ( + payload["readback"][ + "latest_visible_cd_harbor_repair_blocked_by_no_matching_awoooi_host" + ] + is False + ) + assert ( + payload["readback"]["latest_visible_cd_harbor_repair_lane_classifier"] + == "cd_harbor_repair_requires_110_controlled_lane" + ) assert payload["readback"]["latest_visible_cd_harbor_public_route_blocked"] is True assert payload["rollups"]["current_main_cd_harbor_public_route_blocked"] is True assert ( @@ -707,6 +723,55 @@ def test_build_readback_classifies_harbor_public_route_blocker() -> None: ] == "not_110_host" ) + assert ( + payload["rollups"][ + "current_main_cd_harbor_repair_requires_110_controlled_lane" + ] + is True + ) + + +def test_harbor_repair_no_matching_explains_non110_cd_skip_reason() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_harbor_repair_waiting_with_workflow_no_matching(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={"jobs": [], "total_count": 0}, + latest_cd_build_log_http_status=200, + latest_cd_build_log_text=_harbor_blocked_log(), + ) + + assert payload["status"] == "blocked_harbor_110_repair_no_matching_runner" + assert ( + payload["readback"][ + "latest_visible_cd_harbor_controlled_repair_skip_reason" + ] + == "not_110_host" + ) + assert ( + payload["readback"][ + "latest_visible_cd_harbor_repair_requires_110_controlled_lane" + ] + is True + ) + assert ( + payload["readback"][ + "latest_visible_cd_harbor_repair_blocked_by_no_matching_awoooi_host" + ] + is True + ) + assert ( + payload["readback"]["latest_visible_cd_harbor_repair_lane_classifier"] + == "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host" + ) + assert ( + payload["rollups"][ + "current_main_cd_harbor_repair_lane_classifier" + ] + == "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host" + ) def test_build_readback_prefers_visible_blocked_over_stale_jobs() -> None: