From 5e629efa443b29646779ea0fa6cbcc6fbb4b9e60 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 1 Jul 2026 08:40:53 +0800 Subject: [PATCH] feat(recovery): accept non110 runner readiness receipt --- ..._controlled_writeback_executor_readback.py | 40 ++++ ...or_registry_controlled_recovery_receipt.py | 218 +++++++++++++++++- ...trolled_writeback_executor_readback_api.py | 14 +- ...awoooi_priority_work_order_readback_api.py | 14 +- ...or_registry_controlled_recovery_receipt.py | 96 +++++++- 5 files changed, 366 insertions(+), 16 deletions(-) diff --git a/apps/api/src/services/ai_agent_log_controlled_writeback_executor_readback.py b/apps/api/src/services/ai_agent_log_controlled_writeback_executor_readback.py index a715d56d..41a819b3 100644 --- a/apps/api/src/services/ai_agent_log_controlled_writeback_executor_readback.py +++ b/apps/api/src/services/ai_agent_log_controlled_writeback_executor_readback.py @@ -450,6 +450,17 @@ def _harbor_recovery_receipt_inputs() -> list[dict[str, Any]]: ), **metadata_boundary, }, + { + "input_id": "non110_runner_readiness_output", + "source": "check-awoooi-non110-runner-readiness.sh", + "required_when": "before_retrying_current_cd_or_harbor_repair_queue", + "purpose": ( + "classify non110 controlled runner config, binary, registration, " + "service, active service, autostart path, rollback, pressure, " + "and safe-next-step blockers without reading .runner content" + ), + **metadata_boundary, + }, { "input_id": "public_registry_v2_http_status", "source": "public registry /v2/ verifier", @@ -506,6 +517,16 @@ def _harbor_recovery_receipt_output_contract() -> list[dict[str, Any]]: ), **metadata_boundary, }, + { + "output_id": "non110_runner_readiness", + "source": "harbor-registry-controlled-recovery-receipt.readback.non110_runner_readiness", + "required_when": "before_retrying_current_cd_or_harbor_repair_queue", + "purpose": ( + "write back non110 runner registration, service, active service, " + "autostart path, blocker count, and safe-next-step readiness" + ), + **metadata_boundary, + }, { "output_id": "gitea_actions_queue", "source": "harbor-registry-controlled-recovery-receipt.readback.gitea_actions_queue", @@ -575,6 +596,25 @@ def _queue_readback_normalizer_contract() -> list[dict[str, Any]]: ], "learning_targets": ["km", "rag", "playbook", "mcp", "verifier"], }, + { + "field_id": "current_cd_workflow_runner_readiness", + "purpose": "classify whether current CD is waiting on a non110 runner label", + "writes_blockers": [ + "gitea_queue_current_cd_no_matching_runner", + "gitea_queue_current_cd_waiting_for_runner_or_queue", + ], + "learning_targets": ["km", "rag", "playbook", "mcp", "verifier"], + }, + { + "field_id": "controlled_profile_no_matching_runner_labels", + "purpose": ( + "classify all controlled workflow labels with no matching online runner" + ), + "writes_blockers": [ + "gitea_queue_controlled_profile_no_matching_runner_labels", + ], + "learning_targets": ["km", "rag", "playbook", "mcp", "verifier"], + }, ] diff --git a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py index f1e143ef..8159e800 100644 --- a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py @@ -30,6 +30,9 @@ def validate_harbor_registry_controlled_recovery_receipt( controlled_cd_lane_output = _text( receipt_payload.get("controlled_cd_lane_readiness_output") ) + non110_runner_output = _text( + receipt_payload.get("non110_runner_readiness_output") + ) ssh_diagnosis = _parse_ssh_publickey_diagnosis_output(ssh_diagnosis_output) ssh_local = _parse_ssh_local_repair_output(ssh_local_output) @@ -38,6 +41,7 @@ def validate_harbor_registry_controlled_recovery_receipt( controlled_cd_lane = _parse_controlled_cd_lane_readiness_output( controlled_cd_lane_output ) + non110_runner = _parse_non110_runner_readiness_output(non110_runner_output) verifier = _post_apply_verifier(receipt_payload) gitea_queue = _gitea_queue_readback( receipt_payload.get("gitea_actions_queue_readback") @@ -52,6 +56,7 @@ def validate_harbor_registry_controlled_recovery_receipt( watchdog_check=watchdog_check, watchdog_repair=watchdog_repair, controlled_cd_lane=controlled_cd_lane, + non110_runner=non110_runner, verifier=verifier, gitea_queue=gitea_queue, deploy_marker=deploy_marker, @@ -62,6 +67,7 @@ def validate_harbor_registry_controlled_recovery_receipt( watchdog_check=watchdog_check, watchdog_repair=watchdog_repair, controlled_cd_lane=controlled_cd_lane, + non110_runner=non110_runner, verifier=verifier, gitea_queue=gitea_queue, deploy_marker=deploy_marker, @@ -73,10 +79,13 @@ def validate_harbor_registry_controlled_recovery_receipt( watchdog_check=watchdog_check, watchdog_repair=watchdog_repair, controlled_cd_lane=controlled_cd_lane, + non110_runner=non110_runner, verifier=verifier, + gitea_queue=gitea_queue, ) control_path_readiness = _control_path_readiness( ssh_diagnosis=ssh_diagnosis, + non110_runner=non110_runner, verifier=verifier, gitea_queue=gitea_queue, ) @@ -101,6 +110,7 @@ def validate_harbor_registry_controlled_recovery_receipt( "controlled_cd_lane_readiness_output": _text_stats( controlled_cd_lane_output ), + "non110_runner_readiness_output": _text_stats(non110_runner_output), "gitea_actions_queue_readback": { "provided": gitea_queue["receipt_seen"], "metadata_only": True, @@ -119,6 +129,7 @@ def validate_harbor_registry_controlled_recovery_receipt( "watchdog_check": watchdog_check, "watchdog_repair": watchdog_repair, "controlled_cd_lane_readiness": controlled_cd_lane, + "non110_runner_readiness": non110_runner, "post_apply_verifier": verifier, "gitea_actions_queue": gitea_queue, "deploy_marker": deploy_marker, @@ -137,6 +148,7 @@ def validate_harbor_registry_controlled_recovery_receipt( "validate_harbor_watchdog_check_receipt", "validate_harbor_watchdog_repair_once_receipt", "validate_110_controlled_cd_lane_readiness_receipt", + "validate_non110_runner_readiness_receipt", "validate_public_and_internal_registry_v2_verifier", "km_rag_mcp_playbook_metadata_writeback", "retry_gitea_cd_after_registry_v2_green", @@ -202,6 +214,30 @@ def validate_harbor_registry_controlled_recovery_receipt( "controlled_cd_lane_safe_next_step": controlled_cd_lane[ "safe_next_step" ], + "non110_runner_readiness_receipt_seen": non110_runner[ + "receipt_seen" + ], + "non110_runner_ready": non110_runner["non110_runner_ready"], + "non110_runner_blocker_count": non110_runner["blocker_count"], + "non110_runner_safe_next_step": non110_runner["safe_next_step"], + "non110_runner_ready_config_count": non110_runner[ + "ready_config_count" + ], + "non110_runner_ready_binary_count": non110_runner[ + "ready_binary_count" + ], + "non110_runner_ready_registration_count": non110_runner[ + "ready_registration_count" + ], + "non110_runner_ready_service_count": non110_runner[ + "ready_service_count" + ], + "non110_runner_ready_active_service_count": non110_runner[ + "ready_active_service_count" + ], + "non110_runner_ready_autostart_path_count": non110_runner[ + "ready_autostart_path_count" + ], "post_apply_verifier_ready": verifier["registry_v2_ready"], "gitea_queue_readback_seen": gitea_queue["receipt_seen"], "gitea_queue_blocker_count": gitea_queue["blocker_count"], @@ -295,6 +331,9 @@ def validate_harbor_registry_controlled_recovery_receipt( "control_path_readiness_awoooi_host_unavailable": ( control_path_readiness["awoooi_host_runner_unavailable"] ), + "control_path_readiness_non110_runner_unavailable": ( + control_path_readiness["non110_runner_unavailable"] + ), "control_path_readiness_registry_v2_public_ready": ( control_path_readiness["registry_v2_public_ready"] ), @@ -331,6 +370,7 @@ def validate_harbor_registry_controlled_recovery_receipt( def _control_path_readiness( *, ssh_diagnosis: dict[str, Any], + non110_runner: dict[str, Any], verifier: dict[str, Any], gitea_queue: dict[str, Any], ) -> dict[str, Any]: @@ -351,13 +391,33 @@ def _control_path_readiness( ) cd_jobs_stale = bool(gitea_queue["cd_run_jobs_stale_or_mismatched"]) cd_jobs_head_sha_mismatch = bool(gitea_queue["cd_run_jobs_head_sha_mismatch"]) + current_cd_waiting = bool(gitea_queue["current_cd_waiting_for_runner_or_queue"]) + current_cd_no_matching_runner = bool( + gitea_queue["current_cd_no_matching_runner_label"] + ) + controlled_profile_no_matching_runner = ( + gitea_queue["controlled_profile_no_matching_runner_label_count"] > 0 + ) + non110_runner_not_ready = bool( + non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"] + ) runner_timeout = bool(ssh_diagnosis["runner_systemctl_show_timeout_seen"]) node_high_load = bool(ssh_diagnosis["node_high_load_seen"]) publickey_offer_timeout = bool(ssh_diagnosis["publickey_offer_timeout_seen"]) server_accepts_key_then_timeout = bool( ssh_diagnosis["server_accepts_key_then_timeout_seen"] ) - awoooi_host_unavailable = queue_no_matching_runner or queue_jobs_stale + awoooi_host_unavailable = ( + queue_no_matching_runner + or queue_jobs_stale + or current_cd_no_matching_runner + or controlled_profile_no_matching_runner + ) + non110_runner_unavailable = ( + non110_runner_not_ready + or current_cd_no_matching_runner + or controlled_profile_no_matching_runner + ) registry_v2_ready = public_registry_ready and internal_registry_ready signal_ids = _control_path_signal_ids( node_high_load=node_high_load, @@ -371,6 +431,10 @@ def _control_path_readiness( cd_harbor_repair_blocked_by_awoooi_host=( cd_harbor_repair_blocked_by_awoooi_host ), + current_cd_waiting=current_cd_waiting, + current_cd_no_matching_runner=current_cd_no_matching_runner, + controlled_profile_no_matching_runner=controlled_profile_no_matching_runner, + non110_runner_not_ready=non110_runner_not_ready, cd_jobs_head_sha_mismatch=cd_jobs_head_sha_mismatch, cd_jobs_stale=cd_jobs_stale, public_registry_ready=public_registry_ready, @@ -381,6 +445,7 @@ def _control_path_readiness( runner_timeout=runner_timeout, ssh_timeout=publickey_offer_timeout or server_accepts_key_then_timeout, awoooi_host_unavailable=awoooi_host_unavailable, + non110_runner_unavailable=non110_runner_unavailable, registry_v2_ready=registry_v2_ready, signal_ids=signal_ids, ) @@ -400,6 +465,19 @@ def _control_path_readiness( "ssh_server_accepts_key_then_timeout": server_accepts_key_then_timeout, "runner_systemctl_show_timeout": runner_timeout, "awoooi_host_runner_unavailable": awoooi_host_unavailable, + "non110_runner_unavailable": non110_runner_unavailable, + "non110_runner_readiness_receipt_seen": non110_runner["receipt_seen"], + "non110_runner_ready": non110_runner["non110_runner_ready"], + "non110_runner_blocker_count": non110_runner["blocker_count"], + "non110_runner_blockers": non110_runner["blockers"], + "non110_runner_safe_next_step": non110_runner["safe_next_step"], + "current_cd_no_matching_runner_label": gitea_queue[ + "current_cd_no_matching_runner_label" + ], + "current_cd_waiting_for_runner_or_queue": current_cd_waiting, + "controlled_profile_no_matching_runner_label_count": gitea_queue[ + "controlled_profile_no_matching_runner_label_count" + ], "harbor_110_repair_no_matching_runner": queue_no_matching_runner, "harbor_110_repair_no_matching_runner_label": gitea_queue[ "harbor_110_repair_no_matching_runner_label" @@ -442,6 +520,10 @@ def _control_path_signal_ids( queue_jobs_cross_workflow: bool, cd_harbor_repair_requires_110_lane: bool, cd_harbor_repair_blocked_by_awoooi_host: bool, + current_cd_waiting: bool, + current_cd_no_matching_runner: bool, + controlled_profile_no_matching_runner: bool, + non110_runner_not_ready: bool, cd_jobs_head_sha_mismatch: bool, cd_jobs_stale: bool, public_registry_ready: bool, @@ -468,6 +550,14 @@ def _control_path_signal_ids( ) if cd_harbor_repair_requires_110_lane: signal_ids.append("gitea_queue_cd_harbor_repair_requires_110_controlled_lane") + if current_cd_no_matching_runner: + signal_ids.append("gitea_queue_current_cd_no_matching_runner") + if current_cd_waiting: + signal_ids.append("gitea_queue_current_cd_waiting_for_runner_or_queue") + if controlled_profile_no_matching_runner: + signal_ids.append("gitea_queue_controlled_profile_no_matching_runner_labels") + if non110_runner_not_ready: + signal_ids.append("non110_runner_readiness_receipt_not_ready") if cd_jobs_head_sha_mismatch: signal_ids.append("gitea_queue_cd_jobs_head_sha_mismatch") if cd_jobs_stale: @@ -485,6 +575,7 @@ def _control_path_readiness_status( runner_timeout: bool, ssh_timeout: bool, awoooi_host_unavailable: bool, + non110_runner_unavailable: bool, registry_v2_ready: bool, signal_ids: list[str], ) -> str: @@ -492,6 +583,8 @@ def _control_path_readiness_status( return "ready" if node_high_load and awoooi_host_unavailable: return "blocked_110_high_load_and_awoooi_host_control_path_unavailable" + if non110_runner_unavailable: + return "blocked_non110_controlled_runner_unavailable" if awoooi_host_unavailable: return "blocked_awoooi_host_runner_queue_unavailable" if node_high_load: @@ -510,6 +603,8 @@ def _control_path_safe_next_action(*, status: str) -> str: return "hold_110_capacity_protection_then_rerun_readonly_awoooi_host_and_registry_verifiers" if status == "blocked_awoooi_host_runner_queue_unavailable": return "run_awoooi_host_lane_readiness_verifier_before_controlled_lane_restore" + if status == "blocked_non110_controlled_runner_unavailable": + return "submit_non110_runner_readiness_receipt_then_restore_non110_controlled_lane" if status == "blocked_110_node_high_load": return "wait_for_110_load_to_normalize_then_rerun_readonly_control_path_probe" if status == "blocked_110_ssh_or_runner_control_path_timeout": @@ -526,7 +621,9 @@ def _local_console_phase_readback( watchdog_check: dict[str, Any], watchdog_repair: dict[str, Any], controlled_cd_lane: dict[str, Any], + non110_runner: dict[str, Any], verifier: dict[str, Any], + gitea_queue: dict[str, Any], ) -> dict[str, Any]: phases = [ _phase( @@ -570,6 +667,15 @@ def _local_console_phase_readback( ), "controlled_cd_lane_readiness", ), + _phase( + "verify_non110_runner_lane", + _non110_runner_phase_status( + non110_runner=non110_runner, + verifier=verifier, + gitea_queue=gitea_queue, + ), + "non110_runner_readiness", + ), ] completed_statuses = {"ready", "skipped_not_required"} return { @@ -662,6 +768,27 @@ def _controlled_cd_lane_phase_status( ) +def _non110_runner_phase_status( + *, + non110_runner: dict[str, Any], + verifier: dict[str, Any], + gitea_queue: dict[str, Any], +) -> str: + if non110_runner["receipt_seen"]: + if non110_runner["non110_runner_ready"]: + return "ready" + return "blocked_non110_runner_readiness_receipt_not_ready" + if ( + gitea_queue["current_cd_waiting_for_runner_or_queue"] + or gitea_queue["current_cd_no_matching_runner_label"] + or gitea_queue["controlled_profile_no_matching_runner_label_count"] > 0 + ): + return "blocked_waiting_non110_runner_readiness_receipt" + if verifier["registry_v2_ready"]: + return "skipped_not_required" + return "blocked_waiting_non110_runner_or_registry_readiness_receipt" + + def _parse_ssh_publickey_diagnosis_output(output: str) -> dict[str, Any]: fields = _parse_key_values(output) marker_seen = "AWOOOI_110_SSH_PUBLICKEY_AUTH_DIAGNOSIS" in output @@ -863,6 +990,65 @@ def _parse_controlled_cd_lane_readiness_output(output: str) -> dict[str, Any]: } +def _parse_non110_runner_readiness_output(output: str) -> dict[str, Any]: + fields = _parse_key_values(output) + marker_seen = "AWOOOI_NON110_RUNNER_READY=" in output + blockers = _prefixed_blockers( + output, + prefix="non110_runner_readiness:", + ) + warning_count = _int_or_none(fields.get("WARNING_COUNT")) or 0 + blocker_count = _int_or_none(fields.get("BLOCKER_COUNT")) + if blocker_count is None: + blocker_count = len(blockers) + ready = _bool_from_field(fields.get("AWOOOI_NON110_RUNNER_READY")) + raw_runner_registration_read = _bool_from_field( + fields.get("raw_runner_registration_read") + ) + runner_token_read = _bool_from_field(fields.get("runner_token_read")) + boundary_blockers = [] + if raw_runner_registration_read: + boundary_blockers.append( + "non110_runner_readiness:raw_runner_registration_read" + ) + if runner_token_read: + boundary_blockers.append("non110_runner_readiness:runner_token_read") + blockers = _unique_strings(blockers + boundary_blockers) + if boundary_blockers and blocker_count == 0: + blocker_count = len(blockers) + return { + "receipt_seen": marker_seen, + "non110_runner_ready": bool( + marker_seen + and ready + and blocker_count == 0 + and not blockers + and not raw_runner_registration_read + and not runner_token_read + ), + "ready_config_count": _int_or_none(fields.get("READY_CONFIG_COUNT")) or 0, + "ready_binary_count": _int_or_none(fields.get("READY_BINARY_COUNT")) or 0, + "ready_registration_count": ( + _int_or_none(fields.get("READY_REGISTRATION_COUNT")) or 0 + ), + "ready_service_count": _int_or_none(fields.get("READY_SERVICE_COUNT")) or 0, + "ready_active_service_count": ( + _int_or_none(fields.get("READY_ACTIVE_SERVICE_COUNT")) or 0 + ), + "ready_autostart_path_count": ( + _int_or_none(fields.get("READY_AUTOSTART_PATH_COUNT")) or 0 + ), + "warning_count": warning_count, + "blocker_count": blocker_count, + "blockers": blockers, + "safe_next_step": str(fields.get("safe_next_step") or ""), + "raw_runner_registration_read": raw_runner_registration_read, + "runner_token_read": runner_token_read, + "metadata_only": True, + "raw_output_returned": False, + } + + def _post_apply_verifier(receipt_payload: dict[str, Any]) -> dict[str, Any]: public_status = _int_or_none(receipt_payload.get("public_registry_v2_http_status")) internal_status = _int_or_none( @@ -1009,6 +1195,7 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: or readback.get("latest_visible_cd_harbor_repair_lane_classifier") or "" ) + controlled_profile_no_matching = bool(controlled_profile_no_matching_labels) cd_jobs_stale = bool( rollups.get("cd_run_jobs_stale_or_mismatched") is True or readback.get("cd_run_jobs_stale_or_mismatched") is True @@ -1055,6 +1242,7 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: current_cd_harbor_repair_blocked_by_awoooi_host=( current_cd_harbor_repair_blocked_by_awoooi_host ), + controlled_profile_no_matching=controlled_profile_no_matching, cd_jobs_stale=cd_jobs_stale, cd_jobs_head_sha_mismatch=cd_jobs_head_sha_mismatch, cd_jobs_run_id_mismatch=cd_jobs_run_id_mismatch, @@ -1067,6 +1255,7 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: no_matching_label=no_matching_label, current_cd_no_matching_label=current_cd_no_matching_label, current_cd_waiting=current_cd_waiting, + controlled_profile_no_matching_labels=controlled_profile_no_matching_labels, blockers=blockers, ) return { @@ -1161,6 +1350,7 @@ def _gitea_queue_normalized_classifier_fields( no_matching_label: str, current_cd_no_matching_label: str, current_cd_waiting: bool, + controlled_profile_no_matching_labels: dict[str, str], blockers: list[str], ) -> list[dict[str, Any]]: fields = [ @@ -1211,6 +1401,20 @@ def _gitea_queue_normalized_classifier_fields( "raw_output_returned": False, } ) + if controlled_profile_no_matching_labels: + fields.append( + { + "field_id": "controlled_profile_no_matching_runner_labels", + "value": controlled_profile_no_matching_labels, + "blockers": [ + item + for item in blockers + if item == "gitea_queue_controlled_profile_no_matching_runner_labels" + ], + "metadata_only": True, + "raw_output_returned": False, + } + ) return fields @@ -1224,6 +1428,7 @@ def _gitea_queue_blockers( current_cd_no_matching_runner: bool, current_cd_harbor_repair_requires_110_lane: bool, current_cd_harbor_repair_blocked_by_awoooi_host: bool, + controlled_profile_no_matching: bool, cd_jobs_stale: bool, cd_jobs_head_sha_mismatch: bool, cd_jobs_run_id_mismatch: bool, @@ -1243,6 +1448,8 @@ def _gitea_queue_blockers( blockers.append("gitea_queue_current_cd_no_matching_runner") if current_cd_waiting: blockers.append("gitea_queue_current_cd_waiting_for_runner_or_queue") + if controlled_profile_no_matching: + blockers.append("gitea_queue_controlled_profile_no_matching_runner_labels") if cd_jobs_head_sha_mismatch: blockers.append("gitea_queue_cd_jobs_head_sha_mismatch") if cd_jobs_run_id_mismatch: @@ -1397,6 +1604,7 @@ def _active_blockers( watchdog_check: dict[str, Any], watchdog_repair: dict[str, Any], controlled_cd_lane: dict[str, Any], + non110_runner: dict[str, Any], verifier: dict[str, Any], gitea_queue: dict[str, Any], deploy_marker: dict[str, Any], @@ -1431,6 +1639,7 @@ def _active_blockers( if watchdog_repair["receipt_seen"] and not watchdog_repair["harbor_ready"]: blockers.append("harbor_watchdog_repair_did_not_restore_local_v2") blockers.extend(_strings(controlled_cd_lane.get("blockers"))) + blockers.extend(_strings(non110_runner.get("blockers"))) if not verifier["public_registry_v2_ready"]: blockers.append("public_registry_v2_verifier_not_green") if not verifier["internal_registry_v2_ready"]: @@ -1447,6 +1656,7 @@ def _status( watchdog_check: dict[str, Any], watchdog_repair: dict[str, Any], controlled_cd_lane: dict[str, Any], + non110_runner: dict[str, Any], verifier: dict[str, Any], gitea_queue: dict[str, Any], deploy_marker: dict[str, Any], @@ -1470,6 +1680,8 @@ def _status( "controlled_cd_lane_ready" ]: return "controlled_cd_lane_readiness_receipt_blocked" + if non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"]: + return "non110_runner_readiness_receipt_blocked" if watchdog_repair["receipt_seen"]: return "harbor_registry_repair_receipt_waiting_registry_v2_verifier" if watchdog_check["receipt_seen"] and watchdog_check["harbor_ready"]: @@ -1492,6 +1704,8 @@ def _safe_next_step(*, status: str) -> str: return "rerun_gitea_cd_then_verify_deploy_marker_and_priority_readback" if status == "controlled_cd_lane_readiness_receipt_blocked": return "fix_controlled_cd_lane_guardrail_blockers_then_rerun_readiness_verifier" + if status == "non110_runner_readiness_receipt_blocked": + return "fix_or_register_non110_runner_then_rerun_readiness_and_queue_verifiers" if status == "harbor_registry_repair_receipt_waiting_registry_v2_verifier": return "rerun_public_and_internal_registry_v2_verifier_before_cd_retry" if status == "harbor_local_registry_ready_waiting_public_registry_v2_verifier": @@ -1517,6 +1731,8 @@ def _current_apply_blocker(*, status: str) -> str: return "deploy_marker_readback_required_after_registry_receipt" if status == "controlled_cd_lane_readiness_receipt_blocked": return "controlled_cd_lane_readiness_required_for_awoooi_host_queue" + if status == "non110_runner_readiness_receipt_blocked": + return "non110_runner_readiness_required_for_current_cd_queue" if status == "harbor_watchdog_check_unhealthy_waiting_repair_once_receipt": return "repair_once_receipt_required_after_unhealthy_check" if status == "ssh_local_repair_receipt_waiting_harbor_watchdog_check": diff --git a/apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py b/apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py index f8707845..1fe156ae 100644 --- a/apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py +++ b/apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py @@ -72,13 +72,13 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): assert payload["rollups"]["current_blocker_local_recovery_package_count"] == 1 assert ( payload["rollups"]["current_blocker_harbor_recovery_receipt_input_count"] - == 9 + == 10 ) assert ( payload["rollups"][ "current_blocker_harbor_recovery_receipt_output_contract_count" ] - == 6 + == 7 ) assert payload["rollups"]["runtime_dispatch_performed"] is False @@ -139,7 +139,7 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): assert current_queue[0]["harbor_recovery_receipt_endpoint"] == ( "/api/v1/agents/harbor-registry-controlled-recovery-receipt" ) - assert current_queue[0]["harbor_recovery_receipt_input_count"] == 9 + assert current_queue[0]["harbor_recovery_receipt_input_count"] == 10 assert [ item["input_id"] for item in current_queue[0]["harbor_recovery_receipt_inputs"] @@ -150,6 +150,7 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): "watchdog_check_output", "watchdog_repair_output", "controlled_cd_lane_readiness_output", + "non110_runner_readiness_output", "public_registry_v2_http_status", "internal_registry_v2_http_status", "deploy_marker_readback", @@ -157,13 +158,14 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): assert current_queue[0]["harbor_recovery_receipt_inputs"][-1][ "expected_schema" ] == "awoooi_production_deploy_readback_blocker_v1" - assert current_queue[0]["harbor_recovery_receipt_output_contract_count"] == 6 + assert current_queue[0]["harbor_recovery_receipt_output_contract_count"] == 7 assert [ item["output_id"] for item in current_queue[0]["harbor_recovery_receipt_output_contract"] ] == [ "control_path_readiness", "controlled_cd_lane_readiness", + "non110_runner_readiness", "gitea_actions_queue", "local_console_phase_readback", "post_apply_verifier", @@ -183,7 +185,7 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): ] for item in current_queue[0]["harbor_recovery_receipt_output_contract"] ) - assert current_queue[0]["queue_readback_normalizer_contract_count"] == 3 + assert current_queue[0]["queue_readback_normalizer_contract_count"] == 5 assert [ item["field_id"] for item in current_queue[0]["queue_readback_normalizer_contract"] @@ -191,6 +193,8 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): "cd_run_jobs_payload_classifier", "harbor_110_repair_jobs_payload_classifier", "latest_visible_harbor_110_repair_no_matching_runner_label", + "current_cd_workflow_runner_readiness", + "controlled_profile_no_matching_runner_labels", ] assert "gitea_queue_cd_jobs_stale_or_mismatched" in current_queue[0][ "queue_readback_normalizer_contract" diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index 2c4347ba..82fbe5fe 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -329,7 +329,7 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu assert evidence["ai_loop_current_blocker_harbor_recovery_receipt_endpoint"] == ( "/api/v1/agents/harbor-registry-controlled-recovery-receipt" ) - assert evidence["ai_loop_current_blocker_harbor_recovery_receipt_input_count"] == 9 + assert evidence["ai_loop_current_blocker_harbor_recovery_receipt_input_count"] == 10 assert evidence["ai_loop_current_blocker_harbor_recovery_receipt_input_ids"] == [ "gitea_actions_queue_readback", "ssh_publickey_diagnosis_output", @@ -337,6 +337,7 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu "watchdog_check_output", "watchdog_repair_output", "controlled_cd_lane_readiness_output", + "non110_runner_readiness_output", "public_registry_v2_http_status", "internal_registry_v2_http_status", "deploy_marker_readback", @@ -348,11 +349,12 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu evidence[ "ai_loop_current_blocker_harbor_recovery_receipt_output_contract_count" ] - == 6 + == 7 ) assert evidence["ai_loop_current_blocker_harbor_recovery_receipt_output_ids"] == [ "control_path_readiness", "controlled_cd_lane_readiness", + "non110_runner_readiness", "gitea_actions_queue", "local_console_phase_readback", "post_apply_verifier", @@ -365,6 +367,8 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu "cd_run_jobs_payload_classifier", "harbor_110_repair_jobs_payload_classifier", "latest_visible_harbor_110_repair_no_matching_runner_label", + "current_cd_workflow_runner_readiness", + "controlled_profile_no_matching_runner_labels", ] assert evidence["ai_loop_current_blocker_queue_readback_normalizer_contract"][0][ "writes_blockers" @@ -402,7 +406,7 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu payload["summary"][ "ai_loop_current_blocker_harbor_recovery_receipt_input_count" ] - == 9 + == 10 ) assert payload["summary"][ "ai_loop_current_blocker_harbor_recovery_receipt_input_ids" @@ -414,7 +418,7 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu payload["summary"][ "ai_loop_current_blocker_harbor_recovery_receipt_output_contract_count" ] - == 6 + == 7 ) assert payload["summary"][ "ai_loop_current_blocker_harbor_recovery_receipt_output_ids" @@ -423,7 +427,7 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu payload["summary"][ "ai_loop_current_blocker_queue_readback_normalizer_contract_count" ] - == 3 + == 5 ) assert payload["summary"][ "ai_loop_current_blocker_queue_readback_normalizer_field_ids" diff --git a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py index 9670fd3b..f415f912 100644 --- a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py @@ -35,8 +35,8 @@ def test_harbor_recovery_receipt_accepts_verified_repair() -> None: assert payload["readback"]["deploy_marker"]["deploy_marker_verified"] is True assert payload["readback"]["deploy_marker"]["raw_output_returned"] is False phase_readback = payload["local_console_phase_readback"] - assert phase_readback["phase_count"] == 5 - assert phase_readback["completed_phase_count"] == 5 + assert phase_readback["phase_count"] == 6 + assert phase_readback["completed_phase_count"] == 6 assert phase_readback["blocked_phase_count"] == 0 assert phase_readback["phase_ids"] == [ "diagnose_ssh_publickey", @@ -44,10 +44,11 @@ def test_harbor_recovery_receipt_accepts_verified_repair() -> None: "repair_ssh_metadata_if_check_confirms_metadata_drift", "repair_harbor_once_if_v2_still_502", "verify_controlled_cd_lane", + "verify_non110_runner_lane", ] assert all(phase["raw_output_returned"] is False for phase in phase_readback["phases"]) - assert payload["rollups"]["local_console_phase_count"] == 5 - assert payload["rollups"]["local_console_completed_phase_count"] == 5 + assert payload["rollups"]["local_console_phase_count"] == 6 + assert payload["rollups"]["local_console_completed_phase_count"] == 6 assert payload["rollups"]["local_console_blocked_phase_count"] == 0 assert payload["rollups"]["deploy_marker_readback_seen"] is True assert payload["rollups"]["deploy_marker_verified"] is True @@ -99,7 +100,7 @@ def test_harbor_recovery_receipt_routes_unhealthy_check_to_repair_once() -> None assert phases["repair_harbor_once_if_v2_still_502"]["status"] == ( "blocked_waiting_harbor_repair_once_receipt" ) - assert payload["rollups"]["local_console_blocked_phase_count"] == 3 + assert payload["rollups"]["local_console_blocked_phase_count"] == 4 assert payload["controlled_apply_policy"]["manual_end_state"] is False @@ -292,6 +293,64 @@ BLOCKER_COUNT=1 assert payload["rollups"]["controlled_cd_lane_blocker_count"] == 1 +def test_harbor_recovery_receipt_surfaces_non110_runner_readiness() -> None: + payload = validate_harbor_registry_controlled_recovery_receipt( + { + "non110_runner_readiness_output": _non110_runner_blocked_output(), + "gitea_actions_queue_readback": _gitea_queue_current_cd_waiting_non110(), + } + ) + + assert payload["status"] == "non110_runner_readiness_receipt_blocked" + assert payload["safe_next_step"] == ( + "fix_or_register_non110_runner_then_rerun_readiness_and_queue_verifiers" + ) + assert "non110_runner_readiness:runner_registration_missing" in payload[ + "active_blockers" + ] + assert "non110_runner_readiness:runner_service_not_active" in payload[ + "active_blockers" + ] + runner = payload["readback"]["non110_runner_readiness"] + assert runner["receipt_seen"] is True + assert runner["non110_runner_ready"] is False + assert runner["ready_config_count"] == 1 + assert runner["ready_binary_count"] == 1 + assert runner["ready_registration_count"] == 0 + assert runner["ready_service_count"] == 1 + assert runner["ready_active_service_count"] == 0 + assert runner["ready_autostart_path_count"] == 1 + assert runner["raw_runner_registration_read"] is False + assert runner["runner_token_read"] is False + assert runner["raw_output_returned"] is False + readiness = payload["readback"]["control_path_readiness"] + assert readiness["status"] == "blocked_non110_controlled_runner_unavailable" + assert readiness["non110_runner_unavailable"] is True + assert readiness["non110_runner_ready"] is False + assert "non110_runner_readiness_receipt_not_ready" in readiness["signal_ids"] + assert ( + "gitea_queue_controlled_profile_no_matching_runner_labels" + in readiness["signal_ids"] + ) + phases = { + phase["phase_id"]: phase + for phase in payload["local_console_phase_readback"]["phases"] + } + assert phases["verify_non110_runner_lane"]["status"] == ( + "blocked_non110_runner_readiness_receipt_not_ready" + ) + assert payload["rollups"]["non110_runner_readiness_receipt_seen"] is True + assert payload["rollups"]["non110_runner_ready"] is False + assert payload["rollups"]["non110_runner_blocker_count"] == 2 + assert ( + payload["rollups"]["control_path_readiness_non110_runner_unavailable"] + is True + ) + assert payload["input_redaction"]["non110_runner_readiness_output"][ + "line_count" + ] > 0 + + def test_harbor_recovery_receipt_surfaces_control_path_readiness_blocker() -> None: diagnosis_output = _ssh_publickey_diagnosis_output().replace( "rc=124 classification=server_accepts_key_then_timeout", @@ -844,6 +903,33 @@ safe_next_step=restore_or_register_awoooi_cd_lane_drain_registration_without_pri """ +def _non110_runner_blocked_output() -> str: + return """ +== audit metadata == +read_only=true +secret_values_collected=false +runner_token_read=false +raw_runner_registration_read=false +== non110 runner registration metadata == +RUNNER_REGISTRATION path=/home/ollama/act-runner-awoooi/.runner present=0 content_read=false +BLOCKER runner_registration_missing +== non110 runner service == +RUNNER_SERVICE unit=awoooi-non110-runner.service installed=1 active=inactive enabled=1 +BLOCKER runner_service_not_active +== verdict == +READY_CONFIG_COUNT=1 +READY_BINARY_COUNT=1 +READY_REGISTRATION_COUNT=0 +READY_SERVICE_COUNT=1 +READY_ACTIVE_SERVICE_COUNT=0 +READY_AUTOSTART_PATH_COUNT=1 +WARNING_COUNT=0 +BLOCKER_COUNT=2 +AWOOOI_NON110_RUNNER_READY=0 +safe_next_step=run_register_awoooi_non110_runner_script_without_printing_token_then_autostart_path_will_enable_service_and_rerun_this_verifier +""" + + def _gitea_queue_no_matching_runner() -> dict: return { "schema_version": "awoooi_public_gitea_actions_queue_readback_v1",