feat(recovery): accept non110 runner readiness receipt
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 37s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 37s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
This commit is contained in:
@@ -450,6 +450,17 @@ def _harbor_recovery_receipt_inputs() -> list[dict[str, Any]]:
|
||||
),
|
||||
**metadata_boundary,
|
||||
},
|
||||
{
|
||||
"input_id": "non110_runner_readiness_output",
|
||||
"source": "check-awoooi-non110-runner-readiness.sh",
|
||||
"required_when": "before_retrying_current_cd_or_harbor_repair_queue",
|
||||
"purpose": (
|
||||
"classify non110 controlled runner config, binary, registration, "
|
||||
"service, active service, autostart path, rollback, pressure, "
|
||||
"and safe-next-step blockers without reading .runner content"
|
||||
),
|
||||
**metadata_boundary,
|
||||
},
|
||||
{
|
||||
"input_id": "public_registry_v2_http_status",
|
||||
"source": "public registry /v2/ verifier",
|
||||
@@ -506,6 +517,16 @@ def _harbor_recovery_receipt_output_contract() -> list[dict[str, Any]]:
|
||||
),
|
||||
**metadata_boundary,
|
||||
},
|
||||
{
|
||||
"output_id": "non110_runner_readiness",
|
||||
"source": "harbor-registry-controlled-recovery-receipt.readback.non110_runner_readiness",
|
||||
"required_when": "before_retrying_current_cd_or_harbor_repair_queue",
|
||||
"purpose": (
|
||||
"write back non110 runner registration, service, active service, "
|
||||
"autostart path, blocker count, and safe-next-step readiness"
|
||||
),
|
||||
**metadata_boundary,
|
||||
},
|
||||
{
|
||||
"output_id": "gitea_actions_queue",
|
||||
"source": "harbor-registry-controlled-recovery-receipt.readback.gitea_actions_queue",
|
||||
@@ -575,6 +596,25 @@ def _queue_readback_normalizer_contract() -> list[dict[str, Any]]:
|
||||
],
|
||||
"learning_targets": ["km", "rag", "playbook", "mcp", "verifier"],
|
||||
},
|
||||
{
|
||||
"field_id": "current_cd_workflow_runner_readiness",
|
||||
"purpose": "classify whether current CD is waiting on a non110 runner label",
|
||||
"writes_blockers": [
|
||||
"gitea_queue_current_cd_no_matching_runner",
|
||||
"gitea_queue_current_cd_waiting_for_runner_or_queue",
|
||||
],
|
||||
"learning_targets": ["km", "rag", "playbook", "mcp", "verifier"],
|
||||
},
|
||||
{
|
||||
"field_id": "controlled_profile_no_matching_runner_labels",
|
||||
"purpose": (
|
||||
"classify all controlled workflow labels with no matching online runner"
|
||||
),
|
||||
"writes_blockers": [
|
||||
"gitea_queue_controlled_profile_no_matching_runner_labels",
|
||||
],
|
||||
"learning_targets": ["km", "rag", "playbook", "mcp", "verifier"],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -30,6 +30,9 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
controlled_cd_lane_output = _text(
|
||||
receipt_payload.get("controlled_cd_lane_readiness_output")
|
||||
)
|
||||
non110_runner_output = _text(
|
||||
receipt_payload.get("non110_runner_readiness_output")
|
||||
)
|
||||
|
||||
ssh_diagnosis = _parse_ssh_publickey_diagnosis_output(ssh_diagnosis_output)
|
||||
ssh_local = _parse_ssh_local_repair_output(ssh_local_output)
|
||||
@@ -38,6 +41,7 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
controlled_cd_lane = _parse_controlled_cd_lane_readiness_output(
|
||||
controlled_cd_lane_output
|
||||
)
|
||||
non110_runner = _parse_non110_runner_readiness_output(non110_runner_output)
|
||||
verifier = _post_apply_verifier(receipt_payload)
|
||||
gitea_queue = _gitea_queue_readback(
|
||||
receipt_payload.get("gitea_actions_queue_readback")
|
||||
@@ -52,6 +56,7 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
watchdog_check=watchdog_check,
|
||||
watchdog_repair=watchdog_repair,
|
||||
controlled_cd_lane=controlled_cd_lane,
|
||||
non110_runner=non110_runner,
|
||||
verifier=verifier,
|
||||
gitea_queue=gitea_queue,
|
||||
deploy_marker=deploy_marker,
|
||||
@@ -62,6 +67,7 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
watchdog_check=watchdog_check,
|
||||
watchdog_repair=watchdog_repair,
|
||||
controlled_cd_lane=controlled_cd_lane,
|
||||
non110_runner=non110_runner,
|
||||
verifier=verifier,
|
||||
gitea_queue=gitea_queue,
|
||||
deploy_marker=deploy_marker,
|
||||
@@ -73,10 +79,13 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
watchdog_check=watchdog_check,
|
||||
watchdog_repair=watchdog_repair,
|
||||
controlled_cd_lane=controlled_cd_lane,
|
||||
non110_runner=non110_runner,
|
||||
verifier=verifier,
|
||||
gitea_queue=gitea_queue,
|
||||
)
|
||||
control_path_readiness = _control_path_readiness(
|
||||
ssh_diagnosis=ssh_diagnosis,
|
||||
non110_runner=non110_runner,
|
||||
verifier=verifier,
|
||||
gitea_queue=gitea_queue,
|
||||
)
|
||||
@@ -101,6 +110,7 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
"controlled_cd_lane_readiness_output": _text_stats(
|
||||
controlled_cd_lane_output
|
||||
),
|
||||
"non110_runner_readiness_output": _text_stats(non110_runner_output),
|
||||
"gitea_actions_queue_readback": {
|
||||
"provided": gitea_queue["receipt_seen"],
|
||||
"metadata_only": True,
|
||||
@@ -119,6 +129,7 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
"watchdog_check": watchdog_check,
|
||||
"watchdog_repair": watchdog_repair,
|
||||
"controlled_cd_lane_readiness": controlled_cd_lane,
|
||||
"non110_runner_readiness": non110_runner,
|
||||
"post_apply_verifier": verifier,
|
||||
"gitea_actions_queue": gitea_queue,
|
||||
"deploy_marker": deploy_marker,
|
||||
@@ -137,6 +148,7 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
"validate_harbor_watchdog_check_receipt",
|
||||
"validate_harbor_watchdog_repair_once_receipt",
|
||||
"validate_110_controlled_cd_lane_readiness_receipt",
|
||||
"validate_non110_runner_readiness_receipt",
|
||||
"validate_public_and_internal_registry_v2_verifier",
|
||||
"km_rag_mcp_playbook_metadata_writeback",
|
||||
"retry_gitea_cd_after_registry_v2_green",
|
||||
@@ -202,6 +214,30 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
"controlled_cd_lane_safe_next_step": controlled_cd_lane[
|
||||
"safe_next_step"
|
||||
],
|
||||
"non110_runner_readiness_receipt_seen": non110_runner[
|
||||
"receipt_seen"
|
||||
],
|
||||
"non110_runner_ready": non110_runner["non110_runner_ready"],
|
||||
"non110_runner_blocker_count": non110_runner["blocker_count"],
|
||||
"non110_runner_safe_next_step": non110_runner["safe_next_step"],
|
||||
"non110_runner_ready_config_count": non110_runner[
|
||||
"ready_config_count"
|
||||
],
|
||||
"non110_runner_ready_binary_count": non110_runner[
|
||||
"ready_binary_count"
|
||||
],
|
||||
"non110_runner_ready_registration_count": non110_runner[
|
||||
"ready_registration_count"
|
||||
],
|
||||
"non110_runner_ready_service_count": non110_runner[
|
||||
"ready_service_count"
|
||||
],
|
||||
"non110_runner_ready_active_service_count": non110_runner[
|
||||
"ready_active_service_count"
|
||||
],
|
||||
"non110_runner_ready_autostart_path_count": non110_runner[
|
||||
"ready_autostart_path_count"
|
||||
],
|
||||
"post_apply_verifier_ready": verifier["registry_v2_ready"],
|
||||
"gitea_queue_readback_seen": gitea_queue["receipt_seen"],
|
||||
"gitea_queue_blocker_count": gitea_queue["blocker_count"],
|
||||
@@ -295,6 +331,9 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
"control_path_readiness_awoooi_host_unavailable": (
|
||||
control_path_readiness["awoooi_host_runner_unavailable"]
|
||||
),
|
||||
"control_path_readiness_non110_runner_unavailable": (
|
||||
control_path_readiness["non110_runner_unavailable"]
|
||||
),
|
||||
"control_path_readiness_registry_v2_public_ready": (
|
||||
control_path_readiness["registry_v2_public_ready"]
|
||||
),
|
||||
@@ -331,6 +370,7 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
def _control_path_readiness(
|
||||
*,
|
||||
ssh_diagnosis: dict[str, Any],
|
||||
non110_runner: dict[str, Any],
|
||||
verifier: dict[str, Any],
|
||||
gitea_queue: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
@@ -351,13 +391,33 @@ def _control_path_readiness(
|
||||
)
|
||||
cd_jobs_stale = bool(gitea_queue["cd_run_jobs_stale_or_mismatched"])
|
||||
cd_jobs_head_sha_mismatch = bool(gitea_queue["cd_run_jobs_head_sha_mismatch"])
|
||||
current_cd_waiting = bool(gitea_queue["current_cd_waiting_for_runner_or_queue"])
|
||||
current_cd_no_matching_runner = bool(
|
||||
gitea_queue["current_cd_no_matching_runner_label"]
|
||||
)
|
||||
controlled_profile_no_matching_runner = (
|
||||
gitea_queue["controlled_profile_no_matching_runner_label_count"] > 0
|
||||
)
|
||||
non110_runner_not_ready = bool(
|
||||
non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"]
|
||||
)
|
||||
runner_timeout = bool(ssh_diagnosis["runner_systemctl_show_timeout_seen"])
|
||||
node_high_load = bool(ssh_diagnosis["node_high_load_seen"])
|
||||
publickey_offer_timeout = bool(ssh_diagnosis["publickey_offer_timeout_seen"])
|
||||
server_accepts_key_then_timeout = bool(
|
||||
ssh_diagnosis["server_accepts_key_then_timeout_seen"]
|
||||
)
|
||||
awoooi_host_unavailable = queue_no_matching_runner or queue_jobs_stale
|
||||
awoooi_host_unavailable = (
|
||||
queue_no_matching_runner
|
||||
or queue_jobs_stale
|
||||
or current_cd_no_matching_runner
|
||||
or controlled_profile_no_matching_runner
|
||||
)
|
||||
non110_runner_unavailable = (
|
||||
non110_runner_not_ready
|
||||
or current_cd_no_matching_runner
|
||||
or controlled_profile_no_matching_runner
|
||||
)
|
||||
registry_v2_ready = public_registry_ready and internal_registry_ready
|
||||
signal_ids = _control_path_signal_ids(
|
||||
node_high_load=node_high_load,
|
||||
@@ -371,6 +431,10 @@ def _control_path_readiness(
|
||||
cd_harbor_repair_blocked_by_awoooi_host=(
|
||||
cd_harbor_repair_blocked_by_awoooi_host
|
||||
),
|
||||
current_cd_waiting=current_cd_waiting,
|
||||
current_cd_no_matching_runner=current_cd_no_matching_runner,
|
||||
controlled_profile_no_matching_runner=controlled_profile_no_matching_runner,
|
||||
non110_runner_not_ready=non110_runner_not_ready,
|
||||
cd_jobs_head_sha_mismatch=cd_jobs_head_sha_mismatch,
|
||||
cd_jobs_stale=cd_jobs_stale,
|
||||
public_registry_ready=public_registry_ready,
|
||||
@@ -381,6 +445,7 @@ def _control_path_readiness(
|
||||
runner_timeout=runner_timeout,
|
||||
ssh_timeout=publickey_offer_timeout or server_accepts_key_then_timeout,
|
||||
awoooi_host_unavailable=awoooi_host_unavailable,
|
||||
non110_runner_unavailable=non110_runner_unavailable,
|
||||
registry_v2_ready=registry_v2_ready,
|
||||
signal_ids=signal_ids,
|
||||
)
|
||||
@@ -400,6 +465,19 @@ def _control_path_readiness(
|
||||
"ssh_server_accepts_key_then_timeout": server_accepts_key_then_timeout,
|
||||
"runner_systemctl_show_timeout": runner_timeout,
|
||||
"awoooi_host_runner_unavailable": awoooi_host_unavailable,
|
||||
"non110_runner_unavailable": non110_runner_unavailable,
|
||||
"non110_runner_readiness_receipt_seen": non110_runner["receipt_seen"],
|
||||
"non110_runner_ready": non110_runner["non110_runner_ready"],
|
||||
"non110_runner_blocker_count": non110_runner["blocker_count"],
|
||||
"non110_runner_blockers": non110_runner["blockers"],
|
||||
"non110_runner_safe_next_step": non110_runner["safe_next_step"],
|
||||
"current_cd_no_matching_runner_label": gitea_queue[
|
||||
"current_cd_no_matching_runner_label"
|
||||
],
|
||||
"current_cd_waiting_for_runner_or_queue": current_cd_waiting,
|
||||
"controlled_profile_no_matching_runner_label_count": gitea_queue[
|
||||
"controlled_profile_no_matching_runner_label_count"
|
||||
],
|
||||
"harbor_110_repair_no_matching_runner": queue_no_matching_runner,
|
||||
"harbor_110_repair_no_matching_runner_label": gitea_queue[
|
||||
"harbor_110_repair_no_matching_runner_label"
|
||||
@@ -442,6 +520,10 @@ def _control_path_signal_ids(
|
||||
queue_jobs_cross_workflow: bool,
|
||||
cd_harbor_repair_requires_110_lane: bool,
|
||||
cd_harbor_repair_blocked_by_awoooi_host: bool,
|
||||
current_cd_waiting: bool,
|
||||
current_cd_no_matching_runner: bool,
|
||||
controlled_profile_no_matching_runner: bool,
|
||||
non110_runner_not_ready: bool,
|
||||
cd_jobs_head_sha_mismatch: bool,
|
||||
cd_jobs_stale: bool,
|
||||
public_registry_ready: bool,
|
||||
@@ -468,6 +550,14 @@ def _control_path_signal_ids(
|
||||
)
|
||||
if cd_harbor_repair_requires_110_lane:
|
||||
signal_ids.append("gitea_queue_cd_harbor_repair_requires_110_controlled_lane")
|
||||
if current_cd_no_matching_runner:
|
||||
signal_ids.append("gitea_queue_current_cd_no_matching_runner")
|
||||
if current_cd_waiting:
|
||||
signal_ids.append("gitea_queue_current_cd_waiting_for_runner_or_queue")
|
||||
if controlled_profile_no_matching_runner:
|
||||
signal_ids.append("gitea_queue_controlled_profile_no_matching_runner_labels")
|
||||
if non110_runner_not_ready:
|
||||
signal_ids.append("non110_runner_readiness_receipt_not_ready")
|
||||
if cd_jobs_head_sha_mismatch:
|
||||
signal_ids.append("gitea_queue_cd_jobs_head_sha_mismatch")
|
||||
if cd_jobs_stale:
|
||||
@@ -485,6 +575,7 @@ def _control_path_readiness_status(
|
||||
runner_timeout: bool,
|
||||
ssh_timeout: bool,
|
||||
awoooi_host_unavailable: bool,
|
||||
non110_runner_unavailable: bool,
|
||||
registry_v2_ready: bool,
|
||||
signal_ids: list[str],
|
||||
) -> str:
|
||||
@@ -492,6 +583,8 @@ def _control_path_readiness_status(
|
||||
return "ready"
|
||||
if node_high_load and awoooi_host_unavailable:
|
||||
return "blocked_110_high_load_and_awoooi_host_control_path_unavailable"
|
||||
if non110_runner_unavailable:
|
||||
return "blocked_non110_controlled_runner_unavailable"
|
||||
if awoooi_host_unavailable:
|
||||
return "blocked_awoooi_host_runner_queue_unavailable"
|
||||
if node_high_load:
|
||||
@@ -510,6 +603,8 @@ def _control_path_safe_next_action(*, status: str) -> str:
|
||||
return "hold_110_capacity_protection_then_rerun_readonly_awoooi_host_and_registry_verifiers"
|
||||
if status == "blocked_awoooi_host_runner_queue_unavailable":
|
||||
return "run_awoooi_host_lane_readiness_verifier_before_controlled_lane_restore"
|
||||
if status == "blocked_non110_controlled_runner_unavailable":
|
||||
return "submit_non110_runner_readiness_receipt_then_restore_non110_controlled_lane"
|
||||
if status == "blocked_110_node_high_load":
|
||||
return "wait_for_110_load_to_normalize_then_rerun_readonly_control_path_probe"
|
||||
if status == "blocked_110_ssh_or_runner_control_path_timeout":
|
||||
@@ -526,7 +621,9 @@ def _local_console_phase_readback(
|
||||
watchdog_check: dict[str, Any],
|
||||
watchdog_repair: dict[str, Any],
|
||||
controlled_cd_lane: dict[str, Any],
|
||||
non110_runner: dict[str, Any],
|
||||
verifier: dict[str, Any],
|
||||
gitea_queue: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
phases = [
|
||||
_phase(
|
||||
@@ -570,6 +667,15 @@ def _local_console_phase_readback(
|
||||
),
|
||||
"controlled_cd_lane_readiness",
|
||||
),
|
||||
_phase(
|
||||
"verify_non110_runner_lane",
|
||||
_non110_runner_phase_status(
|
||||
non110_runner=non110_runner,
|
||||
verifier=verifier,
|
||||
gitea_queue=gitea_queue,
|
||||
),
|
||||
"non110_runner_readiness",
|
||||
),
|
||||
]
|
||||
completed_statuses = {"ready", "skipped_not_required"}
|
||||
return {
|
||||
@@ -662,6 +768,27 @@ def _controlled_cd_lane_phase_status(
|
||||
)
|
||||
|
||||
|
||||
def _non110_runner_phase_status(
|
||||
*,
|
||||
non110_runner: dict[str, Any],
|
||||
verifier: dict[str, Any],
|
||||
gitea_queue: dict[str, Any],
|
||||
) -> str:
|
||||
if non110_runner["receipt_seen"]:
|
||||
if non110_runner["non110_runner_ready"]:
|
||||
return "ready"
|
||||
return "blocked_non110_runner_readiness_receipt_not_ready"
|
||||
if (
|
||||
gitea_queue["current_cd_waiting_for_runner_or_queue"]
|
||||
or gitea_queue["current_cd_no_matching_runner_label"]
|
||||
or gitea_queue["controlled_profile_no_matching_runner_label_count"] > 0
|
||||
):
|
||||
return "blocked_waiting_non110_runner_readiness_receipt"
|
||||
if verifier["registry_v2_ready"]:
|
||||
return "skipped_not_required"
|
||||
return "blocked_waiting_non110_runner_or_registry_readiness_receipt"
|
||||
|
||||
|
||||
def _parse_ssh_publickey_diagnosis_output(output: str) -> dict[str, Any]:
|
||||
fields = _parse_key_values(output)
|
||||
marker_seen = "AWOOOI_110_SSH_PUBLICKEY_AUTH_DIAGNOSIS" in output
|
||||
@@ -863,6 +990,65 @@ def _parse_controlled_cd_lane_readiness_output(output: str) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _parse_non110_runner_readiness_output(output: str) -> dict[str, Any]:
|
||||
fields = _parse_key_values(output)
|
||||
marker_seen = "AWOOOI_NON110_RUNNER_READY=" in output
|
||||
blockers = _prefixed_blockers(
|
||||
output,
|
||||
prefix="non110_runner_readiness:",
|
||||
)
|
||||
warning_count = _int_or_none(fields.get("WARNING_COUNT")) or 0
|
||||
blocker_count = _int_or_none(fields.get("BLOCKER_COUNT"))
|
||||
if blocker_count is None:
|
||||
blocker_count = len(blockers)
|
||||
ready = _bool_from_field(fields.get("AWOOOI_NON110_RUNNER_READY"))
|
||||
raw_runner_registration_read = _bool_from_field(
|
||||
fields.get("raw_runner_registration_read")
|
||||
)
|
||||
runner_token_read = _bool_from_field(fields.get("runner_token_read"))
|
||||
boundary_blockers = []
|
||||
if raw_runner_registration_read:
|
||||
boundary_blockers.append(
|
||||
"non110_runner_readiness:raw_runner_registration_read"
|
||||
)
|
||||
if runner_token_read:
|
||||
boundary_blockers.append("non110_runner_readiness:runner_token_read")
|
||||
blockers = _unique_strings(blockers + boundary_blockers)
|
||||
if boundary_blockers and blocker_count == 0:
|
||||
blocker_count = len(blockers)
|
||||
return {
|
||||
"receipt_seen": marker_seen,
|
||||
"non110_runner_ready": bool(
|
||||
marker_seen
|
||||
and ready
|
||||
and blocker_count == 0
|
||||
and not blockers
|
||||
and not raw_runner_registration_read
|
||||
and not runner_token_read
|
||||
),
|
||||
"ready_config_count": _int_or_none(fields.get("READY_CONFIG_COUNT")) or 0,
|
||||
"ready_binary_count": _int_or_none(fields.get("READY_BINARY_COUNT")) or 0,
|
||||
"ready_registration_count": (
|
||||
_int_or_none(fields.get("READY_REGISTRATION_COUNT")) or 0
|
||||
),
|
||||
"ready_service_count": _int_or_none(fields.get("READY_SERVICE_COUNT")) or 0,
|
||||
"ready_active_service_count": (
|
||||
_int_or_none(fields.get("READY_ACTIVE_SERVICE_COUNT")) or 0
|
||||
),
|
||||
"ready_autostart_path_count": (
|
||||
_int_or_none(fields.get("READY_AUTOSTART_PATH_COUNT")) or 0
|
||||
),
|
||||
"warning_count": warning_count,
|
||||
"blocker_count": blocker_count,
|
||||
"blockers": blockers,
|
||||
"safe_next_step": str(fields.get("safe_next_step") or ""),
|
||||
"raw_runner_registration_read": raw_runner_registration_read,
|
||||
"runner_token_read": runner_token_read,
|
||||
"metadata_only": True,
|
||||
"raw_output_returned": False,
|
||||
}
|
||||
|
||||
|
||||
def _post_apply_verifier(receipt_payload: dict[str, Any]) -> dict[str, Any]:
|
||||
public_status = _int_or_none(receipt_payload.get("public_registry_v2_http_status"))
|
||||
internal_status = _int_or_none(
|
||||
@@ -1009,6 +1195,7 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
|
||||
or readback.get("latest_visible_cd_harbor_repair_lane_classifier")
|
||||
or ""
|
||||
)
|
||||
controlled_profile_no_matching = bool(controlled_profile_no_matching_labels)
|
||||
cd_jobs_stale = bool(
|
||||
rollups.get("cd_run_jobs_stale_or_mismatched") is True
|
||||
or readback.get("cd_run_jobs_stale_or_mismatched") is True
|
||||
@@ -1055,6 +1242,7 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
|
||||
current_cd_harbor_repair_blocked_by_awoooi_host=(
|
||||
current_cd_harbor_repair_blocked_by_awoooi_host
|
||||
),
|
||||
controlled_profile_no_matching=controlled_profile_no_matching,
|
||||
cd_jobs_stale=cd_jobs_stale,
|
||||
cd_jobs_head_sha_mismatch=cd_jobs_head_sha_mismatch,
|
||||
cd_jobs_run_id_mismatch=cd_jobs_run_id_mismatch,
|
||||
@@ -1067,6 +1255,7 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
|
||||
no_matching_label=no_matching_label,
|
||||
current_cd_no_matching_label=current_cd_no_matching_label,
|
||||
current_cd_waiting=current_cd_waiting,
|
||||
controlled_profile_no_matching_labels=controlled_profile_no_matching_labels,
|
||||
blockers=blockers,
|
||||
)
|
||||
return {
|
||||
@@ -1161,6 +1350,7 @@ def _gitea_queue_normalized_classifier_fields(
|
||||
no_matching_label: str,
|
||||
current_cd_no_matching_label: str,
|
||||
current_cd_waiting: bool,
|
||||
controlled_profile_no_matching_labels: dict[str, str],
|
||||
blockers: list[str],
|
||||
) -> list[dict[str, Any]]:
|
||||
fields = [
|
||||
@@ -1211,6 +1401,20 @@ def _gitea_queue_normalized_classifier_fields(
|
||||
"raw_output_returned": False,
|
||||
}
|
||||
)
|
||||
if controlled_profile_no_matching_labels:
|
||||
fields.append(
|
||||
{
|
||||
"field_id": "controlled_profile_no_matching_runner_labels",
|
||||
"value": controlled_profile_no_matching_labels,
|
||||
"blockers": [
|
||||
item
|
||||
for item in blockers
|
||||
if item == "gitea_queue_controlled_profile_no_matching_runner_labels"
|
||||
],
|
||||
"metadata_only": True,
|
||||
"raw_output_returned": False,
|
||||
}
|
||||
)
|
||||
return fields
|
||||
|
||||
|
||||
@@ -1224,6 +1428,7 @@ def _gitea_queue_blockers(
|
||||
current_cd_no_matching_runner: bool,
|
||||
current_cd_harbor_repair_requires_110_lane: bool,
|
||||
current_cd_harbor_repair_blocked_by_awoooi_host: bool,
|
||||
controlled_profile_no_matching: bool,
|
||||
cd_jobs_stale: bool,
|
||||
cd_jobs_head_sha_mismatch: bool,
|
||||
cd_jobs_run_id_mismatch: bool,
|
||||
@@ -1243,6 +1448,8 @@ def _gitea_queue_blockers(
|
||||
blockers.append("gitea_queue_current_cd_no_matching_runner")
|
||||
if current_cd_waiting:
|
||||
blockers.append("gitea_queue_current_cd_waiting_for_runner_or_queue")
|
||||
if controlled_profile_no_matching:
|
||||
blockers.append("gitea_queue_controlled_profile_no_matching_runner_labels")
|
||||
if cd_jobs_head_sha_mismatch:
|
||||
blockers.append("gitea_queue_cd_jobs_head_sha_mismatch")
|
||||
if cd_jobs_run_id_mismatch:
|
||||
@@ -1397,6 +1604,7 @@ def _active_blockers(
|
||||
watchdog_check: dict[str, Any],
|
||||
watchdog_repair: dict[str, Any],
|
||||
controlled_cd_lane: dict[str, Any],
|
||||
non110_runner: dict[str, Any],
|
||||
verifier: dict[str, Any],
|
||||
gitea_queue: dict[str, Any],
|
||||
deploy_marker: dict[str, Any],
|
||||
@@ -1431,6 +1639,7 @@ def _active_blockers(
|
||||
if watchdog_repair["receipt_seen"] and not watchdog_repair["harbor_ready"]:
|
||||
blockers.append("harbor_watchdog_repair_did_not_restore_local_v2")
|
||||
blockers.extend(_strings(controlled_cd_lane.get("blockers")))
|
||||
blockers.extend(_strings(non110_runner.get("blockers")))
|
||||
if not verifier["public_registry_v2_ready"]:
|
||||
blockers.append("public_registry_v2_verifier_not_green")
|
||||
if not verifier["internal_registry_v2_ready"]:
|
||||
@@ -1447,6 +1656,7 @@ def _status(
|
||||
watchdog_check: dict[str, Any],
|
||||
watchdog_repair: dict[str, Any],
|
||||
controlled_cd_lane: dict[str, Any],
|
||||
non110_runner: dict[str, Any],
|
||||
verifier: dict[str, Any],
|
||||
gitea_queue: dict[str, Any],
|
||||
deploy_marker: dict[str, Any],
|
||||
@@ -1470,6 +1680,8 @@ def _status(
|
||||
"controlled_cd_lane_ready"
|
||||
]:
|
||||
return "controlled_cd_lane_readiness_receipt_blocked"
|
||||
if non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"]:
|
||||
return "non110_runner_readiness_receipt_blocked"
|
||||
if watchdog_repair["receipt_seen"]:
|
||||
return "harbor_registry_repair_receipt_waiting_registry_v2_verifier"
|
||||
if watchdog_check["receipt_seen"] and watchdog_check["harbor_ready"]:
|
||||
@@ -1492,6 +1704,8 @@ def _safe_next_step(*, status: str) -> str:
|
||||
return "rerun_gitea_cd_then_verify_deploy_marker_and_priority_readback"
|
||||
if status == "controlled_cd_lane_readiness_receipt_blocked":
|
||||
return "fix_controlled_cd_lane_guardrail_blockers_then_rerun_readiness_verifier"
|
||||
if status == "non110_runner_readiness_receipt_blocked":
|
||||
return "fix_or_register_non110_runner_then_rerun_readiness_and_queue_verifiers"
|
||||
if status == "harbor_registry_repair_receipt_waiting_registry_v2_verifier":
|
||||
return "rerun_public_and_internal_registry_v2_verifier_before_cd_retry"
|
||||
if status == "harbor_local_registry_ready_waiting_public_registry_v2_verifier":
|
||||
@@ -1517,6 +1731,8 @@ def _current_apply_blocker(*, status: str) -> str:
|
||||
return "deploy_marker_readback_required_after_registry_receipt"
|
||||
if status == "controlled_cd_lane_readiness_receipt_blocked":
|
||||
return "controlled_cd_lane_readiness_required_for_awoooi_host_queue"
|
||||
if status == "non110_runner_readiness_receipt_blocked":
|
||||
return "non110_runner_readiness_required_for_current_cd_queue"
|
||||
if status == "harbor_watchdog_check_unhealthy_waiting_repair_once_receipt":
|
||||
return "repair_once_receipt_required_after_unhealthy_check"
|
||||
if status == "ssh_local_repair_receipt_waiting_harbor_watchdog_check":
|
||||
|
||||
@@ -72,13 +72,13 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False):
|
||||
assert payload["rollups"]["current_blocker_local_recovery_package_count"] == 1
|
||||
assert (
|
||||
payload["rollups"]["current_blocker_harbor_recovery_receipt_input_count"]
|
||||
== 9
|
||||
== 10
|
||||
)
|
||||
assert (
|
||||
payload["rollups"][
|
||||
"current_blocker_harbor_recovery_receipt_output_contract_count"
|
||||
]
|
||||
== 6
|
||||
== 7
|
||||
)
|
||||
assert payload["rollups"]["runtime_dispatch_performed"] is False
|
||||
|
||||
@@ -139,7 +139,7 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False):
|
||||
assert current_queue[0]["harbor_recovery_receipt_endpoint"] == (
|
||||
"/api/v1/agents/harbor-registry-controlled-recovery-receipt"
|
||||
)
|
||||
assert current_queue[0]["harbor_recovery_receipt_input_count"] == 9
|
||||
assert current_queue[0]["harbor_recovery_receipt_input_count"] == 10
|
||||
assert [
|
||||
item["input_id"]
|
||||
for item in current_queue[0]["harbor_recovery_receipt_inputs"]
|
||||
@@ -150,6 +150,7 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False):
|
||||
"watchdog_check_output",
|
||||
"watchdog_repair_output",
|
||||
"controlled_cd_lane_readiness_output",
|
||||
"non110_runner_readiness_output",
|
||||
"public_registry_v2_http_status",
|
||||
"internal_registry_v2_http_status",
|
||||
"deploy_marker_readback",
|
||||
@@ -157,13 +158,14 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False):
|
||||
assert current_queue[0]["harbor_recovery_receipt_inputs"][-1][
|
||||
"expected_schema"
|
||||
] == "awoooi_production_deploy_readback_blocker_v1"
|
||||
assert current_queue[0]["harbor_recovery_receipt_output_contract_count"] == 6
|
||||
assert current_queue[0]["harbor_recovery_receipt_output_contract_count"] == 7
|
||||
assert [
|
||||
item["output_id"]
|
||||
for item in current_queue[0]["harbor_recovery_receipt_output_contract"]
|
||||
] == [
|
||||
"control_path_readiness",
|
||||
"controlled_cd_lane_readiness",
|
||||
"non110_runner_readiness",
|
||||
"gitea_actions_queue",
|
||||
"local_console_phase_readback",
|
||||
"post_apply_verifier",
|
||||
@@ -183,7 +185,7 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False):
|
||||
]
|
||||
for item in current_queue[0]["harbor_recovery_receipt_output_contract"]
|
||||
)
|
||||
assert current_queue[0]["queue_readback_normalizer_contract_count"] == 3
|
||||
assert current_queue[0]["queue_readback_normalizer_contract_count"] == 5
|
||||
assert [
|
||||
item["field_id"]
|
||||
for item in current_queue[0]["queue_readback_normalizer_contract"]
|
||||
@@ -191,6 +193,8 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False):
|
||||
"cd_run_jobs_payload_classifier",
|
||||
"harbor_110_repair_jobs_payload_classifier",
|
||||
"latest_visible_harbor_110_repair_no_matching_runner_label",
|
||||
"current_cd_workflow_runner_readiness",
|
||||
"controlled_profile_no_matching_runner_labels",
|
||||
]
|
||||
assert "gitea_queue_cd_jobs_stale_or_mismatched" in current_queue[0][
|
||||
"queue_readback_normalizer_contract"
|
||||
|
||||
@@ -329,7 +329,7 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu
|
||||
assert evidence["ai_loop_current_blocker_harbor_recovery_receipt_endpoint"] == (
|
||||
"/api/v1/agents/harbor-registry-controlled-recovery-receipt"
|
||||
)
|
||||
assert evidence["ai_loop_current_blocker_harbor_recovery_receipt_input_count"] == 9
|
||||
assert evidence["ai_loop_current_blocker_harbor_recovery_receipt_input_count"] == 10
|
||||
assert evidence["ai_loop_current_blocker_harbor_recovery_receipt_input_ids"] == [
|
||||
"gitea_actions_queue_readback",
|
||||
"ssh_publickey_diagnosis_output",
|
||||
@@ -337,6 +337,7 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu
|
||||
"watchdog_check_output",
|
||||
"watchdog_repair_output",
|
||||
"controlled_cd_lane_readiness_output",
|
||||
"non110_runner_readiness_output",
|
||||
"public_registry_v2_http_status",
|
||||
"internal_registry_v2_http_status",
|
||||
"deploy_marker_readback",
|
||||
@@ -348,11 +349,12 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu
|
||||
evidence[
|
||||
"ai_loop_current_blocker_harbor_recovery_receipt_output_contract_count"
|
||||
]
|
||||
== 6
|
||||
== 7
|
||||
)
|
||||
assert evidence["ai_loop_current_blocker_harbor_recovery_receipt_output_ids"] == [
|
||||
"control_path_readiness",
|
||||
"controlled_cd_lane_readiness",
|
||||
"non110_runner_readiness",
|
||||
"gitea_actions_queue",
|
||||
"local_console_phase_readback",
|
||||
"post_apply_verifier",
|
||||
@@ -365,6 +367,8 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu
|
||||
"cd_run_jobs_payload_classifier",
|
||||
"harbor_110_repair_jobs_payload_classifier",
|
||||
"latest_visible_harbor_110_repair_no_matching_runner_label",
|
||||
"current_cd_workflow_runner_readiness",
|
||||
"controlled_profile_no_matching_runner_labels",
|
||||
]
|
||||
assert evidence["ai_loop_current_blocker_queue_readback_normalizer_contract"][0][
|
||||
"writes_blockers"
|
||||
@@ -402,7 +406,7 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu
|
||||
payload["summary"][
|
||||
"ai_loop_current_blocker_harbor_recovery_receipt_input_count"
|
||||
]
|
||||
== 9
|
||||
== 10
|
||||
)
|
||||
assert payload["summary"][
|
||||
"ai_loop_current_blocker_harbor_recovery_receipt_input_ids"
|
||||
@@ -414,7 +418,7 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu
|
||||
payload["summary"][
|
||||
"ai_loop_current_blocker_harbor_recovery_receipt_output_contract_count"
|
||||
]
|
||||
== 6
|
||||
== 7
|
||||
)
|
||||
assert payload["summary"][
|
||||
"ai_loop_current_blocker_harbor_recovery_receipt_output_ids"
|
||||
@@ -423,7 +427,7 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu
|
||||
payload["summary"][
|
||||
"ai_loop_current_blocker_queue_readback_normalizer_contract_count"
|
||||
]
|
||||
== 3
|
||||
== 5
|
||||
)
|
||||
assert payload["summary"][
|
||||
"ai_loop_current_blocker_queue_readback_normalizer_field_ids"
|
||||
|
||||
@@ -35,8 +35,8 @@ def test_harbor_recovery_receipt_accepts_verified_repair() -> None:
|
||||
assert payload["readback"]["deploy_marker"]["deploy_marker_verified"] is True
|
||||
assert payload["readback"]["deploy_marker"]["raw_output_returned"] is False
|
||||
phase_readback = payload["local_console_phase_readback"]
|
||||
assert phase_readback["phase_count"] == 5
|
||||
assert phase_readback["completed_phase_count"] == 5
|
||||
assert phase_readback["phase_count"] == 6
|
||||
assert phase_readback["completed_phase_count"] == 6
|
||||
assert phase_readback["blocked_phase_count"] == 0
|
||||
assert phase_readback["phase_ids"] == [
|
||||
"diagnose_ssh_publickey",
|
||||
@@ -44,10 +44,11 @@ def test_harbor_recovery_receipt_accepts_verified_repair() -> None:
|
||||
"repair_ssh_metadata_if_check_confirms_metadata_drift",
|
||||
"repair_harbor_once_if_v2_still_502",
|
||||
"verify_controlled_cd_lane",
|
||||
"verify_non110_runner_lane",
|
||||
]
|
||||
assert all(phase["raw_output_returned"] is False for phase in phase_readback["phases"])
|
||||
assert payload["rollups"]["local_console_phase_count"] == 5
|
||||
assert payload["rollups"]["local_console_completed_phase_count"] == 5
|
||||
assert payload["rollups"]["local_console_phase_count"] == 6
|
||||
assert payload["rollups"]["local_console_completed_phase_count"] == 6
|
||||
assert payload["rollups"]["local_console_blocked_phase_count"] == 0
|
||||
assert payload["rollups"]["deploy_marker_readback_seen"] is True
|
||||
assert payload["rollups"]["deploy_marker_verified"] is True
|
||||
@@ -99,7 +100,7 @@ def test_harbor_recovery_receipt_routes_unhealthy_check_to_repair_once() -> None
|
||||
assert phases["repair_harbor_once_if_v2_still_502"]["status"] == (
|
||||
"blocked_waiting_harbor_repair_once_receipt"
|
||||
)
|
||||
assert payload["rollups"]["local_console_blocked_phase_count"] == 3
|
||||
assert payload["rollups"]["local_console_blocked_phase_count"] == 4
|
||||
assert payload["controlled_apply_policy"]["manual_end_state"] is False
|
||||
|
||||
|
||||
@@ -292,6 +293,64 @@ BLOCKER_COUNT=1
|
||||
assert payload["rollups"]["controlled_cd_lane_blocker_count"] == 1
|
||||
|
||||
|
||||
def test_harbor_recovery_receipt_surfaces_non110_runner_readiness() -> None:
|
||||
payload = validate_harbor_registry_controlled_recovery_receipt(
|
||||
{
|
||||
"non110_runner_readiness_output": _non110_runner_blocked_output(),
|
||||
"gitea_actions_queue_readback": _gitea_queue_current_cd_waiting_non110(),
|
||||
}
|
||||
)
|
||||
|
||||
assert payload["status"] == "non110_runner_readiness_receipt_blocked"
|
||||
assert payload["safe_next_step"] == (
|
||||
"fix_or_register_non110_runner_then_rerun_readiness_and_queue_verifiers"
|
||||
)
|
||||
assert "non110_runner_readiness:runner_registration_missing" in payload[
|
||||
"active_blockers"
|
||||
]
|
||||
assert "non110_runner_readiness:runner_service_not_active" in payload[
|
||||
"active_blockers"
|
||||
]
|
||||
runner = payload["readback"]["non110_runner_readiness"]
|
||||
assert runner["receipt_seen"] is True
|
||||
assert runner["non110_runner_ready"] is False
|
||||
assert runner["ready_config_count"] == 1
|
||||
assert runner["ready_binary_count"] == 1
|
||||
assert runner["ready_registration_count"] == 0
|
||||
assert runner["ready_service_count"] == 1
|
||||
assert runner["ready_active_service_count"] == 0
|
||||
assert runner["ready_autostart_path_count"] == 1
|
||||
assert runner["raw_runner_registration_read"] is False
|
||||
assert runner["runner_token_read"] is False
|
||||
assert runner["raw_output_returned"] is False
|
||||
readiness = payload["readback"]["control_path_readiness"]
|
||||
assert readiness["status"] == "blocked_non110_controlled_runner_unavailable"
|
||||
assert readiness["non110_runner_unavailable"] is True
|
||||
assert readiness["non110_runner_ready"] is False
|
||||
assert "non110_runner_readiness_receipt_not_ready" in readiness["signal_ids"]
|
||||
assert (
|
||||
"gitea_queue_controlled_profile_no_matching_runner_labels"
|
||||
in readiness["signal_ids"]
|
||||
)
|
||||
phases = {
|
||||
phase["phase_id"]: phase
|
||||
for phase in payload["local_console_phase_readback"]["phases"]
|
||||
}
|
||||
assert phases["verify_non110_runner_lane"]["status"] == (
|
||||
"blocked_non110_runner_readiness_receipt_not_ready"
|
||||
)
|
||||
assert payload["rollups"]["non110_runner_readiness_receipt_seen"] is True
|
||||
assert payload["rollups"]["non110_runner_ready"] is False
|
||||
assert payload["rollups"]["non110_runner_blocker_count"] == 2
|
||||
assert (
|
||||
payload["rollups"]["control_path_readiness_non110_runner_unavailable"]
|
||||
is True
|
||||
)
|
||||
assert payload["input_redaction"]["non110_runner_readiness_output"][
|
||||
"line_count"
|
||||
] > 0
|
||||
|
||||
|
||||
def test_harbor_recovery_receipt_surfaces_control_path_readiness_blocker() -> None:
|
||||
diagnosis_output = _ssh_publickey_diagnosis_output().replace(
|
||||
"rc=124 classification=server_accepts_key_then_timeout",
|
||||
@@ -844,6 +903,33 @@ safe_next_step=restore_or_register_awoooi_cd_lane_drain_registration_without_pri
|
||||
"""
|
||||
|
||||
|
||||
def _non110_runner_blocked_output() -> str:
|
||||
return """
|
||||
== audit metadata ==
|
||||
read_only=true
|
||||
secret_values_collected=false
|
||||
runner_token_read=false
|
||||
raw_runner_registration_read=false
|
||||
== non110 runner registration metadata ==
|
||||
RUNNER_REGISTRATION path=/home/ollama/act-runner-awoooi/.runner present=0 content_read=false
|
||||
BLOCKER runner_registration_missing
|
||||
== non110 runner service ==
|
||||
RUNNER_SERVICE unit=awoooi-non110-runner.service installed=1 active=inactive enabled=1
|
||||
BLOCKER runner_service_not_active
|
||||
== verdict ==
|
||||
READY_CONFIG_COUNT=1
|
||||
READY_BINARY_COUNT=1
|
||||
READY_REGISTRATION_COUNT=0
|
||||
READY_SERVICE_COUNT=1
|
||||
READY_ACTIVE_SERVICE_COUNT=0
|
||||
READY_AUTOSTART_PATH_COUNT=1
|
||||
WARNING_COUNT=0
|
||||
BLOCKER_COUNT=2
|
||||
AWOOOI_NON110_RUNNER_READY=0
|
||||
safe_next_step=run_register_awoooi_non110_runner_script_without_printing_token_then_autostart_path_will_enable_service_and_rerun_this_verifier
|
||||
"""
|
||||
|
||||
|
||||
def _gitea_queue_no_matching_runner() -> dict:
|
||||
return {
|
||||
"schema_version": "awoooi_public_gitea_actions_queue_readback_v1",
|
||||
|
||||
Reference in New Issue
Block a user