feat(agent): add harbor recovery phase receipts
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 34s
CD Pipeline / build-and-deploy (push) Failing after 2m48s
CD Pipeline / post-deploy-checks (push) Has been skipped
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 34s
CD Pipeline / build-and-deploy (push) Failing after 2m48s
CD Pipeline / post-deploy-checks (push) Has been skipped
This commit is contained in:
@@ -43,6 +43,12 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
verifier=verifier,
|
||||
active_blockers=active_blockers,
|
||||
)
|
||||
local_console_phase_readback = _local_console_phase_readback(
|
||||
ssh_local=ssh_local,
|
||||
watchdog_check=watchdog_check,
|
||||
watchdog_repair=watchdog_repair,
|
||||
verifier=verifier,
|
||||
)
|
||||
|
||||
return {
|
||||
"schema_version": _SCHEMA_VERSION,
|
||||
@@ -65,6 +71,7 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
"watchdog_repair": watchdog_repair,
|
||||
"post_apply_verifier": verifier,
|
||||
},
|
||||
"local_console_phase_readback": local_console_phase_readback,
|
||||
"controlled_apply_policy": {
|
||||
"risk_level": "high",
|
||||
"break_glass_required": False,
|
||||
@@ -103,6 +110,13 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
"watchdog_repair_harbor_ready": watchdog_repair["harbor_ready"],
|
||||
"post_apply_verifier_ready": verifier["registry_v2_ready"],
|
||||
"metadata_writeback_contract_ready": True,
|
||||
"local_console_phase_count": local_console_phase_readback["phase_count"],
|
||||
"local_console_completed_phase_count": local_console_phase_readback[
|
||||
"completed_phase_count"
|
||||
],
|
||||
"local_console_blocked_phase_count": local_console_phase_readback[
|
||||
"blocked_phase_count"
|
||||
],
|
||||
},
|
||||
"operation_boundaries": {
|
||||
"raw_output_returned": False,
|
||||
@@ -122,6 +136,111 @@ def validate_harbor_registry_controlled_recovery_receipt(
|
||||
}
|
||||
|
||||
|
||||
def _local_console_phase_readback(
|
||||
*,
|
||||
ssh_local: dict[str, Any],
|
||||
watchdog_check: dict[str, Any],
|
||||
watchdog_repair: dict[str, Any],
|
||||
verifier: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
phases = [
|
||||
_phase(
|
||||
"diagnose_ssh_publickey",
|
||||
_phase_status(
|
||||
ready=ssh_local["receipt_seen"],
|
||||
blocked_status="blocked_waiting_ssh_publickey_diagnosis_receipt",
|
||||
),
|
||||
"ssh_local_repair",
|
||||
),
|
||||
_phase(
|
||||
"preflight_control_path_and_harbor",
|
||||
_phase_status(
|
||||
ready=watchdog_check["receipt_seen"],
|
||||
blocked_status="blocked_waiting_harbor_watchdog_check_receipt",
|
||||
),
|
||||
"watchdog_check",
|
||||
),
|
||||
_phase(
|
||||
"repair_ssh_metadata_if_check_confirms_metadata_drift",
|
||||
_ssh_metadata_phase_status(ssh_local=ssh_local, watchdog_check=watchdog_check),
|
||||
"ssh_local_repair",
|
||||
),
|
||||
_phase(
|
||||
"repair_harbor_once_if_v2_still_502",
|
||||
_harbor_repair_once_phase_status(
|
||||
watchdog_check=watchdog_check,
|
||||
watchdog_repair=watchdog_repair,
|
||||
),
|
||||
"watchdog_repair",
|
||||
),
|
||||
_phase(
|
||||
"verify_controlled_cd_lane",
|
||||
_phase_status(
|
||||
ready=verifier["registry_v2_ready"],
|
||||
blocked_status="blocked_waiting_registry_v2_verifier_green",
|
||||
),
|
||||
"post_apply_verifier",
|
||||
),
|
||||
]
|
||||
completed_statuses = {"ready", "skipped_not_required"}
|
||||
return {
|
||||
"phase_count": len(phases),
|
||||
"completed_phase_count": sum(
|
||||
1 for phase in phases if phase["status"] in completed_statuses
|
||||
),
|
||||
"blocked_phase_count": sum(
|
||||
1 for phase in phases if phase["status"] not in completed_statuses
|
||||
),
|
||||
"phase_ids": [phase["phase_id"] for phase in phases],
|
||||
"phases": phases,
|
||||
"metadata_only": True,
|
||||
"raw_output_returned": False,
|
||||
}
|
||||
|
||||
|
||||
def _phase(phase_id: str, status: str, evidence_key: str) -> dict[str, Any]:
|
||||
return {
|
||||
"phase_id": phase_id,
|
||||
"status": status,
|
||||
"evidence_key": evidence_key,
|
||||
"raw_output_returned": False,
|
||||
}
|
||||
|
||||
|
||||
def _phase_status(*, ready: bool, blocked_status: str) -> str:
|
||||
return "ready" if ready else blocked_status
|
||||
|
||||
|
||||
def _ssh_metadata_phase_status(
|
||||
*,
|
||||
ssh_local: dict[str, Any],
|
||||
watchdog_check: dict[str, Any],
|
||||
) -> str:
|
||||
if ssh_local["receipt_seen"]:
|
||||
if ssh_local["control_channel_metadata_ready"]:
|
||||
return "ready"
|
||||
return "blocked_ssh_metadata_repair_receipt_not_ready"
|
||||
if watchdog_check["receipt_seen"]:
|
||||
return "skipped_not_required"
|
||||
return "blocked_waiting_ssh_metadata_or_harbor_preflight_receipt"
|
||||
|
||||
|
||||
def _harbor_repair_once_phase_status(
|
||||
*,
|
||||
watchdog_check: dict[str, Any],
|
||||
watchdog_repair: dict[str, Any],
|
||||
) -> str:
|
||||
if watchdog_check["receipt_seen"] and watchdog_check["harbor_ready"]:
|
||||
return "skipped_not_required"
|
||||
if watchdog_repair["receipt_seen"] and watchdog_repair["harbor_ready"]:
|
||||
return "ready"
|
||||
if watchdog_repair["receipt_seen"]:
|
||||
return "blocked_harbor_repair_once_receipt_not_green"
|
||||
if watchdog_check["receipt_seen"]:
|
||||
return "blocked_waiting_harbor_repair_once_receipt"
|
||||
return "blocked_waiting_harbor_watchdog_check_receipt"
|
||||
|
||||
|
||||
def _parse_ssh_local_repair_output(output: str) -> dict[str, Any]:
|
||||
fields = _parse_key_values(output)
|
||||
marker_seen = "AWOOOI_110_SSH_PUBLICKEY_AUTH_LOCAL_REPAIR" in output
|
||||
|
||||
@@ -31,6 +31,21 @@ def test_harbor_recovery_receipt_accepts_verified_repair() -> None:
|
||||
] is True
|
||||
assert payload["readback"]["watchdog_repair"]["harbor_ready"] is True
|
||||
assert payload["readback"]["post_apply_verifier"]["registry_v2_ready"] is True
|
||||
phase_readback = payload["local_console_phase_readback"]
|
||||
assert phase_readback["phase_count"] == 5
|
||||
assert phase_readback["completed_phase_count"] == 5
|
||||
assert phase_readback["blocked_phase_count"] == 0
|
||||
assert phase_readback["phase_ids"] == [
|
||||
"diagnose_ssh_publickey",
|
||||
"preflight_control_path_and_harbor",
|
||||
"repair_ssh_metadata_if_check_confirms_metadata_drift",
|
||||
"repair_harbor_once_if_v2_still_502",
|
||||
"verify_controlled_cd_lane",
|
||||
]
|
||||
assert all(phase["raw_output_returned"] is False for phase in phase_readback["phases"])
|
||||
assert payload["rollups"]["local_console_phase_count"] == 5
|
||||
assert payload["rollups"]["local_console_completed_phase_count"] == 5
|
||||
assert payload["rollups"]["local_console_blocked_phase_count"] == 0
|
||||
assert payload["input_redaction"]["raw_output_returned"] is False
|
||||
assert payload["operation_boundaries"]["ssh_used"] is False
|
||||
assert payload["operation_boundaries"]["docker_command_performed"] is False
|
||||
@@ -60,6 +75,21 @@ def test_harbor_recovery_receipt_routes_unhealthy_check_to_repair_once() -> None
|
||||
assert payload["safe_next_step"] == (
|
||||
"run_harbor_watchdog_repair_once_then_submit_receipt_with_verifier"
|
||||
)
|
||||
phases = {
|
||||
phase["phase_id"]: phase
|
||||
for phase in payload["local_console_phase_readback"]["phases"]
|
||||
}
|
||||
assert phases["diagnose_ssh_publickey"]["status"] == (
|
||||
"blocked_waiting_ssh_publickey_diagnosis_receipt"
|
||||
)
|
||||
assert phases["preflight_control_path_and_harbor"]["status"] == "ready"
|
||||
assert phases["repair_ssh_metadata_if_check_confirms_metadata_drift"][
|
||||
"status"
|
||||
] == "skipped_not_required"
|
||||
assert phases["repair_harbor_once_if_v2_still_502"]["status"] == (
|
||||
"blocked_waiting_harbor_repair_once_receipt"
|
||||
)
|
||||
assert payload["rollups"]["local_console_blocked_phase_count"] == 3
|
||||
assert payload["controlled_apply_policy"]["manual_end_state"] is False
|
||||
|
||||
|
||||
|
||||
@@ -50860,3 +50860,21 @@ production browser smoke:
|
||||
|
||||
**下一步**:
|
||||
- commit / push 後讀回 Gitea CD;若仍卡 Harbor / 110,P0 runtime 下一步仍是 110 local console / root shell 按 phase 執行 check-mode / bounded repair,並把 verifier receipt 回寫 AI Loop。
|
||||
|
||||
## 2026-06-30 — 23:01 Harbor recovery receipt local-console phase readback
|
||||
|
||||
**完成內容**:
|
||||
- `harbor-registry-controlled-recovery-receipt` 新增 `local_console_phase_readback`,把 110 local-console recovery 的 5 個 phase 轉成 receipt-side machine-readable readback。
|
||||
- phase 對齊 `diagnose_ssh_publickey`、`preflight_control_path_and_harbor`、`repair_ssh_metadata_if_check_confirms_metadata_drift`、`repair_harbor_once_if_v2_still_502`、`verify_controlled_cd_lane`,並回傳 completed / blocked phase count、metadata-only、raw-output-redacted boundary。
|
||||
- receipt rollups 新增 `local_console_phase_count`、`local_console_completed_phase_count`、`local_console_blocked_phase_count`,讓執行後 receipt 能回接 AI Loop / work-order 的同一批 phase ids。
|
||||
|
||||
**本地驗證結果**:
|
||||
- `DATABASE_URL=postgresql+asyncpg://test:test@localhost:5432/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py apps/api/tests/test_harbor_registry_controlled_recovery_preflight.py apps/api/tests/test_awoooi_priority_work_order_readback_api.py apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py apps/api/tests/test_ai_agent_autonomous_runtime_control.py apps/api/tests/test_ai_agent_autonomous_runtime_control_api.py ops/runner/test_read_public_gitea_actions_queue.py -q`:`51 passed`。
|
||||
- `ruff check`、`py_compile`、`ops/runner/guard-gitea-runner-pressure.py --root .`、`scripts/ci/check-gitea-step-env-secrets.js`、`git diff --check`:通過。
|
||||
|
||||
**live truth**:
|
||||
- latest public queue 轉為 `#4107` Running / `780ad71 fix(runner): classify harbor retrying cd runs`;`#4106` 被新 commit 取消。
|
||||
- `harbor-110-local-repair` `#4108` Waiting,no matching label `awoooi-host`;registry / Harbor health 仍 502。
|
||||
|
||||
**下一步**:
|
||||
- rebase 最新 Gitea main 後 commit / push;若 CD 仍卡 Harbor / 110,P0 runtime 下一步仍是依 phase 在 110 local console / root shell 執行 check-mode / bounded repair,並提交 receipt 驗證 phase readback。
|
||||
|
||||
Reference in New Issue
Block a user