fix(cd): treat deploy evidence as effective success
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled

This commit is contained in:
Your Name
2026-07-02 23:56:17 +08:00
parent 93a160646c
commit 1b3b0fd572
3 changed files with 89 additions and 8 deletions

View File

@@ -53470,13 +53470,15 @@ production browser smoke:
**完成內容**
- 修正 `ops/runner/read-public-gitea-actions-queue.py`:當 Gitea run-level UI 曾短暫顯示 `Blocked`,但 build log 已同時具備 `Job succeeded`、deploy marker pushed、`Production deploy readback matches this build` 與 build-deploy success notification 時,不再把 current main CD 誤判為 blocked。
- 擴大 effective-success 判定:當 run-level UI 仍停在 `Running`,但 build log 已具備完整成功鏈,也視為 `latest_visible_cd_deploy_evidence_succeeded_from_log=true`,同時用 `latest_visible_cd_run_level_lag_after_deploy_success=true` 保留 UI lag 訊號。
- 新增分層欄位:`latest_visible_cd_run_ui_blocked` 保留 UI 殘留訊號,`latest_visible_cd_run_blocked` 只代表仍缺成功部署證據的實際 blocker另回傳 deploy marker SHA、source SHA、production deploy readback matched 與 notification receipt。
- 新增單元測試固定今天的 false-blocked 模式,避免「成功部署但 Gitea UI residual blocked」再次干擾 reboot / CD readiness scorecard。
- 新增單元測試固定今天的 false-blocked / false-running 模式,避免「成功部署但 Gitea UI residual blocked/running」再次干擾 reboot / CD readiness scorecard。
**live readback**
- CD `#4467` 已讀回 `Success`deploy marker 為 `14768cf6f chore(cd): deploy af3a289 [skip ci]`
- queue readback 讀回 `current_main_cd_run_blocked=false``latest_visible_cd_run_status=Success``latest_visible_cd_deploy_marker_sha_from_log=14768cf``latest_visible_cd_production_deploy_readback_matched_from_log=true``latest_visible_cd_build_deploy_success_notification_from_log=true`
- 歷史 `#4466` Harbor repair failure 保留為 `harbor_110_repair_blocked_raw=true`,但因 `harbor_110_repair_historical_after_latest_cd_success=true`,不再阻擋 current CD。
- 後續 CD `#4468` 觀察到 run-level 仍停 `Running`,但 log 已有 deploy marker `93a1606`、production deploy readback matched、success notification 與 `Job succeeded`;已把此模式納入同一個 residual-noise hardening。
**本地驗證結果**
- `python3.11 -m pytest ops/runner/test_read_public_gitea_actions_queue.py -q -p no:cacheprovider``40 passed`

View File

@@ -415,22 +415,30 @@ def build_readback(
)
latest_cd_status = latest_cd_run.get("status", "")
latest_cd_success = latest_cd_status == "Success"
latest_cd_run_ui_blocked = latest_cd_status == "Blocked"
latest_cd_deploy_readback_succeeded_despite_ui_blocked = bool(
latest_cd_run_ui_blocked
and build_log_classifier["job_succeeded"]
latest_cd_deploy_evidence_succeeded = bool(
build_log_classifier["job_succeeded"]
and build_log_classifier["deploy_marker_pushed"]
and build_log_classifier["production_deploy_readback_matched"]
and build_log_classifier["build_deploy_success_notification_sent"]
)
latest_cd_run_ui_blocked = latest_cd_status == "Blocked"
latest_cd_deploy_readback_succeeded_despite_ui_blocked = bool(
latest_cd_run_ui_blocked and latest_cd_deploy_evidence_succeeded
)
latest_cd_run_level_lag_after_deploy_success = bool(
latest_cd_status in {"Waiting", "Running", "Blocked"}
and latest_cd_deploy_evidence_succeeded
)
latest_cd_visible_blocked = bool(
latest_cd_run_ui_blocked
and not latest_cd_deploy_readback_succeeded_despite_ui_blocked
and not latest_cd_deploy_evidence_succeeded
)
latest_cd_effective_success = bool(
latest_cd_success or latest_cd_deploy_readback_succeeded_despite_ui_blocked
latest_cd_success or latest_cd_deploy_evidence_succeeded
)
latest_cd_waiting = bool(
latest_cd_status == "Waiting" and not latest_cd_deploy_evidence_succeeded
)
latest_cd_waiting = latest_cd_status == "Waiting"
host_pressure_waiting_from_stale_jobs = (
cd_jobs_stale_or_mismatched
and latest_cd_status in {"Blocked", "Canceled", "Failure"}
@@ -684,6 +692,12 @@ def build_readback(
"latest_visible_cd_run_waiting": latest_cd_waiting,
"latest_visible_cd_run_blocked": latest_cd_visible_blocked,
"latest_visible_cd_run_ui_blocked": latest_cd_run_ui_blocked,
"latest_visible_cd_deploy_evidence_succeeded_from_log": (
latest_cd_deploy_evidence_succeeded
),
"latest_visible_cd_run_level_lag_after_deploy_success": (
latest_cd_run_level_lag_after_deploy_success
),
"latest_visible_cd_deploy_readback_succeeded_despite_ui_blocked": (
latest_cd_deploy_readback_succeeded_despite_ui_blocked
),
@@ -1052,6 +1066,12 @@ def build_readback(
),
"current_main_cd_run_blocked": latest_cd_visible_blocked,
"current_main_cd_run_ui_blocked": latest_cd_run_ui_blocked,
"current_main_cd_deploy_evidence_succeeded_from_log": (
latest_cd_deploy_evidence_succeeded
),
"current_main_cd_run_level_lag_after_deploy_success": (
latest_cd_run_level_lag_after_deploy_success
),
"current_main_cd_deploy_readback_succeeded_despite_ui_blocked": (
latest_cd_deploy_readback_succeeded_despite_ui_blocked
),

View File

@@ -1267,6 +1267,14 @@ def test_build_readback_downgrades_ui_blocked_after_deploy_readback_success() ->
assert payload["readback"]["latest_visible_cd_run_status"] == "Blocked"
assert payload["readback"]["latest_visible_cd_run_ui_blocked"] is True
assert payload["readback"]["latest_visible_cd_run_blocked"] is False
assert (
payload["readback"]["latest_visible_cd_deploy_evidence_succeeded_from_log"]
is True
)
assert (
payload["readback"]["latest_visible_cd_run_level_lag_after_deploy_success"]
is True
)
assert (
payload["readback"][
"latest_visible_cd_deploy_readback_succeeded_despite_ui_blocked"
@@ -1294,6 +1302,14 @@ def test_build_readback_downgrades_ui_blocked_after_deploy_readback_success() ->
)
assert payload["rollups"]["current_main_cd_run_ui_blocked"] is True
assert payload["rollups"]["current_main_cd_run_blocked"] is False
assert (
payload["rollups"]["current_main_cd_deploy_evidence_succeeded_from_log"]
is True
)
assert (
payload["rollups"]["current_main_cd_run_level_lag_after_deploy_success"]
is True
)
assert (
payload["rollups"][
"current_main_cd_deploy_readback_succeeded_despite_ui_blocked"
@@ -1302,6 +1318,49 @@ def test_build_readback_downgrades_ui_blocked_after_deploy_readback_success() ->
)
def test_build_readback_downgrades_running_lag_after_deploy_success() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_single_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
latest_cd_build_log_http_status=200,
latest_cd_build_log_text=_cd_deploy_success_log(),
)
assert payload["status"] == "no_matching_runner_not_visible"
assert payload["readback"]["latest_visible_cd_run_status"] == "Running"
assert payload["readback"]["latest_visible_cd_run_waiting"] is False
assert payload["readback"]["latest_visible_cd_run_blocked"] is False
assert (
payload["readback"]["latest_visible_cd_deploy_evidence_succeeded_from_log"]
is True
)
assert (
payload["readback"]["latest_visible_cd_run_level_lag_after_deploy_success"]
is True
)
assert (
payload["readback"][
"latest_visible_cd_deploy_readback_succeeded_despite_ui_blocked"
]
is False
)
assert payload["rollups"]["current_main_cd_run_status"] == "Running"
assert payload["rollups"]["current_main_cd_run_waiting"] is False
assert payload["rollups"]["current_main_cd_run_blocked"] is False
assert (
payload["rollups"]["current_main_cd_deploy_evidence_succeeded_from_log"]
is True
)
assert (
payload["rollups"]["current_main_cd_run_level_lag_after_deploy_success"]
is True
)
def test_build_readback_classifies_host_pressure_waiting() -> None:
module = _load_module()
payload = module.build_readback(