fix(cd): treat deploy evidence as effective success
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
This commit is contained in:
@@ -53470,13 +53470,15 @@ production browser smoke:
|
||||
|
||||
**完成內容**:
|
||||
- 修正 `ops/runner/read-public-gitea-actions-queue.py`:當 Gitea run-level UI 曾短暫顯示 `Blocked`,但 build log 已同時具備 `Job succeeded`、deploy marker pushed、`Production deploy readback matches this build` 與 build-deploy success notification 時,不再把 current main CD 誤判為 blocked。
|
||||
- 擴大 effective-success 判定:當 run-level UI 仍停在 `Running`,但 build log 已具備完整成功鏈,也視為 `latest_visible_cd_deploy_evidence_succeeded_from_log=true`,同時用 `latest_visible_cd_run_level_lag_after_deploy_success=true` 保留 UI lag 訊號。
|
||||
- 新增分層欄位:`latest_visible_cd_run_ui_blocked` 保留 UI 殘留訊號,`latest_visible_cd_run_blocked` 只代表仍缺成功部署證據的實際 blocker;另回傳 deploy marker SHA、source SHA、production deploy readback matched 與 notification receipt。
|
||||
- 新增單元測試固定今天的 false-blocked 模式,避免「成功部署但 Gitea UI residual blocked」再次干擾 reboot / CD readiness scorecard。
|
||||
- 新增單元測試固定今天的 false-blocked / false-running 模式,避免「成功部署但 Gitea UI residual blocked/running」再次干擾 reboot / CD readiness scorecard。
|
||||
|
||||
**live readback**:
|
||||
- CD `#4467` 已讀回 `Success`;deploy marker 為 `14768cf6f chore(cd): deploy af3a289 [skip ci]`。
|
||||
- queue readback 讀回 `current_main_cd_run_blocked=false`、`latest_visible_cd_run_status=Success`、`latest_visible_cd_deploy_marker_sha_from_log=14768cf`、`latest_visible_cd_production_deploy_readback_matched_from_log=true`、`latest_visible_cd_build_deploy_success_notification_from_log=true`。
|
||||
- 歷史 `#4466` Harbor repair failure 保留為 `harbor_110_repair_blocked_raw=true`,但因 `harbor_110_repair_historical_after_latest_cd_success=true`,不再阻擋 current CD。
|
||||
- 後續 CD `#4468` 觀察到 run-level 仍停 `Running`,但 log 已有 deploy marker `93a1606`、production deploy readback matched、success notification 與 `Job succeeded`;已把此模式納入同一個 residual-noise hardening。
|
||||
|
||||
**本地驗證結果**:
|
||||
- `python3.11 -m pytest ops/runner/test_read_public_gitea_actions_queue.py -q -p no:cacheprovider`:`40 passed`。
|
||||
|
||||
@@ -415,22 +415,30 @@ def build_readback(
|
||||
)
|
||||
latest_cd_status = latest_cd_run.get("status", "")
|
||||
latest_cd_success = latest_cd_status == "Success"
|
||||
latest_cd_run_ui_blocked = latest_cd_status == "Blocked"
|
||||
latest_cd_deploy_readback_succeeded_despite_ui_blocked = bool(
|
||||
latest_cd_run_ui_blocked
|
||||
and build_log_classifier["job_succeeded"]
|
||||
latest_cd_deploy_evidence_succeeded = bool(
|
||||
build_log_classifier["job_succeeded"]
|
||||
and build_log_classifier["deploy_marker_pushed"]
|
||||
and build_log_classifier["production_deploy_readback_matched"]
|
||||
and build_log_classifier["build_deploy_success_notification_sent"]
|
||||
)
|
||||
latest_cd_run_ui_blocked = latest_cd_status == "Blocked"
|
||||
latest_cd_deploy_readback_succeeded_despite_ui_blocked = bool(
|
||||
latest_cd_run_ui_blocked and latest_cd_deploy_evidence_succeeded
|
||||
)
|
||||
latest_cd_run_level_lag_after_deploy_success = bool(
|
||||
latest_cd_status in {"Waiting", "Running", "Blocked"}
|
||||
and latest_cd_deploy_evidence_succeeded
|
||||
)
|
||||
latest_cd_visible_blocked = bool(
|
||||
latest_cd_run_ui_blocked
|
||||
and not latest_cd_deploy_readback_succeeded_despite_ui_blocked
|
||||
and not latest_cd_deploy_evidence_succeeded
|
||||
)
|
||||
latest_cd_effective_success = bool(
|
||||
latest_cd_success or latest_cd_deploy_readback_succeeded_despite_ui_blocked
|
||||
latest_cd_success or latest_cd_deploy_evidence_succeeded
|
||||
)
|
||||
latest_cd_waiting = bool(
|
||||
latest_cd_status == "Waiting" and not latest_cd_deploy_evidence_succeeded
|
||||
)
|
||||
latest_cd_waiting = latest_cd_status == "Waiting"
|
||||
host_pressure_waiting_from_stale_jobs = (
|
||||
cd_jobs_stale_or_mismatched
|
||||
and latest_cd_status in {"Blocked", "Canceled", "Failure"}
|
||||
@@ -684,6 +692,12 @@ def build_readback(
|
||||
"latest_visible_cd_run_waiting": latest_cd_waiting,
|
||||
"latest_visible_cd_run_blocked": latest_cd_visible_blocked,
|
||||
"latest_visible_cd_run_ui_blocked": latest_cd_run_ui_blocked,
|
||||
"latest_visible_cd_deploy_evidence_succeeded_from_log": (
|
||||
latest_cd_deploy_evidence_succeeded
|
||||
),
|
||||
"latest_visible_cd_run_level_lag_after_deploy_success": (
|
||||
latest_cd_run_level_lag_after_deploy_success
|
||||
),
|
||||
"latest_visible_cd_deploy_readback_succeeded_despite_ui_blocked": (
|
||||
latest_cd_deploy_readback_succeeded_despite_ui_blocked
|
||||
),
|
||||
@@ -1052,6 +1066,12 @@ def build_readback(
|
||||
),
|
||||
"current_main_cd_run_blocked": latest_cd_visible_blocked,
|
||||
"current_main_cd_run_ui_blocked": latest_cd_run_ui_blocked,
|
||||
"current_main_cd_deploy_evidence_succeeded_from_log": (
|
||||
latest_cd_deploy_evidence_succeeded
|
||||
),
|
||||
"current_main_cd_run_level_lag_after_deploy_success": (
|
||||
latest_cd_run_level_lag_after_deploy_success
|
||||
),
|
||||
"current_main_cd_deploy_readback_succeeded_despite_ui_blocked": (
|
||||
latest_cd_deploy_readback_succeeded_despite_ui_blocked
|
||||
),
|
||||
|
||||
@@ -1267,6 +1267,14 @@ def test_build_readback_downgrades_ui_blocked_after_deploy_readback_success() ->
|
||||
assert payload["readback"]["latest_visible_cd_run_status"] == "Blocked"
|
||||
assert payload["readback"]["latest_visible_cd_run_ui_blocked"] is True
|
||||
assert payload["readback"]["latest_visible_cd_run_blocked"] is False
|
||||
assert (
|
||||
payload["readback"]["latest_visible_cd_deploy_evidence_succeeded_from_log"]
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
payload["readback"]["latest_visible_cd_run_level_lag_after_deploy_success"]
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
payload["readback"][
|
||||
"latest_visible_cd_deploy_readback_succeeded_despite_ui_blocked"
|
||||
@@ -1294,6 +1302,14 @@ def test_build_readback_downgrades_ui_blocked_after_deploy_readback_success() ->
|
||||
)
|
||||
assert payload["rollups"]["current_main_cd_run_ui_blocked"] is True
|
||||
assert payload["rollups"]["current_main_cd_run_blocked"] is False
|
||||
assert (
|
||||
payload["rollups"]["current_main_cd_deploy_evidence_succeeded_from_log"]
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
payload["rollups"]["current_main_cd_run_level_lag_after_deploy_success"]
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
payload["rollups"][
|
||||
"current_main_cd_deploy_readback_succeeded_despite_ui_blocked"
|
||||
@@ -1302,6 +1318,49 @@ def test_build_readback_downgrades_ui_blocked_after_deploy_readback_success() ->
|
||||
)
|
||||
|
||||
|
||||
def test_build_readback_downgrades_running_lag_after_deploy_success() -> None:
|
||||
module = _load_module()
|
||||
payload = module.build_readback(
|
||||
actions_html=_actions_html_single_cd_run(),
|
||||
actions_list_http_status=401,
|
||||
actions_list_payload={"message": "token is required"},
|
||||
cd_jobs_http_status=200,
|
||||
cd_jobs_payload={"jobs": [], "total_count": 0},
|
||||
latest_cd_build_log_http_status=200,
|
||||
latest_cd_build_log_text=_cd_deploy_success_log(),
|
||||
)
|
||||
|
||||
assert payload["status"] == "no_matching_runner_not_visible"
|
||||
assert payload["readback"]["latest_visible_cd_run_status"] == "Running"
|
||||
assert payload["readback"]["latest_visible_cd_run_waiting"] is False
|
||||
assert payload["readback"]["latest_visible_cd_run_blocked"] is False
|
||||
assert (
|
||||
payload["readback"]["latest_visible_cd_deploy_evidence_succeeded_from_log"]
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
payload["readback"]["latest_visible_cd_run_level_lag_after_deploy_success"]
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
payload["readback"][
|
||||
"latest_visible_cd_deploy_readback_succeeded_despite_ui_blocked"
|
||||
]
|
||||
is False
|
||||
)
|
||||
assert payload["rollups"]["current_main_cd_run_status"] == "Running"
|
||||
assert payload["rollups"]["current_main_cd_run_waiting"] is False
|
||||
assert payload["rollups"]["current_main_cd_run_blocked"] is False
|
||||
assert (
|
||||
payload["rollups"]["current_main_cd_deploy_evidence_succeeded_from_log"]
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
payload["rollups"]["current_main_cd_run_level_lag_after_deploy_success"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def test_build_readback_classifies_host_pressure_waiting() -> None:
|
||||
module = _load_module()
|
||||
payload = module.build_readback(
|
||||
|
||||
Reference in New Issue
Block a user