wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity
Files
bb61d5ab35a5018ee9e7bfcc01f101a7aaa3a2e4
awoooi/ops/runner/test_read_public_gitea_actions_queue.py
Your Name 12b78453cf
Some checks failed
CD Pipeline / workflow-shape (push) Has been cancelled
Details
CD Pipeline / cancel-stale-cd (push) Has been cancelled
Details
CD Pipeline / tests (push) Has been cancelled
Details
CD Pipeline / build-and-deploy (push) Has been cancelled
Details
CD Pipeline / post-deploy-checks (push) Has been cancelled
Details
fix(runner): prioritize harbor ssh root cause in queue status
2026-07-01 13:35:33 +08:00

1715 lines
65 KiB
Python
Raw Blame History

#!/usr/bin/env python3
from __future__ import annotations
import importlib.util
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "ops/runner/read-public-gitea-actions-queue.py"
def _load_module():
spec = importlib.util.spec_from_file_location(
"read_public_gitea_actions_queue",
SCRIPT,
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _actions_html() -> str:
return """
<span data-tooltip-content="No matching online runner with label: awoooi-non110-ubuntu">
<span><b>ai-technology-watch.yaml #3857</b>:</span>Scheduled</div>
<span data-tooltip-content="Canceled">
<span><b>ai-technology-watch.yaml #3856</b>:</span>Scheduled</div>
<span data-tooltip-content="Waiting">
<span><b>e2e-health.yaml #3855</b>:</span>Scheduled</div>
"""
def _actions_html_full_run_items() -> str:
return """
<div class="flex-list run-list">
<div class="flex-item tw-items-center">
<div class="flex-item-leading">
<span data-tooltip-content="Running"></span>
</div>
<div class="flex-item-main">
<a class="flex-item-title" title="chore(deploy): retry non110 cd after ci image seed" href="/wooo/awoooi/actions/runs/3863">
chore(deploy): retry non110 cd after ci image seed
</a>
<div class="flex-item-body">
<span><b>cd.yaml #3863</b>:</span>Commit
<a href="/wooo/awoooi/commit/a70c6756d9e76c33143676eef82bab7a49ac1839">a70c6756d9</a>
</div>
</div>
</div>
<div class="flex-item tw-items-center">
<div class="flex-item-leading">
<span data-tooltip-content="No matching online runner with label: awoooi-non110-host"></span>
</div>
<div class="flex-item-main">
<a class="flex-item-title" title="fix(ci): restore non110 cd push trigger" href="/wooo/awoooi/actions/runs/3859">
fix(ci): restore non110 cd push trigger
</a>
<div class="flex-item-body">
<span><b>cd.yaml #3859</b>:</span>Commit
<a href="/wooo/awoooi/commit/f3634db18ec3e37381dd634f5f5b80d73f8a8e38">f3634db18e</a>
</div>
</div>
</div>
</div>
"""
def _actions_html_single_cd_run() -> str:
return """
<div class="flex-list run-list">
<div class="flex-item tw-items-center">
<div class="flex-item-leading">
<span data-tooltip-content="Running"></span>
</div>
<div class="flex-item-main">
<a class="flex-item-title" title="chore(deploy): retry non110 cd after ci image seed" href="/wooo/awoooi/actions/runs/3863">
chore(deploy): retry non110 cd after ci image seed
</a>
<div class="flex-item-body">
<span><b>cd.yaml #3863</b>:</span>Commit
<a href="/wooo/awoooi/commit/a70c6756d9e76c33143676eef82bab7a49ac1839">a70c6756d9</a>
</div>
</div>
</div>
</div>
"""
def _cd_workflow_actions_html_single_cd_run() -> str:
return _actions_html_single_cd_run().replace(
"<span><b>cd.yaml #3863</b>:</span>Commit",
"<span><b>#3863</b>:</span>Commit",
)
def _actions_html_cd_running_harbor_repair_waiting() -> str:
return """
<div class="flex-list run-list">
<div class="flex-item tw-items-center">
<div class="flex-item-leading">
<span data-tooltip-content="Running"></span>
</div>
<div class="flex-item-main">
<a class="flex-item-title" title="fix(cd): keep harbor repair workflow on controlled profile" href="/wooo/awoooi/actions/runs/4061">
fix(cd): keep harbor repair workflow on controlled profile
</a>
<div class="flex-item-body">
<span><b>cd.yaml #4061</b>:</span>Commit
<a href="/wooo/awoooi/commit/49a9f73094592150f5d748e6be68a9e49ba1b7df">49a9f7309</a>
</div>
</div>
</div>
<div class="flex-item tw-items-center">
<div class="flex-item-leading">
<span data-tooltip-content="Waiting"></span>
</div>
<div class="flex-item-main">
<a class="flex-item-title" title="fix(cd): schedule bounded 110 harbor repair" href="/wooo/awoooi/actions/runs/4060">
fix(cd): schedule bounded 110 harbor repair
</a>
<div class="flex-item-body">
<span><b>harbor-110-local-repair.yaml #4060</b>:</span>Scheduled
</div>
</div>
</div>
</div>
"""
def _actions_html_cd_failed_harbor_repair_failed() -> str:
return (
_actions_html_cd_running_harbor_repair_waiting()
.replace('data-tooltip-content="Running"', 'data-tooltip-content="Failure"', 1)
.replace('data-tooltip-content="Waiting"', 'data-tooltip-content="Failure"', 1)
.replace("4061", "4211")
.replace("4060", "4212")
)
def _actions_html_harbor_repair_waiting_with_workflow_no_matching() -> str:
return """
<div class="menu">
<a class="item" href="?workflow=cd.yaml&actor=0&status=0">cd.yaml
</a>
<a class="item" href="?workflow=harbor-110-local-repair.yaml&actor=0&status=0">harbor-110-local-repair.yaml
<span data-tooltip-content="No matching online runner with label: awoooi-host"></span>
</a>
</div>
""" + _actions_html_cd_running_harbor_repair_waiting()
def _actions_html_cd_waiting_with_non110_no_matching() -> str:
return """
<div class="menu">
<a class="item" href="?workflow=cd.yaml&actor=0&status=0">cd.yaml
<span data-tooltip-content="No matching online runner with label: awoooi-non110-ubuntu"></span>
</a>
<a class="item" href="?workflow=harbor-110-local-repair.yaml&actor=0&status=0">harbor-110-local-repair.yaml
<span data-tooltip-content="No matching online runner with label: awoooi-non110-host"></span>
</a>
</div>
<div class="flex-list run-list">
<div class="flex-item tw-items-center">
<div class="flex-item-leading">
<span data-tooltip-content="Waiting"></span>
</div>
<div class="flex-item-main">
<a class="flex-item-title" title="feat(agent): expose harbor receipt output contract" href="/wooo/awoooi/actions/runs/4191">
feat(agent): expose harbor receipt output contract
</a>
<div class="flex-item-body">
<span><b>cd.yaml #4191</b>:</span>Commit
<a href="/wooo/awoooi/commit/693a31343bb525b655faffadafbcbff8246b1ff9">693a31343</a>
</div>
</div>
</div>
<div class="flex-item tw-items-center">
<div class="flex-item-leading">
<span data-tooltip-content="Waiting"></span>
</div>
<div class="flex-item-main">
<a class="flex-item-title" title="fix(cd): keep recovery receipts on controlled profile" href="/wooo/awoooi/actions/runs/4190">
fix(cd): keep recovery receipts on controlled profile
</a>
<div class="flex-item-body">
<span><b>harbor-110-local-repair.yaml #4190</b>:</span>Scheduled
</div>
</div>
</div>
</div>
"""
def _actions_html_cd_waiting_harbor_repair_running() -> str:
return """
<div class="flex-list run-list">
<div class="flex-item tw-items-center">
<div class="flex-item-leading">
<span data-tooltip-content="Waiting"></span>
</div>
<div class="flex-item-main">
<a class="flex-item-title" title="fix(runner): print ssh timeout markers in queue summary" href="/wooo/awoooi/actions/runs/4240">
fix(runner): print ssh timeout markers in queue summary
</a>
<div class="flex-item-body">
<span><b>cd.yaml #4240</b>:</span>Commit
<a href="/wooo/awoooi/commit/a5428b6673a670a95fccc060d4543a1dbf2e72b0">a5428b667</a>
</div>
</div>
</div>
<div class="flex-item tw-items-center">
<div class="flex-item-leading">
<span data-tooltip-content="Running"></span>
</div>
<div class="flex-item-main">
<a class="flex-item-title" title="fix(runner): expose ssh offer timeout in closure verifier" href="/wooo/awoooi/actions/runs/4237">
fix(runner): expose ssh offer timeout in closure verifier
</a>
<div class="flex-item-body">
<span><b>harbor-110-local-repair.yaml #4237</b>:</span>Scheduled
</div>
</div>
</div>
</div>
"""
def _actions_html_harbor_repair_only_waiting_with_workflow_no_matching() -> str:
return """
<div class="menu">
<a class="item" href="?workflow=harbor-110-local-repair.yaml&actor=0&status=0">harbor-110-local-repair.yaml
<span data-tooltip-content="No matching online runner with label: awoooi-host"></span>
</a>
</div>
<div class="flex-list run-list">
<div class="flex-item tw-items-center">
<div class="flex-item-leading">
<span data-tooltip-content="Waiting"></span>
</div>
<div class="flex-item-main">
<a class="flex-item-title" title="feat(recovery): surface runner systemd timeout receipt" href="/wooo/awoooi/actions/runs/4173">
feat(recovery): surface runner systemd timeout receipt
</a>
<div class="flex-item-body">
<span><b>harbor-110-local-repair.yaml #4173</b>:</span>Scheduled
</div>
</div>
</div>
</div>
"""
def _actions_html_failed_cd_run() -> str:
return """
<div class="flex-list run-list">
<div class="flex-item tw-items-center">
<div class="flex-item-leading">
<span data-tooltip-content="Failure"></span>
</div>
<div class="flex-item-main">
<a class="flex-item-title" title="fix(runner): flag stale gitea jobs readback" href="/wooo/awoooi/actions/runs/4043">
fix(runner): flag stale gitea jobs readback
</a>
<div class="flex-item-body">
<span><b>cd.yaml #4043</b>:</span>Commit
<a href="/wooo/awoooi/commit/c4fe100620a686b0e62c84113029ec81b040376c">c4fe10062</a>
</div>
</div>
</div>
</div>
"""
def _actions_html_blocked_cd_run() -> str:
return _actions_html_failed_cd_run().replace(
'data-tooltip-content="Failure"',
'data-tooltip-content="Blocked"',
)
def _harbor_blocked_log() -> str:
return """
2026-06-30T11:33:05.5822531Z harbor_login_attempt=1 registry_v2_status=000
2026-06-30T11:33:05.6000000Z harbor_controlled_repair_skipped=not_110_host
2026-06-30T11:33:18.7703396Z harbor_login_attempt=2 registry_v2_status=502
2026-06-30T11:35:30.6670130Z harbor_login_attempt=12 registry_v2_status=502
2026-06-30T11:35:30.6670926Z BLOCKER harbor_registry_public_route_unavailable registry_v2_status=502
2026-06-30T11:35:30.6672047Z Failure - Main Login to Harbor
"""
def _harbor_retrying_unavailable_log() -> str:
return """
2026-06-30T14:49:17.1327272Z harbor_login_attempt=1 registry_v2_status=502
2026-06-30T14:49:17.1423575Z harbor_controlled_repair_skipped=not_110_host
2026-06-30T14:49:30.3496935Z harbor_login_attempt=2 registry_v2_status=502
2026-06-30T14:49:43.5626840Z harbor_login_attempt=3 registry_v2_status=502
"""
def _harbor_110_repair_remote_control_unavailable_log() -> str:
return """
operation_boundary_secret_value_read=false
operation_boundary_docker_daemon_restart_performed=false
operation_boundary_host_reboot_performed=false
operation_boundary_node_drain_performed=false
operation_boundary_remote_ssh_bounded=true
Connection to 192.168.0.110 port 22 timed out
BLOCKED harbor_110_remote_control_channel_unavailable target=wooo@192.168.0.110
harbor_110_remote_ssh_reachable=false
"""
def _harbor_110_repair_publickey_auth_stalled_log() -> str:
return """
operation_boundary_secret_value_read=false
operation_boundary_docker_daemon_restart_performed=false
operation_boundary_host_reboot_performed=false
operation_boundary_node_drain_performed=false
operation_boundary_remote_ssh_bounded=true
harbor_110_remote_ssh_diag_rc=255
harbor_110_remote_ssh_tcp_connected=true
harbor_110_remote_ssh_banner_seen=true
harbor_110_remote_ssh_userauth_service_accept_seen=true
harbor_110_remote_ssh_publickey_offered=true
harbor_110_remote_ssh_publickey_reply_timeout_seen=true
harbor_110_remote_ssh_publickey_auth_stalled=true
SSH_AUTH user=wooo mode=publickey rc=255 classification=publickey_offer_timeout
harbor_110_remote_ssh_server_accepts_key_then_session_timeout=true
SSH_AUTH user=wooo mode=publickey rc=124 classification=server_accepts_key_then_timeout
BLOCKED harbor_110_remote_ssh_publickey_auth_stalled target=wooo@192.168.0.110
harbor_110_remote_ssh_auth_permission_denied=false
harbor_110_remote_ssh_diag_raw_log_printed=false
BLOCKED harbor_110_remote_control_channel_unavailable target=wooo@192.168.0.110
harbor_110_remote_ssh_reachable=false
"""
def _harbor_110_repair_success_jobs() -> dict:
return {
"total_count": 2,
"jobs": [
{
"id": 5821,
"name": "workflow-shape",
"status": "completed",
"conclusion": "success",
"labels": ["awoooi-non110-host"],
"runner_name": "wooo-runner",
"run_id": 4060,
"head_sha": "7c8bb3645bdf1fa5ac1aaa7041c237fce8c19c0e",
},
{
"id": 5822,
"name": "harbor-110-local-repair",
"status": "completed",
"conclusion": "success",
"labels": ["awoooi-host"],
"runner_name": "wooo-runner",
"run_id": 4060,
"head_sha": "7c8bb3645bdf1fa5ac1aaa7041c237fce8c19c0e",
},
],
}
def _harbor_110_repair_stale_code_review_jobs() -> dict:
return {
"total_count": 1,
"jobs": [
{
"id": 5834,
"name": "ai-code-review",
"status": "completed",
"conclusion": "success",
"labels": ["ubuntu-latest"],
"runner_name": "wooo-runner",
"run_id": 4067,
"head_sha": "acaae99986aee2e1f5630984981ccb0f2b676bb8",
},
],
}
def _harbor_110_repair_cross_workflow_jobs() -> dict:
return {
"total_count": 3,
"jobs": [
{
"id": 5901,
"name": "build-and-deploy",
"status": "completed",
"conclusion": "success",
"labels": ["awoooi-host"],
"runner_name": "wooo-runner",
"run_id": 4060,
"head_sha": "f9ad460ff6f3d258bf86da2f30a2d40451234567",
},
{
"id": 5902,
"name": "tests",
"status": "completed",
"conclusion": "success",
"labels": ["awoooi-host"],
"runner_name": "wooo-runner",
"run_id": 4060,
"head_sha": "f9ad460ff6f3d258bf86da2f30a2d40451234567",
},
{
"id": 5903,
"name": "post-deploy-checks",
"status": "completed",
"conclusion": "success",
"labels": ["awoooi-host"],
"runner_name": "wooo-runner",
"run_id": 4060,
"head_sha": "f9ad460ff6f3d258bf86da2f30a2d40451234567",
},
],
}
def _host_pressure_waiting_log() -> str:
return """
2026-06-30T11:48:41.7864172Z ⏳ host web/build/smoke pressure detected (attempt 1/60); waiting 10s
2026-06-30T11:48:41.7918276Z host load5/core 0.935000 > 0.85
2026-06-30T11:52:35.1754675Z ⏳ host web/build/smoke pressure detected (attempt 24/60); waiting 10s
2026-06-30T11:52:35.1827409Z host load5/core 0.931667 > 0.85
"""
def _host_pressure_refused_log() -> str:
return """
2026-06-30T11:58:31.0000000Z ⏳ host web/build/smoke pressure detected (attempt 60/60); waiting 10s
2026-06-30T11:58:31.0000000Z host load5/core 0.920000 > 0.85
2026-06-30T11:58:41.0000000Z ❌ refusing to start AWOOI image build while host web/build/smoke pressure is still active
"""
def _host_pressure_interrupted_postgres_log() -> str:
return """
2026-07-01T00:44:23.7335391Z ⏳ host web/build/smoke pressure detected (attempt 1/60); waiting 10s
2026-07-01T00:44:23.7371812Z k3s-postgres-recovery CPU cores 3.595200 > 2.0
2026-07-01T00:44:33.8647990Z ⏳ host web/build/smoke pressure detected (attempt 2/60); waiting 10s
2026-07-01T00:44:33.8680334Z k3s-postgres-recovery CPU cores 3.595200 > 2.0
2026-07-01T00:44:43.0684981Z signal: interrupt
"""
def test_parse_visible_runs_extracts_no_matching_runner_label() -> None:
module = _load_module()
runs = module.parse_visible_runs(_actions_html())
assert runs[0]["run_id"] == "3857"
assert runs[0]["workflow"] == "ai-technology-watch.yaml"
assert runs[0]["kind"] == "Scheduled"
assert runs[0]["no_matching_runner_label"] == "awoooi-non110-ubuntu"
assert runs[1]["no_matching_runner_label"] == ""
def test_parse_visible_runs_keeps_full_gitea_rows_aligned() -> None:
module = _load_module()
runs = module.parse_visible_runs(_actions_html_full_run_items())
assert runs[0]["run_id"] == "3863"
assert runs[0]["workflow"] == "cd.yaml"
assert runs[0]["kind"] == "Commit"
assert runs[0]["status"] == "Running"
assert runs[0]["title"] == "chore(deploy): retry non110 cd after ci image seed"
assert runs[0]["commit_sha"] == "a70c6756d9e76c33143676eef82bab7a49ac1839"
assert runs[1]["run_id"] == "3859"
assert runs[1]["no_matching_runner_label"] == "awoooi-non110-host"
def test_build_readback_reports_latest_visible_cd_run() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_single_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
)
assert payload["readback"]["latest_visible_cd_run_id"] == "3863"
assert payload["readback"]["latest_visible_cd_run_status"] == "Running"
assert payload["readback"]["latest_visible_cd_run_kind"] == "Commit"
assert payload["readback"]["latest_visible_cd_run_commit_sha"] == (
"a70c6756d9e76c33143676eef82bab7a49ac1839"
)
assert payload["rollups"]["current_main_cd_run_visible"] is True
assert payload["rollups"]["current_main_cd_run_status"] == "Running"
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is False
def test_build_readback_uses_cd_workflow_fallback_when_general_page_is_scheduled_only() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_harbor_repair_only_waiting_with_workflow_no_matching(),
cd_workflow_actions_html=_cd_workflow_actions_html_single_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
)
assert payload["readback"]["actions_page_visible_run_count"] == 1
assert payload["readback"]["cd_workflow_actions_page_visible_run_count"] == 1
assert payload["readback"]["cd_workflow_fallback_used"] is True
assert payload["readback"]["latest_visible_cd_run_id"] == "3863"
assert payload["readback"]["latest_visible_cd_run_status"] == "Running"
assert payload["rollups"]["current_main_cd_run_visible"] is True
assert payload["rollups"]["cd_workflow_fallback_used"] is True
assert payload["readback"]["latest_visible_harbor_110_repair_run_id"] == "4173"
def test_build_readback_surfaces_harbor_110_repair_waiting_run() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_cd_running_harbor_repair_waiting(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
)
assert payload["status"] == "harbor_110_repair_waiting_for_runner_or_queue"
assert payload["readback"]["latest_visible_cd_run_id"] == "4061"
assert payload["readback"]["latest_visible_cd_run_status"] == "Running"
assert payload["readback"]["latest_visible_harbor_110_repair_run_id"] == "4060"
assert (
payload["readback"]["latest_visible_harbor_110_repair_run_status"]
== "Waiting"
)
assert payload["readback"]["latest_visible_harbor_110_repair_waiting"] is True
assert payload["readback"]["latest_visible_harbor_110_repair_running"] is False
assert (
payload["readback"]["latest_visible_harbor_110_repair_status_blocked"]
is False
)
assert payload["readback"]["latest_visible_harbor_110_repair_blocked"] is False
assert payload["rollups"]["harbor_110_repair_run_visible"] is True
assert payload["rollups"]["harbor_110_repair_waiting"] is True
assert payload["rollups"]["harbor_110_repair_blocked"] is False
assert payload["rollups"]["harbor_110_repair_run_status"] == "Waiting"
assert payload["operation_boundaries"]["workflow_dispatch_performed"] is False
def test_build_readback_surfaces_current_cd_non110_waiting() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_cd_waiting_with_non110_no_matching(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
harbor_110_repair_jobs_http_status=200,
harbor_110_repair_jobs_payload={"jobs": [], "total_count": 0},
)
assert payload["status"] == (
"blocked_current_cd_workflow_waiting_for_runner_or_queue"
)
assert payload["readback"]["latest_visible_cd_run_id"] == "4191"
assert payload["readback"]["latest_visible_cd_run_waiting"] is True
assert payload["readback"]["latest_visible_cd_no_matching_runner_label"] == (
"awoooi-non110-ubuntu"
)
assert payload["readback"]["latest_visible_harbor_110_repair_run_id"] == "4190"
assert (
payload["readback"][
"latest_visible_harbor_110_repair_no_matching_runner_label"
]
== "awoooi-non110-host"
)
assert payload["readback"]["controlled_profile_no_matching_runner_labels"] == {
"cd.yaml": "awoooi-non110-ubuntu",
"harbor-110-local-repair.yaml": "awoooi-non110-host",
}
assert payload["rollups"]["current_main_cd_run_waiting"] is True
assert payload["rollups"]["current_main_cd_no_matching_runner_label"] == (
"awoooi-non110-ubuntu"
)
assert payload["rollups"]["controlled_profile_no_matching_runner_label_count"] == 2
def test_build_readback_blocks_cd_waiting_behind_stale_harbor_running_readback() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_cd_waiting_harbor_repair_running(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
harbor_110_repair_jobs_http_status=200,
harbor_110_repair_jobs_payload=_harbor_110_repair_cross_workflow_jobs(),
)
assert payload["status"] == (
"blocked_current_cd_waiting_behind_stale_harbor_110_repair_readback"
)
assert payload["readback"]["latest_visible_cd_run_id"] == "4240"
assert payload["readback"]["latest_visible_cd_run_waiting"] is True
assert payload["readback"]["latest_visible_harbor_110_repair_run_id"] == "4237"
assert payload["readback"]["latest_visible_harbor_110_repair_running"] is True
assert payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] is True
assert (
payload["readback"]["harbor_110_repair_visible_running_jobs_api_stale"]
is True
)
assert (
payload["readback"]["current_cd_waiting_behind_harbor_110_repair_running"]
is True
)
assert (
payload["rollups"]["harbor_110_repair_visible_running_jobs_api_stale"]
is True
)
summary = module._human_summary(payload)
assert "HARBOR_110_REPAIR_VISIBLE_RUNNING_JOBS_API_STALE=1" in summary
assert "CURRENT_CD_WAITING_BEHIND_HARBOR_110_REPAIR_RUNNING=1" in summary
def test_harbor_ssh_blocker_takes_precedence_over_current_cd_waiting() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_cd_waiting_with_non110_no_matching(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
harbor_110_repair_jobs_http_status=200,
harbor_110_repair_jobs_payload={"jobs": [], "total_count": 0},
latest_harbor_110_repair_log_http_status=200,
latest_harbor_110_repair_log_text=(
_harbor_110_repair_publickey_auth_stalled_log()
),
)
assert payload["status"] == "blocked_harbor_110_remote_ssh_publickey_auth_stalled"
assert payload["readback"]["latest_visible_cd_run_waiting"] is True
assert (
payload["readback"][
"latest_visible_harbor_110_repair_remote_ssh_publickey_auth_stalled"
]
is True
)
assert (
payload["readback"][
"latest_visible_harbor_110_repair_remote_ssh_publickey_offer_timeout"
]
is True
)
assert (
payload["rollups"]["harbor_110_repair_remote_ssh_publickey_auth_stalled"]
is True
)
def test_build_readback_classifies_harbor_502_after_110_repair_jobs_success() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_cd_running_harbor_repair_waiting().replace(
'data-tooltip-content="Running"',
'data-tooltip-content="Failure"',
1,
),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
harbor_110_repair_jobs_http_status=200,
harbor_110_repair_jobs_payload=_harbor_110_repair_success_jobs(),
latest_cd_build_log_http_status=200,
latest_cd_build_log_text=_harbor_blocked_log(),
)
assert payload["status"] == (
"blocked_harbor_public_route_unavailable_after_harbor_110_repair_success"
)
assert payload["readback"]["latest_visible_harbor_110_repair_run_id"] == "4060"
assert payload["readback"]["harbor_110_repair_jobs_total_count"] == 2
assert payload["readback"]["harbor_110_repair_jobs_all_success"] is True
assert payload["readback"]["harbor_110_repair_visible_waiting_stale"] is True
assert payload["readback"]["harbor_110_repair_jobs_match_expected_workflow"] is True
assert payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] is False
assert payload["readback"]["harbor_110_repair_jobs_labels"] == [
"awoooi-host",
"awoooi-non110-host",
]
assert payload["readback"]["harbor_110_repair_jobs_runner_names"] == [
"wooo-runner"
]
assert payload["rollups"]["harbor_110_repair_jobs_all_success"] is True
assert payload["rollups"]["harbor_110_repair_visible_waiting_stale"] is True
assert payload["operation_boundaries"]["host_write_performed"] is False
def test_build_readback_rejects_stale_harbor_jobs_but_keeps_waiting_status() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_cd_running_harbor_repair_waiting(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
harbor_110_repair_jobs_http_status=200,
harbor_110_repair_jobs_payload=_harbor_110_repair_stale_code_review_jobs(),
latest_cd_build_log_http_status=200,
latest_cd_build_log_text=_harbor_blocked_log(),
)
assert payload["status"] == "blocked_harbor_110_repair_workflow_waiting"
assert payload["readback"]["harbor_110_repair_jobs_all_success"] is False
assert payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] is True
assert payload["readback"]["harbor_110_repair_jobs_unexpected_names"] == [
"ai-code-review"
]
assert (
payload["readback"]["harbor_110_repair_jobs_payload_classifier"]
== "unexpected_harbor_110_repair_job_names"
)
assert payload["readback"]["harbor_110_repair_jobs_labels"] == ["ubuntu-latest"]
assert payload["rollups"]["harbor_110_repair_jobs_stale_or_mismatched"] is True
def test_build_readback_classifies_cross_workflow_harbor_jobs_payload() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_harbor_repair_waiting_with_workflow_no_matching(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
harbor_110_repair_jobs_http_status=200,
harbor_110_repair_jobs_payload=_harbor_110_repair_cross_workflow_jobs(),
)
assert payload["status"] == "blocked_harbor_110_repair_no_matching_runner"
assert payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] is True
assert (
payload["readback"]["harbor_110_repair_jobs_cross_workflow_mismatch"]
is True
)
assert payload["readback"]["harbor_110_repair_jobs_payload_classifier"] == (
"cd_workflow_jobs_returned_for_harbor_110_repair_run"
)
assert payload["readback"]["harbor_110_repair_jobs_expected_names"] == [
"harbor-110-local-repair",
"workflow-shape",
]
assert payload["readback"]["harbor_110_repair_jobs_unexpected_names"] == [
"build-and-deploy",
"post-deploy-checks",
"tests",
]
assert (
payload["rollups"]["harbor_110_repair_jobs_cross_workflow_mismatch"]
is True
)
def test_build_readback_keeps_visible_harbor_waiting_before_stale_jobs_status() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_cd_running_harbor_repair_waiting(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
harbor_110_repair_jobs_http_status=200,
harbor_110_repair_jobs_payload=_harbor_110_repair_stale_code_review_jobs(),
)
assert payload["status"] == "harbor_110_repair_waiting_for_runner_or_queue"
assert payload["readback"]["harbor_110_repair_jobs_all_success"] is False
assert payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] is True
assert payload["readback"]["latest_visible_harbor_110_repair_waiting"] is True
def test_build_readback_classifies_harbor_repair_remote_control_unavailable() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_cd_running_harbor_repair_waiting().replace(
'data-tooltip-content="Waiting"',
'data-tooltip-content="Failure"',
1,
),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
harbor_110_repair_jobs_http_status=200,
harbor_110_repair_jobs_payload=_harbor_110_repair_stale_code_review_jobs(),
latest_cd_build_log_http_status=200,
latest_cd_build_log_text=_harbor_blocked_log(),
latest_harbor_110_repair_log_http_status=200,
latest_harbor_110_repair_log_text=(
_harbor_110_repair_remote_control_unavailable_log()
),
)
assert payload["status"] == "blocked_harbor_110_remote_control_channel_unavailable"
assert payload["readback"]["latest_visible_harbor_110_repair_log_http_status"] == 200
assert payload["readback"]["latest_visible_harbor_110_repair_failure_classifier"] == (
"harbor_110_remote_control_channel_unavailable"
)
assert (
payload["readback"][
"latest_visible_harbor_110_repair_remote_control_channel_unavailable"
]
is True
)
assert (
payload["readback"]["latest_visible_harbor_110_repair_remote_ssh_reachable"]
is False
)
assert (
payload["readback"]["latest_visible_harbor_110_repair_bounded_ssh_timeout_seen"]
is True
)
assert payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] is True
assert (
payload["rollups"]["harbor_110_repair_remote_control_channel_unavailable"]
is True
)
assert payload["operation_boundaries"]["host_write_performed"] is False
def test_build_readback_classifies_harbor_repair_publickey_auth_stalled() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_cd_running_harbor_repair_waiting().replace(
'data-tooltip-content="Waiting"',
'data-tooltip-content="Failure"',
1,
),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
harbor_110_repair_jobs_http_status=200,
harbor_110_repair_jobs_payload=_harbor_110_repair_stale_code_review_jobs(),
latest_cd_build_log_http_status=200,
latest_cd_build_log_text=_harbor_blocked_log(),
latest_harbor_110_repair_log_http_status=200,
latest_harbor_110_repair_log_text=(
_harbor_110_repair_publickey_auth_stalled_log()
),
)
assert payload["status"] == "blocked_harbor_110_remote_ssh_publickey_auth_stalled"
assert payload["readback"]["latest_visible_harbor_110_repair_failure_classifier"] == (
"harbor_110_remote_ssh_publickey_auth_stalled"
)
assert (
payload["readback"][
"latest_visible_harbor_110_repair_remote_ssh_publickey_auth_stalled"
]
is True
)
assert (
payload["readback"][
"latest_visible_harbor_110_repair_remote_ssh_server_accepts_key_then_session_timeout"
]
is True
)
assert (
payload["readback"][
"latest_visible_harbor_110_repair_remote_ssh_publickey_reply_timeout_seen"
]
is True
)
assert (
payload["readback"][
"latest_visible_harbor_110_repair_remote_ssh_publickey_offer_timeout"
]
is True
)
assert (
payload["readback"][
"latest_visible_harbor_110_repair_remote_ssh_userauth_service_accept_seen"
]
is True
)
assert (
payload["readback"][
"latest_visible_harbor_110_repair_remote_ssh_auth_permission_denied"
]
is False
)
assert (
payload["rollups"]["harbor_110_repair_remote_ssh_publickey_auth_stalled"]
is True
)
assert (
payload["rollups"][
"harbor_110_repair_remote_ssh_publickey_offer_timeout"
]
is True
)
assert (
payload["rollups"][
"harbor_110_repair_remote_ssh_server_accepts_key_then_session_timeout"
]
is True
)
summary = module._human_summary(payload)
assert (
"LATEST_VISIBLE_HARBOR_110_REPAIR_REMOTE_SSH_PUBLICKEY_OFFER_TIMEOUT=True"
in summary
)
assert (
"LATEST_VISIBLE_HARBOR_110_REPAIR_REMOTE_SSH_SERVER_ACCEPTS_KEY_THEN_SESSION_TIMEOUT=True"
in summary
)
assert (
"LATEST_VISIBLE_HARBOR_110_REPAIR_REMOTE_SSH_AUTH_PERMISSION_DENIED=False"
in summary
)
assert payload["operation_boundaries"]["secret_or_runner_token_read"] is False
assert payload["operation_boundaries"]["host_write_performed"] is False
def test_build_readback_surfaces_harbor_110_repair_no_matching_runner() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_harbor_repair_waiting_with_workflow_no_matching(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
)
assert payload["status"] == "blocked_harbor_110_repair_no_matching_runner"
assert payload["readback"]["latest_visible_harbor_110_repair_run_id"] == "4060"
assert (
payload["readback"][
"latest_visible_harbor_110_repair_no_matching_runner_label"
]
== "awoooi-host"
)
assert payload["readback"]["workflow_no_matching_runner_labels"] == {
"harbor-110-local-repair.yaml": "awoooi-host"
}
assert payload["readback"]["no_matching_online_runner_visible"] is True
assert (
payload["readback"]["latest_visible_harbor_110_repair_status_blocked"]
is False
)
assert payload["readback"]["latest_visible_harbor_110_repair_blocked"] is True
assert payload["rollups"]["harbor_110_repair_blocked"] is True
assert (
payload["rollups"]["harbor_110_repair_no_matching_runner_label"]
== "awoooi-host"
)
def test_build_readback_prioritizes_harbor_repair_no_matching_over_stale_jobs() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_harbor_repair_waiting_with_workflow_no_matching(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
harbor_110_repair_jobs_http_status=200,
harbor_110_repair_jobs_payload=_harbor_110_repair_stale_code_review_jobs(),
)
assert payload["status"] == "blocked_harbor_110_repair_no_matching_runner"
assert (
payload["readback"]["latest_visible_harbor_110_repair_no_matching_runner_label"]
== "awoooi-host"
)
assert payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] is True
def test_build_readback_classifies_harbor_public_route_blocker() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_failed_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
latest_cd_build_log_http_status=200,
latest_cd_build_log_text=_harbor_blocked_log(),
)
assert payload["status"] == "blocked_harbor_public_route_unavailable"
assert payload["readback"]["latest_visible_cd_run_id"] == "4043"
assert payload["readback"]["latest_visible_cd_run_status"] == "Failure"
assert payload["readback"]["latest_visible_cd_failure_classifier"] == (
"harbor_registry_public_route_unavailable"
)
assert payload["readback"]["latest_visible_cd_failure_status_code"] == "502"
assert payload["readback"]["latest_visible_cd_harbor_login_attempt_count"] == 12
assert (
payload["readback"][
"latest_visible_cd_harbor_controlled_repair_attempted"
]
is True
)
assert (
payload["readback"][
"latest_visible_cd_harbor_controlled_repair_skip_reason"
]
== "not_110_host"
)
assert (
payload["readback"][
"latest_visible_cd_harbor_repair_requires_110_controlled_lane"
]
is True
)
assert (
payload["readback"][
"latest_visible_cd_harbor_repair_blocked_by_no_matching_awoooi_host"
]
is False
)
assert (
payload["readback"]["latest_visible_cd_harbor_repair_lane_classifier"]
== "cd_harbor_repair_requires_110_controlled_lane"
)
assert payload["readback"]["latest_visible_cd_harbor_public_route_blocked"] is True
assert payload["rollups"]["current_main_cd_harbor_public_route_blocked"] is True
assert (
payload["rollups"][
"current_main_cd_harbor_controlled_repair_skip_reason"
]
== "not_110_host"
)
assert (
payload["rollups"][
"current_main_cd_harbor_repair_requires_110_controlled_lane"
]
is True
)
def test_harbor_repair_no_matching_explains_non110_cd_skip_reason() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_harbor_repair_waiting_with_workflow_no_matching(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
latest_cd_build_log_http_status=200,
latest_cd_build_log_text=_harbor_blocked_log(),
)
assert payload["status"] == "blocked_harbor_110_repair_no_matching_runner"
assert (
payload["readback"][
"latest_visible_cd_harbor_controlled_repair_skip_reason"
]
== "not_110_host"
)
assert (
payload["readback"][
"latest_visible_cd_harbor_repair_requires_110_controlled_lane"
]
is True
)
assert (
payload["readback"][
"latest_visible_cd_harbor_repair_blocked_by_no_matching_awoooi_host"
]
is True
)
assert (
payload["readback"]["latest_visible_cd_harbor_repair_lane_classifier"]
== "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host"
)
assert (
payload["rollups"][
"current_main_cd_harbor_repair_lane_classifier"
]
== "cd_harbor_repair_requires_110_controlled_lane_no_matching_awoooi_host"
)
def test_build_readback_prefers_visible_blocked_over_stale_jobs() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_blocked_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={
"jobs": [
{
"run_id": 4043,
"head_sha": "528d2c54e266b2362f7cc004aaa3378406e889c4",
"conclusion": "success",
}
],
"total_count": 1,
},
)
assert payload["status"] == "blocked_latest_visible_cd_run"
assert payload["readback"]["latest_visible_cd_run_status"] == "Blocked"
assert payload["readback"]["latest_visible_cd_run_blocked"] is True
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
assert payload["rollups"]["current_main_cd_run_blocked"] is True
def test_build_readback_classifies_host_pressure_waiting() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_single_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
latest_cd_tests_log_http_status=200,
latest_cd_tests_log_text=_host_pressure_waiting_log(),
)
assert payload["status"] == "blocked_host_web_build_pressure"
assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == (
"host_web_build_pressure_waiting"
)
assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 24
assert payload["readback"]["latest_visible_cd_host_pressure_attempt_limit"] == 60
assert (
payload["readback"]["latest_visible_cd_host_pressure_latest_load5_per_core"]
== "0.931667"
)
assert payload["readback"]["latest_visible_cd_host_pressure_load5_threshold"] == (
"0.85"
)
assert payload["readback"]["latest_visible_cd_host_pressure_waiting"] is True
assert payload["readback"]["latest_visible_cd_host_pressure_refused"] is False
assert payload["rollups"]["current_main_cd_host_pressure_waiting"] is True
def test_host_pressure_refusal_takes_status_precedence_over_stale_jobs() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_failed_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={
"jobs": [
{
"run_id": 4047,
"head_sha": "df89bdf00b9413b4db61b7f2d9bbca57e9fc9923",
"conclusion": "success",
}
],
"total_count": 1,
},
latest_cd_tests_log_http_status=200,
latest_cd_tests_log_text=_host_pressure_refused_log(),
)
assert payload["status"] == "blocked_host_web_build_pressure"
assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == (
"host_web_build_pressure_refused"
)
assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 60
assert payload["readback"]["latest_visible_cd_host_pressure_refused"] is True
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
def test_stale_waiting_host_pressure_log_does_not_hide_failed_cd_jobs_payload() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_failed_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={
"jobs": [
{
"run_id": 4043,
"head_sha": "df89bdf00b9413b4db61b7f2d9bbca57e9fc9923",
"conclusion": "success",
}
],
"total_count": 1,
},
latest_cd_tests_log_http_status=200,
latest_cd_tests_log_text=_host_pressure_waiting_log(),
)
assert payload["status"] == "cd_jobs_stale_or_mismatched"
assert payload["readback"]["latest_visible_cd_run_status"] == "Failure"
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == (
"host_web_build_pressure_waiting_from_stale_jobs_payload"
)
assert payload["readback"]["latest_visible_cd_host_pressure_waiting"] is False
assert (
payload["readback"]["latest_visible_cd_host_pressure_log_stale_or_mismatched"]
is True
)
assert payload["rollups"]["current_main_cd_host_pressure_waiting"] is False
assert (
payload["rollups"]["current_main_cd_host_pressure_log_stale_or_mismatched"]
is True
)
def test_interrupted_host_pressure_reports_postgres_recovery_cpu() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_failed_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
latest_cd_tests_log_http_status=200,
latest_cd_tests_log_text=_host_pressure_interrupted_postgres_log(),
)
assert payload["status"] == "blocked_host_web_build_pressure"
assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == (
"host_web_build_pressure_interrupted"
)
assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 2
assert payload["readback"]["latest_visible_cd_host_pressure_interrupted"] is True
assert (
payload["readback"]["latest_visible_cd_postgres_recovery_cpu_pressure"]
is True
)
assert (
payload["readback"]["latest_visible_cd_postgres_recovery_cpu_cores"]
== "3.595200"
)
assert (
payload["readback"]["latest_visible_cd_postgres_recovery_cpu_threshold"]
== "2.0"
)
assert payload["rollups"]["current_main_cd_host_pressure_interrupted"] is True
assert (
payload["rollups"]["current_main_cd_postgres_recovery_cpu_pressure"]
is True
)
def test_build_readback_flags_stale_cd_jobs_api_payload() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_single_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={
"jobs": [
{
"run_id": 3863,
"name": "tests",
"head_sha": "b17a28c29375afd207a90d05b4898daaa0f3f730",
"conclusion": "success",
}
],
"total_count": 1,
},
)
assert payload["status"] == "cd_jobs_stale_or_mismatched"
assert payload["readback"]["latest_visible_cd_run_commit_sha"] == (
"a70c6756d9e76c33143676eef82bab7a49ac1839"
)
assert payload["readback"]["cd_run_jobs_head_shas"] == [
"b17a28c29375afd207a90d05b4898daaa0f3f730"
]
assert payload["readback"]["cd_run_jobs_expected_run_id"] == "3863"
assert payload["readback"]["cd_run_jobs_expected_head_sha"] == (
"a70c6756d9e76c33143676eef82bab7a49ac1839"
)
assert payload["readback"]["cd_run_jobs_head_sha_matches_visible"] is False
assert payload["readback"]["cd_run_jobs_run_id_matches_visible"] is True
assert payload["readback"]["cd_run_jobs_head_sha_mismatch"] is True
assert payload["readback"]["cd_run_jobs_run_id_mismatch"] is False
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
assert payload["readback"]["cd_run_jobs_payload_classifier"] == (
"cd_jobs_api_head_sha_mismatch_for_visible_cd_run"
)
assert payload["rollups"]["cd_run_jobs_stale_or_mismatched"] is True
assert payload["rollups"]["cd_run_jobs_payload_classifier"] == (
"cd_jobs_api_head_sha_mismatch_for_visible_cd_run"
)
def test_build_readback_classifies_cross_run_cd_jobs_api_payload() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_single_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={
"jobs": [
{
"run_id": 4172,
"name": "build-and-deploy",
"head_sha": "26b67d11f7b7de4f9c9d95c01bb1dacf4000e887",
"conclusion": "success",
}
],
"total_count": 1,
},
)
assert payload["status"] == "cd_jobs_stale_or_mismatched"
assert payload["readback"]["cd_run_jobs_head_sha_mismatch"] is True
assert payload["readback"]["cd_run_jobs_run_id_mismatch"] is True
assert payload["readback"]["cd_run_jobs_payload_classifier"] == (
"cd_jobs_api_head_sha_and_run_id_mismatch_for_visible_cd_run"
)
assert payload["rollups"]["cd_run_jobs_head_sha_mismatch"] is True
assert payload["rollups"]["cd_run_jobs_run_id_mismatch"] is True
def test_harbor_blocker_takes_status_precedence_over_stale_jobs_payload() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_failed_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={
"jobs": [
{
"run_id": 4045,
"head_sha": "f4fb078100000000000000000000000000000000",
"conclusion": "success",
}
],
"total_count": 1,
},
latest_cd_build_log_http_status=200,
latest_cd_build_log_text=_harbor_blocked_log(),
)
assert payload["status"] == "blocked_harbor_public_route_unavailable"
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
assert payload["readback"]["latest_visible_cd_failure_classifier"] == (
"harbor_registry_public_route_unavailable"
)
def test_harbor_repair_waiting_takes_actionable_precedence_after_harbor_blocker() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_cd_running_harbor_repair_waiting(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
harbor_110_repair_jobs_http_status=200,
harbor_110_repair_jobs_payload=_harbor_110_repair_cross_workflow_jobs(),
latest_cd_build_log_http_status=200,
latest_cd_build_log_text=_harbor_blocked_log(),
)
assert payload["status"] == "blocked_harbor_110_repair_workflow_waiting"
assert payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] is True
assert payload["readback"]["harbor_110_repair_jobs_cross_workflow_mismatch"] is True
assert payload["readback"]["latest_visible_harbor_110_repair_waiting"] is True
assert (
payload["readback"][
"harbor_110_repair_waiting_after_cd_harbor_blocker"
]
is True
)
assert (
payload["rollups"]["harbor_110_repair_waiting_after_cd_harbor_blocker"]
is True
)
assert (
payload["readback"]["latest_visible_harbor_110_repair_status_blocked"]
is False
)
assert payload["readback"]["latest_visible_harbor_110_repair_blocked"] is True
assert payload["rollups"]["harbor_110_repair_blocked"] is True
def test_harbor_repair_failure_takes_precedence_over_stale_jobs_payload() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_cd_failed_harbor_repair_failed(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
harbor_110_repair_jobs_http_status=200,
harbor_110_repair_jobs_payload=_harbor_110_repair_cross_workflow_jobs(),
latest_cd_build_log_http_status=200,
latest_cd_build_log_text=_harbor_blocked_log(),
)
assert payload["status"] == "blocked_harbor_110_repair_failed"
assert payload["readback"]["latest_visible_harbor_110_repair_run_id"] == "4212"
assert payload["readback"]["latest_visible_harbor_110_repair_failed"] is True
assert payload["readback"]["latest_visible_harbor_110_repair_blocked"] is True
assert payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] is True
assert (
payload["readback"]["harbor_110_repair_visible_failure_jobs_api_stale"]
is True
)
assert payload["rollups"]["harbor_110_repair_failed"] is True
assert (
payload["rollups"]["harbor_110_repair_visible_failure_jobs_api_stale"]
is True
)
def test_harbor_retrying_unavailable_marks_inflight_blocker_before_final_failure() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_cd_running_harbor_repair_waiting(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
latest_cd_build_log_http_status=200,
latest_cd_build_log_text=_harbor_retrying_unavailable_log(),
)
assert payload["status"] == "blocked_harbor_110_repair_workflow_waiting"
assert payload["readback"]["latest_visible_cd_failure_classifier"] == ""
assert payload["readback"]["latest_visible_cd_inflight_classifier"] == (
"harbor_registry_public_route_unavailable_pending_retry"
)
assert (
payload["readback"]["latest_visible_cd_harbor_public_route_blocked"]
is False
)
assert (
payload["readback"][
"latest_visible_cd_harbor_public_route_retrying_unavailable"
]
is True
)
assert (
payload["readback"]["latest_visible_cd_harbor_latest_registry_v2_status"]
== "502"
)
assert (
payload["readback"][
"harbor_110_repair_waiting_after_cd_harbor_blocker"
]
is True
)
assert payload["rollups"]["current_main_cd_harbor_public_route_blocked"] is False
assert (
payload["rollups"][
"current_main_cd_harbor_public_route_retrying_unavailable"
]
is True
)
def test_derive_jobs_api_url_tracks_latest_visible_run_id() -> None:
module = _load_module()
assert module.DEFAULT_CD_RUN_JOBS_API_URL == ""
assert module.derive_jobs_api_url(
"https://gitea.wooo.work/api/v1/repos/wooo/awoooi/actions/runs?limit=10",
"4043",
) == "https://gitea.wooo.work/api/v1/repos/wooo/awoooi/actions/runs/4043/jobs"
def test_build_readback_sanitizes_actions_api_internal_url() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html(),
actions_list_http_status=401,
actions_list_payload={
"message": "token is required",
"url": "http://192.168.0.110:3001/api/swagger",
},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
)
text = json.dumps(payload, sort_keys=True)
assert payload["schema_version"] == module.SCHEMA_VERSION
assert payload["status"] == "blocked_no_matching_online_runner"
assert payload["readback"]["actions_list_without_token_message"] == (
"token is required"
)
assert payload["readback"]["latest_visible_no_matching_runner_run_id"] == "3857"
assert payload["readback"]["latest_visible_no_matching_runner_label"] == (
"awoooi-non110-ubuntu"
)
assert payload["rollups"]["actions_list_requires_token"] is True
assert payload["operation_boundaries"]["github_api_used"] is False
assert "192.168.0." not in text
def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None:
html_path = tmp_path / "actions.html"
list_path = tmp_path / "actions-list.json"
jobs_path = tmp_path / "jobs.json"
log_path = tmp_path / "build.log"
tests_log_path = tmp_path / "tests.log"
repair_log_path = tmp_path / "harbor-repair.log"
html_path.write_text(_actions_html(), encoding="utf-8")
list_path.write_text(
json.dumps(
{
"message": "token is required",
"url": "http://192.168.0.110:3001/api/swagger",
}
),
encoding="utf-8",
)
jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8")
log_path.write_text("", encoding="utf-8")
tests_log_path.write_text("", encoding="utf-8")
repair_log_path.write_text("", encoding="utf-8")
result = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--actions-html-file",
str(html_path),
"--actions-list-json-file",
str(list_path),
"--actions-list-http-status",
"401",
"--cd-run-jobs-json-file",
str(jobs_path),
"--cd-run-jobs-http-status",
"200",
"--cd-build-job-log-file",
str(log_path),
"--cd-build-job-log-http-status",
"200",
"--cd-tests-job-log-file",
str(tests_log_path),
"--cd-tests-job-log-http-status",
"200",
"--harbor-110-repair-job-log-file",
str(repair_log_path),
"--harbor-110-repair-job-log-http-status",
"200",
"--json",
],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
assert result.returncode == 0, result.stdout + result.stderr
payload = json.loads(result.stdout)
assert payload["readback"]["actions_page_visible_run_count"] == 3
assert payload["readback"]["cd_run_jobs_total_count"] == 0
assert payload["readback"]["latest_visible_no_matching_runner_label"] == (
"awoooi-non110-ubuntu"
)
assert "192.168.0." not in result.stdout
def test_cli_json_uses_cd_workflow_fallback_fixture_without_network(
tmp_path: Path,
) -> None:
html_path = tmp_path / "actions.html"
cd_workflow_html_path = tmp_path / "cd-workflow-actions.html"
list_path = tmp_path / "actions-list.json"
jobs_path = tmp_path / "jobs.json"
log_path = tmp_path / "build.log"
tests_log_path = tmp_path / "tests.log"
html_path.write_text(
_actions_html_harbor_repair_only_waiting_with_workflow_no_matching(),
encoding="utf-8",
)
cd_workflow_html_path.write_text(
_cd_workflow_actions_html_single_cd_run(),
encoding="utf-8",
)
list_path.write_text(json.dumps({"message": "token is required"}), encoding="utf-8")
jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8")
log_path.write_text("", encoding="utf-8")
tests_log_path.write_text("", encoding="utf-8")
result = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--actions-html-file",
str(html_path),
"--cd-workflow-actions-html-file",
str(cd_workflow_html_path),
"--actions-list-json-file",
str(list_path),
"--actions-list-http-status",
"401",
"--cd-run-jobs-json-file",
str(jobs_path),
"--cd-run-jobs-http-status",
"200",
"--cd-build-job-log-file",
str(log_path),
"--cd-build-job-log-http-status",
"200",
"--cd-tests-job-log-file",
str(tests_log_path),
"--cd-tests-job-log-http-status",
"200",
"--json",
],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
assert result.returncode == 0, result.stdout + result.stderr
payload = json.loads(result.stdout)
assert payload["readback"]["latest_visible_cd_run_id"] == "3863"
assert payload["readback"]["cd_workflow_fallback_used"] is True
assert payload["rollups"]["current_main_cd_run_visible"] is True
assert payload["operation_boundaries"]["workflow_dispatch_performed"] is False
def test_cli_json_classifies_harbor_blocker_from_fixture_log(tmp_path: Path) -> None:
html_path = tmp_path / "actions.html"
list_path = tmp_path / "actions-list.json"
jobs_path = tmp_path / "jobs.json"
log_path = tmp_path / "build.log"
tests_log_path = tmp_path / "tests.log"
html_path.write_text(_actions_html_failed_cd_run(), encoding="utf-8")
list_path.write_text(json.dumps({"message": "token is required"}), encoding="utf-8")
jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8")
log_path.write_text(_harbor_blocked_log(), encoding="utf-8")
tests_log_path.write_text("", encoding="utf-8")
result = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--actions-html-file",
str(html_path),
"--actions-list-json-file",
str(list_path),
"--actions-list-http-status",
"401",
"--cd-run-jobs-json-file",
str(jobs_path),
"--cd-run-jobs-http-status",
"200",
"--cd-build-job-log-file",
str(log_path),
"--cd-build-job-log-http-status",
"200",
"--cd-tests-job-log-file",
str(tests_log_path),
"--cd-tests-job-log-http-status",
"200",
"--json",
],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
assert result.returncode == 0, result.stdout + result.stderr
payload = json.loads(result.stdout)
assert payload["status"] == "blocked_harbor_public_route_unavailable"
assert payload["readback"]["latest_visible_cd_failure_status_code"] == "502"
assert "HARBOR_PASSWORD" not in result.stdout
def test_cli_json_classifies_host_pressure_from_fixture_log(tmp_path: Path) -> None:
html_path = tmp_path / "actions.html"
list_path = tmp_path / "actions-list.json"
jobs_path = tmp_path / "jobs.json"
build_log_path = tmp_path / "build.log"
tests_log_path = tmp_path / "tests.log"
html_path.write_text(_actions_html_single_cd_run(), encoding="utf-8")
list_path.write_text(json.dumps({"message": "token is required"}), encoding="utf-8")
jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8")
build_log_path.write_text("", encoding="utf-8")
tests_log_path.write_text(_host_pressure_waiting_log(), encoding="utf-8")
result = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--actions-html-file",
str(html_path),
"--actions-list-json-file",
str(list_path),
"--actions-list-http-status",
"401",
"--cd-run-jobs-json-file",
str(jobs_path),
"--cd-run-jobs-http-status",
"200",
"--cd-build-job-log-file",
str(build_log_path),
"--cd-build-job-log-http-status",
"200",
"--cd-tests-job-log-file",
str(tests_log_path),
"--cd-tests-job-log-http-status",
"200",
"--json",
],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
assert result.returncode == 0, result.stdout + result.stderr
payload = json.loads(result.stdout)
assert payload["status"] == "blocked_host_web_build_pressure"
assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 24
assert "HARBOR_PASSWORD" not in result.stdout
Reference in New Issue View Git Blame Copy Permalink