fix(reboot): expose public maintenance fallback in api scorecard
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 37s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped

This commit is contained in:
ogt
2026-07-02 21:41:56 +08:00
parent 5b0c77495b
commit f973cd12b2
4 changed files with 144 additions and 15 deletions

View File

@@ -173,11 +173,17 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
host_boot_detection = _dict(scorecard.get("host_boot_detection"))
post_reboot_readiness = _dict(scorecard.get("post_reboot_readiness"))
stockplatform = _dict(scorecard.get("stockplatform_data_freshness"))
public_maintenance = _build_public_maintenance_fallback_readback(
scorecard.get("public_maintenance_fallback")
)
windows99 = _dict(scorecard.get("windows99_vmware_autostart"))
windows99_management = _dict(scorecard.get("windows99_management_channel"))
windows99_ssh_batch = _dict(windows99_management.get("ssh_batch"))
source_controls = _dict(scorecard.get("source_controls"))
active_blockers = _strings(scorecard.get("active_blockers"))
active_blockers = _unique_strings(
_strings(scorecard.get("active_blockers"))
+ _strings(public_maintenance.get("blockers"))
)
required_checks = {
"source_controls_present": all(source_controls.values()),
"required_hosts_observed": not _strings(host_boot_detection.get("missing_hosts")),
@@ -196,6 +202,9 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"host_188_service_green": post_reboot_readiness.get("host_188_service_green") is True,
"stockplatform_freshness_ok": stockplatform.get("freshness_status") == "ok",
"stockplatform_ingestion_ok": stockplatform.get("ingestion_status") in {"ok", "unknown"},
"public_maintenance_fallback_runtime_ready": (
public_maintenance.get("ready") is True
),
"fresh_reboot_window_observed": not _strings(host_boot_detection.get("stale_hosts")),
"can_claim_slo": scorecard.get("can_claim_all_services_recovered_within_target")
is True,
@@ -291,6 +300,23 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"required"
)
is True,
"public_maintenance_runtime_readback_present": (
public_maintenance.get("runtime_readback_present") is True
),
"public_maintenance_fallback_ready": (
public_maintenance.get("ready") is True
),
"public_route_raw_5xx_without_fallback_count": _int(
public_maintenance.get("raw_5xx_without_fallback_count")
),
"public_route_unreachable_without_external_fallback_count": _int(
public_maintenance.get(
"route_unreachable_without_external_fallback_count"
)
),
"public_maintenance_fallback_route_count": _int(
public_maintenance.get("maintenance_fallback_route_count")
),
"controlled_service_data_backup_readback_present": (
controlled_service_data_backup_readback["readback_present"] is True
),
@@ -438,6 +464,15 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"readiness_percent": readiness_percent,
"blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only,
"latest_verify_only_metric_present": latest_verify_only_metric_present,
"public_maintenance_runtime_readback_present": rollups[
"public_maintenance_runtime_readback_present"
],
"public_maintenance_fallback_ready": rollups[
"public_maintenance_fallback_ready"
],
"public_route_raw_5xx_without_fallback_count": rollups[
"public_route_raw_5xx_without_fallback_count"
],
"controlled_service_data_backup_readback_status": rollups[
"controlled_service_data_backup_readback_status"
],
@@ -477,6 +512,7 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"host_boot_detection": host_boot_detection,
"post_reboot_readiness": post_reboot_readiness,
"stockplatform_data_freshness": stockplatform,
"public_maintenance_fallback": public_maintenance,
"windows99_vmware_autostart": windows99,
"windows99_verify_collection": windows99_verify_collection,
"windows99_management_channel": windows99_management,
@@ -499,6 +535,66 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
}
def _build_public_maintenance_fallback_readback(value: Any) -> dict[str, Any]:
public_maintenance = _dict(value)
if not public_maintenance:
return {
"runtime_readback_present": False,
"ready": False,
"target_url_count": 0,
"route_count": 0,
"raw_5xx_without_fallback_count": 0,
"route_unreachable_without_external_fallback_count": 0,
"maintenance_fallback_route_count": 0,
"raw_5xx_without_fallback_urls": [],
"route_unreachable_without_external_fallback_urls": [],
"maintenance_fallback_urls": [],
"routes": [],
"blockers": ["public_maintenance_fallback_runtime_readback_missing"],
}
raw_5xx_count = _int(public_maintenance.get("raw_5xx_without_fallback_count"))
unreachable_count = _int(
public_maintenance.get("route_unreachable_without_external_fallback_count")
)
blockers = _strings(public_maintenance.get("blockers"))
if (
raw_5xx_count
and "public_route_raw_5xx_without_maintenance_fallback" not in blockers
):
blockers.append("public_route_raw_5xx_without_maintenance_fallback")
if (
unreachable_count
and "public_route_unreachable_without_external_l1_fallback" not in blockers
):
blockers.append("public_route_unreachable_without_external_l1_fallback")
return {
"runtime_readback_present": True,
"schema_version": str(public_maintenance.get("schema_version") or "unknown"),
"generated_at": str(public_maintenance.get("generated_at") or ""),
"ready": public_maintenance.get("ready") is True and not blockers,
"target_url_count": _int(public_maintenance.get("target_url_count")),
"route_count": _int(public_maintenance.get("route_count")),
"raw_5xx_without_fallback_count": raw_5xx_count,
"route_unreachable_without_external_fallback_count": unreachable_count,
"maintenance_fallback_route_count": _int(
public_maintenance.get("maintenance_fallback_route_count")
),
"raw_5xx_without_fallback_urls": _strings(
public_maintenance.get("raw_5xx_without_fallback_urls")
),
"route_unreachable_without_external_fallback_urls": _strings(
public_maintenance.get("route_unreachable_without_external_fallback_urls")
),
"maintenance_fallback_urls": _strings(
public_maintenance.get("maintenance_fallback_urls")
),
"routes": _list_of_dicts(public_maintenance.get("routes")),
"blockers": _unique_strings(blockers),
}
def _build_controlled_service_data_backup_readback(
*,
post_reboot_readiness: dict[str, Any],
@@ -893,6 +989,12 @@ def _strings(value: Any) -> list[str]:
return [str(item) for item in value if item is not None]
def _list_of_dicts(value: Any) -> list[dict[str, Any]]:
if not isinstance(value, list):
return []
return [item for item in value if isinstance(item, dict)]
def _unique_strings(values: list[str]) -> list[str]:
seen: set[str] = set()
unique: list[str] = []

View File

@@ -69,7 +69,7 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order():
"blockers_open"
)
assert payload["mainline_execution_state"]["active_p0_immediate_apply_gap_count"] == 0
assert payload["mainline_execution_state"]["active_p0_readiness_percent"] == 43
assert payload["mainline_execution_state"]["active_p0_readiness_percent"] == 40
assert (
payload["mainline_execution_state"][
"controlled_service_data_backup_readback_present"

View File

@@ -24,6 +24,7 @@ EXPECTED_REBOOT_SLO_BLOCKERS = [
"wazuh_dashboard_degraded",
"windows99_remote_execution_channel_unavailable",
"windows99_vmware_autostart_readback_missing",
"public_maintenance_fallback_runtime_readback_missing",
]
EXPECTED_DRILL_PREFLIGHT_BLOCKERS = [
@@ -95,8 +96,8 @@ def test_reboot_auto_recovery_slo_scorecard_endpoint_overlays_live_stockplatform
assert payload["required_checks"]["stockplatform_freshness_ok"] is False
assert payload["required_checks"]["stockplatform_ingestion_ok"] is False
assert payload["required_checks"]["product_data_green"] is False
assert payload["readiness_percent"] == 21
assert payload["active_blocker_count"] == 15
assert payload["readiness_percent"] == 20
assert payload["active_blocker_count"] == 16
assert "product_data_green_not_1" in payload["active_blockers"]
assert "stockplatform_freshness_core_margin_short_daily_missing" in payload[
"active_blockers"
@@ -149,8 +150,8 @@ def _assert_reboot_slo_payload(payload: dict):
assert payload["safe_next_step"] == EXPECTED_WINDOWS99_NEXT_STEP
assert payload["next_safe_action"] == payload["safe_next_step"]
assert payload["can_claim_all_services_recovered_within_target"] is False
assert payload["active_blocker_count"] == 11
assert payload["readiness_percent"] == 43
assert payload["active_blocker_count"] == 12
assert payload["readiness_percent"] == 40
assert payload["service_green"] is False
assert payload["product_data_green"] is True
assert payload["backup_core_green"] is False
@@ -167,6 +168,14 @@ def _assert_reboot_slo_payload(payload: dict):
assert payload["latest_verify_only_metric_last_run_timestamp"] == 0
assert payload["stockplatform_freshness_status"] == "ok"
assert payload["stockplatform_ingestion_status"] == "ok"
assert payload["required_checks"][
"public_maintenance_fallback_runtime_ready"
] is False
assert payload["public_maintenance_fallback"]["runtime_readback_present"] is False
assert payload["public_maintenance_fallback"]["ready"] is False
assert payload["public_maintenance_fallback"]["blockers"] == [
"public_maintenance_fallback_runtime_readback_missing"
]
assert payload["windows99_vmware_verify_ready"] is False
assert payload["windows99_update_no_auto_reboot_ready"] is False
assert payload["windows99_remote_execution_channel_ready"] is False
@@ -196,10 +205,13 @@ def _assert_reboot_slo_payload(payload: dict):
)
assert payload["readback"]["safe_next_step"] == EXPECTED_WINDOWS99_NEXT_STEP
assert payload["readback"]["next_safe_action"] == payload["safe_next_step"]
assert payload["readback"]["active_blocker_count"] == 11
assert payload["readback"]["readiness_percent"] == 43
assert payload["readback"]["active_blocker_count"] == 12
assert payload["readback"]["readiness_percent"] == 40
assert payload["readback"]["blocked_by_fresh_reboot_window_only"] is False
assert payload["readback"]["latest_verify_only_metric_present"] is False
assert payload["readback"]["public_maintenance_runtime_readback_present"] is False
assert payload["readback"]["public_maintenance_fallback_ready"] is False
assert payload["readback"]["public_route_raw_5xx_without_fallback_count"] == 0
assert payload["readback"]["windows99_vmware_verify_ready"] is False
assert payload["readback"]["windows99_update_no_auto_reboot_ready"] is False
assert payload["readback"]["windows99_verify_collection_status"] == (
@@ -211,8 +223,8 @@ def _assert_reboot_slo_payload(payload: dict):
)
assert payload["readback"]["windows99_remote_execution_channel_ready"] is False
assert payload["readback"]["windows99_ssh_batch_status"] == "permission_denied"
assert payload["rollups"]["active_blocker_count"] == 11
assert payload["rollups"]["readiness_percent"] == 43
assert payload["rollups"]["active_blocker_count"] == 12
assert payload["rollups"]["readiness_percent"] == 40
assert payload["rollups"]["observed_host_count"] == 7
assert payload["rollups"]["missing_host_count"] == 0
assert payload["rollups"]["unreachable_host_count"] == 1
@@ -229,6 +241,9 @@ def _assert_reboot_slo_payload(payload: dict):
assert payload["rollups"]["stockplatform_ingestion_status"] == "ok"
assert payload["rollups"]["stockplatform_freshness_blocker_count"] == 0
assert payload["rollups"]["stockplatform_ingestion_blocker_count"] == 0
assert payload["rollups"]["public_maintenance_runtime_readback_present"] is False
assert payload["rollups"]["public_maintenance_fallback_ready"] is False
assert payload["rollups"]["public_route_raw_5xx_without_fallback_count"] == 0
assert (
payload["rollups"]["controlled_service_data_backup_readback_present"]
is True
@@ -282,8 +297,8 @@ def _assert_reboot_slo_payload(payload: dict):
"event_and_probe"
)
assert progress["primary_blocker"] == "reboot_event_required_host_unreachable"
assert progress["active_blocker_count"] == 11
assert progress["readiness_percent"] == 43
assert progress["active_blocker_count"] == 12
assert progress["readiness_percent"] == 40
assert progress["next_safe_action"] == payload["safe_next_step"]
assert progress["fixed_triage_order"][0] == "99_vmware_autostart_and_vm_power"
service_data_backup = payload["controlled_service_data_backup_readback"]
@@ -451,8 +466,8 @@ def _assert_drill_preflight_payload(payload: dict):
}
current = payload["current_readback"]
assert current["scorecard_status"] == "blocked_reboot_auto_recovery_slo_not_ready"
assert current["readiness_percent"] == 43
assert current["active_blocker_count"] == 11
assert current["readiness_percent"] == 40
assert current["active_blocker_count"] == 12
assert current["active_blockers"] == EXPECTED_REBOOT_SLO_BLOCKERS
assert current["latest_verify_only_metric_ready"] == 0
assert current["latest_verify_only_metric_blocker_count"] == 0
@@ -477,7 +492,7 @@ def _assert_drill_preflight_payload(payload: dict):
assert readback["workplan_id"] == "P0-006"
assert readback["preflight_ready"] is False
assert readback["target_selector"]["required_host_count"] == 7
assert readback["current_readback"]["active_blocker_count"] == 11
assert readback["current_readback"]["active_blocker_count"] == 12
assert readback["safe_next_step"] == payload["safe_next_step"]
rollups = payload["rollups"]
assert rollups["preflight_ready"] is False

View File

@@ -10,6 +10,18 @@
**仍維持**
- 未使用 GitHub / `gh` / GitHub API未讀 secret / token / `.env` / raw sessions / SQLite / auth未 workflow_dispatch未重啟主機 / VM / Docker / Nginx / K3s / DB / firewall未切 DNS / CDN / provider。
## 2026-07-02 — 21:41 P0-006 public maintenance fallback API readback parity
**完成內容**
- `apps/api/src/services/reboot_auto_recovery_slo_scorecard.py` 已同步 script scorecard 的 public maintenance fallback gate缺 runtime readback 時 fail-closed 加入 `public_maintenance_fallback_runtime_readback_missing`,並上卷到 `required_checks` / `rollups` / `readback` / `public_maintenance_fallback`
- `awoooi-priority-work-order-readback` 上卷 readiness 已同步新 required check目前 P0-006 readiness 由 `43` 收斂為 `40`,但 primary blocker 仍是 host/windows99/reboot SLO不會把主線帶回 StockPlatform。
**驗證**
- `DATABASE_URL=sqlite+aiosqlite:////tmp/awoooi-codex-api-test.db PYTHONPATH=apps/api python3.11 -m pytest -q apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py apps/api/tests/test_awoooi_priority_work_order_readback_api.py apps/api/tests/test_stockplatform_public_api_runtime_readback.py apps/api/tests/test_stockplatform_public_api_controlled_recovery_preflight.py scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py apps/api/tests/test_platform_read_model_fallbacks.py -p no:cacheprovider``58 passed`
**仍維持**
- 未讀 secret / token / `.env` / raw sessions / SQLite / auth未使用 GitHub / gh未 SSH、未 Docker / K8s / firewall / Wazuh 寫入、未重啟 host / VM / service、未切 DNS / CDN / provider。
## 2026-07-02 — 21:38 P0-006 public maintenance fallback test fixture closure
**完成內容**