fix(reboot): expose public maintenance fallback in api scorecard
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 37s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 37s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
This commit is contained in:
@@ -173,11 +173,17 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
host_boot_detection = _dict(scorecard.get("host_boot_detection"))
|
||||
post_reboot_readiness = _dict(scorecard.get("post_reboot_readiness"))
|
||||
stockplatform = _dict(scorecard.get("stockplatform_data_freshness"))
|
||||
public_maintenance = _build_public_maintenance_fallback_readback(
|
||||
scorecard.get("public_maintenance_fallback")
|
||||
)
|
||||
windows99 = _dict(scorecard.get("windows99_vmware_autostart"))
|
||||
windows99_management = _dict(scorecard.get("windows99_management_channel"))
|
||||
windows99_ssh_batch = _dict(windows99_management.get("ssh_batch"))
|
||||
source_controls = _dict(scorecard.get("source_controls"))
|
||||
active_blockers = _strings(scorecard.get("active_blockers"))
|
||||
active_blockers = _unique_strings(
|
||||
_strings(scorecard.get("active_blockers"))
|
||||
+ _strings(public_maintenance.get("blockers"))
|
||||
)
|
||||
required_checks = {
|
||||
"source_controls_present": all(source_controls.values()),
|
||||
"required_hosts_observed": not _strings(host_boot_detection.get("missing_hosts")),
|
||||
@@ -196,6 +202,9 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
"host_188_service_green": post_reboot_readiness.get("host_188_service_green") is True,
|
||||
"stockplatform_freshness_ok": stockplatform.get("freshness_status") == "ok",
|
||||
"stockplatform_ingestion_ok": stockplatform.get("ingestion_status") in {"ok", "unknown"},
|
||||
"public_maintenance_fallback_runtime_ready": (
|
||||
public_maintenance.get("ready") is True
|
||||
),
|
||||
"fresh_reboot_window_observed": not _strings(host_boot_detection.get("stale_hosts")),
|
||||
"can_claim_slo": scorecard.get("can_claim_all_services_recovered_within_target")
|
||||
is True,
|
||||
@@ -291,6 +300,23 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
"required"
|
||||
)
|
||||
is True,
|
||||
"public_maintenance_runtime_readback_present": (
|
||||
public_maintenance.get("runtime_readback_present") is True
|
||||
),
|
||||
"public_maintenance_fallback_ready": (
|
||||
public_maintenance.get("ready") is True
|
||||
),
|
||||
"public_route_raw_5xx_without_fallback_count": _int(
|
||||
public_maintenance.get("raw_5xx_without_fallback_count")
|
||||
),
|
||||
"public_route_unreachable_without_external_fallback_count": _int(
|
||||
public_maintenance.get(
|
||||
"route_unreachable_without_external_fallback_count"
|
||||
)
|
||||
),
|
||||
"public_maintenance_fallback_route_count": _int(
|
||||
public_maintenance.get("maintenance_fallback_route_count")
|
||||
),
|
||||
"controlled_service_data_backup_readback_present": (
|
||||
controlled_service_data_backup_readback["readback_present"] is True
|
||||
),
|
||||
@@ -438,6 +464,15 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
"readiness_percent": readiness_percent,
|
||||
"blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only,
|
||||
"latest_verify_only_metric_present": latest_verify_only_metric_present,
|
||||
"public_maintenance_runtime_readback_present": rollups[
|
||||
"public_maintenance_runtime_readback_present"
|
||||
],
|
||||
"public_maintenance_fallback_ready": rollups[
|
||||
"public_maintenance_fallback_ready"
|
||||
],
|
||||
"public_route_raw_5xx_without_fallback_count": rollups[
|
||||
"public_route_raw_5xx_without_fallback_count"
|
||||
],
|
||||
"controlled_service_data_backup_readback_status": rollups[
|
||||
"controlled_service_data_backup_readback_status"
|
||||
],
|
||||
@@ -477,6 +512,7 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
"host_boot_detection": host_boot_detection,
|
||||
"post_reboot_readiness": post_reboot_readiness,
|
||||
"stockplatform_data_freshness": stockplatform,
|
||||
"public_maintenance_fallback": public_maintenance,
|
||||
"windows99_vmware_autostart": windows99,
|
||||
"windows99_verify_collection": windows99_verify_collection,
|
||||
"windows99_management_channel": windows99_management,
|
||||
@@ -499,6 +535,66 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _build_public_maintenance_fallback_readback(value: Any) -> dict[str, Any]:
|
||||
public_maintenance = _dict(value)
|
||||
if not public_maintenance:
|
||||
return {
|
||||
"runtime_readback_present": False,
|
||||
"ready": False,
|
||||
"target_url_count": 0,
|
||||
"route_count": 0,
|
||||
"raw_5xx_without_fallback_count": 0,
|
||||
"route_unreachable_without_external_fallback_count": 0,
|
||||
"maintenance_fallback_route_count": 0,
|
||||
"raw_5xx_without_fallback_urls": [],
|
||||
"route_unreachable_without_external_fallback_urls": [],
|
||||
"maintenance_fallback_urls": [],
|
||||
"routes": [],
|
||||
"blockers": ["public_maintenance_fallback_runtime_readback_missing"],
|
||||
}
|
||||
|
||||
raw_5xx_count = _int(public_maintenance.get("raw_5xx_without_fallback_count"))
|
||||
unreachable_count = _int(
|
||||
public_maintenance.get("route_unreachable_without_external_fallback_count")
|
||||
)
|
||||
blockers = _strings(public_maintenance.get("blockers"))
|
||||
if (
|
||||
raw_5xx_count
|
||||
and "public_route_raw_5xx_without_maintenance_fallback" not in blockers
|
||||
):
|
||||
blockers.append("public_route_raw_5xx_without_maintenance_fallback")
|
||||
if (
|
||||
unreachable_count
|
||||
and "public_route_unreachable_without_external_l1_fallback" not in blockers
|
||||
):
|
||||
blockers.append("public_route_unreachable_without_external_l1_fallback")
|
||||
|
||||
return {
|
||||
"runtime_readback_present": True,
|
||||
"schema_version": str(public_maintenance.get("schema_version") or "unknown"),
|
||||
"generated_at": str(public_maintenance.get("generated_at") or ""),
|
||||
"ready": public_maintenance.get("ready") is True and not blockers,
|
||||
"target_url_count": _int(public_maintenance.get("target_url_count")),
|
||||
"route_count": _int(public_maintenance.get("route_count")),
|
||||
"raw_5xx_without_fallback_count": raw_5xx_count,
|
||||
"route_unreachable_without_external_fallback_count": unreachable_count,
|
||||
"maintenance_fallback_route_count": _int(
|
||||
public_maintenance.get("maintenance_fallback_route_count")
|
||||
),
|
||||
"raw_5xx_without_fallback_urls": _strings(
|
||||
public_maintenance.get("raw_5xx_without_fallback_urls")
|
||||
),
|
||||
"route_unreachable_without_external_fallback_urls": _strings(
|
||||
public_maintenance.get("route_unreachable_without_external_fallback_urls")
|
||||
),
|
||||
"maintenance_fallback_urls": _strings(
|
||||
public_maintenance.get("maintenance_fallback_urls")
|
||||
),
|
||||
"routes": _list_of_dicts(public_maintenance.get("routes")),
|
||||
"blockers": _unique_strings(blockers),
|
||||
}
|
||||
|
||||
|
||||
def _build_controlled_service_data_backup_readback(
|
||||
*,
|
||||
post_reboot_readiness: dict[str, Any],
|
||||
@@ -893,6 +989,12 @@ def _strings(value: Any) -> list[str]:
|
||||
return [str(item) for item in value if item is not None]
|
||||
|
||||
|
||||
def _list_of_dicts(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _unique_strings(values: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
unique: list[str] = []
|
||||
|
||||
@@ -69,7 +69,7 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order():
|
||||
"blockers_open"
|
||||
)
|
||||
assert payload["mainline_execution_state"]["active_p0_immediate_apply_gap_count"] == 0
|
||||
assert payload["mainline_execution_state"]["active_p0_readiness_percent"] == 43
|
||||
assert payload["mainline_execution_state"]["active_p0_readiness_percent"] == 40
|
||||
assert (
|
||||
payload["mainline_execution_state"][
|
||||
"controlled_service_data_backup_readback_present"
|
||||
|
||||
@@ -24,6 +24,7 @@ EXPECTED_REBOOT_SLO_BLOCKERS = [
|
||||
"wazuh_dashboard_degraded",
|
||||
"windows99_remote_execution_channel_unavailable",
|
||||
"windows99_vmware_autostart_readback_missing",
|
||||
"public_maintenance_fallback_runtime_readback_missing",
|
||||
]
|
||||
|
||||
EXPECTED_DRILL_PREFLIGHT_BLOCKERS = [
|
||||
@@ -95,8 +96,8 @@ def test_reboot_auto_recovery_slo_scorecard_endpoint_overlays_live_stockplatform
|
||||
assert payload["required_checks"]["stockplatform_freshness_ok"] is False
|
||||
assert payload["required_checks"]["stockplatform_ingestion_ok"] is False
|
||||
assert payload["required_checks"]["product_data_green"] is False
|
||||
assert payload["readiness_percent"] == 21
|
||||
assert payload["active_blocker_count"] == 15
|
||||
assert payload["readiness_percent"] == 20
|
||||
assert payload["active_blocker_count"] == 16
|
||||
assert "product_data_green_not_1" in payload["active_blockers"]
|
||||
assert "stockplatform_freshness_core_margin_short_daily_missing" in payload[
|
||||
"active_blockers"
|
||||
@@ -149,8 +150,8 @@ def _assert_reboot_slo_payload(payload: dict):
|
||||
assert payload["safe_next_step"] == EXPECTED_WINDOWS99_NEXT_STEP
|
||||
assert payload["next_safe_action"] == payload["safe_next_step"]
|
||||
assert payload["can_claim_all_services_recovered_within_target"] is False
|
||||
assert payload["active_blocker_count"] == 11
|
||||
assert payload["readiness_percent"] == 43
|
||||
assert payload["active_blocker_count"] == 12
|
||||
assert payload["readiness_percent"] == 40
|
||||
assert payload["service_green"] is False
|
||||
assert payload["product_data_green"] is True
|
||||
assert payload["backup_core_green"] is False
|
||||
@@ -167,6 +168,14 @@ def _assert_reboot_slo_payload(payload: dict):
|
||||
assert payload["latest_verify_only_metric_last_run_timestamp"] == 0
|
||||
assert payload["stockplatform_freshness_status"] == "ok"
|
||||
assert payload["stockplatform_ingestion_status"] == "ok"
|
||||
assert payload["required_checks"][
|
||||
"public_maintenance_fallback_runtime_ready"
|
||||
] is False
|
||||
assert payload["public_maintenance_fallback"]["runtime_readback_present"] is False
|
||||
assert payload["public_maintenance_fallback"]["ready"] is False
|
||||
assert payload["public_maintenance_fallback"]["blockers"] == [
|
||||
"public_maintenance_fallback_runtime_readback_missing"
|
||||
]
|
||||
assert payload["windows99_vmware_verify_ready"] is False
|
||||
assert payload["windows99_update_no_auto_reboot_ready"] is False
|
||||
assert payload["windows99_remote_execution_channel_ready"] is False
|
||||
@@ -196,10 +205,13 @@ def _assert_reboot_slo_payload(payload: dict):
|
||||
)
|
||||
assert payload["readback"]["safe_next_step"] == EXPECTED_WINDOWS99_NEXT_STEP
|
||||
assert payload["readback"]["next_safe_action"] == payload["safe_next_step"]
|
||||
assert payload["readback"]["active_blocker_count"] == 11
|
||||
assert payload["readback"]["readiness_percent"] == 43
|
||||
assert payload["readback"]["active_blocker_count"] == 12
|
||||
assert payload["readback"]["readiness_percent"] == 40
|
||||
assert payload["readback"]["blocked_by_fresh_reboot_window_only"] is False
|
||||
assert payload["readback"]["latest_verify_only_metric_present"] is False
|
||||
assert payload["readback"]["public_maintenance_runtime_readback_present"] is False
|
||||
assert payload["readback"]["public_maintenance_fallback_ready"] is False
|
||||
assert payload["readback"]["public_route_raw_5xx_without_fallback_count"] == 0
|
||||
assert payload["readback"]["windows99_vmware_verify_ready"] is False
|
||||
assert payload["readback"]["windows99_update_no_auto_reboot_ready"] is False
|
||||
assert payload["readback"]["windows99_verify_collection_status"] == (
|
||||
@@ -211,8 +223,8 @@ def _assert_reboot_slo_payload(payload: dict):
|
||||
)
|
||||
assert payload["readback"]["windows99_remote_execution_channel_ready"] is False
|
||||
assert payload["readback"]["windows99_ssh_batch_status"] == "permission_denied"
|
||||
assert payload["rollups"]["active_blocker_count"] == 11
|
||||
assert payload["rollups"]["readiness_percent"] == 43
|
||||
assert payload["rollups"]["active_blocker_count"] == 12
|
||||
assert payload["rollups"]["readiness_percent"] == 40
|
||||
assert payload["rollups"]["observed_host_count"] == 7
|
||||
assert payload["rollups"]["missing_host_count"] == 0
|
||||
assert payload["rollups"]["unreachable_host_count"] == 1
|
||||
@@ -229,6 +241,9 @@ def _assert_reboot_slo_payload(payload: dict):
|
||||
assert payload["rollups"]["stockplatform_ingestion_status"] == "ok"
|
||||
assert payload["rollups"]["stockplatform_freshness_blocker_count"] == 0
|
||||
assert payload["rollups"]["stockplatform_ingestion_blocker_count"] == 0
|
||||
assert payload["rollups"]["public_maintenance_runtime_readback_present"] is False
|
||||
assert payload["rollups"]["public_maintenance_fallback_ready"] is False
|
||||
assert payload["rollups"]["public_route_raw_5xx_without_fallback_count"] == 0
|
||||
assert (
|
||||
payload["rollups"]["controlled_service_data_backup_readback_present"]
|
||||
is True
|
||||
@@ -282,8 +297,8 @@ def _assert_reboot_slo_payload(payload: dict):
|
||||
"event_and_probe"
|
||||
)
|
||||
assert progress["primary_blocker"] == "reboot_event_required_host_unreachable"
|
||||
assert progress["active_blocker_count"] == 11
|
||||
assert progress["readiness_percent"] == 43
|
||||
assert progress["active_blocker_count"] == 12
|
||||
assert progress["readiness_percent"] == 40
|
||||
assert progress["next_safe_action"] == payload["safe_next_step"]
|
||||
assert progress["fixed_triage_order"][0] == "99_vmware_autostart_and_vm_power"
|
||||
service_data_backup = payload["controlled_service_data_backup_readback"]
|
||||
@@ -451,8 +466,8 @@ def _assert_drill_preflight_payload(payload: dict):
|
||||
}
|
||||
current = payload["current_readback"]
|
||||
assert current["scorecard_status"] == "blocked_reboot_auto_recovery_slo_not_ready"
|
||||
assert current["readiness_percent"] == 43
|
||||
assert current["active_blocker_count"] == 11
|
||||
assert current["readiness_percent"] == 40
|
||||
assert current["active_blocker_count"] == 12
|
||||
assert current["active_blockers"] == EXPECTED_REBOOT_SLO_BLOCKERS
|
||||
assert current["latest_verify_only_metric_ready"] == 0
|
||||
assert current["latest_verify_only_metric_blocker_count"] == 0
|
||||
@@ -477,7 +492,7 @@ def _assert_drill_preflight_payload(payload: dict):
|
||||
assert readback["workplan_id"] == "P0-006"
|
||||
assert readback["preflight_ready"] is False
|
||||
assert readback["target_selector"]["required_host_count"] == 7
|
||||
assert readback["current_readback"]["active_blocker_count"] == 11
|
||||
assert readback["current_readback"]["active_blocker_count"] == 12
|
||||
assert readback["safe_next_step"] == payload["safe_next_step"]
|
||||
rollups = payload["rollups"]
|
||||
assert rollups["preflight_ready"] is False
|
||||
|
||||
@@ -10,6 +10,18 @@
|
||||
**仍維持**:
|
||||
- 未使用 GitHub / `gh` / GitHub API;未讀 secret / token / `.env` / raw sessions / SQLite / auth;未 workflow_dispatch;未重啟主機 / VM / Docker / Nginx / K3s / DB / firewall;未切 DNS / CDN / provider。
|
||||
|
||||
## 2026-07-02 — 21:41 P0-006 public maintenance fallback API readback parity
|
||||
|
||||
**完成內容**:
|
||||
- `apps/api/src/services/reboot_auto_recovery_slo_scorecard.py` 已同步 script scorecard 的 public maintenance fallback gate:缺 runtime readback 時 fail-closed 加入 `public_maintenance_fallback_runtime_readback_missing`,並上卷到 `required_checks` / `rollups` / `readback` / `public_maintenance_fallback`。
|
||||
- `awoooi-priority-work-order-readback` 上卷 readiness 已同步新 required check:目前 P0-006 readiness 由 `43` 收斂為 `40`,但 primary blocker 仍是 host/windows99/reboot SLO,不會把主線帶回 StockPlatform。
|
||||
|
||||
**驗證**:
|
||||
- `DATABASE_URL=sqlite+aiosqlite:////tmp/awoooi-codex-api-test.db PYTHONPATH=apps/api python3.11 -m pytest -q apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py apps/api/tests/test_awoooi_priority_work_order_readback_api.py apps/api/tests/test_stockplatform_public_api_runtime_readback.py apps/api/tests/test_stockplatform_public_api_controlled_recovery_preflight.py scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py apps/api/tests/test_platform_read_model_fallbacks.py -p no:cacheprovider`:`58 passed`。
|
||||
|
||||
**仍維持**:
|
||||
- 未讀 secret / token / `.env` / raw sessions / SQLite / auth;未使用 GitHub / gh;未 SSH、未 Docker / K8s / firewall / Wazuh 寫入、未重啟 host / VM / service、未切 DNS / CDN / provider。
|
||||
|
||||
## 2026-07-02 — 21:38 P0-006 public maintenance fallback test fixture closure
|
||||
|
||||
**完成內容**:
|
||||
|
||||
Reference in New Issue
Block a user