wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

test(reboot): require maintenance fallback readback
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 58s
Details
CD Pipeline / build-and-deploy (push) Successful in 4m45s
Details
CD Pipeline / post-deploy-checks (push) Successful in 1m47s
Details

Browse Source
This commit is contained in:
Your Name
2026-07-02 21:27:08 +08:00
parent 0beb3e4f10
commit 5d5bc86fed
2 changed files with 99 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py
View File

@@ -91,6 +91,45 @@ VMWARE_AUTOSTART_VERIFY_READY=1
"""
PUBLIC_MAINTENANCE_GREEN = {
"schema_version": "awoooi_public_maintenance_fallback_probe_v1",
"generated_at": "2026-06-29T14:30:00+08:00",
"runtime_readback_present": True,
"ready": True,
"target_url_count": 2,
"route_count": 2,
"raw_5xx_without_fallback_count": 0,
"route_unreachable_without_external_fallback_count": 0,
"maintenance_fallback_route_count": 0,
"raw_5xx_without_fallback_urls": [],
"route_unreachable_without_external_fallback_urls": [],
"maintenance_fallback_urls": [],
"blockers": [],
"routes": [
{
"url": "https://awoooi.wooo.work/api/v1/health",
"http_status": 200,
"status": "normal_origin_serving",
"fallback_header": "",
"maintenance_body_detected": False,
"maintenance_fallback_serving": False,
"raw_5xx_without_fallback": False,
"route_unreachable_without_external_fallback": False,
},
{
"url": "https://awoooi.wooo.work/",
"http_status": 200,
"status": "normal_origin_serving",
"fallback_header": "",
"maintenance_body_detected": False,
"maintenance_fallback_serving": False,
"raw_5xx_without_fallback": False,
"route_unreachable_without_external_fallback": False,
},
],
}
WINDOWS99_MANAGEMENT_BLOCKED = {
"schema_version": "windows99_management_channel_readback_v1",
"generated_at": "2026-07-02T15:20:00+08:00",
@@ -138,12 +177,14 @@ def run_scorecard(
summary: str,
probe: str = HOST_PROBE_GREEN,
windows99: str = WINDOWS99_VMWARE_GREEN,
public_maintenance: dict | None = PUBLIC_MAINTENANCE_GREEN,
windows99_management: str | None = None,
) -> dict:
summary_path = tmp_path / "summary.txt"
probe_path = tmp_path / "probe.txt"
reboot_event_path = tmp_path / "reboot-event.json"
windows99_path = tmp_path / "windows99-vmware.txt"
public_maintenance_path = tmp_path / "public-maintenance.json"
summary_path.write_text(summary, encoding="utf-8")
probe_path.write_text(probe, encoding="utf-8")
reboot_event_path.write_text(json.dumps(REBOOT_EVENT_GREEN), encoding="utf-8")
@@ -163,6 +204,12 @@ def run_scorecard(
"--generated-at",
"2026-06-29T14:30:00+08:00",
]
if public_maintenance is not None:
public_maintenance_path.write_text(
json.dumps(public_maintenance),
encoding="utf-8",
)
args.extend(["--public-maintenance-file", str(public_maintenance_path)])
if windows99_management is not None:
windows99_management_path.write_text(windows99_management, encoding="utf-8")
args.extend(["--windows99-management-file", str(windows99_management_path)])
@@ -185,11 +232,16 @@ def run_scorecard_with_host_pressure(
reboot_event_path = tmp_path / "reboot-event.json"
host_pressure_path = tmp_path / "host-pressure.json"
windows99_path = tmp_path / "windows99-vmware.txt"
public_maintenance_path = tmp_path / "public-maintenance.json"
summary_path.write_text(summary, encoding="utf-8")
probe_path.write_text(HOST_PROBE_GREEN, encoding="utf-8")
reboot_event_path.write_text(json.dumps(REBOOT_EVENT_GREEN), encoding="utf-8")
host_pressure_path.write_text(json.dumps(host_pressure), encoding="utf-8")
windows99_path.write_text(WINDOWS99_VMWARE_GREEN, encoding="utf-8")
public_maintenance_path.write_text(
json.dumps(PUBLIC_MAINTENANCE_GREEN),
encoding="utf-8",
)
result = subprocess.run(
[
sys.executable,
@@ -202,6 +254,8 @@ def run_scorecard_with_host_pressure(
str(reboot_event_path),
"--host-pressure-file",
str(host_pressure_path),
"--public-maintenance-file",
str(public_maintenance_path),
"--windows99-vmware-file",
str(windows99_path),
"--generated-at",
@@ -227,12 +281,17 @@ def run_scorecard_with_stock(
freshness_path = tmp_path / "freshness.json"
ingestion_path = tmp_path / "ingestion.json"
windows99_path = tmp_path / "windows99-vmware.txt"
public_maintenance_path = tmp_path / "public-maintenance.json"
summary_path.write_text(summary, encoding="utf-8")
probe_path.write_text(HOST_PROBE_GREEN, encoding="utf-8")
reboot_event_path.write_text(json.dumps(REBOOT_EVENT_GREEN), encoding="utf-8")
freshness_path.write_text(json.dumps(freshness), encoding="utf-8")
ingestion_path.write_text(json.dumps(ingestion), encoding="utf-8")
windows99_path.write_text(WINDOWS99_VMWARE_GREEN, encoding="utf-8")
public_maintenance_path.write_text(
json.dumps(PUBLIC_MAINTENANCE_GREEN),
encoding="utf-8",
)
result = subprocess.run(
[
sys.executable,
@@ -247,6 +306,8 @@ def run_scorecard_with_stock(
str(freshness_path),
"--stock-ingestion-file",
str(ingestion_path),
"--public-maintenance-file",
str(public_maintenance_path),
"--windows99-vmware-file",
str(windows99_path),
"--generated-at",
@@ -288,7 +349,11 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) -
is False
)
assert payload["required_checks"]["windows99_management_channel_ready"] is True
assert payload["required_checks"]["public_maintenance_fallback_runtime_ready"] is True
assert payload["rollups"]["source_controls_present"] is True
assert payload["rollups"]["public_maintenance_runtime_readback_present"] is True
assert payload["rollups"]["public_maintenance_fallback_ready"] is True
assert payload["rollups"]["public_route_raw_5xx_without_fallback_count"] == 0
assert payload["rollups"]["windows99_vmware_verify_ready"] is True
assert payload["rollups"]["windows99_update_no_auto_reboot_ready"] is True
assert payload["rollups"]["windows99_verify_collection_status"] == (
@@ -333,6 +398,25 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) -
] is True
def test_missing_public_maintenance_runtime_readback_fails_closed(tmp_path: Path) -> None:
payload = run_scorecard(
tmp_path,
GREEN_SUMMARY,
public_maintenance=None,
)
assert payload["status"] == "blocked_reboot_auto_recovery_slo_not_ready"
assert payload["can_claim_all_services_recovered_within_target"] is False
assert "public_maintenance_fallback_runtime_readback_missing" in payload[
"active_blockers"
]
assert (
payload["required_checks"]["public_maintenance_fallback_runtime_ready"]
is False
)
assert payload["current_phase"] == "public_maintenance_fallback_blocked"
def test_missing_probe_fails_closed(tmp_path: Path) -> None:
payload = run_scorecard(tmp_path, GREEN_SUMMARY, probe="")