test(reboot): require maintenance fallback readback
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 58s
CD Pipeline / build-and-deploy (push) Successful in 4m45s
CD Pipeline / post-deploy-checks (push) Successful in 1m47s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 58s
CD Pipeline / build-and-deploy (push) Successful in 4m45s
CD Pipeline / post-deploy-checks (push) Successful in 1m47s
This commit is contained in:
@@ -1,3 +1,18 @@
|
||||
## 2026-07-02 — 21:38 P0-006 public maintenance fallback test fixture closure
|
||||
|
||||
**完成內容**:
|
||||
- Gitea CD `#4450` 失敗點定位為 `test_green_summary_and_recent_all_host_probe_can_claim_slo`:新增 public maintenance runtime readback fail-closed gate 後,既有 green SLO fixture 未提供 `--public-maintenance-file`,因此仍用舊契約宣稱 ready。
|
||||
- 修正 `scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py`:新增 `PUBLIC_MAINTENANCE_GREEN` fixture,所有 green host / Stock / host-pressure helper 都帶入 public fallback runtime readback。
|
||||
- 新增 `test_missing_public_maintenance_runtime_readback_fails_closed`,明確鎖住缺 runtime probe 時 active blocker 必須包含 `public_maintenance_fallback_runtime_readback_missing`,不得回到 source-only ready。
|
||||
|
||||
**驗證**:
|
||||
- `python3.11 -m pytest scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py -q -p no:cacheprovider`:`18 passed`。
|
||||
- `python3.11 -m py_compile scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py scripts/reboot-recovery/public-maintenance-fallback-probe.py`:通過。
|
||||
- `git diff --check`:通過。
|
||||
|
||||
**仍維持**:
|
||||
- 未使用 GitHub / `gh` / GitHub API;未讀 secret / token / `.env` / raw sessions / SQLite / auth;未 workflow_dispatch;未重啟主機 / VM / Docker / Nginx / K3s / DB / firewall;未切 DNS / CDN / provider。
|
||||
|
||||
## 2026-07-02 — 21:24 P0-006 public maintenance fallback runtime gate
|
||||
|
||||
**完成內容**:
|
||||
|
||||
@@ -91,6 +91,45 @@ VMWARE_AUTOSTART_VERIFY_READY=1
|
||||
"""
|
||||
|
||||
|
||||
PUBLIC_MAINTENANCE_GREEN = {
|
||||
"schema_version": "awoooi_public_maintenance_fallback_probe_v1",
|
||||
"generated_at": "2026-06-29T14:30:00+08:00",
|
||||
"runtime_readback_present": True,
|
||||
"ready": True,
|
||||
"target_url_count": 2,
|
||||
"route_count": 2,
|
||||
"raw_5xx_without_fallback_count": 0,
|
||||
"route_unreachable_without_external_fallback_count": 0,
|
||||
"maintenance_fallback_route_count": 0,
|
||||
"raw_5xx_without_fallback_urls": [],
|
||||
"route_unreachable_without_external_fallback_urls": [],
|
||||
"maintenance_fallback_urls": [],
|
||||
"blockers": [],
|
||||
"routes": [
|
||||
{
|
||||
"url": "https://awoooi.wooo.work/api/v1/health",
|
||||
"http_status": 200,
|
||||
"status": "normal_origin_serving",
|
||||
"fallback_header": "",
|
||||
"maintenance_body_detected": False,
|
||||
"maintenance_fallback_serving": False,
|
||||
"raw_5xx_without_fallback": False,
|
||||
"route_unreachable_without_external_fallback": False,
|
||||
},
|
||||
{
|
||||
"url": "https://awoooi.wooo.work/",
|
||||
"http_status": 200,
|
||||
"status": "normal_origin_serving",
|
||||
"fallback_header": "",
|
||||
"maintenance_body_detected": False,
|
||||
"maintenance_fallback_serving": False,
|
||||
"raw_5xx_without_fallback": False,
|
||||
"route_unreachable_without_external_fallback": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
WINDOWS99_MANAGEMENT_BLOCKED = {
|
||||
"schema_version": "windows99_management_channel_readback_v1",
|
||||
"generated_at": "2026-07-02T15:20:00+08:00",
|
||||
@@ -138,12 +177,14 @@ def run_scorecard(
|
||||
summary: str,
|
||||
probe: str = HOST_PROBE_GREEN,
|
||||
windows99: str = WINDOWS99_VMWARE_GREEN,
|
||||
public_maintenance: dict | None = PUBLIC_MAINTENANCE_GREEN,
|
||||
windows99_management: str | None = None,
|
||||
) -> dict:
|
||||
summary_path = tmp_path / "summary.txt"
|
||||
probe_path = tmp_path / "probe.txt"
|
||||
reboot_event_path = tmp_path / "reboot-event.json"
|
||||
windows99_path = tmp_path / "windows99-vmware.txt"
|
||||
public_maintenance_path = tmp_path / "public-maintenance.json"
|
||||
summary_path.write_text(summary, encoding="utf-8")
|
||||
probe_path.write_text(probe, encoding="utf-8")
|
||||
reboot_event_path.write_text(json.dumps(REBOOT_EVENT_GREEN), encoding="utf-8")
|
||||
@@ -163,6 +204,12 @@ def run_scorecard(
|
||||
"--generated-at",
|
||||
"2026-06-29T14:30:00+08:00",
|
||||
]
|
||||
if public_maintenance is not None:
|
||||
public_maintenance_path.write_text(
|
||||
json.dumps(public_maintenance),
|
||||
encoding="utf-8",
|
||||
)
|
||||
args.extend(["--public-maintenance-file", str(public_maintenance_path)])
|
||||
if windows99_management is not None:
|
||||
windows99_management_path.write_text(windows99_management, encoding="utf-8")
|
||||
args.extend(["--windows99-management-file", str(windows99_management_path)])
|
||||
@@ -185,11 +232,16 @@ def run_scorecard_with_host_pressure(
|
||||
reboot_event_path = tmp_path / "reboot-event.json"
|
||||
host_pressure_path = tmp_path / "host-pressure.json"
|
||||
windows99_path = tmp_path / "windows99-vmware.txt"
|
||||
public_maintenance_path = tmp_path / "public-maintenance.json"
|
||||
summary_path.write_text(summary, encoding="utf-8")
|
||||
probe_path.write_text(HOST_PROBE_GREEN, encoding="utf-8")
|
||||
reboot_event_path.write_text(json.dumps(REBOOT_EVENT_GREEN), encoding="utf-8")
|
||||
host_pressure_path.write_text(json.dumps(host_pressure), encoding="utf-8")
|
||||
windows99_path.write_text(WINDOWS99_VMWARE_GREEN, encoding="utf-8")
|
||||
public_maintenance_path.write_text(
|
||||
json.dumps(PUBLIC_MAINTENANCE_GREEN),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
@@ -202,6 +254,8 @@ def run_scorecard_with_host_pressure(
|
||||
str(reboot_event_path),
|
||||
"--host-pressure-file",
|
||||
str(host_pressure_path),
|
||||
"--public-maintenance-file",
|
||||
str(public_maintenance_path),
|
||||
"--windows99-vmware-file",
|
||||
str(windows99_path),
|
||||
"--generated-at",
|
||||
@@ -227,12 +281,17 @@ def run_scorecard_with_stock(
|
||||
freshness_path = tmp_path / "freshness.json"
|
||||
ingestion_path = tmp_path / "ingestion.json"
|
||||
windows99_path = tmp_path / "windows99-vmware.txt"
|
||||
public_maintenance_path = tmp_path / "public-maintenance.json"
|
||||
summary_path.write_text(summary, encoding="utf-8")
|
||||
probe_path.write_text(HOST_PROBE_GREEN, encoding="utf-8")
|
||||
reboot_event_path.write_text(json.dumps(REBOOT_EVENT_GREEN), encoding="utf-8")
|
||||
freshness_path.write_text(json.dumps(freshness), encoding="utf-8")
|
||||
ingestion_path.write_text(json.dumps(ingestion), encoding="utf-8")
|
||||
windows99_path.write_text(WINDOWS99_VMWARE_GREEN, encoding="utf-8")
|
||||
public_maintenance_path.write_text(
|
||||
json.dumps(PUBLIC_MAINTENANCE_GREEN),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
@@ -247,6 +306,8 @@ def run_scorecard_with_stock(
|
||||
str(freshness_path),
|
||||
"--stock-ingestion-file",
|
||||
str(ingestion_path),
|
||||
"--public-maintenance-file",
|
||||
str(public_maintenance_path),
|
||||
"--windows99-vmware-file",
|
||||
str(windows99_path),
|
||||
"--generated-at",
|
||||
@@ -288,7 +349,11 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) -
|
||||
is False
|
||||
)
|
||||
assert payload["required_checks"]["windows99_management_channel_ready"] is True
|
||||
assert payload["required_checks"]["public_maintenance_fallback_runtime_ready"] is True
|
||||
assert payload["rollups"]["source_controls_present"] is True
|
||||
assert payload["rollups"]["public_maintenance_runtime_readback_present"] is True
|
||||
assert payload["rollups"]["public_maintenance_fallback_ready"] is True
|
||||
assert payload["rollups"]["public_route_raw_5xx_without_fallback_count"] == 0
|
||||
assert payload["rollups"]["windows99_vmware_verify_ready"] is True
|
||||
assert payload["rollups"]["windows99_update_no_auto_reboot_ready"] is True
|
||||
assert payload["rollups"]["windows99_verify_collection_status"] == (
|
||||
@@ -333,6 +398,25 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) -
|
||||
] is True
|
||||
|
||||
|
||||
def test_missing_public_maintenance_runtime_readback_fails_closed(tmp_path: Path) -> None:
|
||||
payload = run_scorecard(
|
||||
tmp_path,
|
||||
GREEN_SUMMARY,
|
||||
public_maintenance=None,
|
||||
)
|
||||
|
||||
assert payload["status"] == "blocked_reboot_auto_recovery_slo_not_ready"
|
||||
assert payload["can_claim_all_services_recovered_within_target"] is False
|
||||
assert "public_maintenance_fallback_runtime_readback_missing" in payload[
|
||||
"active_blockers"
|
||||
]
|
||||
assert (
|
||||
payload["required_checks"]["public_maintenance_fallback_runtime_ready"]
|
||||
is False
|
||||
)
|
||||
assert payload["current_phase"] == "public_maintenance_fallback_blocked"
|
||||
|
||||
|
||||
def test_missing_probe_fails_closed(tmp_path: Path) -> None:
|
||||
payload = run_scorecard(tmp_path, GREEN_SUMMARY, probe="")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user