fix(reboot): bound slo prometheus readback latency
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 58s
CD Pipeline / build-and-deploy (push) Successful in 5m57s
CD Pipeline / post-deploy-checks (push) Has been cancelled

This commit is contained in:
Your Name
2026-07-03 01:08:28 +08:00
parent 4c897d23c9
commit e1f232b153
2 changed files with 91 additions and 23 deletions

View File

@@ -8,6 +8,7 @@ from fastapi.testclient import TestClient
from src.api.v1 import agents
from src.api.v1.agents import router
from src.services import reboot_auto_recovery_slo_scorecard as reboot_slo_scorecard
from src.services.reboot_auto_recovery_drill_preflight import (
load_latest_reboot_auto_recovery_drill_preflight,
)
@@ -296,6 +297,69 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
)
def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch):
monkeypatch.delenv("AWOOOI_REBOOT_SLO_PROMETHEUS_TIMEOUT_SECONDS", raising=False)
calls = []
def fake_query_prometheus_vector(query_url, query, *, timeout_seconds):
calls.append((query_url, query, timeout_seconds))
active_metric = "awoooi_reboot_auto_recovery_slo_active_blocker"
return [
*[
{
"metric": {"__name__": active_metric, "blocker": blocker},
"value": [1783010479, "1"],
}
for blocker in PROMETHEUS_RUNTIME_BLOCKERS
],
{
"metric": {
"__name__": "awoooi_reboot_auto_recovery_slo_primary_blocker",
"blocker": "reboot_event_required_host_unreachable",
},
"value": [1783010479, "1"],
},
{
"metric": {
"__name__": "awoooi_reboot_auto_recovery_slo_blocker_count",
},
"value": [1783010479, "7"],
},
{
"metric": {"__name__": "awoooi_reboot_auto_recovery_slo_ready"},
"value": [1783010479, "0"],
},
{
"metric": {
"__name__": "awoooi_reboot_auto_recovery_slo_last_run_timestamp",
},
"value": [1783010479, "1783010479"],
},
]
monkeypatch.setattr(
reboot_slo_scorecard,
"_query_prometheus_vector",
fake_query_prometheus_vector,
)
payload = reboot_slo_scorecard._load_reboot_slo_prometheus_metric_readback(
prometheus_query_url="http://prometheus.example/api/v1/query"
)
assert len(calls) == 1
assert calls[0][0] == "http://prometheus.example/api/v1/query"
assert calls[0][2] == 1.0
assert "awoooi_reboot_auto_recovery_slo_active_blocker" in calls[0][1]
assert "awoooi_reboot_auto_recovery_slo_last_run_timestamp" in calls[0][1]
assert payload["readback_present"] is True
assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS
assert payload["active_blocker_count"] == 7
assert payload["primary_blocker"] == "reboot_event_required_host_unreachable"
assert payload["ready"] is False
assert payload["last_run_timestamp"] == 1783010479
def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(monkeypatch):
monkeypatch.setattr(
agents,