fix(reboot): surface windows99 vmware alias metrics

apps/api/src/services/reboot_auto_recovery_slo_scorecard.py

@@ -44,6 +44,13 @@ _PROMETHEUS_RUNTIME_QUERIES = {
     "blocker_count": "awoooi_reboot_auto_recovery_slo_blocker_count",
     "ready": "awoooi_reboot_auto_recovery_slo_ready",
     "last_run_timestamp": "awoooi_reboot_auto_recovery_slo_last_run_timestamp",
+    "windows99_vmware_readback_present": "awoooi_windows99_vmware_readback_present",
+    "windows99_vmware_verify_ready": "awoooi_windows99_vmware_verify_ready",
+    "windows99_windows_update_ready": (
+        "awoooi_windows99_vmware_windows_update_no_auto_reboot_ready"
+    ),
+    "windows99_missing_vmx_alias": "awoooi_windows99_vmware_missing_vmx_alias",
+    "windows99_powered_off_alias": "awoooi_windows99_vmware_powered_off_alias",
 }
 _PROMETHEUS_RUNTIME_COMBINED_QUERY = (
     '{__name__=~"'
@@ -261,8 +268,22 @@ def _load_reboot_slo_prometheus_metric_readback(
     scalar_results = {
         key: results_by_name.get(query, [])
         for key, query in _PROMETHEUS_RUNTIME_QUERIES.items()
-        if key not in {"active_blocker", "primary_blocker"}
+        if key
+        not in {
+            "active_blocker",
+            "primary_blocker",
+            "windows99_missing_vmx_alias",
+            "windows99_powered_off_alias",
+        }
     }
+    missing_vmx_results = results_by_name.get(
+        _PROMETHEUS_RUNTIME_QUERIES["windows99_missing_vmx_alias"],
+        [],
+    )
+    powered_off_results = results_by_name.get(
+        _PROMETHEUS_RUNTIME_QUERIES["windows99_powered_off_alias"],
+        [],
+    )
     active_blockers = _unique_strings(
         [
             str(_dict(result.get("metric")).get("blocker") or "")
@@ -289,6 +310,35 @@ def _load_reboot_slo_prometheus_metric_readback(
         scalar_results.get("last_run_timestamp") or [],
         default=0,
     )
+    windows99_readback_present = (
+        _prometheus_first_int(
+            scalar_results.get("windows99_vmware_readback_present") or [],
+            default=0,
+        )
+        == 1
+    )
+    windows99_verify_ready = (
+        _prometheus_first_int(
+            scalar_results.get("windows99_vmware_verify_ready") or [],
+            default=0,
+        )
+        == 1
+    )
+    windows99_update_ready = (
+        _prometheus_first_int(
+            scalar_results.get("windows99_windows_update_ready") or [],
+            default=0,
+        )
+        == 1
+    )
+    windows99_missing_vmx_aliases = _prometheus_label_values(
+        missing_vmx_results,
+        "alias",
+    )
+    windows99_powered_off_aliases = _prometheus_label_values(
+        powered_off_results,
+        "alias",
+    )
     return {
         "schema_version": "reboot_slo_prometheus_metric_readback_v1",
         "readback_present": bool(combined_results),
@@ -299,6 +349,11 @@ def _load_reboot_slo_prometheus_metric_readback(
         "primary_blocker": primary_blocker,
         "ready": ready,
         "last_run_timestamp": last_run_timestamp,
+        "windows99_vmware_readback_present": windows99_readback_present,
+        "windows99_vmware_verify_ready": windows99_verify_ready,
+        "windows99_windows_update_no_auto_reboot_ready": windows99_update_ready,
+        "windows99_missing_vmx_aliases": windows99_missing_vmx_aliases,
+        "windows99_powered_off_aliases": windows99_powered_off_aliases,
     }
 
 
@@ -329,6 +384,19 @@ def _prometheus_first_int(results: list[dict[str, Any]], *, default: int) -> int
         return default
 
 
+def _prometheus_label_values(
+    results: list[dict[str, Any]],
+    label_name: str,
+) -> list[str]:
+    return _unique_strings(
+        [
+            str(_dict(result.get("metric")).get(label_name) or "")
+            for result in results
+            if str(_dict(result.get("metric")).get(label_name) or "")
+        ]
+    )
+
+
 def _annotate_prometheus_metric_readback(
     payload: dict[str, Any],
     metric_readback: dict[str, Any],
@@ -367,6 +435,59 @@ def _annotate_prometheus_metric_readback(
         "runtime_metric_last_run_timestamp"
     ]
     rollups["runtime_metric_source_control_reconciled_blocker_count"] = 0
+    _apply_prometheus_windows99_vmware_readback(payload, metric_readback)
+
+
+def _apply_prometheus_windows99_vmware_readback(
+    payload: dict[str, Any],
+    metric_readback: dict[str, Any],
+) -> None:
+    if metric_readback.get("readback_present") is not True:
+        return
+    if metric_readback.get("windows99_vmware_readback_present") is not True:
+        return
+
+    active_blockers = _strings(metric_readback.get("active_blockers"))
+    missing_vmx_aliases = _strings(metric_readback.get("windows99_missing_vmx_aliases"))
+    powered_off_aliases = _strings(metric_readback.get("windows99_powered_off_aliases"))
+    windows99_blockers = [
+        blocker for blocker in active_blockers if blocker.startswith("windows99_")
+    ]
+    verify_ready = metric_readback.get("windows99_vmware_verify_ready") is True
+    update_ready = (
+        metric_readback.get("windows99_windows_update_no_auto_reboot_ready") is True
+    )
+    windows99 = _dict(payload.setdefault("windows99_vmware_autostart", {}))
+    windows99.update(
+        {
+            "readback_present": True,
+            "verify_ready": verify_ready,
+            "vmrun_present": True,
+            "config_ready": not missing_vmx_aliases,
+            "power_ready": not powered_off_aliases,
+            "windows_update_no_auto_reboot_ready": update_ready,
+            "missing_vmx_aliases": missing_vmx_aliases,
+            "powered_off_aliases": powered_off_aliases,
+            "blockers": windows99_blockers,
+            "runtime_metric_overlay_applied": True,
+        }
+    )
+    payload["windows99_vmware_verify_ready"] = verify_ready
+    payload["windows99_update_no_auto_reboot_ready"] = update_ready
+
+    readback = _dict(payload.setdefault("readback", {}))
+    readback["windows99_vmware_verify_ready"] = verify_ready
+    readback["windows99_update_no_auto_reboot_ready"] = update_ready
+    readback["windows99_runtime_metric_overlay_applied"] = True
+    readback["windows99_missing_vmx_aliases"] = missing_vmx_aliases
+    readback["windows99_powered_off_aliases"] = powered_off_aliases
+
+    rollups = _dict(payload.setdefault("rollups", {}))
+    rollups["windows99_vmware_verify_ready"] = verify_ready
+    rollups["windows99_update_no_auto_reboot_ready"] = update_ready
+    rollups["windows99_runtime_metric_overlay_applied"] = True
+    rollups["windows99_missing_vmx_alias_count"] = len(missing_vmx_aliases)
+    rollups["windows99_powered_off_alias_count"] = len(powered_off_aliases)
 
 
 def _reconcile_prometheus_metric_active_blockers_with_source_controls(

apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py

@@ -87,7 +87,8 @@ PROMETHEUS_RUNTIME_BLOCKERS = [
     "host_unreachable_after_reboot",
     "host_uptime_unknown",
     "reboot_event_required_host_unreachable",
-    "windows99_vmware_autostart_readback_missing",
+    "windows99_vmware_guest_power_not_ready",
+    "windows99_vmware_vmx_missing",
 ]
 PROMETHEUS_SOURCE_RECONCILED_BLOCKER = (
     "conversation_event_hot_path_index_migration_source_missing"
@@ -104,10 +105,15 @@ PROMETHEUS_RUNTIME_READBACK = {
     "source": "prometheus_query_api",
     "query_url": "http://192.168.0.110:9090/api/v1/query",
     "active_blockers": PROMETHEUS_RUNTIME_BLOCKERS,
-    "active_blocker_count": 7,
+    "active_blocker_count": 8,
     "primary_blocker": "reboot_event_required_host_unreachable",
     "ready": False,
     "last_run_timestamp": 1783010479,
+    "windows99_vmware_readback_present": True,
+    "windows99_vmware_verify_ready": False,
+    "windows99_windows_update_no_auto_reboot_ready": True,
+    "windows99_missing_vmx_aliases": ["111"],
+    "windows99_powered_off_aliases": ["111", "112", "120", "121", "188"],
 }
 
 
@@ -264,7 +270,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
 
     assert payload["runtime_scorecard_readback_present"] is False
     assert payload["runtime_metric_readback_present"] is True
-    assert payload["runtime_metric_active_blocker_count"] == 7
+    assert payload["runtime_metric_active_blocker_count"] == 8
     assert payload["runtime_metric_last_run_timestamp"] == 1783010479
     assert (
         payload["runtime_metric_source_control_reconciled_blockers"]
@@ -272,7 +278,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
     )
     assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 1
     assert payload["active_blockers"] == PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS
-    assert payload["active_blocker_count"] == 6
+    assert payload["active_blocker_count"] == 7
     assert payload["readiness_percent"] == 47
     assert payload["primary_blocker"] == "reboot_event_required_host_unreachable"
     assert payload["next_safe_action"] == (
@@ -281,7 +287,22 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
     assert "backup_core_green_not_1" not in payload["active_blockers"]
     assert "service_green_not_1" not in payload["active_blockers"]
     assert PROMETHEUS_SOURCE_RECONCILED_BLOCKER not in payload["active_blockers"]
-    assert payload["active_blocker_action_matrix"]["item_count"] == 6
+    assert payload["active_blocker_action_matrix"]["item_count"] == 7
+    assert payload["windows99_vmware_autostart"]["readback_present"] is True
+    assert payload["windows99_vmware_autostart"]["missing_vmx_aliases"] == ["111"]
+    assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [
+        "111",
+        "112",
+        "120",
+        "121",
+        "188",
+    ]
+    assert payload["windows99_vmware_autostart"]["blockers"] == [
+        "windows99_vmware_guest_power_not_ready",
+        "windows99_vmware_vmx_missing",
+    ]
+    assert payload["readback"]["windows99_runtime_metric_overlay_applied"] is True
+    assert payload["rollups"]["windows99_missing_vmx_alias_count"] == 1
     action_by_blocker = {
         item["blocker"]: item
         for item in payload["active_blocker_action_matrix"]["items"]
@@ -291,12 +312,12 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
         "host_cpu_pressure",
         0,
     ) == 0
-    assert payload["readback"]["active_blocker_count"] == 6
+    assert payload["readback"]["active_blocker_count"] == 7
     assert payload["readback"]["runtime_metric_readback_present"] is True
     assert payload["readback"][
         "runtime_metric_source_control_reconciled_blocker_count"
     ] == 1
-    assert payload["rollups"]["active_blocker_count"] == 6
+    assert payload["rollups"]["active_blocker_count"] == 7
     assert payload["rollups"]["runtime_metric_readback_present"] is True
     assert payload["rollups"][
         "runtime_metric_source_control_reconciled_blocker_count"
@@ -324,7 +345,7 @@ def test_reboot_auto_recovery_slo_scorecard_keeps_prometheus_source_missing_when
     )
 
     assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS
-    assert payload["active_blocker_count"] == 7
+    assert payload["active_blocker_count"] == 8
     assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 0
     action_by_blocker = {
         item["blocker"]: item
@@ -372,7 +393,7 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch):
                 "metric": {
                     "__name__": "awoooi_reboot_auto_recovery_slo_blocker_count",
                 },
-                "value": [1783010479, "7"],
+                "value": [1783010479, str(len(PROMETHEUS_RUNTIME_BLOCKERS))],
             },
             {
                 "metric": {"__name__": "awoooi_reboot_auto_recovery_slo_ready"},
@@ -384,6 +405,43 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch):
                 },
                 "value": [1783010479, "1783010479"],
             },
+            {
+                "metric": {
+                    "__name__": "awoooi_windows99_vmware_readback_present",
+                },
+                "value": [1783010479, "1"],
+            },
+            {
+                "metric": {
+                    "__name__": "awoooi_windows99_vmware_verify_ready",
+                },
+                "value": [1783010479, "0"],
+            },
+            {
+                "metric": {
+                    "__name__": (
+                        "awoooi_windows99_vmware_windows_update_no_auto_reboot_ready"
+                    ),
+                },
+                "value": [1783010479, "1"],
+            },
+            {
+                "metric": {
+                    "__name__": "awoooi_windows99_vmware_missing_vmx_alias",
+                    "alias": "111",
+                },
+                "value": [1783010479, "1"],
+            },
+            *[
+                {
+                    "metric": {
+                        "__name__": "awoooi_windows99_vmware_powered_off_alias",
+                        "alias": alias,
+                    },
+                    "value": [1783010479, "1"],
+                }
+                for alias in ["111", "112", "120", "121", "188"]
+            ],
         ]
 
     monkeypatch.setattr(
@@ -403,10 +461,18 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch):
     assert "awoooi_reboot_auto_recovery_slo_last_run_timestamp" in calls[0][1]
     assert payload["readback_present"] is True
     assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS
-    assert payload["active_blocker_count"] == 7
+    assert payload["active_blocker_count"] == len(PROMETHEUS_RUNTIME_BLOCKERS)
     assert payload["primary_blocker"] == "reboot_event_required_host_unreachable"
     assert payload["ready"] is False
     assert payload["last_run_timestamp"] == 1783010479
+    assert payload["windows99_missing_vmx_aliases"] == ["111"]
+    assert payload["windows99_powered_off_aliases"] == [
+        "111",
+        "112",
+        "120",
+        "121",
+        "188",
+    ]
 
 
 def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(monkeypatch):

scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh

@@ -175,6 +175,52 @@ payload=json.load(open(sys.argv[1], encoding="utf-8"))
 print(payload.get("public_maintenance_fallback", {}).get("route_unreachable_without_external_fallback_count", 0) or 0)
 PY
 )"
+windows99_vmware_metrics="$(python3 - "$scorecard_file" <<'PY'
+import json, sys
+
+payload = json.load(open(sys.argv[1], encoding="utf-8"))
+windows99 = payload.get("windows99_vmware_autostart") or {}
+
+
+def label(value: object) -> str:
+    text = str(value or "")
+    return (
+        text.replace("\\", "\\\\")
+        .replace("\n", "\\n")
+        .replace('"', '\\"')
+    )
+
+
+def gauge(name: str, value: bool) -> None:
+    print(f'{name}{{scope="windows99"}} {1 if value else 0}')
+
+
+gauge(
+    "awoooi_windows99_vmware_readback_present",
+    windows99.get("readback_present") is True,
+)
+gauge(
+    "awoooi_windows99_vmware_verify_ready",
+    windows99.get("verify_ready") is True,
+)
+gauge(
+    "awoooi_windows99_vmware_windows_update_no_auto_reboot_ready",
+    windows99.get("windows_update_no_auto_reboot_ready") is True,
+)
+for alias in windows99.get("missing_vmx_aliases") or []:
+    if alias:
+        print(
+            "awoooi_windows99_vmware_missing_vmx_alias"
+            f'{{scope="windows99",alias="{label(alias)}"}} 1'
+        )
+for alias in windows99.get("powered_off_aliases") or []:
+    if alias:
+        print(
+            "awoooi_windows99_vmware_powered_off_alias"
+            f'{{scope="windows99",alias="{label(alias)}"}} 1'
+        )
+PY
+)"
 active_blocker_metrics="$(python3 - "$scorecard_file" "$TARGET_MINUTES" <<'PY'
 import json, sys
 
@@ -258,7 +304,20 @@ awoooi_public_route_raw_5xx_without_maintenance_fallback{scope="public_routes"}
 # HELP awoooi_public_route_unreachable_without_external_l1_fallback Public routes unreachable without external fallback proof.
 # TYPE awoooi_public_route_unreachable_without_external_l1_fallback gauge
 awoooi_public_route_unreachable_without_external_l1_fallback{scope="public_routes"} $public_unreachable_count
+# HELP awoooi_windows99_vmware_readback_present Whether Windows99 VMware verifier stdout was collected.
+# TYPE awoooi_windows99_vmware_readback_present gauge
+# HELP awoooi_windows99_vmware_verify_ready Whether Windows99 VMware autostart verifier is fully green.
+# TYPE awoooi_windows99_vmware_verify_ready gauge
+# HELP awoooi_windows99_vmware_windows_update_no_auto_reboot_ready Whether Windows Update no-auto-reboot policy is verified.
+# TYPE awoooi_windows99_vmware_windows_update_no_auto_reboot_ready gauge
+# HELP awoooi_windows99_vmware_missing_vmx_alias Missing required Windows99 VMX aliases.
+# TYPE awoooi_windows99_vmware_missing_vmx_alias gauge
+# HELP awoooi_windows99_vmware_powered_off_alias Required Windows99 VMware guests observed powered off.
+# TYPE awoooi_windows99_vmware_powered_off_alias gauge
 METRICS
+if [ -n "$windows99_vmware_metrics" ]; then
+  printf '%s\n' "$windows99_vmware_metrics" >>"$tmp_metric"
+fi
 if [ -n "$active_blocker_metrics" ]; then
   printf '%s\n' "$active_blocker_metrics" >>"$tmp_metric"
 fi

scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py

@@ -109,6 +109,8 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None:
     assert "scorecard_args+=(--windows99-vmware-file" in text
     assert "POST_REBOOT_READINESS_TIMEOUT_SECONDS" in text
     assert "POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1" in text
+    assert "awoooi_windows99_vmware_missing_vmx_alias" in text
+    assert "awoooi_windows99_vmware_powered_off_alias" in text
     assert "active_blocker_metrics" in text
     assert 'for blocker in payload.get("active_blockers") or []' in text
     assert "awoooi_reboot_auto_recovery_slo_active_blocker" in text