fix(agent99): expose verified recovery lifecycle

2026-07-11 20:19:21 +08:00
parent 244bad7259
commit 0a671bd46a
6 changed files with 203 additions and 33 deletions
--- a/agent99-control-plane.ps1
+++ b/agent99-control-plane.ps1
@@ -407,7 +407,43 @@ function Format-AgentTelegramText {
  $lines += "Agent99 事件卡 / Incident Card"
  $lines += "嚴重度 Severity: $severityLabel"

-  if ($EventType -match "^performance_") {
+  if ($EventType -eq "recovery_slo_result") {
+    $runId = [string](Get-AgentObjectValue $Data "runId" "unknown")
+    $scope = [string](Get-AgentObjectValue $Data "scope" "service_recovery")
+    $elapsedSeconds = [double](Get-AgentObjectValue $Data "elapsedSeconds" 0)
+    $targetMinutes = [int](Get-AgentObjectValue $Data "targetMinutes" 10)
+    $completed = Convert-AgentBool (Get-AgentObjectValue $Data "completed" $false)
+    $withinSlo = Convert-AgentBool (Get-AgentObjectValue $Data "withinSlo" $false)
+    $coordinatorVerified = Convert-AgentBool (Get-AgentObjectValue $Data "coordinatorVerified" $false)
+    $rebootSloClaimed = Convert-AgentBool (Get-AgentObjectValue $Data "rebootSloClaimed" $false)
+    $phases = if ($Data -and $Data.PSObject.Properties["phases"]) { @($Data.phases) } else { @() }
+    $failedPhases = @($phases | Where-Object { [string]$_.status -ne "ok" })
+    $statusLabel = if ($completed -and $withinSlo) {
+      "已恢復且符合目標 / recovered within target"
+    } elseif ($completed) {
+      "已恢復但超過目標 / recovered late"
+    } else {
+      "恢復未完成 / recovery incomplete"
+    }
+    $scopeLabel = if ($scope -eq "full_host_reboot") { "全主機重啟 / full-host reboot" } else { "服務恢復 / service recovery" }
+    $phaseSummary = @($phases | ForEach-Object {
+      "$([string]$_.name)=$([string]$_.status)@$([math]::Round([double]$_.elapsedSeconds, 1))s"
+    }) -join ", "
+    $lines += "事件 Event: 主機與服務恢復 / recovery lifecycle"
+    $lines += "結果 Result: $statusLabel"
+    $lines += "範圍 Scope: $scopeLabel"
+    $lines += "耗時 Duration: $elapsedSeconds 秒；目標 Target: $targetMinutes 分鐘"
+    $lines += "驗證 Verifier: coordinator=$coordinatorVerified, rebootSloClaimed=$rebootSloClaimed"
+    $lines += "階段 Phases: $(Limit-AgentTextLine $phaseSummary 500)"
+    $lines += "Run ID: $runId"
+    if ($failedPhases.Count -gt 0) {
+      $lines += "Agent99 動作 Action: 保留維護狀態並持續處理失敗階段：$(@($failedPhases | ForEach-Object { $_.name }) -join ', ')。"
+    } elseif ($scope -eq "service_recovery" -and -not $rebootSloClaimed) {
+      $lines += "Agent99 動作 Action: 服務 verifier 已通過；本輪不是 fresh reboot drill，不宣稱全主機重啟 SLA。"
+    } else {
+      $lines += "Agent99 動作 Action: verifier 已通過，完成事件閉環並保留 recurrence/KM receipt。"
+    }
+  } elseif ($EventType -match "^performance_") {
    $hostName = Get-AgentObjectValue $Data "host" "unknown"
    $reasons = if ($Data -and $Data.PSObject.Properties["reasons"]) { @($Data.reasons) } else { @() }
    $disk = Get-AgentObjectValue $Data "diskUsedPercent" $null
@@ -650,7 +686,10 @@ function New-AgentTelegramCardImage {
    $visualCardsEnabled = [bool]$Config.telegram.visualCardsEnabled
  }
  if (-not $visualCardsEnabled) { return $null }
-  $forceVisual = [bool]($EventType -match "^operator_command_" -and (Test-AgentSreOperatorResultAlertEnabled $Data))
+  $forceVisual = [bool](
+    $EventType -eq "recovery_slo_result" -or
+    ($EventType -match "^operator_command_" -and (Test-AgentSreOperatorResultAlertEnabled $Data))
+  )
  if (($Severity -notin @("warning", "critical")) -and -not $forceVisual) { return $null }

  $cardDir = Join-Path $EvidenceDir "telegram-cards"
@@ -660,8 +699,9 @@ function New-AgentTelegramCardImage {

  try {
    Add-Type -AssemblyName System.Drawing -ErrorAction Stop
-    $width = 1040
-    $height = 620
+    $isRecoveryCard = [bool]($EventType -eq "recovery_slo_result")
+    $width = if ($isRecoveryCard) { 1200 } else { 1040 }
+    $height = if ($isRecoveryCard) { 760 } else { 620 }
    $bitmap = New-Object System.Drawing.Bitmap -ArgumentList $width, $height
    $graphics = [System.Drawing.Graphics]::FromImage($bitmap)
    $graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias
@@ -688,18 +728,88 @@ function New-AgentTelegramCardImage {
    $graphics.FillRectangle($bgBrush, 0, 0, $width, $height)
    $graphics.FillRectangle($panelBrush, 36, 34, $width - 72, $height - 68)
    $graphics.FillRectangle($accentBrush, 36, 34, 14, $height - 68)
-    $graphics.DrawString("Agent99 事件卡 / Incident Card", $titleFont, $textBrush, 72, 58)
+    $cardTitle = if ($isRecoveryCard) { "Agent99 恢復狀態 / Recovery Status" } else { "Agent99 事件卡 / Incident Card" }
+    $graphics.DrawString($cardTitle, $titleFont, $textBrush, 72, 58)
    $graphics.DrawString(("嚴重度 Severity: " + $Severity.ToUpperInvariant()), $labelFont, $accentBrush, 72, 106)
    $graphics.DrawString(("事件 Event: " + (Limit-AgentTextLine $EventType 76)), $labelFont, $mutedBrush, 72, 134)

-    $bodyLines = @($Text -split "`n" | Where-Object { $_ -and $_ -notmatch "^Agent99 事件卡 / Incident Card$" } | Select-Object -First 12)
-    $y = 184
-    foreach ($line in $bodyLines) {
-      $graphics.DrawString((Limit-AgentTextLine $line 96), $bodyFont, $textBrush, 72, $y)
-      $y += 32
-      if ($y -gt 545) { break }
+    if ($isRecoveryCard) {
+      $completed = Convert-AgentBool (Get-AgentObjectValue $Data "completed" $false)
+      $withinSlo = Convert-AgentBool (Get-AgentObjectValue $Data "withinSlo" $false)
+      $scope = [string](Get-AgentObjectValue $Data "scope" "service_recovery")
+      $elapsedSeconds = [double](Get-AgentObjectValue $Data "elapsedSeconds" 0)
+      $targetMinutes = [int](Get-AgentObjectValue $Data "targetMinutes" 10)
+      $runId = [string](Get-AgentObjectValue $Data "runId" "unknown")
+      $coordinatorVerified = Convert-AgentBool (Get-AgentObjectValue $Data "coordinatorVerified" $false)
+      $statusText = if ($completed -and $withinSlo) {
+        "已恢復 / RECOVERED"
+      } elseif ($completed) {
+        "已恢復但逾時 / RECOVERED LATE"
+      } else {
+        "尚未恢復 / ACTION REQUIRED"
+      }
+      $statusColor = if ($completed -and $withinSlo) {
+        [System.Drawing.Color]::FromArgb(22, 101, 52)
+      } elseif ($completed) {
+        [System.Drawing.Color]::FromArgb(180, 83, 9)
+      } else {
+        [System.Drawing.Color]::FromArgb(190, 18, 60)
+      }
+      $statusBrush = New-Object System.Drawing.SolidBrush -ArgumentList $statusColor
+      $statusFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 23, ([System.Drawing.FontStyle]::Bold)
+      $scopeText = if ($scope -eq "full_host_reboot") { "全主機重啟" } else { "服務恢復" }
+      $graphics.DrawString($statusText, $statusFont, $statusBrush, 72, 176)
+      $graphics.DrawString(("範圍 " + $scopeText + "    耗時 " + $elapsedSeconds + " 秒    目標 " + $targetMinutes + " 分鐘"), $bodyFont, $textBrush, 72, 220)
+      $graphics.DrawString(("Verifier: coordinator=" + $coordinatorVerified + "    Run: " + (Limit-AgentTextLine $runId 72)), $smallFont, $mutedBrush, 72, 256)
+      $graphics.DrawString("恢復階段 / RECOVERY PHASES", $labelFont, $mutedBrush, 72, 296)
+
+      $phaseY = 330
+      $phaseIndex = 0
+      $phases = if ($Data -and $Data.PSObject.Properties["phases"]) { @($Data.phases) } else { @() }
+      foreach ($phase in @($phases | Select-Object -First 7)) {
+        $phaseIndex += 1
+        $phaseName = [string]$phase.name
+        $phaseLabel = switch ($phaseName) {
+          "vm_start" { "VMware 虛擬主機" }
+          "host_reachability" { "主機網路與 SSH" }
+          "host112_guest_readiness" { "Kali 112 Console 與安全服務" }
+          "harbor_registry" { "Harbor 與 Registry" }
+          "k3s_workloads" { "K3s 與 AWOOOI workloads" }
+          "public_routes" { "網站與公開路由" }
+          "full_sop_scorecard" { "Full-stack SOP scorecard" }
+          default { $phaseName }
+        }
+        $phaseOk = [bool]([string]$phase.status -eq "ok")
+        $phaseColor = if ($phaseOk) { [System.Drawing.Color]::FromArgb(22, 101, 52) } else { [System.Drawing.Color]::FromArgb(190, 18, 60) }
+        $phaseBrush = New-Object System.Drawing.SolidBrush -ArgumentList $phaseColor
+        $rowBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(244, 246, 248))
+        $graphics.FillRectangle($rowBrush, 72, $phaseY, $width - 144, 38)
+        $graphics.FillRectangle($phaseBrush, 72, $phaseY, 10, 38)
+        $graphics.DrawString(("$phaseIndex. " + (Limit-AgentTextLine $phaseLabel 48)), $bodyFont, $textBrush, 98, ($phaseY + 5))
+        $phaseState = if ($phaseOk) { "VERIFIED" } else { "FAILED" }
+        $phaseDuration = [math]::Round([double]$phase.elapsedSeconds, 1)
+        $graphics.DrawString(("$phaseState   ${phaseDuration}s"), $labelFont, $phaseBrush, ($width - 300), ($phaseY + 8))
+        $phaseBrush.Dispose()
+        $rowBrush.Dispose()
+        $phaseY += 46
+      }
+      if ($phases.Count -eq 0) {
+        $graphics.DrawString("尚無 phase evidence，事件不可視為完成。", $bodyFont, $accentBrush, 72, 338)
+      }
+      $graphics.DrawString("同一 run 保留 Detect -> Apply -> Verify -> Notify receipt", $smallFont, $mutedBrush, 72, 706)
+      $statusBrush.Dispose()
+      $statusFont.Dispose()
+    } else {
+      $bodyLines = @($Text -split "`n" | Where-Object { $_ -and $_ -notmatch "^Agent99 事件卡 / Incident Card$" } | Select-Object -First 12)
+      $y = 184
+      foreach ($line in $bodyLines) {
+        $graphics.DrawString((Limit-AgentTextLine $line 96), $bodyFont, $textBrush, 72, $y)
+        $y += 32
+        if ($y -gt 545) { break }
+      }
    }
-    $graphics.DrawString(("產生時間 Generated: " + (Get-Date -Format "yyyy-MM-dd HH:mm:ss zzz")), $smallFont, $mutedBrush, 72, 570)
+    $footerY = if ($isRecoveryCard) { 724 } else { 570 }
+    $graphics.DrawString(("產生時間 Generated: " + (Get-Date -Format "yyyy-MM-dd HH:mm:ss zzz")), $smallFont, $mutedBrush, 72, $footerY)
    $bitmap.Save($imagePath, [System.Drawing.Imaging.ImageFormat]::Png)

    $graphics.Dispose()
@@ -981,7 +1091,7 @@ function Send-AgentTelegram {
 	    enabled = [bool]$enabled
 	    eventType = $EventType
 	    severity = $Severity
-	    messageFormat = "incident_card_zh_tw_v3"
+	    messageFormat = "incident_card_zh_tw_v4"
 	    configured = [bool]($token -and $chatId)
 	    tokenEnv = if ($tokenSecret) { $tokenSecret.name } else { $null }
 	    chatIdEnv = if ($chatSecret) { $chatSecret.name } else { $null }
@@ -1158,7 +1268,8 @@ function Record-AgentEvent {
      Write-AgentLog "telegram_sre_operator_success_enabled type=$EventType mode=$operatorMode id=$operatorId alertId=$operatorAlertId"
    }
  }
-  $shouldAlert = ($Severity -in @("warning", "critical")) -or $operatorCommandAlert -or ($Alert -and $Severity -ne "info") -or ($alertAll -and ($Severity -ne "info" -or $alertInfo))
+  $lifecycleInfoAlert = [bool]($Alert -and $Severity -eq "info" -and $EventType -in @("recovery_slo_result"))
+  $shouldAlert = ($Severity -in @("warning", "critical")) -or $operatorCommandAlert -or $lifecycleInfoAlert -or ($Alert -and $Severity -ne "info") -or ($alertAll -and ($Severity -ne "info" -or $alertInfo))
  if ($shouldAlert) {
    $dedupeMinutes = 30
    if ($Config.telegram -and $Config.telegram.PSObject.Properties["dedupeMinutes"]) {
@@ -1166,7 +1277,7 @@ function Record-AgentEvent {
    }
    $dedupeParts = @($EventType, $Severity)
    if ($Data) {
-      foreach ($propName in @("host", "name", "path", "reason", "target", "status", "mode", "alertKind", "source")) {
+      foreach ($propName in @("host", "name", "path", "reason", "target", "status", "mode", "alertKind", "source", "scope", "completed", "withinSlo", "rebootSloClaimed")) {
        if ($Data.PSObject.Properties[$propName]) {
          $dedupeParts += "$propName=$($Data.PSObject.Properties[$propName].Value)"
        }
@@ -1196,6 +1307,17 @@ function Record-AgentEvent {
      $ageMinutes = ((Get-Date) - (Get-Item $dedupePath).LastWriteTime).TotalMinutes
      if ($ageMinutes -lt $dedupeMinutes) {
        Write-AgentLog "telegram_dedupe_skip type=$EventType severity=$Severity ageMinutes=$([math]::Round($ageMinutes, 2)) cooldown=$dedupeMinutes"
+        $script:TelegramAttempts += [pscustomobject]@{
+          timestamp = (Get-Date -Format o)
+          eventType = $EventType
+          severity = $Severity
+          messageFormat = "incident_card_zh_tw_v4"
+          sent = $false
+          suppressed = $true
+          reason = "dedupe_window"
+          dedupeAgeMinutes = [math]::Round($ageMinutes, 2)
+          dedupeMinutes = $dedupeMinutes
+        }
        return
      }
    }
--- a/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py
+++ b/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py
@@ -31,7 +31,7 @@ def test_agent99_enterprise_work_items_loader_returns_complete_scope() -> None:
        "agent99_enterprise_ai_automation_work_items_v1"
    )
    assert payload["scope_complete"] is False
-    assert payload["current_p0"]["id"] == "AG99-P0-007"
+    assert payload["current_p0"]["id"] == "AG99-P0-001"
    assert payload["summary"] == {
        "host_count": 7,
        "product_count": 12,
@@ -40,21 +40,27 @@ def test_agent99_enterprise_work_items_loader_returns_complete_scope() -> None:
        "p0_count": 14,
        "p1_count": 8,
        "p2_count": 3,
-        "in_progress_count": 7,
+        "in_progress_count": 9,
        "in_progress_blocked_count": 0,
-        "planned_count": 16,
-        "complete_count": 2,
+        "planned_count": 15,
+        "complete_count": 1,
        "current_p0_count": 1,
    }
-    assert payload["next_execution_order"][0] == "AG99-P0-007"
-    assert payload["next_execution_order"][-1] == "AG99-P0-014"
+    assert payload["next_execution_order"][:5] == [
+        "AG99-P0-001",
+        "AG99-P0-006",
+        "AG99-P0-003",
+        "AG99-P0-004",
+        "AG99-P0-005",
+    ]
+    assert payload["next_execution_order"][-1] == "AG99-P0-011"


 def test_agent99_enterprise_projection_is_bounded() -> None:
    payload = load_latest_agent99_enterprise_ai_automation_work_items()
    projection = build_agent99_enterprise_priority_projection(payload)

-    assert projection["current_p0"]["id"] == "AG99-P0-007"
+    assert projection["current_p0"]["id"] == "AG99-P0-001"
    assert projection["coverage"] == {
        "host_count": 7,
        "product_count": 12,
@@ -73,7 +79,7 @@ def test_priority_readback_projects_agent99_enterprise_order() -> None:
    payload = load_latest_awoooi_priority_work_order_readback()
    projection = payload["agent99_enterprise_ai_automation"]

-    assert projection["current_p0"]["id"] == "AG99-P0-007"
+    assert projection["current_p0"]["id"] == "AG99-P0-001"
    assert projection["work_item_counts"]["p0"] == 14
    assert (
        payload["source_refs"]["agent99_enterprise_ai_automation_work_items_api"]
@@ -118,7 +124,7 @@ def test_agent99_enterprise_work_items_endpoint_returns_snapshot() -> None:

    assert response.status_code == 200
    payload = response.json()
-    assert payload["current_p0"]["id"] == "AG99-P0-007"
+    assert payload["current_p0"]["id"] == "AG99-P0-001"
    assert payload["summary"]["host_count"] == 7
    assert payload["summary"]["product_count"] == 12
    assert payload["summary"]["public_surface_count"] == 23
--- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py
+++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py
@@ -43,6 +43,30 @@ def test_agent99_auto_recovery_is_single_flight_and_slo_measured() -> None:
    assert '$host112Recovery -and $host112Recovery.verified' in source


+def test_agent99_recovery_completion_has_human_lifecycle_receipt() -> None:
+    source = CONTROL.read_text(encoding="utf-8")
+
+    assert '$EventType -eq "recovery_slo_result"' in source
+    assert '主機與服務恢復 / recovery lifecycle' in source
+    assert '已恢復且符合目標 / recovered within target' in source
+    assert '$lifecycleInfoAlert' in source
+    assert '$EventType -in @("recovery_slo_result")' in source
+    assert 'Agent99 恢復狀態 / Recovery Status' in source
+    assert '恢復階段 / RECOVERY PHASES' in source
+    assert 'Detect -> Apply -> Verify -> Notify receipt' in source
+
+
+def test_agent99_records_deduped_telegram_as_an_explicit_attempt() -> None:
+    source = CONTROL.read_text(encoding="utf-8")
+    dedupe = source[source.index('if ($dedupeMinutes -gt 0 -and (Test-Path $dedupePath))') :]
+    dedupe = dedupe[: dedupe.index('Set-Content -Path $dedupePath')]
+
+    assert '$script:TelegramAttempts +=' in dedupe
+    assert 'suppressed = $true' in dedupe
+    assert 'reason = "dedupe_window"' in dedupe
+    assert 'dedupeAgeMinutes' in dedupe
+
+
 def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> None:
    source = CONTROL.read_text(encoding="utf-8")
    function = source[source.index("function Convert-AgentHost112GuestReadback") :]
--- a/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md
+++ b/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md
@@ -1,6 +1,6 @@
 # Agent99 全域 AI 自動化主計畫

-> 建立時間：2026-07-10 15:05 CST；最近更新：2026-07-11 17:40 CST
+> 建立時間：2026-07-10 15:05 CST；最近更新：2026-07-11 20:14 CST
 > 狀態：P0 執行中，尚未達成全域 AI 自動化
 > 機器可讀總帳：`docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json`
 > 歷史需求來源：`docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md`
@@ -97,13 +97,13 @@

 ## 5. 更新後的唯一執行順序

-2026-07-11 事故優先序覆寫後，機器可讀執行順序固定為：`AG99-P0-007 → P0-008 → P0-009 → P0-010 → P0-003 → P0-004 → P0-005 → P0-006 → P0-011 → P0-012 → P0-013 → P0-014`。下表的數字是穩定 work-item ID，不再被誤當目前執行順位；`next_execution_order` 才是 runtime projection 的唯一順位來源，且必須包含每個未完成 P0 恰好一次。
+2026-07-11 20:14 live audit 重新校正後，機器可讀執行順序固定為：`AG99-P0-001 → P0-006 → P0-003 → P0-004 → P0-005 → P0-007 → P0-008 → P0-009 → P0-013 → P0-012 → P0-014 → P0-010 → P0-011`。先關閉 Agent99 假綠與看不見的執行結果，再擴充 ingress、evidence/candidate、verifier 與各 domain 自動化。下表的數字是穩定 work-item ID，不再被誤當目前執行順位；`next_execution_order` 才是 runtime projection 的唯一順位來源，且必須包含每個未完成 P0 恰好一次。

 ### Wave 0：先修正 AI 決策真相

 | 順序 | ID | 工作 | 狀態 | 驗收 |
 | --- | --- | --- | --- | --- |
-| 1 | AG99-P0-001 | 修正 `exitCode=0 => resolved` 假綠，對齊 99 runtime script 與 Gitea source SHA | **完成** | Production 99 outcome self-test 通過；runtime manifest `7/7` matched、drift `0`；operator replay 為 verified resolved，alert replay 缺 source-event resolved 時維持 verifying |
+| 1 | AG99-P0-001 | 修正 `exitCode=0 => resolved`、evidence parse 假綠，對齊 99 runtime script 與 source SHA | **重新開啟；current P0** | Production 99 outcome self-test 通過；required evidence 必須 fresh 且可解析；Recover 必須有 sent 或明確 suppressed receipt；runtime manifest matched、drift `0` |
 | 2 | AG99-P0-002 | 修正 alert taxonomy、wrong-domain routing、bridge-before-grouping、actionable Telegram card 漏接與重複事件 | **完成；99 runtime `34e32f6e`、14/14 matched** | Alertmanager、AI card、ACTION REQUIRED、CI/CD failure 共用 dispatcher；`suggestedMode`/kind 優先；5 分鐘先 group；同 fingerprint 只建立一個 active incident；保留 acceptance receipt |
 | 3 | AG99-P0-003 | 強化 ingress：relay token 必填、Telegram chat+sender allowlist、idempotency/single consumer | 待辦 | 未認證 LAN request 拒絕；producer `auto_repair`/policy 被保留；所有 request 可追 trace |
 | 4 | AG99-P0-004 | 新增 EvidenceCollect / RepairCandidate / MCP lane | 待辦 | 缺 evidence 的事件會建立可查 candidate，不再錯送 Perf/AwoooRepair；`INC-*` 可從收件追到 candidate |
@@ -113,8 +113,8 @@
 | 順序 | ID | 工作 | 狀態 | 驗收 |
 | --- | --- | --- | --- | --- |
 | 5 | AG99-P0-005 | 模式專屬 verifier、真 rollback、cooldown/circuit-breaker | 待辦 | Recover/Perf/LoadShed/Harbor/AWOOOI/backup/provider 各有 post-condition；失敗不寫成功 cooldown |
-| 6 | AG99-P0-006 | Agent99 → AWOOOI/AwoooP/IWOOOS completion callback | 待辦 | incident/Run/Work Item/Verifier/recurrence 同一 trace 收斂；Telegram 可讀回報只從結果物件產生 |
-| 7 | AG99-P0-007 | 全主機 10 分鐘 reboot state machine | **進行中，current P0；本輪 breach 後已 late recovery，L1/L2 runtime 已整合，L0/600 秒重測待完成** | L0 off-LAN detection/OOB power、L1 Agent99/VMware、L2 110 full-SOP 必須以同一 incident 閉環；首可用/全綠時間與 blocker/ETA 可查 |
+| 6 | AG99-P0-006 | Agent99 → AWOOOI/AwoooP/IWOOOS completion callback | **進行中；先關閉 Recover 可見生命週期 receipt** | incident/Run/Work Item/Verifier/recurrence 同一 trace 收斂；Telegram 可讀回報只從結果物件產生 |
+| 7 | AG99-P0-007 | 全主機 10 分鐘 reboot state machine | **進行中；本輪 breach 後已 late recovery，L1/L2 runtime 已整合，L0/600 秒重測待完成** | L0 off-LAN detection/OOB power、L1 Agent99/VMware、L2 110 full-SOP 必須以同一 incident 閉環；首可用/全綠時間與 blocker/ETA 可查 |
 | 8 | AG99-P0-008 | 99 VMware/VMX/autostart/lock 與 Windows Update no-auto-reboot | **本次 boot runtime green；controlled apply/rollback/verifier 完成，待下次 reboot 驗證 startup 時序** | 110/188/112/120/121 VMX、autostart task、guest running 與 no-auto-reboot verifier green；111 由實體 Mac readback 驗證 |

 ### Wave 2：擴至所有產品與運維面
@@ -191,7 +191,14 @@
 - Windows 99 runtime 已部署 `442b319bb80e009291ced7afa4f704cbe3b64886`：manifest `14/14`、mismatch `0`、task `9/9`、SRE relay listening、Startup-Recovery `PT20M`。VMware controlled receipt `windows99-vmware-autostart-apply-20260711-173505.json` 回報 pre/post VMX process `5/5`、五 guests running、process-aware duplicate-start guard true、沒有 reboot/VM power change/secret read。
 - 110 artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-172623` 顯示七主機 reachable、service/data/freshness/backup/188 hygiene/edge/VMware 全綠；只保留本輪時間窗三個 blocker。production-principal `agent99-Recover-20260711-173543.json` 在 `10.3s` 完成 service recovery、coordinator verified，且 `rebootSloClaimed=false`。
 - 全部 23 個 public surfaces 已從 110 驗證為預期 2xx/3xx，SignOz bridge 也已受控恢復。這證明「目前已恢復」與「Agent99 已整合 full SOP」，不證明下一次全主機重啟會在 600 秒內完成。
- L0 external supervisor、可驗證 OOB power 與外部 maintenance origin 仍只到 source contract，未有 production runtime receipt。current P0 因此仍是 `AG99-P0-007`；下一次 drill 只有在 L0/L1/L2 使用同一 incident 且 600 秒內 blocker=0 才能 terminal complete。
+- L0 external supervisor、可驗證 OOB power 與外部 maintenance origin 仍只到 source contract，未有 production runtime receipt。17:40 當時的 current P0 因此是 `AG99-P0-007`；下一次 drill 只有在 L0/L1/L2 使用同一 incident 且 600 秒內 blocker=0 才能 terminal complete。
+
+### 2026-07-11 20:14 Agent99 可見閉環 live audit
+
+- Windows 99 runtime revision `d8c7a81a2a03e72f61116212354657f5e38e00a2` 為 `14/14` matched，九個必要排程皆 healthy、relay listener=1、五台 VMware guests 與五台服務主機皆可達，Harbor/AWOOOI/五個 public checks 與 backup health 皆綠。Agent99 並非完全沒有工作。
+- 最新 production-principal Recover 在 10.9 秒內完成 service recovery，七個 phase 與 full-SOP coordinator verifier 都有 evidence；但 `telegramAttemptCount=0`，所以 operator 看不到「Agent99 做了什麼、是否完成」。
+- 同一 preflight 對最新 SRE/Telegram inbox safe summary 顯示 `RuntimeException`，卻仍因檔案時間新而回傳綠燈。這是明確 false-green regression，不可用更多 dashboard 掩蓋。
+- current P0 因此重新定位為 `AG99-P0-001`，第二順位是 `AG99-P0-006`：先把 required evidence 改成 fresh + parseable、Recover 成功事件形成繁中生命週期卡與 phase timeline、dedupe 也留下 receipt；驗證後才回到 secure ingress、candidate、verifier 與 reboot/各 domain 主線。

 ## 7. 工作項強制欄位與治理

@@ -208,4 +215,4 @@

 ## 8. 立即下一步

-目前唯一 P0 仍是 `AG99-P0-007`，不另拆支線。L1 Agent99/VMware 與 L2 110 full-SOP 已有 runtime receipts；下一步只收斂剩餘 L0：把 external supervisor 部署到 99/110/188/LAN 之外，接上可驗證 OOB power、外部 maintenance、durable incident 與 Agent99 resume callback，再用同一 incident 做 600 秒 E2E 重測。之後依序前進：`P0-008 → P0-009 → P0-010 → P0-003 → P0-004 → P0-005 → P0-006 → P0-011 → P0-012 → P0-013 → P0-014`。main push 只在唯一 writer slot 進行；沒有 slot 時保留 local commit，不讓推送協調打斷 runtime 主線。
+目前唯一 P0 是 `AG99-P0-001`。立即動作是把 parse-aware preflight、Recover 生命週期卡、phase timeline 與 dedupe receipt 原子部署到 99，再由 production principal 跑一次 Recover；只有 runtime manifest matched、preflight parse-clean、Telegram sent 或 dedupe-suppressed receipt 明確存在才前進。後續固定依序：`P0-006 → P0-003 → P0-004 → P0-005 → P0-007 → P0-008 → P0-009 → P0-013 → P0-012 → P0-014 → P0-010 → P0-011`。main push 只在唯一 writer slot 進行；沒有 slot 時保留 local commit，不讓推送協調打斷 runtime 主線。
--- a/scripts/reboot-recovery/agent99-live-preflight.ps1
+++ b/scripts/reboot-recovery/agent99-live-preflight.ps1
@@ -86,6 +86,7 @@ function Get-EvidenceReadback {
  $fresh = [bool]($file -and $ageMinutes -le $MaxAgeMinutes)
  $safeSummary = $null
  $parseError = ""
+  $contentHealthy = $true
  if ($file) {
    try {
      $rawEvidence = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json
@@ -180,13 +181,17 @@ function Get-EvidenceReadback {
        } else { $null }
        }
      }
+      if ($Name -in @("sre_alert_inbox", "telegram_inbox")) {
+        $contentHealthy = [bool]($safeSummary -and $safeSummary.ok)
+      }
    } catch {
      $parseError = $_.Exception.GetType().Name
+      $contentHealthy = $false
    }
  }

  $parsed = [bool]($file -and -not $parseError)
-  $usable = [bool]($fresh -and $parsed)
+  $usable = [bool]($fresh -and $parsed -and $contentHealthy)
  return [pscustomobject]@{
    name = $Name
    pattern = $Pattern
@@ -197,6 +202,7 @@ function Get-EvidenceReadback {
    maxAgeMinutes = $MaxAgeMinutes
    fresh = $fresh
    parsed = $parsed
+    contentHealthy = $contentHealthy
    usable = $usable
    healthy = [bool](-not $Required -or $usable)
    parseError = $parseError
@@ -283,6 +289,7 @@ $taskFailures = @($tasks | Where-Object { -not $_.healthy })
 $requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.healthy })
 $requiredEvidenceStale = @($evidence | Where-Object { $_.required -and -not $_.fresh })
 $requiredEvidenceParseFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and -not $_.parsed })
+$requiredEvidenceContentFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and $_.parsed -and -not $_.contentHealthy })
 $selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1
 $selfHealthSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthSeverity } else { "unknown" }
 $alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1
@@ -296,6 +303,7 @@ if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy"
 if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" }
 if ($requiredEvidenceStale.Count -gt 0) { $blockingReasons += "required_evidence_stale" }
 if ($requiredEvidenceParseFailures.Count -gt 0) { $blockingReasons += "required_evidence_parse_failed" }
+if ($requiredEvidenceContentFailures.Count -gt 0) { $blockingReasons += "required_evidence_content_unhealthy" }
 if ($selfHealthSeverity -ne "ok") { $blockingReasons += "self_health_not_ok" }
 if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" }
 if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" }
@@ -322,6 +330,7 @@ $result = [pscustomobject]@{
    usableRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.usable }).Count
    requiredEvidenceFailureCount = $requiredEvidenceFailures.Count
    requiredEvidenceParseFailureCount = $requiredEvidenceParseFailures.Count
+    requiredEvidenceContentFailureCount = $requiredEvidenceContentFailures.Count
    selfHealthSeverity = $selfHealthSeverity
    alertIncomingCount = [int]$alertIncoming.count
    alertRunningCount = [int]$alertRunning.count
--- a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py
+++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py
@@ -122,11 +122,13 @@ def test_live_preflight_rejects_fresh_but_unparseable_required_evidence() -> Non
    preflight = read("scripts/reboot-recovery/agent99-live-preflight.ps1")

    assert '$parsed = [bool]($file -and -not $parseError)' in preflight
-    assert '$usable = [bool]($fresh -and $parsed)' in preflight
+    assert '$usable = [bool]($fresh -and $parsed -and $contentHealthy)' in preflight
    assert 'healthy = [bool](-not $Required -or $usable)' in preflight
    assert '$_.required -and -not $_.healthy' in preflight
    assert 'required_evidence_parse_failed' in preflight
    assert 'requiredEvidenceParseFailureCount' in preflight
+    assert 'required_evidence_content_unhealthy' in preflight
+    assert 'requiredEvidenceContentFailureCount' in preflight


 def test_live_preflight_adapts_inbox_evidence_without_raw_message_readback() -> None: