#!/usr/bin/env bash
# Read-only audit for AWOOOI reboot-recovery readiness artifacts.

set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd "$ROOT_DIR"

RUN_LIVE=0
NO_COLOR=0

for arg in "$@"; do
  case "$arg" in
    --live)
      RUN_LIVE=1
      ;;
    --no-color)
      NO_COLOR=1
      ;;
    -h|--help)
      cat <<'USAGE'
Usage: bash scripts/reboot-recovery/reboot-recovery-readiness-audit.sh [--live] [--no-color]

Checks repo-side SOP/script/Ansible/alert/CI readiness. With --live, also runs
the read-only full-stack cold-start gate.
USAGE
      exit 0
      ;;
    *)
      echo "Unknown argument: $arg" >&2
      exit 2
      ;;
  esac
done

if [ "$NO_COLOR" = "1" ]; then
  green=""
  yellow=""
  red=""
  reset=""
else
  green="$(printf '\033[32m')"
  yellow="$(printf '\033[33m')"
  red="$(printf '\033[31m')"
  reset="$(printf '\033[0m')"
fi

pass=0
warn=0
fail=0

ok() {
  pass=$((pass + 1))
  printf "%sOK%s %s\n" "$green" "$reset" "$*"
}

warning() {
  warn=$((warn + 1))
  printf "%sWARN%s %s\n" "$yellow" "$reset" "$*"
}

blocked() {
  fail=$((fail + 1))
  printf "%sBLOCKED%s %s\n" "$red" "$reset" "$*"
}

require_file() {
  local path="$1"
  local label="$2"
  if [ -f "$path" ]; then
    ok "$label exists: $path"
  else
    blocked "$label missing: $path"
  fi
}

require_dir() {
  local path="$1"
  local label="$2"
  if [ -d "$path" ]; then
    ok "$label exists: $path"
  else
    blocked "$label missing: $path"
  fi
}

require_pattern() {
  local pattern="$1"
  local path="$2"
  local label="$3"
  if rg -q "$pattern" "$path"; then
    ok "$label present in $path"
  else
    blocked "$label missing in $path"
  fi
}

forbid_pattern() {
  local pattern="$1"
  local path="$2"
  local label="$3"
  if rg -q "$pattern" "$path"; then
    blocked "$label forbidden pattern present in $path"
  else
    ok "$label forbidden pattern absent in $path"
  fi
}

run_with_retries() {
  local attempts="$1"
  local interval="$2"
  shift 2
  local attempt=1
  while true; do
    if "$@"; then
      return 0
    fi
    if [ "$attempt" -ge "$attempts" ]; then
      return 1
    fi
    sleep "$interval"
    attempt=$((attempt + 1))
  done
}

echo "AWOOOI reboot-recovery readiness audit"
date
echo

echo "== SOP and baseline =="
require_file docs/runbooks/FULL-STACK-COLD-START-SOP.md "Full-stack cold-start SOP"
require_file docs/runbooks/REBOOT-RECOVERY-SOP.md "Legacy reboot recovery SOP"
require_file docs/runbooks/HOST-RESOURCE-BASELINE-110-188.md "110/188 resource baseline"
require_file docs/runbooks/ANSIBLE-OPERATING-MODEL.md "Ansible operating model"
require_file docs/runbooks/OFFSITE-BACKUP-ESCROW-RUNBOOK.md "Offsite backup and credential escrow runbook"
require_file ops/reboot-recovery/full-stack-cold-start-baseline.yml "Machine-readable baseline"
require_file ops/reboot-recovery/full-stack-backup-baseline.yml "Machine-readable backup baseline"
require_pattern "P0-NETWORK" ops/reboot-recovery/full-stack-cold-start-baseline.yml "P0-NETWORK phase"
require_pattern "P2-SCHEDULES" ops/reboot-recovery/full-stack-cold-start-baseline.yml "P2-SCHEDULES phase"
require_pattern "P3-RUNNER-CD" ops/reboot-recovery/full-stack-cold-start-baseline.yml "P3-RUNNER-CD phase"
require_pattern "host_service_config_backup_success_under_48h" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Config backup gate"
require_pattern "backup_domains" ops/reboot-recovery/full-stack-backup-baseline.yml "Backup domain inventory"
require_pattern "credential_escrow" ops/reboot-recovery/full-stack-backup-baseline.yml "Credential escrow backup domain"
require_pattern "external_dns_and_public_routes" ops/reboot-recovery/full-stack-backup-baseline.yml "External DNS/public route backup domain"
require_pattern "backup_repositories_and_integrity" ops/reboot-recovery/full-stack-backup-baseline.yml "Backup repository integrity domain"
require_pattern "source_of_truth_and_ops_memory" ops/reboot-recovery/full-stack-backup-baseline.yml "Source-of-truth and ops memory backup domain"
require_pattern "live_visibility_checks" ops/reboot-recovery/full-stack-backup-baseline.yml "Backup live alert visibility contract"
require_pattern "dr_offsite_scorecard" ops/reboot-recovery/full-stack-backup-baseline.yml "Strict DR offsite scorecard gate"
require_pattern "dr_offsite_operator_checklist" ops/reboot-recovery/full-stack-backup-baseline.yml "DR offsite operator checklist gate"
require_pattern "strict_dr_exit_conditions" ops/reboot-recovery/full-stack-backup-baseline.yml "Strict DR exit conditions"

echo
echo "== Scripts =="
require_file scripts/reboot-recovery/full-stack-cold-start-check.sh "Authoritative cold-start gate"
require_file scripts/reboot-recovery/full-stack-recovery-scorecard.sh "Full-stack recovery scorecard"
require_file scripts/reboot-recovery/120-fsck-maintenance-checklist.sh "120 filesystem maintenance checklist"
require_file scripts/reboot-recovery/dr-offsite-operator-checklist.sh "DR offsite operator checklist"
require_file scripts/reboot-recovery/wait-dr-offsite-ready.sh "DR offsite post-marker convergence waiter"
require_file scripts/reboot-recovery/cold-start-textfile-exporter.sh "Cold-start textfile exporter"
require_pattern "NODE_FS_ERROR_EVENTS" scripts/reboot-recovery/full-stack-cold-start-check.sh "K3s node filesystem event gate"
require_pattern "CHECK_WATCH_MAX_ATTEMPTS" scripts/reboot-recovery/cold-start-textfile-exporter.sh "Cold-start textfile exporter retries transient route failures"
require_pattern "max-attempts" scripts/reboot-recovery/cold-start-textfile-exporter.sh "Cold-start textfile exporter uses watch mode"
require_pattern "awoooi_cold_start_blocker_reason" scripts/reboot-recovery/cold-start-textfile-exporter.sh "Cold-start blocker reason metric"
require_pattern "host_unreachable" scripts/reboot-recovery/cold-start-textfile-exporter.sh "Cold-start host unreachable blocker reason"
require_pattern "Do not run fsck against the mounted root filesystem" scripts/reboot-recovery/120-fsck-maintenance-checklist.sh "120 fsck checklist online fsck prohibition"
require_file scripts/reboot-recovery/install-cold-start-monitor-110.sh "110 monitor installer"
require_file scripts/reboot-recovery/reboot-recovery-readiness-audit.sh "Readiness audit"
require_file scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh "Cold-start monitor deploy parity check"
require_file scripts/reboot-recovery/p3-controlled-release-gate.sh "P3 controlled release gate"
require_file scripts/ops/ansible-validate.sh "Ansible validation script"
require_file scripts/ops/bootstrap-ansible-validation-env.sh "Ansible validation venv bootstrap script"
require_file scripts/ops/doc-secrets-sanity-check.py "Documentation secrets sanity check"
require_file scripts/ops/docker-stats-textfile-exporter.py "Docker stats textfile exporter"
require_file scripts/ops/systemd-units-textfile-exporter.py "Systemd units textfile exporter"
require_file scripts/ops/storage-health-textfile-exporter.py "Storage health textfile exporter"
require_file scripts/ops/backup-health-textfile-exporter.py "Backup health textfile exporter"
require_file scripts/ops/backup-alert-label-contract-check.py "Backup alert label contract check"
require_file scripts/ops/backup-alert-live-visibility-check.py "Backup alert live visibility check"
require_file scripts/ops/recovery-scorecard-contract-check.py "Recovery scorecard contract check"
require_file scripts/backup/common.sh "Backup shared library"
require_file scripts/backup/backup-all.sh "Full backup orchestrator"
require_file scripts/backup/backup-status.sh "Daily backup Telegram heartbeat"
require_file scripts/backup/backup-gitea.sh "Gitea backup script"
require_file scripts/backup/backup-harbor.sh "Harbor backup script"
require_file scripts/backup/backup-momo.sh "110 momo restic backup script"
require_file scripts/backup/backup-awoooi.sh "AWOOOI daily DB backup"
require_file scripts/backup/backup-awoooi-frequent.sh "AWOOOI high-frequency DB backup"
require_file scripts/backup/backup-langfuse.sh "Langfuse backup script"
require_file scripts/backup/backup-monitoring.sh "Monitoring backup script"
require_file scripts/backup/backup-signoz.sh "SignOz backup script"
require_file scripts/backup/backup-open-webui.sh "Open-WebUI backup script"
require_file scripts/backup/backup-clawbot.sh "ClawBot backup script"
require_file scripts/backup/backup-configs.sh "Host/service config backup"
require_file scripts/backup/backup-momo-188-pg.sh "188 momo PostgreSQL backup script"
require_file scripts/backup/backup-sentry.sh "Sentry dedicated data backup"
require_file scripts/backup/backup-ai-artifacts.sh "AI artifacts and Ollama manifest backup"
require_file scripts/backup/backup-public-routes.sh "Public routes DNS/TLS evidence backup"
require_file scripts/backup/configure-offsite-rclone.sh "Offsite Google Drive/rclone host-local config helper"
require_pattern "create-root-remote" scripts/backup/configure-offsite-rclone.sh "Offsite Google Drive root-scoped remote helper"
require_pattern "gdrive_awoooi_restic" docs/runbooks/OFFSITE-BACKUP-ESCROW-RUNBOOK.md "Offsite Google Drive root-scoped remote runbook"
require_file scripts/backup/configure-offsite-b2.sh "Offsite B2 legacy config helper"
require_file scripts/backup/sync-offsite-backups.sh "Offsite backup copy controller"
require_file scripts/backup/backup-offsite-readiness-gate.sh "Offsite backup readiness gate"
require_file scripts/backup/offsite-escrow-evidence-report.sh "Offsite and escrow evidence report"
require_file scripts/backup/verify-offsite-full-sync.sh "Offsite full sync remote verifier"
require_file scripts/backup/mark-credential-escrow-verified.sh "Credential escrow verification marker helper"
require_pattern "missing-commands" scripts/backup/mark-credential-escrow-verified.sh "Credential escrow missing command template"
require_pattern "dry-run" scripts/backup/mark-credential-escrow-verified.sh "Credential escrow marker dry-run validation"
require_pattern "placeholder" scripts/backup/mark-credential-escrow-verified.sh "Credential escrow placeholder rejection"
require_pattern "BACKUP_COMMON_QUIET" scripts/backup/mark-credential-escrow-verified.sh "Credential escrow command template stays quiet"
require_pattern "TEXTFILE_REFRESHED" scripts/backup/mark-credential-escrow-verified.sh "Credential escrow marker refreshes backup health textfile"
require_pattern "credential escrow missing command template" scripts/backup/offsite-escrow-evidence-report.sh "Credential escrow report command template"
require_pattern "dr-offsite-operator-checklist.sh --require-dr" scripts/reboot-recovery/wait-dr-offsite-ready.sh "DR offsite waiter final checklist gate"
require_pattern "backup-alert-live-visibility-check.py" scripts/reboot-recovery/wait-dr-offsite-ready.sh "DR offsite waiter alert visibility gate"
require_pattern "recovery-scorecard-contract-check.py" scripts/reboot-recovery/wait-dr-offsite-ready.sh "DR offsite waiter Prometheus recording-rule gate"
require_file scripts/backup/check-backup-integrity.sh "Backup integrity and restore drill script"
require_file scripts/backup/enforce-latest-only-retention.sh "Latest-only backup retention enforcer"
require_file scripts/ops/backup-from-110.sh "188 backup-from-110 script"
require_file scripts/cron_backup_restore_test.sh "Velero restore dry-run script"
forbid_pattern "^[[:space:]]*rclone[[:space:]]+sync" scripts/backup/backup-gitea.sh "Gitea backup direct rclone sync"
forbid_pattern "^[[:space:]]*rclone[[:space:]]+sync" scripts/backup/backup-harbor.sh "Harbor backup direct rclone sync"
forbid_pattern "^[[:space:]]*rclone[[:space:]]+sync" scripts/backup/backup-awoooi.sh "AWOOOI backup direct rclone sync"

echo
echo "== Ansible =="
require_dir infra/ansible/roles/cold-start-monitor "cold-start-monitor role"
require_dir infra/ansible/roles/runner-guardrails "runner-guardrails role"
require_dir infra/ansible/roles/host-textfile-exporters "host-textfile-exporters role"
require_file infra/ansible/roles/nginx/templates/188-internal-tools-https.conf.j2 "188 internal tools HTTPS template"
require_pattern "cold_start_monitor" infra/ansible/playbooks/110-devops.yml "110 cold-start monitor tag"
require_pattern "runner_guardrails" infra/ansible/playbooks/110-devops.yml "110 runner guardrails tag"
require_pattern "textfile_exporters" infra/ansible/playbooks/110-devops.yml "110 textfile exporters tag"
require_pattern "backup_jobs" infra/ansible/playbooks/110-devops.yml "110 backup jobs tag"
require_pattern "common.sh" infra/ansible/playbooks/110-devops.yml "110 backup shared library deploy"
require_pattern "backup-status.sh" infra/ansible/playbooks/110-devops.yml "110 daily backup Telegram heartbeat deploy"
require_pattern "AWOOOI daily backup Telegram heartbeat" infra/ansible/playbooks/110-devops.yml "110 daily backup Telegram heartbeat cron"
require_pattern "backup-gitea.sh" infra/ansible/playbooks/110-devops.yml "110 Gitea backup deploy"
require_pattern "backup-harbor.sh" infra/ansible/playbooks/110-devops.yml "110 Harbor backup deploy"
require_pattern "backup-momo.sh" infra/ansible/playbooks/110-devops.yml "110 momo backup deploy"
require_pattern "backup-awoooi.sh" infra/ansible/playbooks/110-devops.yml "110 AWOOOI backup deploy"
require_pattern "backup-configs.sh" infra/ansible/playbooks/110-devops.yml "110 config backup deploy"
require_pattern "offsite-escrow-evidence-report.sh" infra/ansible/playbooks/110-devops.yml "110 offsite evidence report deploy"
require_pattern "offsite-escrow-evidence-report.sh --no-color" infra/ansible/playbooks/110-devops.yml "110 offsite evidence report cron"
require_pattern "verify-offsite-full-sync.sh" infra/ansible/playbooks/110-devops.yml "110 offsite full sync verifier deploy"
require_pattern "verify-offsite-full-sync.sh --write-textfile" infra/ansible/playbooks/110-devops.yml "110 offsite full sync verifier cron"
require_pattern "offsite_escrow_evidence_report" scripts/ops/backup-health-textfile-exporter.py "110 offsite evidence report cron metric"
require_pattern "offsite_full_sync_verify" scripts/ops/backup-health-textfile-exporter.py "110 offsite full sync verifier cron metric"
require_pattern "awoooi_backup_dr_next_step_info" scripts/ops/backup-health-textfile-exporter.py "110 DR next-step textfile metric"
require_pattern "awoooi_backup_offsite_partial_fresh" scripts/ops/backup-health-textfile-exporter.py "110 partial offsite sync textfile metric"
require_pattern "awoooi_backup_offsite_full_sync_enabled" scripts/ops/backup-health-textfile-exporter.py "110 full offsite sync enable marker metric"
require_pattern "awoooi_backup_retention_latest_only" scripts/ops/backup-health-textfile-exporter.py "110 latest-only retention textfile metric"
require_pattern "awoooi_backup_cron_active_duplicate_count" scripts/ops/backup-health-textfile-exporter.py "110 backup cron duplicate textfile metric"
require_pattern "awoooi_backup_cron_singular_entry_ok" scripts/ops/backup-health-textfile-exporter.py "110 backup cron singular textfile metric"
require_pattern "textfile_exporters" infra/ansible/playbooks/188-ai-web.yml "188 textfile exporters tag"
require_pattern "backup-momo-188-pg.sh" infra/ansible/playbooks/188-ai-web.yml "188 momo PostgreSQL backup deploy"
require_pattern "/home/ollama/bin/momo-pg-backup.sh" infra/ansible/playbooks/188-ai-web.yml "188 host-owned momo backup entrypoint"
forbid_pattern "/home/ollama/momo-pro/scripts/pg_backup.sh" infra/ansible/playbooks/188-ai-web.yml "188 app-directory momo backup cron"
require_pattern "/home/ollama/bin/momo-pg-backup.sh" scripts/ops/backup-health-textfile-exporter.py "188 backup health executable entrypoint"
require_pattern "AWOOOI momo PostgreSQL daily backup" infra/ansible/playbooks/188-ai-web.yml "188 momo PostgreSQL backup cron"
require_pattern "188-internal-tools-https.conf.j2" infra/ansible/playbooks/nginx-sync.yml "188 HTTPS route sync"

echo
echo "== Monitoring and CI =="
require_pattern "cold_start_recovery_alerts" ops/monitoring/alerts-unified.yml "Cold-start alert group"
require_pattern "PrometheusRuleDriftGuardFailed" ops/monitoring/alerts-unified.yml "Prometheus rule drift guard failure alert"
require_pattern "PrometheusRuleDriftAutoRepaired" ops/monitoring/alerts-unified.yml "Prometheus rule drift repaired alert"
require_pattern "awoooi_prometheus_rule_drift_guard_missing_required_count" ops/monitoring/alerts-unified.yml "Prometheus rule drift guard missing-required metric alert"
require_pattern "ColdStartRecoveryBlocked" ops/monitoring/alerts-unified.yml "ColdStartRecoveryBlocked alert"
require_pattern "K3sNodeFilesystemErrorGateBlocked" ops/monitoring/alerts-unified.yml "K3s node filesystem blocker alert"
require_pattern "ColdStartHost120Unreachable" ops/monitoring/alerts-unified.yml "120 host unreachable cold-start alert"
require_pattern "awoooi_cold_start_blocker_reason" ops/monitoring/alerts-unified.yml "Cold-start blocker reason alert metric"
require_pattern "docker_container_cpu_cores" ops/monitoring/alerts-unified.yml "Docker CPU alert metric"
require_pattern "systemd_unit_watchdog_seconds" ops/monitoring/alerts-unified.yml "Systemd watchdog alert metric"
require_pattern "awoooi_host_storage_error_count" ops/monitoring/alerts-unified.yml "Storage health alert metric"
require_pattern "awoooi_backup_job_fresh" ops/monitoring/alerts-unified.yml "Backup freshness alert metric"
require_pattern "awoooi_backup_integrity_fresh" ops/monitoring/alerts-unified.yml "Backup integrity alert metric"
require_pattern "awoooi_backup_offsite_configured" ops/monitoring/alerts-unified.yml "Backup offsite alert metric"
require_pattern "awoooi_backup_credential_escrow_fresh" ops/monitoring/alerts-unified.yml "Backup credential escrow alert metric"
require_pattern "BackupRetentionPolicyNotLatestOnly" ops/monitoring/alerts-unified.yml "Backup latest-only retention alert"
require_pattern "BackupSnapshotRetentionExceeded" ops/monitoring/alerts-unified.yml "Backup snapshot count retention alert"
require_pattern "BackupScheduleDuplicateActiveEntries" ops/monitoring/alerts-unified.yml "Backup duplicate cron alert"
require_pattern "BackupScheduleSingletonMismatch" ops/monitoring/alerts-unified.yml "Backup singleton cron alert"
require_pattern "BackupOffsiteFullVerifyFailed" ops/monitoring/alerts-unified.yml "Backup offsite full verify alert"
require_pattern "BackupOffsiteRemoteSnapshotRetentionExceeded" ops/monitoring/alerts-unified.yml "Backup offsite remote snapshot retention alert"
require_pattern "BackupRestoreTestStale" ops/monitoring/alerts-unified.yml "Backup restore stale alert"
require_pattern "BackupOffsiteCopyNotConfigured" ops/monitoring/alerts-unified.yml "Backup offsite not configured alert"
require_pattern "BackupCredentialEscrowEvidenceMissing" ops/monitoring/alerts-unified.yml "Backup credential escrow evidence alert"
require_pattern "awoooi_recovery_core_ready" ops/monitoring/alerts-unified.yml "Recovery core ready recording rule"
require_pattern "awoooi_recovery_dr_offsite_ready" ops/monitoring/alerts-unified.yml "Recovery DR offsite ready recording rule"
require_pattern '\$labels\.exported_job' ops/monitoring/alerts-unified.yml "Backup alert exported_job annotation label"
require_pattern "ansible-validate.sh" .gitea/workflows/ansible-lint.yml "Gitea Ansible validation workflow"
require_pattern "bootstrap-ansible-validation-env.sh" .gitea/workflows/ansible-lint.yml "Gitea Ansible validation bootstrap workflow"
require_pattern "doc-secrets-sanity-check.py" .gitea/workflows/ansible-lint.yml "Gitea documentation secrets validation workflow"
require_pattern "backup-alert-label-contract-check.py" .gitea/workflows/ansible-lint.yml "Gitea backup alert label contract trigger"
require_pattern "backup-alert-live-visibility-check.py" .gitea/workflows/ansible-lint.yml "Gitea backup alert live visibility trigger"
require_pattern "recovery-scorecard-contract-check.py" .gitea/workflows/ansible-lint.yml "Gitea recovery scorecard contract trigger"
require_pattern "full-stack-recovery-scorecard.sh" .gitea/workflows/ansible-lint.yml "Gitea recovery scorecard trigger"
require_pattern "dr-offsite-operator-checklist.sh" .gitea/workflows/ansible-lint.yml "Gitea DR offsite checklist trigger"
require_pattern "scripts/reboot-recovery/\\*\\*" .gitea/workflows/ansible-lint.yml "Gitea reboot recovery scripts trigger"
require_pattern "verify-cold-start-monitor-deploy.sh" .gitea/workflows/ansible-lint.yml "Gitea cold-start deploy parity trigger"
require_pattern "docs/\\*\\*" .gitea/workflows/ansible-lint.yml "Gitea all-docs validation trigger"
require_pattern "\\.gitea/workflows/\\*\\*" .gitea/workflows/ansible-lint.yml "Gitea workflow self-validation trigger"

echo
echo "== Local validation =="
if bash scripts/ops/ansible-validate.sh >/tmp/awoooi-ansible-validate.log 2>&1; then
  ok "scripts/ops/ansible-validate.sh passed"
else
  blocked "scripts/ops/ansible-validate.sh failed; see /tmp/awoooi-ansible-validate.log"
fi

if python3 scripts/ops/doc-secrets-sanity-check.py >/tmp/awoooi-doc-secrets-sanity.log 2>&1; then
  ok "documentation secrets sanity check passed"
else
  blocked "documentation secrets sanity check failed; see /tmp/awoooi-doc-secrets-sanity.log"
fi

if python3 scripts/ops/backup-alert-label-contract-check.py >/tmp/awoooi-backup-alert-label-contract.log 2>&1; then
  ok "backup alert label contract check passed"
else
  blocked "backup alert label contract check failed; see /tmp/awoooi-backup-alert-label-contract.log"
fi

if python3 scripts/ops/recovery-scorecard-contract-check.py >/tmp/awoooi-recovery-scorecard-contract.log 2>&1; then
  ok "recovery scorecard contract check passed"
else
  blocked "recovery scorecard contract check failed; see /tmp/awoooi-recovery-scorecard-contract.log"
fi

if command -v ansible-playbook >/dev/null 2>&1; then
  ok "ansible-playbook available locally"
else
  warning "ansible-playbook unavailable locally; CI/ops host must run syntax-check"
fi

if [ "$RUN_LIVE" = "1" ]; then
  echo
  echo "== Live read-only cold-start gate =="
  if ssh wooo@192.168.0.110 "test -x /home/wooo/scripts/prometheus-rule-drift-guard.sh && test -f /home/wooo/monitoring/alerts-unified.canonical.yml && crontab -l 2>/dev/null | grep -q 'AWOOOI Prometheus rule drift guard'" >/tmp/awoooi-prometheus-rule-drift-guard-live.log 2>&1; then
    ok "live 110 Prometheus rule drift guard installed"
  else
    blocked "live 110 Prometheus rule drift guard missing; run bash scripts/ops/deploy-alerts.sh"
  fi

  set +e
  SSH_BATCH_MODE=yes bash scripts/reboot-recovery/full-stack-cold-start-check.sh --monitor-read-only --no-color --watch --interval 1 --max-attempts 1 >/tmp/awoooi-cold-start-live.log 2>&1
  cold_start_rc=$?
  set -e
  cold_start_summary=$(grep -E '^PASS=[0-9]+ WARN=[0-9]+ BLOCKED=[0-9]+' /tmp/awoooi-cold-start-live.log | tail -1 || true)
  cold_start_blocked=$(awk -F'BLOCKED=' '/^PASS=/ {print $2}' <<<"$cold_start_summary")
  if [ "$cold_start_rc" -eq 0 ]; then
    ok "live read-only cold-start gate passed"
    if python3 - <<'PY' >/tmp/awoooi-cold-start-alert-live-state.log 2>&1
import json
import sys
import urllib.parse
import urllib.request

expr = 'ALERTS{alertname=~"ColdStart.*",alertstate="firing"}'
url = "http://192.168.0.110:9090/api/v1/query?" + urllib.parse.urlencode({"query": expr})
payload = json.load(urllib.request.urlopen(url, timeout=8))
if payload.get("status") != "success":
    print(f"Prometheus query failed: {payload}", file=sys.stderr)
    sys.exit(1)
rows = payload.get("data", {}).get("result") or []
if rows:
    names = sorted({(row.get("metric") or {}).get("alertname", "unknown") for row in rows})
    print("Cold-start alerts still firing after GREEN gate: " + ", ".join(names), file=sys.stderr)
    sys.exit(1)
print("COLD_START_ALERT_LIVE_STATE_OK")
PY
    then
      ok "live Prometheus cold-start alerts cleared after green gate"
    else
      blocked "live Prometheus cold-start alerts still firing after green gate; see /tmp/awoooi-cold-start-alert-live-state.log"
    fi
  elif [ "${cold_start_blocked:-1}" = "0" ]; then
    warning "live read-only cold-start gate degraded but not blocked: ${cold_start_summary:-summary unavailable}"
  else
    blocked "live read-only cold-start gate failed: ${cold_start_summary:-summary unavailable}; see /tmp/awoooi-cold-start-live.log"
  fi

  if python3 scripts/ops/backup-alert-label-contract-check.py --prometheus-url http://192.168.0.110:9090 >/tmp/awoooi-backup-alert-label-contract-live.log 2>&1; then
    ok "live Prometheus backup alert label contract passed"
  else
    blocked "live Prometheus backup alert label contract failed; run bash scripts/ops/deploy-alerts.sh, wait 1-2 evaluation cycles, then recheck /tmp/awoooi-backup-alert-label-contract-live.log"
  fi

  if run_with_retries 3 20 \
    python3 scripts/ops/recovery-scorecard-contract-check.py \
      --prometheus-url http://192.168.0.110:9090 \
      --expect-core-ready \
    >/tmp/awoooi-recovery-scorecard-contract-live.log 2>&1; then
    ok "live Prometheus recovery scorecard contract passed"
  elif [ "${cold_start_blocked:-1}" != "0" ]; then
    blocked "live Prometheus recovery scorecard contract correctly cannot be core-ready while cold-start is blocked: ${cold_start_summary:-summary unavailable}; resolve the first cold-start blocker before expecting awoooi_recovery_core_ready=1"
  else
    blocked "live Prometheus recovery scorecard contract failed; run bash scripts/ops/deploy-alerts.sh, wait 1-2 evaluation cycles, then recheck /tmp/awoooi-recovery-scorecard-contract-live.log"
  fi

  if python3 scripts/ops/backup-alert-live-visibility-check.py \
    --prometheus-url http://192.168.0.110:9090 \
    --alertmanager-url http://192.168.0.110:9093 \
    >/tmp/awoooi-backup-alert-live-visibility.log 2>&1; then
    ok "live backup gap alert visibility passed"
  else
    blocked "live backup gap alert visibility failed; if gap metrics exist but alerts are missing, run bash scripts/ops/deploy-alerts.sh, wait for the 1m alert window, then recheck /tmp/awoooi-backup-alert-live-visibility.log"
  fi

  if bash scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh >/tmp/awoooi-cold-start-monitor-deploy-parity.log 2>&1; then
    ok "live 110 cold-start monitor deploy parity passed"
  elif [ "${cold_start_blocked:-1}" != "0" ]; then
    blocked "live 110 cold-start monitor deploy parity cannot be green while cold-start is blocked: ${cold_start_summary:-summary unavailable}; see /tmp/awoooi-cold-start-monitor-deploy-parity.log"
  else
    blocked "live 110 cold-start monitor deploy parity failed; see /tmp/awoooi-cold-start-monitor-deploy-parity.log"
  fi

  if ssh -o BatchMode=yes -o ConnectTimeout=8 wooo@192.168.0.110 '
      set -eu
      test -x /backup/scripts/backup-status.sh
      crontab -l | grep -q "/backup/scripts/backup-status.sh"
      test -f /backup/state/backup-status-last-notified
    ' >/tmp/awoooi-backup-heartbeat-live.log 2>&1; then
    ok "live 110 daily backup Telegram heartbeat installed and has notification marker"
  else
    blocked "live 110 daily backup Telegram heartbeat check failed; see /tmp/awoooi-backup-heartbeat-live.log"
  fi

  if ssh -o BatchMode=yes -o ConnectTimeout=8 wooo@192.168.0.110 '
      set -eu
      crontab -l | awk "
        NF && \$0 !~ /^[[:space:]]*#/ {
          count[\$0]++
        }
        END {
          for (line in count) {
            if (count[line] > 1) {
              print count[line] \"x \" line
              bad = 1
            }
          }
          exit bad
        }
      "
    ' >/tmp/awoooi-110-cron-duplicate-live.log 2>&1; then
    ok "live 110 crontab has no exact duplicate active entries"
  else
    blocked "live 110 crontab has duplicate active entries; see /tmp/awoooi-110-cron-duplicate-live.log"
  fi

  if ssh -o BatchMode=yes -o ConnectTimeout=8 wooo@192.168.0.110 '
      set -eu
      cron="$(crontab -l)"
      bad=0
      check_count() {
        pattern="$1"
        expected="$2"
        count="$(printf "%s\n" "$cron" | awk -v p="$pattern" "index(\$0, p) && \$0 !~ /^[[:space:]]*#/ {c++} END {print c + 0}")"
        if [ "$count" != "$expected" ]; then
          printf "%s expected=%s actual=%s\n" "$pattern" "$expected" "$count"
          bad=1
        fi
      }
      check_count "/home/wooo/scripts/backup-health-textfile-exporter.py" 1
      check_count "/backup/scripts/sync-offsite-backups.sh --mode status" 1
      check_count "/backup/scripts/offsite-escrow-evidence-report.sh --no-color" 1
      check_count "/backup/scripts/sync-offsite-backups.sh --mode sync" 1
      check_count "/backup/scripts/verify-offsite-full-sync.sh --write-textfile" 1
      exit "$bad"
    ' >/tmp/awoooi-110-backup-cron-count-live.log 2>&1; then
    ok "live 110 backup/offsite cron entries are singular"
  else
    blocked "live 110 backup/offsite cron entry count mismatch; see /tmp/awoooi-110-backup-cron-count-live.log"
  fi

  if ssh -o BatchMode=yes -o ConnectTimeout=8 wooo@192.168.0.110 '
      set -eu
      ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 ollama@192.168.0.188 "
        test -x /home/ollama/bin/momo-pg-backup.sh
        crontab -l | grep -q /home/ollama/bin/momo-pg-backup.sh
        ! crontab -l | grep -q /home/ollama/momo-pro/scripts/pg_backup.sh
        grep -q \"momo-pg-backup.sh\" /home/ollama/node_exporter_textfiles/backup_health.prom
      "
    ' >/tmp/awoooi-188-momo-host-backup-live.log 2>&1; then
    ok "live 188 momo backup uses host-owned executable entrypoint"
  else
    blocked "live 188 momo host-owned backup entrypoint check failed; see /tmp/awoooi-188-momo-host-backup-live.log"
  fi

  if ssh -o BatchMode=yes -o ConnectTimeout=8 wooo@192.168.0.110 '
      set -eu
      ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 ollama@192.168.0.188 "
        policy=\$(docker inspect -f \"{{.HostConfig.RestartPolicy.Name}}\" momo-db)
        health=\$(docker inspect -f \"{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}\" momo-db)
        state=\$(docker inspect -f \"{{.State.Status}}\" momo-db)
        printf \"momo-db state=%s health=%s restart=%s\n\" \"\$state\" \"\$health\" \"\$policy\"
        test \"\$state\" = running
        test \"\$health\" = healthy
        case \"\$policy\" in
          always|unless-stopped) exit 0 ;;
          *) exit 1 ;;
        esac
      "
    ' >/tmp/awoooi-188-momo-db-restart-policy-live.log 2>&1; then
    ok "live 188 momo-db restart policy and health passed"
  else
    blocked "live 188 momo-db restart policy or health failed; run docker update --restart unless-stopped momo-db and see /tmp/awoooi-188-momo-db-restart-policy-live.log"
  fi
else
  warning "live cold-start gate skipped; pass --live to verify runtime state"
fi

echo
echo "== Summary =="
echo "PASS=$pass WARN=$warn BLOCKED=$fail"

if [ "$fail" -gt 0 ]; then
  echo "Result: NOT READY. Fix BLOCKED items before relying on reboot automation."
  exit 1
fi

if [ "$warn" -gt 0 ]; then
  echo "Result: READY WITH WARNINGS. Core SOP exists, but hardening remains."
  exit 0
fi

echo "Result: READY. Reboot recovery SOP, scripts, monitoring, and CI gates are present."
