#!/usr/bin/env bash
# 120 root filesystem 維護前只讀檢查。
# 本腳本不會 reboot、drain、cordon、fsck、刪檔或修改遠端狀態。

set -uo pipefail

REMOTE_120="${REMOTE_120:-wooo@192.168.0.120}"
REMOTE_110="${REMOTE_110:-wooo@192.168.0.110}"
SSH_BATCH_MODE="${SSH_BATCH_MODE:-yes}"
SSH_STRICT_HOST_KEY_CHECKING="${SSH_STRICT_HOST_KEY_CHECKING:-accept-new}"
NO_COLOR=0

usage() {
  cat <<'USAGE'
Usage: bash scripts/reboot-recovery/120-fsck-maintenance-checklist.sh [--no-color]

Read-only pre-maintenance checklist for host 120 filesystem repair.
It prints evidence and manual console steps only; it never runs fsck online.

Environment:
  REMOTE_120=wooo@192.168.0.120
  REMOTE_110=wooo@192.168.0.110
  SSH_BATCH_MODE=yes
  SSH_STRICT_HOST_KEY_CHECKING=accept-new
USAGE
}

while [ "$#" -gt 0 ]; do
  case "$1" in
    --no-color)
      NO_COLOR=1
      shift
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "Unknown argument: $1" >&2
      usage >&2
      exit 64
      ;;
  esac
done

if [ "$NO_COLOR" = "1" ]; then
  green=""
  yellow=""
  red=""
  blue=""
  reset=""
else
  green="$(printf '\033[32m')"
  yellow="$(printf '\033[33m')"
  red="$(printf '\033[31m')"
  blue="$(printf '\033[34m')"
  reset="$(printf '\033[0m')"
fi

PASS=0
WARN=0
BLOCKED=0

ssh_opts=(-o BatchMode="$SSH_BATCH_MODE" -o ConnectTimeout=8 -o StrictHostKeyChecking="$SSH_STRICT_HOST_KEY_CHECKING")

section() {
  printf "\n%s=== %s ===%s\n" "$blue" "$1" "$reset"
}

ok() {
  PASS=$((PASS + 1))
  printf "%sOK%s %s\n" "$green" "$reset" "$*"
}

warn() {
  WARN=$((WARN + 1))
  printf "%sWARN%s %s\n" "$yellow" "$reset" "$*"
}

blocked() {
  BLOCKED=$((BLOCKED + 1))
  printf "%sBLOCKED%s %s\n" "$red" "$reset" "$*"
}

ssh_cmd() {
  local target="$1"
  local command="$2"
  ssh "${ssh_opts[@]}" "$target" "$command"
}

echo "AWOOOI 120 filesystem maintenance checklist"
date '+%Y-%m-%d %H:%M:%S %Z'
echo "Scope: 120 root LV fsck readiness. 112 Kali is intentionally skipped."

section "120 host state"
if out=$(ssh_cmd "$REMOTE_120" '
hostname
uptime
systemctl is-system-running || true
findmnt -n -o SOURCE,FSTYPE,OPTIONS /
test -r /proc/mounts && awk "\$2 == \"/\" {print \"ROOT_MOUNT_OPTIONS \" \$4}" /proc/mounts
' 2>&1); then
  echo "$out"
  ok "120 SSH and host state readable"
  grep -q ' rw,' <<<"$out" && ok "120 root filesystem currently writable" || blocked "120 root filesystem is not confirmed writable"
else
  blocked "120 host state unavailable"
  echo "$out"
fi

section "K3s service and API state"
if out=$(ssh_cmd "$REMOTE_120" '
kcmd() {
  sudo -n kubectl "$@"
}
echo "K3S_ACTIVE $(systemctl is-active k3s 2>/dev/null || true)"
echo "KEEPALIVED_ACTIVE $(systemctl is-active keepalived 2>/dev/null || true)"
kcmd get nodes -o wide
non_running="$(kcmd get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers 2>/dev/null || true)"
echo "NON_RUNNING_PODS $(printf "%s\n" "$non_running" | awk "NF {count++} END {print count+0}")"
printf "%s\n" "$non_running"
kcmd get --raw /readyz >/dev/null && echo "READYZ ok" || echo "READYZ failed"
kcmd get --raw /livez >/dev/null && echo "LIVEZ ok" || echo "LIVEZ failed"
' 2>&1); then
  echo "$out"
  grep -q 'K3S_ACTIVE active' <<<"$out" && ok "120 k3s active" || blocked "120 k3s not active"
  grep -q 'KEEPALIVED_ACTIVE active' <<<"$out" && ok "120 keepalived active" || warn "120 keepalived not active"
  grep -q 'NON_RUNNING_PODS 0' <<<"$out" && ok "K3s has no non-running/non-succeeded pods" || warn "K3s has non-running/non-succeeded pods"
  grep -q 'READYZ ok' <<<"$out" && ok "K3s readyz passed" || blocked "K3s readyz failed"
  grep -q 'LIVEZ ok' <<<"$out" && ok "K3s livez passed" || blocked "K3s livez failed"
else
  blocked "K3s API check unavailable"
  echo "$out"
fi

section "120 filesystem blocker evidence"
if out=$(ssh_cmd "$REMOTE_120" '
kcmd() {
  sudo -n kubectl "$@"
}
events="$(kcmd get events -A --field-selector involvedObject.kind=Node --sort-by=.lastTimestamp --no-headers 2>/dev/null | grep -Ei "EXT4-fs error|Buffer I/O error|I/O error|Structure needs cleaning|deleted inode" || true)"
echo "NODE_FS_ERROR_EVENTS $(printf "%s\n" "$events" | awk "NF {count++} END {print count+0}")"
printf "%s\n" "$events"
' 2>&1); then
  echo "$out"
  if grep -q 'NODE_FS_ERROR_EVENTS 0' <<<"$out"; then
    ok "K3s Node filesystem error events absent"
  else
    blocked "120 still has K3s Node filesystem error events; do not declare reboot safe before offline fsck"
  fi
else
  blocked "120 filesystem event evidence unavailable"
  echo "$out"
fi

section "Backup and restore evidence"
if out=$(ssh_cmd "$REMOTE_120" '
kcmd() {
  sudo -n kubectl "$@"
}
kcmd get schedules,backups -n velero 2>/dev/null || true
' 2>&1); then
  echo "$out"
  grep -q 'schedule.velero.io/daily-awoooi-prod' <<<"$out" && ok "Velero daily schedule exists" || warn "Velero daily schedule not confirmed"
  grep -Eq 'daily-awoooi-prod-[0-9]+' <<<"$out" && ok "Velero backup history visible" || warn "Velero backup history not confirmed"
else
  warn "Velero evidence unavailable"
  echo "$out"
fi

if out=$(ssh_cmd "$REMOTE_110" '
test -x /backup/scripts/offsite-escrow-evidence-report.sh
/backup/scripts/offsite-escrow-evidence-report.sh --no-color
' 2>&1); then
  echo "$out"
  grep -q 'FULL_MARKER_PRESENT=1' <<<"$out" && ok "110 offsite full marker present" || warn "110 offsite full marker not confirmed"
  grep -q 'ESCROW_MISSING_COUNT=0' <<<"$out" && ok "credential escrow complete" || warn "credential escrow still has manual gaps"
else
  warn "110 offsite/escrow evidence unavailable"
  echo "$out"
fi

section "Public route smoke check"
route_fail=0
for domain in awoooi.wooo.work mo.wooo.work gitea.wooo.work harbor.wooo.work registry.wooo.work sentry.wooo.work signoz.wooo.work; do
  code="$(curl -LsS -o /dev/null -w '%{http_code}' --max-time 12 "https://${domain}/" 2>/dev/null || echo 000)"
  printf 'PUBLIC_ROUTE_TLS %s %s\n' "$domain" "$code"
  case "$code" in
    2??|3??|4??) ;;
    *) route_fail=1 ;;
  esac
done
if [ "$route_fail" -eq 0 ]; then
  ok "public HTTPS routes respond with verified TLS"
else
  blocked "one or more public HTTPS routes failed verified TLS check"
fi

section "Manual console-only fsck procedure"
cat <<'STEPS'
Do not run fsck against the mounted root filesystem.

Maintenance window sequence:
1. 確認本腳本的 Public route、Velero、offsite marker 與 K3s API 證據已保存。
2. 暫停非必要 deploy / runner / AI auto-repair full execution，只保留 observe-only 告警。
3. 透過主機 console、rescue mode 或 initramfs 停在 120，不要在線上 root mount 狀態執行 fsck。
4. 在 console/rescue 執行：
   fsck -f /dev/mapper/ubuntu--vg-ubuntu--lv
5. 若 fsck 要求互動修復，逐項確認；完成後 reboot 120。
6. 回到 SSH 後執行：
   SSH_BATCH_MODE=yes bash scripts/reboot-recovery/full-stack-cold-start-check.sh --monitor-read-only --no-color --watch --interval 10 --max-attempts 6
   SSH_BATCH_MODE=yes bash scripts/reboot-recovery/reboot-recovery-readiness-audit.sh --live --no-color
7. 只有在 NODE_FS_ERROR_EVENTS=0、public TLS gate 通過、Prometheus scorecard core ready 後，才解除維護狀態。
STEPS

echo
echo "PASS=$PASS WARN=$WARN BLOCKED=$BLOCKED"
if [ "$BLOCKED" -gt 0 ]; then
  echo "Result: MAINTENANCE REQUIRED. 120 filesystem risk is still blocking reboot confidence."
  exit 1
fi
if [ "$WARN" -gt 0 ]; then
  echo "Result: READY WITH WARNINGS for scheduled manual fsck."
  exit 0
fi
echo "Result: READY for scheduled manual fsck."
