awoooi/scripts/backup/offsite-escrow-evidence-report.sh

#!/bin/bash
# =============================================================================
# WOOO AIOps - Offsite backup / credential escrow evidence report
# 2026-05-06 ogt + Codex: 產出可交接、可告警研判的紅acted 證據摘要。
#
# 預設只讀，不觸發 rclone remote status、不上傳、不寫 success marker。
# 如需確認 Google Drive/rclone remote 可列出，明確加 --include-remote-status。
# =============================================================================

set -euo pipefail

BACKUP_BASE="${BACKUP_BASE:-/backup}"
SCRIPTS_DIR="${BACKUP_SCRIPTS_DIR:-${BACKUP_BASE}/scripts}"
OFFSITE_DIR="${BACKUP_OFFSITE_STATUS_DIR:-${BACKUP_BASE}/offsite}"
ESCROW_DIR="${BACKUP_ESCROW_EVIDENCE_DIR:-${BACKUP_BASE}/escrow-evidence}"
TEXTFILE_PROM="${BACKUP_HEALTH_PROM:-/home/wooo/node_exporter_textfiles/backup_health.prom}"
INCLUDE_REMOTE_STATUS=0
NO_COLOR=0

CONFIG_B2_SCRIPT="${SCRIPTS_DIR}/configure-offsite-b2.sh"
CONFIG_RCLONE_SCRIPT="${SCRIPTS_DIR}/configure-offsite-rclone.sh"
READINESS_SCRIPT="${SCRIPTS_DIR}/backup-offsite-readiness-gate.sh"
SYNC_SCRIPT="${SCRIPTS_DIR}/sync-offsite-backups.sh"
ESCROW_SCRIPT="${SCRIPTS_DIR}/mark-credential-escrow-verified.sh"

usage() {
    cat <<'USAGE'
Usage:
  offsite-escrow-evidence-report.sh [--no-color]
  offsite-escrow-evidence-report.sh --include-remote-status [--no-color]

Rules:
  - Default mode is read-only and does not query remote provider.
  - --include-remote-status runs sync-offsite-backups.sh --mode status only.
  - This report must never print credential values.
  - Use it after reboot, after Google Drive/rclone config, after small sync, and before full sync review.
USAGE
}

while [ "$#" -gt 0 ]; do
    case "$1" in
        --include-remote-status)
            INCLUDE_REMOTE_STATUS=1
            shift
            ;;
        --no-color)
            NO_COLOR=1
            shift
            ;;
        -h|--help)
            usage
            exit 0
            ;;
        *)
            echo "Unknown argument: $1" >&2
            usage >&2
            exit 2
            ;;
    esac
done

if [ "${NO_COLOR}" = "1" ]; then
    green=""
    yellow=""
    red=""
    reset=""
else
    green="$(printf '\033[32m')"
    yellow="$(printf '\033[33m')"
    red="$(printf '\033[31m')"
    reset="$(printf '\033[0m')"
fi

redact_output() {
    sed -E \
        -e '/CONFIGURED=/! s/^([[:space:]]*(export[[:space:]]+)?[A-Za-z_][A-Za-z0-9_]*(KEY|TOKEN|PASSWORD|SECRET)[A-Za-z0-9_]*=).*/\1<redacted>/I' \
        -e '/CONFIGURED=/! s/^([[:space:]]*B2_APPLICATION_KEY=).*/\1<redacted>/'
}

section() {
    echo
    echo "== $* =="
}

tool_status() {
    local title="$1"
    shift
    local rc=0
    local output=""
    section "${title}"
    if output="$("$@" 2>&1)"; then
        printf "%sOK%s rc=0 command=%s\n" "${green}" "${reset}" "$*"
    else
        rc=$?
        printf "%sWARN%s rc=%s command=%s\n" "${yellow}" "${reset}" "${rc}" "$*"
    fi
    printf "%s\n" "${output}" | redact_output
    return "${rc}"
}

marker_timestamp() {
    local path="$1"
    [ -f "${path}" ] || {
        echo 0
        return
    }
    awk -F= '/^timestamp=/ {print int($2); found=1; exit} END {if (!found) print 0}' "${path}" 2>/dev/null || echo 0
}

marker_state() {
    local label="$1"
    local path="$2"
    local ts
    ts="$(marker_timestamp "${path}")"
    if [ "${ts}" -gt 0 ]; then
        printf "%sOK%s %s present timestamp=%s path=%s\n" "${green}" "${reset}" "${label}" "${ts}" "${path}"
        return 0
    fi
    printf "%sWARN%s %s missing path=%s\n" "${yellow}" "${reset}" "${label}" "${path}"
    return 1
}

script_state() {
    local path="$1"
    if [ -x "${path}" ]; then
        printf "%sOK%s script executable: %s\n" "${green}" "${reset}" "${path}"
        return 0
    fi
    printf "%sBLOCKED%s script missing or not executable: %s\n" "${red}" "${reset}" "${path}"
    return 1
}

AWOOOI_OFFSITE_ESCROW_REPORT_VERSION="2026-05-19.v2"
echo "AWOOOI offsite / credential escrow evidence report"
date
echo "REPORT_VERSION=${AWOOOI_OFFSITE_ESCROW_REPORT_VERSION}"
echo "BACKUP_BASE=${BACKUP_BASE}"
echo "SCRIPTS_DIR=${SCRIPTS_DIR}"
echo "INCLUDE_REMOTE_STATUS=${INCLUDE_REMOTE_STATUS}"

section "script presence"
missing_scripts=0
for path in "${CONFIG_RCLONE_SCRIPT}" "${READINESS_SCRIPT}" "${SYNC_SCRIPT}" "${ESCROW_SCRIPT}"; do
    script_state "${path}" || missing_scripts=$((missing_scripts + 1))
done
[ -x "${CONFIG_B2_SCRIPT}" ] && script_state "${CONFIG_B2_SCRIPT}" || true

config_rc=99
readiness_rc=99
remote_rc=0
escrow_rc=99
rclone_ready=0
b2_ready=0
offsite_ready=0
readiness_blocked=0
escrow_missing=0

if [ -x "${CONFIG_RCLONE_SCRIPT}" ]; then
    config_output="$("${CONFIG_RCLONE_SCRIPT}" --status 2>&1)" || config_rc=$?
    [ "${config_rc}" = "99" ] && config_rc=0
    section "rclone local config status"
    printf "RC=%s command=%s --status\n" "${config_rc}" "${CONFIG_RCLONE_SCRIPT}"
    printf "%s\n" "${config_output}" | redact_output
    if grep -q "RCLONE_REMOTE_CONFIGURED=1" <<<"${config_output}"; then
        rclone_ready=1
    fi
fi

if [ -x "${CONFIG_B2_SCRIPT}" ]; then
    b2_output="$("${CONFIG_B2_SCRIPT}" --status 2>&1)" || true
    section "legacy b2 local config status"
    printf "RC=0 command=%s --status\n" "${CONFIG_B2_SCRIPT}"
    printf "%s\n" "${b2_output}" | redact_output
    if grep -q "B2_ACCOUNT_ID_CONFIGURED=1" <<<"${b2_output}" \
        && grep -q "B2_APPLICATION_KEY_CONFIGURED=1" <<<"${b2_output}" \
        && grep -q "B2_BUCKET_CONFIGURED=1" <<<"${b2_output}"; then
        b2_ready=1
    fi
fi
if [ "${rclone_ready}" -eq 1 ] || [ "${b2_ready}" -eq 1 ]; then
    offsite_ready=1
fi

if [ -x "${READINESS_SCRIPT}" ]; then
    tool_status "offsite readiness status" "${READINESS_SCRIPT}" --status --no-color || readiness_rc=$?
    [ "${readiness_rc}" = "99" ] && readiness_rc=0
    if "${READINESS_SCRIPT}" --status --require-configured --no-color >/tmp/awoooi-offsite-evidence-readiness-require.log 2>&1; then
        readiness_blocked=0
    else
        readiness_blocked=1
    fi
fi

if [ "${INCLUDE_REMOTE_STATUS}" = "1" ] && [ -x "${SYNC_SCRIPT}" ]; then
    tool_status "offsite remote status" "${SYNC_SCRIPT}" --mode status || remote_rc=$?
fi

if [ -x "${ESCROW_SCRIPT}" ]; then
    escrow_output="$("${ESCROW_SCRIPT}" --status 2>&1)" || escrow_rc=$?
    [ "${escrow_rc}" = "99" ] && escrow_rc=0
    section "credential escrow status"
    printf "RC=%s command=%s --status\n" "${escrow_rc}" "${ESCROW_SCRIPT}"
    printf "%s\n" "${escrow_output}" | redact_output
    escrow_missing="$(grep -c " missing" <<<"${escrow_output}" || true)"
    if [ "${escrow_missing}" -gt 0 ]; then
        section "credential escrow missing command template"
        echo "以下命令只接受非 secret evidence-id；請把 EVIDENCE_ID_FOR_* 換成密碼管理器項目 ID、工單 ID、sealed envelope ID 或 recovery checklist ID。"
        echo "直接執行 placeholder 會被拒絕；可先加 --dry-run 驗證 evidence-id，不會寫 marker。"
        BACKUP_COMMON_QUIET=1 "${ESCROW_SCRIPT}" --missing-commands | redact_output
    fi
fi

section "offsite markers"
partial_marker=0
full_marker=0
marker_state "partial offsite marker" "${OFFSITE_DIR}/b2-partial-last-success" && partial_marker=1 || true
marker_state "full offsite marker" "${OFFSITE_DIR}/b2-last-success" && full_marker=1 || true
marker_state "partial offsite marker (rclone)" "${OFFSITE_DIR}/rclone-partial-last-success" && partial_marker=1 || true
marker_state "full offsite marker (rclone)" "${OFFSITE_DIR}/rclone-last-success" && full_marker=1 || true

section "prometheus textfile evidence"
if [ -r "${TEXTFILE_PROM}" ]; then
    grep -E 'awoooi_backup_offsite_|awoooi_backup_credential_escrow_' "${TEXTFILE_PROM}" | redact_output || true
else
    printf "%sWARN%s backup health textfile missing or unreadable: %s\n" "${yellow}" "${reset}" "${TEXTFILE_PROM}"
fi

section "next step"
if [ "${missing_scripts}" -gt 0 ]; then
    echo "NEXT_STEP=deploy_backup_jobs_with_ansible"
    echo "DETAIL=先套用 110-devops.yml --tags backup_jobs，補齊 /backup/scripts。"
elif [ "${offsite_ready}" -ne 1 ]; then
    echo "NEXT_STEP=configure_google_drive_rclone_on_110_tty"
    echo "DETAIL=在 110 本機執行 configure-offsite-rclone.sh --interactive；完成 Google Drive OAuth 後，只把非 secret remote 設定寫入 offsite.env。"
elif [ "${readiness_blocked}" -ne 0 ]; then
    echo "NEXT_STEP=fix_offsite_readiness_blockers"
    echo "DETAIL=先看 backup-offsite-readiness-gate.sh --status --require-configured --no-color 的 BLOCKED 項目。"
elif [ "${partial_marker}" -ne 1 ]; then
    echo "NEXT_STEP=run_small_dry_run_then_partial_sync"
    echo "DETAIL=先跑 backup-offsite-readiness-gate.sh --dry-run-small，再只同步 ai-artifacts public-routes。"
elif [ "${escrow_missing}" -gt 0 ]; then
    echo "NEXT_STEP=complete_credential_escrow_review"
    echo "DETAIL=人工確認金庫可用後，用 mark-credential-escrow-verified.sh 寫非 secret evidence-id marker。"
elif [ "${full_marker}" -ne 1 ]; then
    echo "NEXT_STEP=pre_full_sync_review"
    echo "DETAIL=低峰窗口前跑 backup-offsite-readiness-gate.sh --pre-full-sync --require-configured --require-escrow --no-color。"
else
    echo "NEXT_STEP=offsite_and_escrow_ready"
    echo "DETAIL=維持每日 status、每週 integrity check、每月 restore drill 與 escrow review。"
fi

section "summary"
echo "SCRIPT_MISSING_COUNT=${missing_scripts}"
echo "OFFSITE_CONFIGURED=${offsite_ready}"
echo "RCLONE_CONFIGURED=${rclone_ready}"
echo "B2_CONFIGURED=${b2_ready}"
echo "READINESS_REQUIRE_CONFIGURED_BLOCKED=${readiness_blocked}"
echo "REMOTE_STATUS_INCLUDED=${INCLUDE_REMOTE_STATUS}"
echo "REMOTE_STATUS_RC=${remote_rc}"
echo "ESCROW_MISSING_COUNT=${escrow_missing}"
echo "PARTIAL_MARKER_PRESENT=${partial_marker}"
echo "FULL_MARKER_PRESENT=${full_marker}"