#!/bin/bash # ============================================================================= # WOOO AIOps - Offsite backup / credential escrow evidence report # 2026-05-06 ogt + Codex: 產出可交接、可告警研判的紅acted 證據摘要。 # # 預設只讀,不觸發 rclone remote status、不上傳、不寫 success marker。 # 如需確認 Google Drive/rclone remote 可列出,明確加 --include-remote-status。 # ============================================================================= set -euo pipefail BACKUP_BASE="${BACKUP_BASE:-/backup}" SCRIPTS_DIR="${BACKUP_SCRIPTS_DIR:-${BACKUP_BASE}/scripts}" OFFSITE_DIR="${BACKUP_OFFSITE_STATUS_DIR:-${BACKUP_BASE}/offsite}" ESCROW_DIR="${BACKUP_ESCROW_EVIDENCE_DIR:-${BACKUP_BASE}/escrow-evidence}" TEXTFILE_PROM="${BACKUP_HEALTH_PROM:-/home/wooo/node_exporter_textfiles/backup_health.prom}" INCLUDE_REMOTE_STATUS=0 NO_COLOR=0 CONFIG_B2_SCRIPT="${SCRIPTS_DIR}/configure-offsite-b2.sh" CONFIG_RCLONE_SCRIPT="${SCRIPTS_DIR}/configure-offsite-rclone.sh" READINESS_SCRIPT="${SCRIPTS_DIR}/backup-offsite-readiness-gate.sh" SYNC_SCRIPT="${SCRIPTS_DIR}/sync-offsite-backups.sh" ESCROW_SCRIPT="${SCRIPTS_DIR}/mark-credential-escrow-verified.sh" usage() { cat <<'USAGE' Usage: offsite-escrow-evidence-report.sh [--no-color] offsite-escrow-evidence-report.sh --include-remote-status [--no-color] Rules: - Default mode is read-only and does not query remote provider. - --include-remote-status runs sync-offsite-backups.sh --mode status only. - This report must never print credential values. - Use it after reboot, after Google Drive/rclone config, after small sync, and before full sync review. USAGE } while [ "$#" -gt 0 ]; do case "$1" in --include-remote-status) INCLUDE_REMOTE_STATUS=1 shift ;; --no-color) NO_COLOR=1 shift ;; -h|--help) usage exit 0 ;; *) echo "Unknown argument: $1" >&2 usage >&2 exit 2 ;; esac done if [ "${NO_COLOR}" = "1" ]; then green="" yellow="" red="" reset="" else green="$(printf '\033[32m')" yellow="$(printf '\033[33m')" red="$(printf '\033[31m')" reset="$(printf '\033[0m')" fi redact_output() { sed -E \ -e '/CONFIGURED=/! s/^([[:space:]]*(export[[:space:]]+)?[A-Za-z_][A-Za-z0-9_]*(KEY|TOKEN|PASSWORD|SECRET)[A-Za-z0-9_]*=).*/\1/I' \ -e '/CONFIGURED=/! s/^([[:space:]]*B2_APPLICATION_KEY=).*/\1/' } section() { echo echo "== $* ==" } tool_status() { local title="$1" shift local rc=0 local output="" section "${title}" if output="$("$@" 2>&1)"; then printf "%sOK%s rc=0 command=%s\n" "${green}" "${reset}" "$*" else rc=$? printf "%sWARN%s rc=%s command=%s\n" "${yellow}" "${reset}" "${rc}" "$*" fi printf "%s\n" "${output}" | redact_output return "${rc}" } marker_timestamp() { local path="$1" [ -f "${path}" ] || { echo 0 return } awk -F= '/^timestamp=/ {print int($2); found=1; exit} END {if (!found) print 0}' "${path}" 2>/dev/null || echo 0 } marker_state() { local label="$1" local path="$2" local ts ts="$(marker_timestamp "${path}")" if [ "${ts}" -gt 0 ]; then printf "%sOK%s %s present timestamp=%s path=%s\n" "${green}" "${reset}" "${label}" "${ts}" "${path}" return 0 fi printf "%sWARN%s %s missing path=%s\n" "${yellow}" "${reset}" "${label}" "${path}" return 1 } script_state() { local path="$1" if [ -x "${path}" ]; then printf "%sOK%s script executable: %s\n" "${green}" "${reset}" "${path}" return 0 fi printf "%sBLOCKED%s script missing or not executable: %s\n" "${red}" "${reset}" "${path}" return 1 } AWOOOI_OFFSITE_ESCROW_REPORT_VERSION="2026-05-19.v2" echo "AWOOOI offsite / credential escrow evidence report" date echo "REPORT_VERSION=${AWOOOI_OFFSITE_ESCROW_REPORT_VERSION}" echo "BACKUP_BASE=${BACKUP_BASE}" echo "SCRIPTS_DIR=${SCRIPTS_DIR}" echo "INCLUDE_REMOTE_STATUS=${INCLUDE_REMOTE_STATUS}" section "script presence" missing_scripts=0 for path in "${CONFIG_RCLONE_SCRIPT}" "${READINESS_SCRIPT}" "${SYNC_SCRIPT}" "${ESCROW_SCRIPT}"; do script_state "${path}" || missing_scripts=$((missing_scripts + 1)) done [ -x "${CONFIG_B2_SCRIPT}" ] && script_state "${CONFIG_B2_SCRIPT}" || true config_rc=99 readiness_rc=99 remote_rc=0 escrow_rc=99 rclone_ready=0 b2_ready=0 offsite_ready=0 readiness_blocked=0 escrow_missing=0 if [ -x "${CONFIG_RCLONE_SCRIPT}" ]; then config_output="$("${CONFIG_RCLONE_SCRIPT}" --status 2>&1)" || config_rc=$? [ "${config_rc}" = "99" ] && config_rc=0 section "rclone local config status" printf "RC=%s command=%s --status\n" "${config_rc}" "${CONFIG_RCLONE_SCRIPT}" printf "%s\n" "${config_output}" | redact_output if grep -q "RCLONE_REMOTE_CONFIGURED=1" <<<"${config_output}"; then rclone_ready=1 fi fi if [ -x "${CONFIG_B2_SCRIPT}" ]; then b2_output="$("${CONFIG_B2_SCRIPT}" --status 2>&1)" || true section "legacy b2 local config status" printf "RC=0 command=%s --status\n" "${CONFIG_B2_SCRIPT}" printf "%s\n" "${b2_output}" | redact_output if grep -q "B2_ACCOUNT_ID_CONFIGURED=1" <<<"${b2_output}" \ && grep -q "B2_APPLICATION_KEY_CONFIGURED=1" <<<"${b2_output}" \ && grep -q "B2_BUCKET_CONFIGURED=1" <<<"${b2_output}"; then b2_ready=1 fi fi if [ "${rclone_ready}" -eq 1 ] || [ "${b2_ready}" -eq 1 ]; then offsite_ready=1 fi if [ -x "${READINESS_SCRIPT}" ]; then tool_status "offsite readiness status" "${READINESS_SCRIPT}" --status --no-color || readiness_rc=$? [ "${readiness_rc}" = "99" ] && readiness_rc=0 if "${READINESS_SCRIPT}" --status --require-configured --no-color >/tmp/awoooi-offsite-evidence-readiness-require.log 2>&1; then readiness_blocked=0 else readiness_blocked=1 fi fi if [ "${INCLUDE_REMOTE_STATUS}" = "1" ] && [ -x "${SYNC_SCRIPT}" ]; then tool_status "offsite remote status" "${SYNC_SCRIPT}" --mode status || remote_rc=$? fi if [ -x "${ESCROW_SCRIPT}" ]; then escrow_output="$("${ESCROW_SCRIPT}" --status 2>&1)" || escrow_rc=$? [ "${escrow_rc}" = "99" ] && escrow_rc=0 section "credential escrow status" printf "RC=%s command=%s --status\n" "${escrow_rc}" "${ESCROW_SCRIPT}" printf "%s\n" "${escrow_output}" | redact_output escrow_missing="$(grep -c " missing" <<<"${escrow_output}" || true)" if [ "${escrow_missing}" -gt 0 ]; then section "credential escrow missing command template" echo "以下命令只接受非 secret evidence-id;請把 EVIDENCE_ID_FOR_* 換成密碼管理器項目 ID、工單 ID、sealed envelope ID 或 recovery checklist ID。" echo "直接執行 placeholder 會被拒絕;可先加 --dry-run 驗證 evidence-id,不會寫 marker。" BACKUP_COMMON_QUIET=1 "${ESCROW_SCRIPT}" --missing-commands | redact_output fi fi section "offsite markers" partial_marker=0 full_marker=0 marker_state "partial offsite marker" "${OFFSITE_DIR}/b2-partial-last-success" && partial_marker=1 || true marker_state "full offsite marker" "${OFFSITE_DIR}/b2-last-success" && full_marker=1 || true marker_state "partial offsite marker (rclone)" "${OFFSITE_DIR}/rclone-partial-last-success" && partial_marker=1 || true marker_state "full offsite marker (rclone)" "${OFFSITE_DIR}/rclone-last-success" && full_marker=1 || true section "prometheus textfile evidence" if [ -r "${TEXTFILE_PROM}" ]; then grep -E 'awoooi_backup_offsite_|awoooi_backup_credential_escrow_' "${TEXTFILE_PROM}" | redact_output || true else printf "%sWARN%s backup health textfile missing or unreadable: %s\n" "${yellow}" "${reset}" "${TEXTFILE_PROM}" fi section "next step" if [ "${missing_scripts}" -gt 0 ]; then echo "NEXT_STEP=deploy_backup_jobs_with_ansible" echo "DETAIL=先套用 110-devops.yml --tags backup_jobs,補齊 /backup/scripts。" elif [ "${offsite_ready}" -ne 1 ]; then echo "NEXT_STEP=configure_google_drive_rclone_on_110_tty" echo "DETAIL=在 110 本機執行 configure-offsite-rclone.sh --interactive;完成 Google Drive OAuth 後,只把非 secret remote 設定寫入 offsite.env。" elif [ "${readiness_blocked}" -ne 0 ]; then echo "NEXT_STEP=fix_offsite_readiness_blockers" echo "DETAIL=先看 backup-offsite-readiness-gate.sh --status --require-configured --no-color 的 BLOCKED 項目。" elif [ "${partial_marker}" -ne 1 ]; then echo "NEXT_STEP=run_small_dry_run_then_partial_sync" echo "DETAIL=先跑 backup-offsite-readiness-gate.sh --dry-run-small,再只同步 ai-artifacts public-routes。" elif [ "${escrow_missing}" -gt 0 ]; then echo "NEXT_STEP=complete_credential_escrow_review" echo "DETAIL=人工確認金庫可用後,用 mark-credential-escrow-verified.sh 寫非 secret evidence-id marker。" elif [ "${full_marker}" -ne 1 ]; then echo "NEXT_STEP=pre_full_sync_review" echo "DETAIL=低峰窗口前跑 backup-offsite-readiness-gate.sh --pre-full-sync --require-configured --require-escrow --no-color。" else echo "NEXT_STEP=offsite_and_escrow_ready" echo "DETAIL=維持每日 status、每週 integrity check、每月 restore drill 與 escrow review。" fi section "summary" echo "SCRIPT_MISSING_COUNT=${missing_scripts}" echo "OFFSITE_CONFIGURED=${offsite_ready}" echo "RCLONE_CONFIGURED=${rclone_ready}" echo "B2_CONFIGURED=${b2_ready}" echo "READINESS_REQUIRE_CONFIGURED_BLOCKED=${readiness_blocked}" echo "REMOTE_STATUS_INCLUDED=${INCLUDE_REMOTE_STATUS}" echo "REMOTE_STATUS_RC=${remote_rc}" echo "ESCROW_MISSING_COUNT=${escrow_missing}" echo "PARTIAL_MARKER_PRESENT=${partial_marker}" echo "FULL_MARKER_PRESENT=${full_marker}"