#!/usr/bin/env bash
# docker-health-monitor.sh
# Sprint 5.2 Plan A: 感知 + 自動修復 + 回報
#
# 部署: cron */5 * * * * /opt/awoooi-ops/docker-health-monitor.sh >> /var/log/docker-health-monitor.log 2>&1
# 設定: /etc/awoooi-ops/secrets.env
# 撰寫: Claude Sonnet 4.6 / 2026-04-09 Asia/Taipei
#
# 修復分級 (ADR-060 統帥裁示):
#   自動 docker restart: 一般應用容器 (非 DB/Redis/監控棧)
#   自動 docker start:   prometheus / grafana / alertmanager (保護 WAL)
#   僅告警 (禁止重啟):   postgres / redis / clickhouse / DB 類
#
# 流程: Detect → AutoRepair → Report (Intent→Action→Result 三段)

set -euo pipefail

# ─── 環境載入 ───────────────────────────────────────────────────────────────
SECRETS_FILE="/etc/awoooi-ops/secrets.env"
if [[ -f "$SECRETS_FILE" ]]; then
    # shellcheck source=/dev/null
    source "$SECRETS_FILE"
fi

: "${AWOOOI_API_URL:=https://awoooi.wooo.work}"
: "${TELEGRAM_BOT_TOKEN:=}"
: "${SRE_GROUP_CHAT_ID:=-1003711974679}"
: "${LOG_FILE:=/var/log/docker-health-monitor.log}"
: "${SEND_COOLDOWN_SECONDS:=300}"
: "${COOLDOWN_DIR:=/tmp/docker-health-monitor-cooldown}"
: "${EXCLUDE_CONTAINERS:=signoz-telemetrystore-migrator,signoz-clickhouse,signoz-init-clickhouse,gitea-runner}"

mkdir -p "$COOLDOWN_DIR"

# ─── 禁止自動重啟的容器 (模式匹配) ─────────────────────────────────────────
# DB / Cache / 監控棧核心 — 僅告警，不自動重啟
READONLY_PATTERNS=(
    "postgres" "momo-db" "langfuse-db" "harbor-db" "sentry-postgres"
    "redis" "harbor-redis" "sentry-redis" "signoz-clickhouse"
)

# 監控棧容器：用 docker start（非 restart），保護 WAL
SAFE_START_PATTERNS=(
    "prometheus" "grafana" "alertmanager"
)

# ─── 工具函數 ────────────────────────────────────────────────────────────────
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S %z')] $*"
}

is_in_cooldown() {
    local container="$1"
    local cooldown_file="${COOLDOWN_DIR}/${container}.cooldown"
    if [[ -f "$cooldown_file" ]]; then
        local last_sent now elapsed
        last_sent=$(cat "$cooldown_file")
        now=$(date +%s)
        elapsed=$(( now - last_sent ))
        if (( elapsed < SEND_COOLDOWN_SECONDS )); then
            log "COOLDOWN: ${container} 距上次處理 ${elapsed}s，跳過（冷卻 ${SEND_COOLDOWN_SECONDS}s）"
            return 0
        fi
    fi
    return 1
}

set_cooldown() {
    local container="$1"
    date +%s > "${COOLDOWN_DIR}/${container}.cooldown"
}

# 判斷容器是否符合模式清單
matches_pattern() {
    local name="$1"
    shift
    local patterns=("$@")
    for pattern in "${patterns[@]}"; do
        case "$name" in
            *"${pattern}"*) return 0 ;;
        esac
    done
    return 1
}

# ─── Telegram 直發 Fallback ──────────────────────────────────────────────────
send_telegram_direct() {
    local message="$1"
    [[ -z "$TELEGRAM_BOT_TOKEN" || -z "$SRE_GROUP_CHAT_ID" ]] && return 0
    curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
        -H "Content-Type: application/json" \
        -d "{\"chat_id\":\"${SRE_GROUP_CHAT_ID}\",\"text\":\"${message}\",\"parse_mode\":\"HTML\"}" \
        > /dev/null 2>&1 || true
}

# ─── 回報到 AWOOOI API ───────────────────────────────────────────────────────
report_to_awoooi() {
    local container="$1"
    local detected_status="$2"   # unhealthy | exited | dead
    local repair_action="$3"     # restarted | started | alert_only | failed
    local repair_result="$4"     # success | failed | skipped
    local hostname
    hostname=$(hostname)
    local now_ts
    now_ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ')

    # auto_repair label 根據實際動作設定
    local auto_repair_label="false"
    [[ "$repair_result" == "success" ]] && auto_repair_label="true"

    local payload
    payload=$(cat <<JSON
{
  "version": "4",
  "groupKey": "docker-health-${hostname}-${container}",
  "status": "firing",
  "alerts": [{
    "status": "firing",
    "labels": {
      "alertname": "DockerContainerUnhealthy",
      "container": "${container}",
      "host": "${hostname}",
      "layer": "docker",
      "severity": "warning",
      "auto_repair": "${auto_repair_label}",
      "repair_action": "${repair_action}",
      "repair_result": "${repair_result}",
      "source": "docker-health-monitor"
    },
    "annotations": {
      "summary": "容器 ${container} 狀態=${detected_status} → 修復=${repair_action}(${repair_result})",
      "description": "主機 ${hostname} | 偵測=${detected_status} | 動作=${repair_action} | 結果=${repair_result}"
    },
    "startsAt": "${now_ts}"
  }]
}
JSON
)

    local http_code
    http_code=$(curl -s -o /dev/null -w "%{http_code}" \
        -X POST "${AWOOOI_API_URL}/api/v1/webhooks/alertmanager" \
        -H "Content-Type: application/json" \
        -d "$payload" \
        --connect-timeout 10 \
        --max-time 30 2>/dev/null) || http_code="0"

    if [[ "$http_code" == "200" || "$http_code" == "202" ]]; then
        log "REPORTED: ${container} repair=${repair_action}(${repair_result}) → API (${http_code})"
    else
        log "WARN: API 回應 ${http_code}，Fallback Telegram"
        local emoji="🔧"
        [[ "$repair_result" == "failed" ]] && emoji="❌"
        [[ "$repair_action" == "alert_only" ]] && emoji="⚠️"
        send_telegram_direct "${emoji} [docker-health-monitor]&#10;主機: ${hostname}&#10;容器: ${container}&#10;狀態: ${detected_status}&#10;修復: ${repair_action} → ${repair_result}&#10;(API 不可達)"
    fi
}

# ─── 自動修復邏輯 ────────────────────────────────────────────────────────────
attempt_repair() {
    local container="$1"
    local detected_status="$2"

    # 1. 禁止重啟清單
    if matches_pattern "$container" "${READONLY_PATTERNS[@]}"; then
        log "ALERT_ONLY: ${container} 在禁止重啟清單，僅告警"
        report_to_awoooi "$container" "$detected_status" "alert_only" "skipped"
        return
    fi

    # 2. 監控棧 — 用 docker start（保護 WAL）
    if matches_pattern "$container" "${SAFE_START_PATTERNS[@]}"; then
        log "SAFE_START: ${container} 屬監控棧，執行 docker start"
        if docker start "$container" >> /dev/null 2>&1; then
            log "SUCCESS: docker start ${container}"
            report_to_awoooi "$container" "$detected_status" "started" "success"
        else
            log "FAILED: docker start ${container}"
            report_to_awoooi "$container" "$detected_status" "started" "failed"
        fi
        return
    fi

    # 3. 一般容器 — docker restart
    log "AUTO_REPAIR: docker restart ${container}"
    if docker restart "$container" >> /dev/null 2>&1; then
        log "SUCCESS: docker restart ${container}"
        report_to_awoooi "$container" "$detected_status" "restarted" "success"
    else
        log "FAILED: docker restart ${container}"
        report_to_awoooi "$container" "$detected_status" "restarted" "failed"
    fi
}

# ─── 核心：掃描所有容器 ─────────────────────────────────────────────────────
check_containers() {
    local hostname
    hostname=$(hostname)

    while IFS=$'\t' read -r container_id container_name state health; do
        [[ -z "$container_name" ]] && continue

        # 排除清單
        local excluded=false
        IFS=',' read -ra EXCLUDES <<< "$EXCLUDE_CONTAINERS"
        for pattern in "${EXCLUDES[@]}"; do
            pattern="${pattern// /}"
            [[ -z "$pattern" ]] && continue
            # shellcheck disable=SC2254
            case "$container_name" in
                $pattern) excluded=true; break ;;
            esac
        done
        $excluded && continue

        local needs_action=false
        local detected_status=""

        if [[ "$state" == "exited" || "$state" == "dead" ]]; then
            needs_action=true
            detected_status="$state"
        fi
        if [[ "$health" == "unhealthy" ]]; then
            needs_action=true
            detected_status="unhealthy"
        fi

        if $needs_action; then
            log "DETECTED: ${container_name} state=${state} health=${health} on ${hostname}"
            is_in_cooldown "$container_name" && continue
            set_cooldown "$container_name"
            attempt_repair "$container_name" "$detected_status"
        fi
    done < <(docker ps -a --format '{{.ID}}\t{{.Names}}\t{{.State}}\t{{.Status}}' | \
        awk -F'\t' '{
            health = ""
            if ($4 ~ /\(unhealthy\)/) health = "unhealthy"
            else if ($4 ~ /\(healthy\)/) health = "healthy"
            print $1 "\t" $2 "\t" $3 "\t" health
        }')
}

# ─── Main ───────────────────────────────────────────────────────────────────
main() {
    log "=== docker-health-monitor 啟動 (自動修復模式) on $(hostname) ==="
    check_containers
    log "=== 掃描完成 ==="
}

main "$@"
