#!/bin/bash
# =============================================================================
# MOMO Pro System 統一健康監控腳本
#
# 功能：整合所有監控項目
#   1. Docker 網路健康（502 Bad Gateway 預防）
#   2. Harbor Registry 服務（K8s ImagePullBackOff 預防）
#   3. Google Drive 認證（自動匯入功能）
#   4. 應用服務健康檢查
#
# 用法：
#   每 5 分鐘執行一次（透過 cron）
#   */5 * * * * /home/wooo/scripts/system_health_monitor.sh >> /var/log/system_health_monitor.log 2>&1
#
# 作者：Claude Code
# 日期：2026-01-28
# =============================================================================

set -e

# =============================================================================
# 配置區
# =============================================================================

# 服務端點
HEALTH_URL="https://mo.wooo.work/health"
GOOGLE_DRIVE_URL="https://mo.wooo.work/api/test_drive_connection"
HARBOR_URL="http://127.0.0.1:5050/api/v2.0/health"

# Docker 容器
APP_CONTAINER="momo-pro-system"
SCHEDULER_CONTAINER="momo-scheduler"

# Harbor 目錄
HARBOR_DIR="/home/wooo/devops/harbor/harbor"

# Telegram 配置
TELEGRAM_BOT_TOKEN="<TELEGRAM_BOT_TOKEN>"
TELEGRAM_CHAT_ID="5619078117"

# 超時設定（秒）
CURL_TIMEOUT=15

# 狀態檔案目錄
STATE_DIR="/tmp/momo_monitor_state"
mkdir -p "${STATE_DIR}"

# =============================================================================
# 函數定義
# =============================================================================

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
}

send_telegram() {
    local message="$1"
    curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
        -d chat_id="${TELEGRAM_CHAT_ID}" \
        -d text="${message}" \
        -d parse_mode="HTML" \
        > /dev/null 2>&1 || true
}

# 檢查是否在告警冷卻期
should_alert() {
    local key="$1"
    local cooldown="${2:-3600}"  # 預設 1 小時
    local state_file="${STATE_DIR}/${key}"

    if [ -f "${state_file}" ]; then
        local last_alert=$(cat "${state_file}" 2>/dev/null || echo "0")
        local now=$(date +%s)
        local diff=$((now - last_alert))
        if [ ${diff} -lt ${cooldown} ]; then
            return 1
        fi
    fi
    return 0
}

record_alert() {
    local key="$1"
    date +%s > "${STATE_DIR}/${key}"
}

clear_alert() {
    local key="$1"
    rm -f "${STATE_DIR}/${key}"
}

# =============================================================================
# 監控項目 1: Docker 網路健康
# =============================================================================

check_docker_network() {
    log "[Docker 網路] 開始檢查..."

    # 檢查外部健康端點
    local http_code
    http_code=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout ${CURL_TIMEOUT} "${HEALTH_URL}" 2>/dev/null || echo "000")

    if [ "${http_code}" = "200" ]; then
        log "[Docker 網路] ✅ 正常 (HTTP ${http_code})"
        clear_alert "docker_network"
        return 0
    fi

    log "[Docker 網路] ❌ 異常 (HTTP ${http_code})"

    # 診斷：檢查容器內部
    local internal_check
    internal_check=$(docker exec ${APP_CONTAINER} curl -s -o /dev/null -w "%{http_code}" --connect-timeout 5 http://127.0.0.1:80/health 2>/dev/null || echo "000")

    if [ "${internal_check}" = "200" ]; then
        # 容器內部正常但外部不通 → Docker 網路問題
        log "[Docker 網路] 診斷: 容器內部正常，Docker 網路轉發失效"

        if should_alert "docker_network" 1800; then
            send_telegram "🔴 <b>Docker 網路轉發失效</b>

症狀: mo.wooo.work 無法存取 (HTTP ${http_code})
診斷: 容器內部正常，端口映射失效
動作: 正在自動重啟 Docker 服務...
時間: $(date '+%Y-%m-%d %H:%M:%S')"
            record_alert "docker_network"
        fi

        # 自動修復：重啟 Docker
        log "[Docker 網路] 執行自動修復: 重啟 Docker 服務"
        sudo systemctl restart docker
        sleep 60

        # 驗證修復
        http_code=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout ${CURL_TIMEOUT} "${HEALTH_URL}" 2>/dev/null || echo "000")
        if [ "${http_code}" = "200" ]; then
            send_telegram "🟢 <b>Docker 網路已恢復</b>

動作: Docker 服務重啟成功
時間: $(date '+%Y-%m-%d %H:%M:%S')"
            clear_alert "docker_network"
        else
            send_telegram "🔴 <b>Docker 網路修復失敗</b>

狀態: 重啟後服務仍未恢復
需要: 人工介入處理
SSH: ssh wooo@192.168.0.110"
        fi
    else
        # 容器內部也異常
        log "[Docker 網路] 診斷: 容器內部也異常，可能是應用問題"

        if should_alert "docker_network" 1800; then
            send_telegram "🔴 <b>MOMO 應用服務異常</b>

症狀: mo.wooo.work 無法存取 (HTTP ${http_code})
診斷: 容器內部服務也異常
動作: 正在重啟容器...
時間: $(date '+%Y-%m-%d %H:%M:%S')"
            record_alert "docker_network"
        fi

        # 嘗試重啟容器
        docker restart ${APP_CONTAINER} ${SCHEDULER_CONTAINER}
        sleep 30
    fi

    return 1
}

# =============================================================================
# 監控項目 2: Harbor Registry 服務
# =============================================================================

check_harbor() {
    log "[Harbor] 開始檢查..."

    # 檢查 Harbor 健康狀態
    local http_code
    http_code=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout ${CURL_TIMEOUT} "${HARBOR_URL}" 2>/dev/null || echo "000")

    if [ "${http_code}" = "200" ]; then
        log "[Harbor] ✅ 正常 (HTTP ${http_code})"
        clear_alert "harbor"
        return 0
    fi

    log "[Harbor] ❌ 異常 (HTTP ${http_code})"

    # 檢查 Harbor 容器是否運行
    if ! docker ps --format '{{.Names}}' | grep -q "^harbor-core$"; then
        log "[Harbor] 診斷: Harbor 容器未運行"

        if should_alert "harbor" 1800; then
            send_telegram "🔴 <b>Harbor Registry 服務停止</b>

症狀: Harbor 容器未運行
影響: K8s Pods 將無法拉取映像
動作: 正在自動啟動 Harbor...
時間: $(date '+%Y-%m-%d %H:%M:%S')"
            record_alert "harbor"
        fi

        # 自動修復：啟動 Harbor
        log "[Harbor] 執行自動修復: 啟動 Harbor"
        cd "${HARBOR_DIR}"
        docker compose up -d
        sleep 60

        # 驗證修復
        http_code=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout ${CURL_TIMEOUT} "${HARBOR_URL}" 2>/dev/null || echo "000")
        if [ "${http_code}" = "200" ]; then
            send_telegram "🟢 <b>Harbor Registry 已恢復</b>

動作: Docker Compose 啟動成功
時間: $(date '+%Y-%m-%d %H:%M:%S')

⚠️ K8s Pods 可能需要重啟：
<code>sudo kubectl rollout restart deployment/momo-app deployment/momo-scheduler -n momo</code>"
            clear_alert "harbor"
        else
            send_telegram "🔴 <b>Harbor 啟動失敗</b>

需要: 人工介入處理
SSH: ssh wooo@192.168.0.110
目錄: ${HARBOR_DIR}"
        fi
    fi

    return 1
}

# =============================================================================
# 監控項目 3: Google Drive 認證
# =============================================================================

check_google_drive() {
    log "[Google Drive] 開始檢查..."

    # 測試 Google Drive API 連線
    local response
    response=$(curl -s --connect-timeout ${CURL_TIMEOUT} \
        -X POST "${GOOGLE_DRIVE_URL}" \
        -H "Content-Type: application/json" \
        -d '{}' 2>/dev/null || echo '{"success":false}')

    if echo "${response}" | grep -q '"success":true'; then
        log "[Google Drive] ✅ 認證正常"
        clear_alert "google_drive"
        return 0
    fi

    log "[Google Drive] ❌ 認證失敗"

    # Google Drive 認證無法自動修復，只能發送告警
    if should_alert "google_drive" 3600; then
        send_telegram "🔴 <b>Google Drive 認證失敗</b>

症狀: OAuth Token 已過期或被撤銷
影響: 當日業績自動匯入功能停止

<b>需要人工處理：</b>
1. 在本機重新認證：
<code>cd /Users/ogt/momo-pro-system
rm -f config/google_token.pickle
python3 -c \"from services.google_drive_service import drive_service; drive_service.authenticate()\"</code>

2. 更新 Docker 容器：
<code>scp config/google_*.* wooo@192.168.0.110:/tmp/
ssh wooo@192.168.0.110 'docker cp /tmp/google_credentials.json momo-pro-system:/app/config/ && docker cp /tmp/google_token.pickle momo-pro-system:/app/config/ && docker restart momo-pro-system momo-scheduler'</code>

時間: $(date '+%Y-%m-%d %H:%M:%S')"
        record_alert "google_drive"
    fi

    return 1
}

# =============================================================================
# 主程式
# =============================================================================

main() {
    log "========== 開始系統健康檢查 =========="

    local all_healthy=true

    # 執行所有檢查
    check_docker_network || all_healthy=false
    check_harbor || all_healthy=false
    check_google_drive || all_healthy=false

    if [ "${all_healthy}" = true ]; then
        log "========== 所有服務正常 =========="
    else
        log "========== 部分服務異常，已嘗試修復或發送告警 =========="
    fi
}

# 執行主程式
main "$@"