ewoooc/scripts/startup_services.sh

#!/bin/bash
# 開機自動啟動腳本
# 部署位置: 192.168.0.110:/home/wooo/scripts/startup_services.sh
# systemd service: /etc/systemd/system/momo-startup.service

set -e

LOG_FILE="/var/log/momo_startup.log"
TELEGRAM_BOT_TOKEN="8075645931:AAH-EGKMo8ZC4QJs-Nc1_0s92xHrGdQvdpg"
TELEGRAM_CHAT_ID="5619078117"

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

send_telegram() {
    local message="$1"
    curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
        -d chat_id="${TELEGRAM_CHAT_ID}" \
        -d text="${message}" \
        -d parse_mode="HTML" > /dev/null 2>&1
}

wait_for_docker() {
    log "等待 Docker 啟動..."
    for i in {1..30}; do
        if docker info > /dev/null 2>&1; then
            log "Docker 已就緒"
            return 0
        fi
        sleep 2
    done
    log "ERROR: Docker 啟動超時"
    return 1
}

wait_for_k3s() {
    log "等待 K3s 啟動..."
    for i in {1..60}; do
        if kubectl get nodes > /dev/null 2>&1; then
            log "K3s 已就緒"
            return 0
        fi
        sleep 2
    done
    log "ERROR: K3s 啟動超時"
    return 1
}

start_harbor() {
    log "啟動 Harbor Registry..."
    cd /home/wooo/devops/harbor/harbor

    # 使用 docker compose v2
    if docker compose up -d 2>&1; then
        log "Harbor 啟動命令已發送"

        # 等待 Harbor 就緒
        for i in {1..30}; do
            if docker ps --filter "name=harbor-core" --format "{{.Status}}" | grep -q "healthy"; then
                log "Harbor 已就緒"
                return 0
            fi
            sleep 3
        done
        log "WARNING: Harbor 可能未完全啟動"
        return 1
    else
        log "ERROR: Harbor 啟動失敗"
        return 1
    fi
}

start_monitoring() {
    log "啟動監控服務..."
    cd /home/wooo/monitoring

    if [ -f "docker-compose.yml" ]; then
        docker compose up -d 2>&1 || docker-compose up -d 2>&1
        log "監控服務已啟動"
    else
        log "找不到監控服務配置"
    fi
}

start_gitlab() {
    log "啟動 GitLab..."

    # 檢查 GitLab 容器是否存在
    if docker ps -a --format "{{.Names}}" | grep -q "gitlab"; then
        docker start gitlab 2>/dev/null || true
        docker start gitlab-runner 2>/dev/null || true
        log "GitLab 已啟動"
    else
        log "GitLab 容器不存在，跳過"
    fi
}

start_n8n() {
    log "啟動 n8n..."

    if docker ps -a --format "{{.Names}}" | grep -q "n8n"; then
        docker start momo-n8n 2>/dev/null || docker start n8n 2>/dev/null || true
        log "n8n 已啟動"
    else
        log "n8n 容器不存在，跳過"
    fi
}

check_momo_pods() {
    log "檢查 MOMO K8s Pods..."

    # 等待 pods 啟動
    for i in {1..60}; do
        local app_status=$(kubectl get pods -n momo -l app=momo-app -o jsonpath='{.items[0].status.phase}' 2>/dev/null)
        local pg_status=$(kubectl get pods -n momo -l app=momo-postgres -o jsonpath='{.items[0].status.phase}' 2>/dev/null)

        if [[ "$app_status" == "Running" ]] && [[ "$pg_status" == "Running" ]]; then
            log "MOMO Pods 已就緒"
            return 0
        fi

        # 如果 ErrImagePull，嘗試重新拉取
        if kubectl get pods -n momo 2>/dev/null | grep -q "ErrImagePull\|ImagePullBackOff"; then
            log "偵測到映像拉取錯誤，重啟 pods..."
            kubectl delete pod -l app=momo-app -n momo 2>/dev/null || true
            kubectl delete pod -l app=momo-scheduler -n momo 2>/dev/null || true
        fi

        sleep 5
    done

    log "WARNING: MOMO Pods 啟動超時"
    return 1
}

verify_services() {
    log "驗證所有服務..."

    local all_ok=true
    local status_report=""

    # 檢查 Harbor
    if docker ps --filter "name=harbor-core" --format "{{.Status}}" | grep -q "healthy"; then
        status_report="${status_report}• Harbor: ✅ 正常\n"
    else
        status_report="${status_report}• Harbor: ❌ 異常\n"
        all_ok=false
    fi

    # 檢查 MOMO App
    local health_code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 https://mo.wooo.work/health 2>/dev/null)
    if [[ "$health_code" == "200" ]]; then
        status_report="${status_report}• MOMO App: ✅ 正常\n"
    else
        status_report="${status_report}• MOMO App: ❌ HTTP ${health_code}\n"
        all_ok=false
    fi

    # 檢查 PostgreSQL
    local pg_status=$(kubectl get pods -n momo -l app=momo-postgres -o jsonpath='{.items[0].status.phase}' 2>/dev/null)
    if [[ "$pg_status" == "Running" ]]; then
        status_report="${status_report}• PostgreSQL: ✅ 正常\n"
    else
        status_report="${status_report}• PostgreSQL: ❌ ${pg_status}\n"
        all_ok=false
    fi

    # 檢查 GitLab
    if docker ps --filter "name=gitlab" --format "{{.Status}}" | grep -q "Up"; then
        status_report="${status_report}• GitLab: ✅ 正常\n"
    else
        status_report="${status_report}• GitLab: ⚠️ 未運行\n"
    fi

    # 檢查 n8n
    if docker ps --filter "name=n8n" --format "{{.Status}}" | grep -q "Up"; then
        status_report="${status_report}• n8n: ✅ 正常\n"
    else
        status_report="${status_report}• n8n: ⚠️ 未運行\n"
    fi

    log "服務狀態:\n${status_report}"

    if [[ "$all_ok" == "true" ]]; then
        send_telegram "🟢 <b>UAT 伺服器開機完成</b>

所有服務已正常啟動！

$(echo -e "$status_report")

時間: $(date '+%Y-%m-%d %H:%M:%S')"
        return 0
    else
        send_telegram "🟠 <b>UAT 伺服器開機完成 (部分服務異常)</b>

$(echo -e "$status_report")

請檢查異常服務
SSH: ssh wooo@192.168.0.110

時間: $(date '+%Y-%m-%d %H:%M:%S')"
        return 1
    fi
}

main() {
    log "========== 開機啟動服務開始 =========="

    # 1. 等待 Docker
    if ! wait_for_docker; then
        send_telegram "🔴 <b>UAT 開機失敗</b>

Docker 啟動超時
需要人工介入

時間: $(date '+%Y-%m-%d %H:%M:%S')"
        exit 1
    fi

    # 2. 啟動 Harbor (K8s 需要 Registry)
    start_harbor

    # 3. 等待 K3s
    if ! wait_for_k3s; then
        send_telegram "🔴 <b>UAT 開機失敗</b>

K3s 啟動超時
需要人工介入

時間: $(date '+%Y-%m-%d %H:%M:%S')"
        exit 1
    fi

    # 4. 啟動其他 Docker 服務
    start_gitlab
    start_n8n
    start_monitoring

    # 5. 等待並檢查 MOMO Pods
    check_momo_pods

    # 6. 最終驗證
    sleep 30  # 給服務一些時間穩定
    verify_services

    log "========== 開機啟動服務完成 =========="
}

# 執行主程式
main "$@"