#!/bin/bash # ============================================================================= # MOMO Pro System - CI/CD 完整驗證腳本 # 用途: 驗證完整的 CI/CD 流程和一鍵部署自動化 # ============================================================================= set -e # 顏色定義 RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' BOLD='\033[1m' # 配置 K3S_HOST="${K3S_HOST:-192.168.0.110}" K3S_USER="${K3S_USER:-wooo}" PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # 統計 TOTAL_TESTS=0 PASSED_TESTS=0 FAILED_TESTS=0 log_section() { echo "" echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}" echo -e "${CYAN} $1${NC}" echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}" } log_test() { echo -e "${BLUE}[TEST]${NC} $1" TOTAL_TESTS=$((TOTAL_TESTS + 1)) } log_pass() { echo -e "${GREEN} ✅ PASS${NC}: $1" PASSED_TESTS=$((PASSED_TESTS + 1)) } log_fail() { echo -e "${RED} ❌ FAIL${NC}: $1" FAILED_TESTS=$((FAILED_TESTS + 1)) } log_skip() { echo -e "${YELLOW} ⏭️ SKIP${NC}: $1" } # ============================================================================= # 1. 本地環境檢查 # ============================================================================= verify_local_environment() { log_section "1. 本地環境檢查" # Docker log_test "Docker 是否安裝" if command -v docker &> /dev/null; then log_pass "Docker 已安裝 ($(docker --version | head -1))" else log_fail "Docker 未安裝" fi # Git log_test "Git 是否安裝" if command -v git &> /dev/null; then log_pass "Git 已安裝 ($(git --version))" else log_fail "Git 未安裝" fi # SSH log_test "SSH 是否可用" if command -v ssh &> /dev/null; then log_pass "SSH 可用" else log_fail "SSH 不可用" fi # 專案目錄 log_test "專案目錄結構" local required_files=("app.py" "Dockerfile" "requirements.txt" "config.py") local missing_files=() for file in "${required_files[@]}"; do if [[ ! -f "$PROJECT_ROOT/$file" ]]; then missing_files+=("$file") fi done if [[ ${#missing_files[@]} -eq 0 ]]; then log_pass "所有必要檔案存在" else log_fail "缺少檔案: ${missing_files[*]}" fi } # ============================================================================= # 2. SSH 連線測試 # ============================================================================= verify_ssh_connection() { log_section "2. SSH 連線測試" log_test "SSH 連線到 K3s 主機 (${K3S_HOST})" if ssh -o ConnectTimeout=10 -o BatchMode=yes "${K3S_USER}@${K3S_HOST}" "echo 'SSH OK'" &>/dev/null; then log_pass "SSH 連線成功" else log_fail "SSH 連線失敗" return 1 fi log_test "K3s 主機 Docker 狀態" if ssh "${K3S_USER}@${K3S_HOST}" "docker info" &>/dev/null; then log_pass "Docker 運行正常" else log_fail "Docker 未運行或無法訪問" fi log_test "K3s 叢集狀態" if ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get nodes" &>/dev/null; then log_pass "K3s 叢集正常" else log_fail "K3s 叢集無法訪問" fi } # ============================================================================= # 3. K8s 服務狀態 # ============================================================================= verify_k8s_services() { log_section "3. K8s 服務狀態" log_test "momo namespace 存在" if ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get namespace momo" &>/dev/null; then log_pass "momo namespace 存在" else log_fail "momo namespace 不存在" return 1 fi log_test "momo-app Pod 狀態" local app_status=$(ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get pod -n momo -l app=momo-app -o jsonpath='{.items[0].status.phase}'" 2>/dev/null) if [[ "$app_status" == "Running" ]]; then log_pass "momo-app 運行中" else log_fail "momo-app 狀態異常: $app_status" fi log_test "momo-postgres Pod 狀態" local pg_status=$(ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get pod -n momo momo-postgres-0 -o jsonpath='{.status.phase}'" 2>/dev/null) if [[ "$pg_status" == "Running" ]]; then log_pass "momo-postgres 運行中" else log_fail "momo-postgres 狀態異常: $pg_status" fi log_test "momo-scheduler Pod 狀態" local sched_status=$(ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get pod -n momo -l app=momo-scheduler -o jsonpath='{.items[0].status.phase}'" 2>/dev/null) if [[ "$sched_status" == "Running" ]]; then log_pass "momo-scheduler 運行中" else log_fail "momo-scheduler 狀態異常: $sched_status" fi } # ============================================================================= # 4. 應用健康檢查 # ============================================================================= verify_app_health() { log_section "4. 應用健康檢查" log_test "HTTPS 端點可訪問 (mo.wooo.work)" local health_response=$(curl -s -m 10 https://mo.wooo.work/health 2>/dev/null) if echo "$health_response" | grep -q "healthy"; then log_pass "應用健康 ($health_response)" else log_fail "應用健康檢查失敗" fi log_test "資料庫連線" local db_test=$(ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl exec -n momo momo-postgres-0 -- psql -U momo -d momo_analytics -c 'SELECT 1'" 2>/dev/null) if echo "$db_test" | grep -q "1"; then log_pass "資料庫連線正常" else log_fail "資料庫連線失敗" fi } # ============================================================================= # 5. CI/CD 檔案驗證 # ============================================================================= verify_cicd_files() { log_section "5. CI/CD 檔案驗證" log_test ".gitlab-ci-simple.yml 存在" if [[ -f "$PROJECT_ROOT/.gitlab-ci-simple.yml" ]]; then log_pass "簡化版 CI 配置存在" else log_fail "簡化版 CI 配置不存在" fi log_test "build-and-deploy.sh 存在且可執行" if [[ -x "$PROJECT_ROOT/scripts/deploy/build-and-deploy.sh" ]]; then log_pass "部署腳本存在且可執行" else log_fail "部署腳本不存在或不可執行" fi log_test "deploy/deploy.sh 存在且可執行" if [[ -x "$PROJECT_ROOT/deploy/deploy.sh" ]]; then log_pass "一鍵部署腳本存在且可執行" else log_fail "一鍵部署腳本不存在或不可執行" fi log_test "K8s 配置檔案完整" local k8s_files=("00-namespace.yaml" "01-secrets.yaml" "02-configmap.yaml" "03-postgres.yaml" "04-momo-app.yaml" "05-scheduler.yaml") local missing_k8s=() for file in "${k8s_files[@]}"; do if [[ ! -f "$PROJECT_ROOT/k8s/$file" ]]; then missing_k8s+=("$file") fi done if [[ ${#missing_k8s[@]} -eq 0 ]]; then log_pass "所有 K8s 配置存在" else log_fail "缺少 K8s 配置: ${missing_k8s[*]}" fi } # ============================================================================= # 6. 自動啟動驗證 # ============================================================================= verify_auto_start() { log_section "6. 系統自動啟動驗證" log_test "momo-startup-complete.service 已啟用" local service_status=$(ssh "${K3S_USER}@${K3S_HOST}" "systemctl is-enabled momo-startup-complete.service 2>/dev/null" || echo "not-found") if [[ "$service_status" == "enabled" ]]; then log_pass "自動啟動服務已啟用" else log_fail "自動啟動服務未啟用 ($service_status)" fi log_test "啟動腳本存在" local script_exists=$(ssh "${K3S_USER}@${K3S_HOST}" "test -f /home/wooo/momo_pro_system/scripts/tools/system_startup_complete.sh && echo 'yes'" 2>/dev/null) if [[ "$script_exists" == "yes" ]]; then log_pass "啟動腳本存在" else log_fail "啟動腳本不存在" fi } # ============================================================================= # 7. 端對端部署測試(可選) # ============================================================================= verify_e2e_deploy() { log_section "7. 端對端部署測試(模擬)" log_test "本地 Docker 建置" echo " 執行: docker build -t momo-pro-system:test ." if docker build -t momo-pro-system:test "$PROJECT_ROOT" -q &>/dev/null; then log_pass "本地 Docker 建置成功" docker rmi momo-pro-system:test &>/dev/null || true else log_fail "本地 Docker 建置失敗" fi log_test "rsync 同步測試(dry-run)" if rsync -avzn --delete --exclude='.git' --exclude='venv' --exclude='__pycache__' "$PROJECT_ROOT/" "${K3S_USER}@${K3S_HOST}:/tmp/rsync-test/" &>/dev/null; then log_pass "rsync 同步可行" else log_fail "rsync 同步失敗" fi } # ============================================================================= # 8. 監控系統驗證 # ============================================================================= verify_monitoring() { log_section "8. 監控系統驗證" log_test "Prometheus 運行中" local prom_pods=$(ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get pods -n monitoring -l app.kubernetes.io/name=prometheus -o jsonpath='{.items[*].status.phase}'" 2>/dev/null) if echo "$prom_pods" | grep -q "Running"; then log_pass "Prometheus 運行中" else log_skip "Prometheus 未部署或未運行" fi log_test "Alertmanager 運行中" local am_pods=$(ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get pods -n monitoring -l app.kubernetes.io/name=alertmanager -o jsonpath='{.items[*].status.phase}'" 2>/dev/null) if echo "$am_pods" | grep -q "Running"; then log_pass "Alertmanager 運行中" else log_skip "Alertmanager 未部署或未運行" fi } # ============================================================================= # 輸出總結 # ============================================================================= print_summary() { log_section "驗證結果總結" echo "" echo -e " ${BOLD}總測試數:${NC} $TOTAL_TESTS" echo -e " ${GREEN}通過:${NC} $PASSED_TESTS" echo -e " ${RED}失敗:${NC} $FAILED_TESTS" echo "" if [[ $FAILED_TESTS -eq 0 ]]; then echo -e "${GREEN}═══════════════════════════════════════════════════════════════${NC}" echo -e "${GREEN} ✅ 所有驗證通過!CI/CD 流程準備就緒${NC}" echo -e "${GREEN}═══════════════════════════════════════════════════════════════${NC}" return 0 else echo -e "${RED}═══════════════════════════════════════════════════════════════${NC}" echo -e "${RED} ❌ 有 $FAILED_TESTS 項驗證失敗,請檢查並修復${NC}" echo -e "${RED}═══════════════════════════════════════════════════════════════${NC}" return 1 fi } # ============================================================================= # 主程式 # ============================================================================= main() { echo "" echo -e "${CYAN}╔═══════════════════════════════════════════════════════════════════════════╗${NC}" echo -e "${CYAN}║${NC} ${BOLD}MOMO Pro System - CI/CD 完整驗證${NC} ${CYAN}║${NC}" echo -e "${CYAN}║${NC} ${BOLD}$(date '+%Y-%m-%d %H:%M:%S')${NC} ${CYAN}║${NC}" echo -e "${CYAN}╚═══════════════════════════════════════════════════════════════════════════╝${NC}" verify_local_environment verify_ssh_connection verify_k8s_services verify_app_health verify_cicd_files verify_auto_start verify_e2e_deploy verify_monitoring print_summary } main "$@"