1b4f3a7bbe
Some checks failed
CD Pipeline / deploy (push) Failing after 59s
- 建立 Gitea Actions CD pipeline (.gitea/workflows/cd.yaml) - 部署模式: rsync Python 檔案至 188 → docker restart (volume mount) - Dockerfile/requirements 變動時自動重建 Docker image - 部署通知: Telegram (開始/成功/失敗) - 健康檢查: https://mo.wooo.work/health (最多 5 次重試) - 同步最新 CLAUDE.md / ADR-008 / memory (2026-04-19) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
350 lines
13 KiB
Bash
Executable File
350 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
||
# =============================================================================
|
||
# MOMO Pro System - CI/CD 完整驗證腳本
|
||
# 用途: 驗證完整的 CI/CD 流程和一鍵部署自動化
|
||
# =============================================================================
|
||
|
||
set -e
|
||
|
||
# 顏色定義
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
BLUE='\033[0;34m'
|
||
CYAN='\033[0;36m'
|
||
NC='\033[0m'
|
||
BOLD='\033[1m'
|
||
|
||
# 配置
|
||
K3S_HOST="${K3S_HOST:-192.168.0.110}"
|
||
K3S_USER="${K3S_USER:-wooo}"
|
||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||
|
||
# 統計
|
||
TOTAL_TESTS=0
|
||
PASSED_TESTS=0
|
||
FAILED_TESTS=0
|
||
|
||
log_section() {
|
||
echo ""
|
||
echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}"
|
||
echo -e "${CYAN} $1${NC}"
|
||
echo -e "${CYAN}═══════════════════════════════════════════════════════════════${NC}"
|
||
}
|
||
|
||
log_test() {
|
||
echo -e "${BLUE}[TEST]${NC} $1"
|
||
TOTAL_TESTS=$((TOTAL_TESTS + 1))
|
||
}
|
||
|
||
log_pass() {
|
||
echo -e "${GREEN} ✅ PASS${NC}: $1"
|
||
PASSED_TESTS=$((PASSED_TESTS + 1))
|
||
}
|
||
|
||
log_fail() {
|
||
echo -e "${RED} ❌ FAIL${NC}: $1"
|
||
FAILED_TESTS=$((FAILED_TESTS + 1))
|
||
}
|
||
|
||
log_skip() {
|
||
echo -e "${YELLOW} ⏭️ SKIP${NC}: $1"
|
||
}
|
||
|
||
# =============================================================================
|
||
# 1. 本地環境檢查
|
||
# =============================================================================
|
||
verify_local_environment() {
|
||
log_section "1. 本地環境檢查"
|
||
|
||
# Docker
|
||
log_test "Docker 是否安裝"
|
||
if command -v docker &> /dev/null; then
|
||
log_pass "Docker 已安裝 ($(docker --version | head -1))"
|
||
else
|
||
log_fail "Docker 未安裝"
|
||
fi
|
||
|
||
# Git
|
||
log_test "Git 是否安裝"
|
||
if command -v git &> /dev/null; then
|
||
log_pass "Git 已安裝 ($(git --version))"
|
||
else
|
||
log_fail "Git 未安裝"
|
||
fi
|
||
|
||
# SSH
|
||
log_test "SSH 是否可用"
|
||
if command -v ssh &> /dev/null; then
|
||
log_pass "SSH 可用"
|
||
else
|
||
log_fail "SSH 不可用"
|
||
fi
|
||
|
||
# 專案目錄
|
||
log_test "專案目錄結構"
|
||
local required_files=("app.py" "Dockerfile" "requirements.txt" "config.py")
|
||
local missing_files=()
|
||
for file in "${required_files[@]}"; do
|
||
if [[ ! -f "$PROJECT_ROOT/$file" ]]; then
|
||
missing_files+=("$file")
|
||
fi
|
||
done
|
||
if [[ ${#missing_files[@]} -eq 0 ]]; then
|
||
log_pass "所有必要檔案存在"
|
||
else
|
||
log_fail "缺少檔案: ${missing_files[*]}"
|
||
fi
|
||
}
|
||
|
||
# =============================================================================
|
||
# 2. SSH 連線測試
|
||
# =============================================================================
|
||
verify_ssh_connection() {
|
||
log_section "2. SSH 連線測試"
|
||
|
||
log_test "SSH 連線到 K3s 主機 (${K3S_HOST})"
|
||
if ssh -o ConnectTimeout=10 -o BatchMode=yes "${K3S_USER}@${K3S_HOST}" "echo 'SSH OK'" &>/dev/null; then
|
||
log_pass "SSH 連線成功"
|
||
else
|
||
log_fail "SSH 連線失敗"
|
||
return 1
|
||
fi
|
||
|
||
log_test "K3s 主機 Docker 狀態"
|
||
if ssh "${K3S_USER}@${K3S_HOST}" "docker info" &>/dev/null; then
|
||
log_pass "Docker 運行正常"
|
||
else
|
||
log_fail "Docker 未運行或無法訪問"
|
||
fi
|
||
|
||
log_test "K3s 叢集狀態"
|
||
if ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get nodes" &>/dev/null; then
|
||
log_pass "K3s 叢集正常"
|
||
else
|
||
log_fail "K3s 叢集無法訪問"
|
||
fi
|
||
}
|
||
|
||
# =============================================================================
|
||
# 3. K8s 服務狀態
|
||
# =============================================================================
|
||
verify_k8s_services() {
|
||
log_section "3. K8s 服務狀態"
|
||
|
||
log_test "momo namespace 存在"
|
||
if ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get namespace momo" &>/dev/null; then
|
||
log_pass "momo namespace 存在"
|
||
else
|
||
log_fail "momo namespace 不存在"
|
||
return 1
|
||
fi
|
||
|
||
log_test "momo-app Pod 狀態"
|
||
local app_status=$(ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get pod -n momo -l app=momo-app -o jsonpath='{.items[0].status.phase}'" 2>/dev/null)
|
||
if [[ "$app_status" == "Running" ]]; then
|
||
log_pass "momo-app 運行中"
|
||
else
|
||
log_fail "momo-app 狀態異常: $app_status"
|
||
fi
|
||
|
||
log_test "momo-postgres Pod 狀態"
|
||
local pg_status=$(ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get pod -n momo momo-postgres-0 -o jsonpath='{.status.phase}'" 2>/dev/null)
|
||
if [[ "$pg_status" == "Running" ]]; then
|
||
log_pass "momo-postgres 運行中"
|
||
else
|
||
log_fail "momo-postgres 狀態異常: $pg_status"
|
||
fi
|
||
|
||
log_test "momo-scheduler Pod 狀態"
|
||
local sched_status=$(ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get pod -n momo -l app=momo-scheduler -o jsonpath='{.items[0].status.phase}'" 2>/dev/null)
|
||
if [[ "$sched_status" == "Running" ]]; then
|
||
log_pass "momo-scheduler 運行中"
|
||
else
|
||
log_fail "momo-scheduler 狀態異常: $sched_status"
|
||
fi
|
||
}
|
||
|
||
# =============================================================================
|
||
# 4. 應用健康檢查
|
||
# =============================================================================
|
||
verify_app_health() {
|
||
log_section "4. 應用健康檢查"
|
||
|
||
log_test "HTTPS 端點可訪問 (mo.wooo.work)"
|
||
local health_response=$(curl -s -m 10 https://mo.wooo.work/health 2>/dev/null)
|
||
if echo "$health_response" | grep -q "healthy"; then
|
||
log_pass "應用健康 ($health_response)"
|
||
else
|
||
log_fail "應用健康檢查失敗"
|
||
fi
|
||
|
||
log_test "資料庫連線"
|
||
local db_test=$(ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl exec -n momo momo-postgres-0 -- psql -U momo -d momo_analytics -c 'SELECT 1'" 2>/dev/null)
|
||
if echo "$db_test" | grep -q "1"; then
|
||
log_pass "資料庫連線正常"
|
||
else
|
||
log_fail "資料庫連線失敗"
|
||
fi
|
||
}
|
||
|
||
# =============================================================================
|
||
# 5. CI/CD 檔案驗證
|
||
# =============================================================================
|
||
verify_cicd_files() {
|
||
log_section "5. CI/CD 檔案驗證"
|
||
|
||
log_test ".gitlab-ci-simple.yml 存在"
|
||
if [[ -f "$PROJECT_ROOT/.gitlab-ci-simple.yml" ]]; then
|
||
log_pass "簡化版 CI 配置存在"
|
||
else
|
||
log_fail "簡化版 CI 配置不存在"
|
||
fi
|
||
|
||
log_test "build-and-deploy.sh 存在且可執行"
|
||
if [[ -x "$PROJECT_ROOT/scripts/deploy/build-and-deploy.sh" ]]; then
|
||
log_pass "部署腳本存在且可執行"
|
||
else
|
||
log_fail "部署腳本不存在或不可執行"
|
||
fi
|
||
|
||
log_test "deploy/deploy.sh 存在且可執行"
|
||
if [[ -x "$PROJECT_ROOT/deploy/deploy.sh" ]]; then
|
||
log_pass "一鍵部署腳本存在且可執行"
|
||
else
|
||
log_fail "一鍵部署腳本不存在或不可執行"
|
||
fi
|
||
|
||
log_test "K8s 配置檔案完整"
|
||
local k8s_files=("00-namespace.yaml" "01-secrets.yaml" "02-configmap.yaml" "03-postgres.yaml" "04-momo-app.yaml" "05-scheduler.yaml")
|
||
local missing_k8s=()
|
||
for file in "${k8s_files[@]}"; do
|
||
if [[ ! -f "$PROJECT_ROOT/k8s/$file" ]]; then
|
||
missing_k8s+=("$file")
|
||
fi
|
||
done
|
||
if [[ ${#missing_k8s[@]} -eq 0 ]]; then
|
||
log_pass "所有 K8s 配置存在"
|
||
else
|
||
log_fail "缺少 K8s 配置: ${missing_k8s[*]}"
|
||
fi
|
||
}
|
||
|
||
# =============================================================================
|
||
# 6. 自動啟動驗證
|
||
# =============================================================================
|
||
verify_auto_start() {
|
||
log_section "6. 系統自動啟動驗證"
|
||
|
||
log_test "momo-startup-complete.service 已啟用"
|
||
local service_status=$(ssh "${K3S_USER}@${K3S_HOST}" "systemctl is-enabled momo-startup-complete.service 2>/dev/null" || echo "not-found")
|
||
if [[ "$service_status" == "enabled" ]]; then
|
||
log_pass "自動啟動服務已啟用"
|
||
else
|
||
log_fail "自動啟動服務未啟用 ($service_status)"
|
||
fi
|
||
|
||
log_test "啟動腳本存在"
|
||
local script_exists=$(ssh "${K3S_USER}@${K3S_HOST}" "test -f /home/wooo/momo_pro_system/scripts/tools/system_startup_complete.sh && echo 'yes'" 2>/dev/null)
|
||
if [[ "$script_exists" == "yes" ]]; then
|
||
log_pass "啟動腳本存在"
|
||
else
|
||
log_fail "啟動腳本不存在"
|
||
fi
|
||
}
|
||
|
||
# =============================================================================
|
||
# 7. 端對端部署測試（可選）
|
||
# =============================================================================
|
||
verify_e2e_deploy() {
|
||
log_section "7. 端對端部署測試（模擬）"
|
||
|
||
log_test "本地 Docker 建置"
|
||
echo " 執行: docker build -t momo-pro-system:test ."
|
||
if docker build -t momo-pro-system:test "$PROJECT_ROOT" -q &>/dev/null; then
|
||
log_pass "本地 Docker 建置成功"
|
||
docker rmi momo-pro-system:test &>/dev/null || true
|
||
else
|
||
log_fail "本地 Docker 建置失敗"
|
||
fi
|
||
|
||
log_test "rsync 同步測試（dry-run）"
|
||
if rsync -avzn --delete --exclude='.git' --exclude='venv' --exclude='__pycache__' "$PROJECT_ROOT/" "${K3S_USER}@${K3S_HOST}:/tmp/rsync-test/" &>/dev/null; then
|
||
log_pass "rsync 同步可行"
|
||
else
|
||
log_fail "rsync 同步失敗"
|
||
fi
|
||
}
|
||
|
||
# =============================================================================
|
||
# 8. 監控系統驗證
|
||
# =============================================================================
|
||
verify_monitoring() {
|
||
log_section "8. 監控系統驗證"
|
||
|
||
log_test "Prometheus 運行中"
|
||
local prom_pods=$(ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get pods -n monitoring -l app.kubernetes.io/name=prometheus -o jsonpath='{.items[*].status.phase}'" 2>/dev/null)
|
||
if echo "$prom_pods" | grep -q "Running"; then
|
||
log_pass "Prometheus 運行中"
|
||
else
|
||
log_skip "Prometheus 未部署或未運行"
|
||
fi
|
||
|
||
log_test "Alertmanager 運行中"
|
||
local am_pods=$(ssh "${K3S_USER}@${K3S_HOST}" "sudo kubectl get pods -n monitoring -l app.kubernetes.io/name=alertmanager -o jsonpath='{.items[*].status.phase}'" 2>/dev/null)
|
||
if echo "$am_pods" | grep -q "Running"; then
|
||
log_pass "Alertmanager 運行中"
|
||
else
|
||
log_skip "Alertmanager 未部署或未運行"
|
||
fi
|
||
}
|
||
|
||
# =============================================================================
|
||
# 輸出總結
|
||
# =============================================================================
|
||
print_summary() {
|
||
log_section "驗證結果總結"
|
||
|
||
echo ""
|
||
echo -e " ${BOLD}總測試數:${NC} $TOTAL_TESTS"
|
||
echo -e " ${GREEN}通過:${NC} $PASSED_TESTS"
|
||
echo -e " ${RED}失敗:${NC} $FAILED_TESTS"
|
||
echo ""
|
||
|
||
if [[ $FAILED_TESTS -eq 0 ]]; then
|
||
echo -e "${GREEN}═══════════════════════════════════════════════════════════════${NC}"
|
||
echo -e "${GREEN} ✅ 所有驗證通過！CI/CD 流程準備就緒${NC}"
|
||
echo -e "${GREEN}═══════════════════════════════════════════════════════════════${NC}"
|
||
return 0
|
||
else
|
||
echo -e "${RED}═══════════════════════════════════════════════════════════════${NC}"
|
||
echo -e "${RED} ❌ 有 $FAILED_TESTS 項驗證失敗，請檢查並修復${NC}"
|
||
echo -e "${RED}═══════════════════════════════════════════════════════════════${NC}"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# =============================================================================
|
||
# 主程式
|
||
# =============================================================================
|
||
main() {
|
||
echo ""
|
||
echo -e "${CYAN}╔═══════════════════════════════════════════════════════════════════════════╗${NC}"
|
||
echo -e "${CYAN}║${NC} ${BOLD}MOMO Pro System - CI/CD 完整驗證${NC} ${CYAN}║${NC}"
|
||
echo -e "${CYAN}║${NC} ${BOLD}$(date '+%Y-%m-%d %H:%M:%S')${NC} ${CYAN}║${NC}"
|
||
echo -e "${CYAN}╚═══════════════════════════════════════════════════════════════════════════╝${NC}"
|
||
|
||
verify_local_environment
|
||
verify_ssh_connection
|
||
verify_k8s_services
|
||
verify_app_health
|
||
verify_cicd_files
|
||
verify_auto_start
|
||
verify_e2e_deploy
|
||
verify_monitoring
|
||
|
||
print_summary
|
||
}
|
||
|
||
main "$@"
|