#!/bin/bash
# 192.168.0.110 重開機自動恢復腳本
# 2026-04-05 ogt: 根據第二次重開機事故建立
# 部署位置: /usr/local/bin/awoooi-startup-110.sh (on 192.168.0.110)
# systemd unit: /etc/systemd/system/awoooi-startup-110.service
#
# 已知問題處理:
# - Docker BoltDB 損壞 (network/files/local-kv.db, volumes/metadata.db)
# - 舊容器使用已不存在的 Docker network (需要 docker rm -f 全部)
# - Harbor nginx 依賴 harbor-log (需要等 harbor-log healthy 後才 compose up)

set -uo pipefail
LOG="/var/log/awoooi-startup-110.log"
exec > >(tee -a "$LOG") 2>&1

log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }

log "=== 192.168.0.110 啟動序列開始 ==="

# ──────────────────────────────────────────────
# STEP 1: Docker 修復（若 BoltDB 損壞）
# ──────────────────────────────────────────────
log "[1/5] 檢查 Docker..."

if ! systemctl is-active docker >/dev/null 2>&1; then
    log "Docker 未啟動，嘗試啟動..."
    systemctl start docker || true
    sleep 8
fi

if ! systemctl is-active docker >/dev/null 2>&1; then
    log "Docker 啟動失敗，修復 BoltDB..."
    # 清除所有已知 BoltDB 損壞點
    NETWORK_DB="/var/lib/docker/network/files/local-kv.db"
    VOLUMES_DB="/var/lib/docker/volumes/metadata.db"

    if [ -f "$NETWORK_DB" ]; then
        cp "$NETWORK_DB" "${NETWORK_DB}.bak.$(date +%Y%m%d%H%M%S)"
        rm -f "$NETWORK_DB"
        log "清除損壞的 local-kv.db"
    fi
    if [ -f "$VOLUMES_DB" ]; then
        cp "$VOLUMES_DB" "${VOLUMES_DB}.bak.$(date +%Y%m%d%H%M%S)"
        rm -f "$VOLUMES_DB"
        log "清除損壞的 metadata.db"
    fi
    # 清除 buildkit 快取（也可能損壞）
    find /var/lib/docker/buildkit -name "*.db" -delete 2>/dev/null || true

    systemctl start docker
    sleep 8
    systemctl is-active docker && log "✅ Docker 修復成功" || { log "❌ Docker 修復失敗"; exit 1; }
else
    log "✅ Docker 已 active"
fi

# ──────────────────────────────────────────────
# STEP 2: 清除孤兒容器（舊容器的 network 已不存在）
# ──────────────────────────────────────────────
log "[2/5] 清除孤兒容器..."
STALE=$(docker ps -a --format "{{.Names}}\t{{.Status}}" | grep "Exited (128)\|Exited (137)" | awk '{print $1}')
if [ -n "$STALE" ]; then
    log "發現孤兒容器: $(echo $STALE | tr '\n' ' ')"
    echo "$STALE" | xargs docker rm -f 2>/dev/null || true
    docker network prune -f 2>/dev/null || true
    log "✅ 孤兒容器清除完成"
else
    log "✅ 無孤兒容器"
fi

# ──────────────────────────────────────────────
# STEP 3: Harbor（嚴格時序：harbor-log first）
# 2026-04-05 Claude Code: 修復 Race Condition —
#   舊邏輯：compose up -d 全部 → 等 harbor-log healthy → compose up -d 全部
#   問題：第一次 up 時 harbor-core/db/portal 嘗試連 syslog:1514 失敗 exit，
#         restart:always 重試直到放棄，第二次 compose up 無效
#   新邏輯：只啟動 harbor-log → 等 healthy → 啟動其餘全部
# 首席架構師 Review I1: 使用 lockfile 防止 watchdog 並行修復
# ──────────────────────────────────────────────
log "[3/5] 啟動 Harbor..."
HARBOR_DIR="/home/wooo/harbor/harbor"
HARBOR_LOCKFILE="/var/lock/harbor-repair.lock"
if [ -f "$HARBOR_DIR/docker-compose.yml" ]; then
    cd "$HARBOR_DIR"

    # I1: 持有 lockfile，防止 harbor-watchdog 同時修復
    exec 9>"$HARBOR_LOCKFILE"
    flock -w 30 9 || { log "⚠️  無法取得 Harbor lock，跳過（watchdog 可能正在修復）"; return 0; }

    # Phase 1: 清除任何 Exited 狀態的 Harbor 容器（避免 restart:always 干擾）
    # 使用 --filter 取代 grep/awk（更精確，不受欄位格式影響）
    HARBOR_EXITED=$(docker ps -a --filter name="harbor-" --filter status=exited --format "{{.Names}}" 2>/dev/null || true)
    if [ -n "$HARBOR_EXITED" ]; then
        log "Phase 1: 清除 Exited Harbor 容器: $(echo "$HARBOR_EXITED" | tr '\n' ' ')"
        echo "$HARBOR_EXITED" | xargs docker rm -f 2>&1 | tail -3 || true
    else
        log "Phase 1: 無 Exited 容器"
    fi

    # Phase 2: 只啟動 harbor-log（syslog 接收端，其他容器的前置依賴）
    log "Phase 2: 啟動 harbor-log..."
    docker compose up -d harbor-log 2>&1 | tail -3

    # Phase 3: 等待 harbor-log healthy（最多 90 秒）
    log "Phase 3: 等待 harbor-log healthy（最多 90s）..."
    for i in $(seq 1 18); do
        STATUS=$(docker inspect --format='{{.State.Health.Status}}' harbor-log 2>/dev/null || echo "missing")
        if [ "$STATUS" = "healthy" ]; then
            log "✅ harbor-log healthy (${i}x5s)"
            break
        fi
        sleep 5
    done

    STATUS=$(docker inspect --format='{{.State.Health.Status}}' harbor-log 2>/dev/null || echo "unknown")
    if [ "$STATUS" = "healthy" ]; then
        # Phase 4: harbor-log 就緒後才啟動其餘組件
        log "Phase 4: 啟動 Harbor 全組件..."
        docker compose up -d 2>&1 | tail -5

        # S1: 等 30s 讓 harbor-core 完成初始化（之前 5s 不夠）
        sleep 30
        # 注意：Harbor /v2/ 回 401（需認證），不能用 -f flag（會把 401 當失敗）
        HTTP=$(curl -s --max-time 5 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/v2/ 2>/dev/null || echo "0")
        if [ "$HTTP" = "401" ] || [ "$HTTP" = "200" ]; then
            log "✅ Harbor 就緒 (HTTP $HTTP)"
        else
            log "⚠️  Harbor API 未就緒 (HTTP $HTTP)，harbor-watchdog 會在 60s 後自動修復"
        fi
    else
        log "❌ harbor-log 未 healthy (status=$STATUS)，跳過其餘 Harbor 容器"
        log "   harbor-watchdog 會在 60s 後自動偵測並修復"
    fi
else
    log "⚠️  找不到 Harbor compose 檔案"
fi

# ──────────────────────────────────────────────
# STEP 4: 其他服務（Gitea, Langfuse, Monitoring）
# ──────────────────────────────────────────────
log "[4/6] 啟動其他服務（Gitea, Langfuse, Monitoring）..."

GITEA_DIR="/home/wooo/gitea"
if [ -f "$GITEA_DIR/docker-compose.yml" ]; then
    cd "$GITEA_DIR"
    docker compose up -d 2>&1 | tail -3
    log "✅ Gitea 啟動指令已發送"
fi

LANGFUSE_DIR="/home/wooo/langfuse"
if [ -f "$LANGFUSE_DIR/docker-compose.yml" ]; then
    cd "$LANGFUSE_DIR"
    docker compose up -d 2>&1 | tail -3
    log "✅ Langfuse 啟動指令已發送"
fi

MONITORING_DIR="/home/wooo/monitoring"
if [ -f "$MONITORING_DIR/docker-compose.yml" ]; then
    cd "$MONITORING_DIR"
    docker compose up -d 2>&1 | tail -3
    log "✅ Monitoring (Prometheus/Grafana/Alertmanager) 啟動指令已發送"
    sleep 10
    # 驗證 Alertmanager 是否啟動
    if curl -sf --max-time 5 http://localhost:9093/-/healthy >/dev/null 2>&1; then
        log "✅ Alertmanager healthy"
    else
        log "⚠️  Alertmanager 尚未就緒，等待 20 秒..."
        sleep 20
        curl -sf --max-time 5 http://localhost:9093/-/healthy >/dev/null 2>&1 && log "✅ Alertmanager 就緒" || log "❌ Alertmanager 未就緒，需手動檢查"
    fi
fi

# ──────────────────────────────────────────────
# STEP 5: SignOz
# ──────────────────────────────────────────────
log "[5/6] 啟動 SignOz..."
SIGNOZ_DIR="/home/wooo/signoz/deploy/docker"
if [ -f "$SIGNOZ_DIR/docker-compose.yaml" ]; then
    cd "$SIGNOZ_DIR"
    docker compose up -d 2>&1 | tail -5
    log "✅ SignOz 啟動指令已發送"
fi

# ──────────────────────────────────────────────
# STEP 6: Gitea Act Runner（CI/CD 核心）
# 2026-04-05 Claude Code: 加入 — 解決重開機後 Gitea runner 離線、CD 失效
# 重要：必須在 Gitea server 啟動後才能啟動 runner
# ──────────────────────────────────────────────
log "[6/6] 啟動 Gitea Act Runner..."
RUNNER_DIR="/home/wooo/act-runner"
RUNNER_SERVICE="gitea-act-runner-host.service"
if [ -x "$RUNNER_DIR/act_runner" ] && [ -f "$RUNNER_DIR/config.yaml" ]; then
    # 若舊的 .runner 配置指向過期 hostname，先清除讓 runner 重新註冊
    RUNNER_FILE="$RUNNER_DIR/data/.runner"
    if [ -f "$RUNNER_FILE" ]; then
        OLD_URL=$(python3 -c "import json; d=json.load(open('$RUNNER_FILE')); print(d.get('address',''))" 2>/dev/null || echo "")
        if [ "$OLD_URL" != "http://192.168.0.110:3001" ]; then
            log "⚠️  runner 配置過期 ($OLD_URL)，清除重新註冊..."
            rm -f "$RUNNER_FILE" || true
        fi
    fi

    # act_runner 預設 shutdown_timeout=0s，會在 daemon 重啟時立刻取消
    # 正在執行的 CD job。這會造成部署實際完成但 Gitea status 失真。
    python3 - "$RUNNER_DIR/config.yaml" <<'PY' || true
import sys
from pathlib import Path

path = Path(sys.argv[1])
lines = path.read_text().splitlines()
if any(line.strip().startswith("shutdown_timeout:") for line in lines):
    lines = [
        "  shutdown_timeout: 1h" if line.strip().startswith("shutdown_timeout:") else line
        for line in lines
    ]
else:
    for idx, line in enumerate(lines):
        if line.strip().startswith("timeout:") and idx > 0:
            lines.insert(idx + 1, "  shutdown_timeout: 1h")
            break
path.write_text("\n".join(lines) + "\n")
PY

    if systemctl list-unit-files "$RUNNER_SERVICE" >/dev/null 2>&1; then
        systemctl enable --now "$RUNNER_SERVICE" >/dev/null 2>&1 || true
    elif ! pgrep -f "$RUNNER_DIR/act_runner daemon" >/dev/null; then
        nohup "$RUNNER_DIR/run-host-runner.sh" >> "$RUNNER_DIR/host-runner.log" 2>&1 &
    fi

    # 已停用 Docker-wrapped runner；避免它搶走 host label job。
    # 若手動執行此 recovery script 時仍有 task container，送 SIGINT
    # 讓 act_runner drain，不再接新 job，並等手上的 job 收尾。
    docker update --restart=no gitea-runner >/dev/null 2>&1 || true
    if docker ps --format '{{.Names}}' | grep -q '^GITEA-ACTIONS-TASK-'; then
        log "⚠️  Gitea Actions task container still running; draining docker-wrapped gitea-runner"
        docker kill --signal=SIGINT gitea-runner >/dev/null 2>&1 || true
    else
        docker stop -t 3700 gitea-runner >/dev/null 2>&1 || true
    fi

    sleep 15

    # 驗證 runner 已連線 Gitea
    if pgrep -f "$RUNNER_DIR/act_runner daemon" >/dev/null; then
        log "✅ Gitea host act_runner 已啟動"
    else
        log "⚠️  Gitea host act_runner 可能尚未啟動，查看: $RUNNER_DIR/host-runner.log"
    fi
else
    log "⚠️  找不到 act-runner binary/config: $RUNNER_DIR"
fi

# ──────────────────────────────────────────────
# STEP 7: Sentry（Error Tracking）
# 2026-04-05 Claude Code: 加入 — 解決重開機後 Sentry 未自動啟動
# 安裝位置: /opt/sentry (2026-03-24 已安裝)
# DSN: awoooi-web :2, awoooi-api :3 (見 memory/project_sentry_full_integration.md)
#
# 已知重開機後損壞問題 (2026-04-05 事故記錄):
#   - sentry-postgres: WAL 損壞 → pg_resetwal -f
#   - sentry-redis: dump.rdb 損壞 → 刪除重建 (redis 是 session cache，可丟失)
#   - sentry-clickhouse: system table parts 損壞 → 刪除對應 store/ 目錄
# ──────────────────────────────────────────────
log "[7/7] 啟動 Sentry..."
SENTRY_DIR="/opt/sentry"
if [ -d "$SENTRY_DIR" ]; then
    cd "$SENTRY_DIR"

    # 修復 Sentry PostgreSQL WAL (若損壞)
    SENTRY_PG_VOL="sentry-postgres"
    PG_RUNNING=$(docker ps --filter name=sentry-self-hosted-postgres-1 --format "{{.Status}}" 2>/dev/null | head -1)
    if echo "$PG_RUNNING" | grep -q "Restarting"; then
        log "⚠️  sentry-postgres 損壞，嘗試 pg_resetwal 修復..."
        docker run --rm -u 999 -v ${SENTRY_PG_VOL}:/var/lib/postgresql/data \
            postgres:14 pg_resetwal -f /var/lib/postgresql/data 2>&1 | head -3 || true
        log "✅ sentry-postgres WAL 修復完成"
    fi

    # 修復 Sentry Redis (若 dump.rdb 損壞)
    REDIS_STATUS=$(docker inspect sentry-self-hosted-redis-1 --format='{{.State.Status}}' 2>/dev/null || echo "missing")
    if [ "$REDIS_STATUS" = "restarting" ]; then
        log "⚠️  sentry-redis 損壞，清除 dump.rdb..."
        docker run --rm --user root -v sentry-redis:/data alpine \
            sh -c 'rm -f /data/dump.rdb && echo cleared' 2>/dev/null || true
        log "✅ sentry-redis dump.rdb 已清除"
    fi

    docker compose up -d 2>&1 | tail -5
    log "✅ Sentry 啟動指令已發送 (啟動約需 2-3 分鐘)"
    sleep 20
    # 非阻塞驗證：Sentry 啟動慢，只做快速健康檢查
    HTTP_CODE=$(curl -sf --max-time 10 -o /dev/null -w "%{http_code}" http://localhost:9000/ 2>/dev/null || echo "0")
    if echo "$HTTP_CODE" | grep -qE "^(200|302|400)$"; then
        log "✅ Sentry 已回應 (HTTP $HTTP_CODE)"
    else
        log "⚠️  Sentry 尚未就緒（正常現象，通常需 2-3 分鐘，HTTP=$HTTP_CODE）"
    fi
else
    log "⚠️  找不到 Sentry 目錄: $SENTRY_DIR"
fi

# ──────────────────────────────────────────────
# 完成
# ──────────────────────────────────────────────
log "=== 192.168.0.110 啟動序列完成 ==="
log "Harbor:       http://192.168.0.110:5000"
log "Gitea:        http://192.168.0.110:3001"
log "Grafana:      http://192.168.0.110:3002"
log "Alertmanager: http://192.168.0.110:9093"
log "Gitea Runner: docker logs gitea-runner"
log "Sentry:       http://192.168.0.110:9000"

exit 0
