# =============================================================================
# AWOOOI CD Pipeline (Gitea Actions - 方案 B)
# =============================================================================
# 流程: Build → Push to Harbor → Deploy to K8s
# 加速措施:
#   1. Docker Layer Cache → Harbor registry cache
#   2. 內部 Mirror → 192.168.0.110:5001 (Harbor Proxy Cache for DockerHub)
# 2026-03-29 Claude Code (ADR-039) - Retry after creating Harbor project

name: CD Pipeline

on:
  push:
    branches: [main]
    paths:
      # 只有實際影響部署的程式碼才觸發 CD
      - 'apps/**'
      - 'k8s/**'
      - '.dockerignore'
      # Dockerfile COPY scripts/ into the API image; keep production ops
      # seed scripts deploy-coupled instead of repo-only.
      - 'scripts/backup/backup-momo-188-pg.sh'
      - 'scripts/ops/notify-awoooi-ops.sh'
      - 'scripts/ops/awooop-seed-auto-repair-canary-playbook.py'
      # Workflow-only changes do not rebuild runtime images. Use workflow_dispatch
      # when an operator explicitly wants to test the CD pipeline itself.
      # docs/、memory/、ADR 等不觸發
      # ops/monitoring/alerts-unified.yml 由 deploy-alerts.yaml 獨立處理 (I3)
  workflow_dispatch:
    # 手動觸發永遠可用（用於補跑、緊急部署）

# 2026-04-02 Claude Code: 改為搶佔模式 — 新 push 立即取消舊 build，只部署最新
# 原理: concurrency group 保證同時只有一個 job 跑；cancel-in-progress:true 讓新的取代舊的
# 解決: 多個 commit 快速連推時不再排隊堆積，且 docker build 卡住時不會阻塞後續部署
# 安全: deploy 步驟本身有 kubectl rollout status 保護，不會出現半部署狀態
concurrency:
  group: cd-deploy-${{ github.ref }}
  cancel-in-progress: true

env:
  HARBOR: 192.168.0.110:5000
  SRE_GROUP_CHAT_ID: "-1003711974679"
  # Harbor Proxy Cache (指向 DockerHub 的內部 Mirror，避免拉取限額)
  HARBOR_MIRROR: 192.168.0.110:5001
  # OTEL CI/CD 監控 (2026-03-31 #46c - 遷移到 Gitea)
  OTEL_EXPORTER_OTLP_ENDPOINT: http://192.168.0.188:24318
  OTEL_SERVICE_NAME: awoooi-cd
  OTEL_RESOURCE_ATTRIBUTES: service.version=${{ github.sha }},deployment.environment=production
  CI_IMAGE: 192.168.0.110:5000/awoooi/ci-runner:act-22.04
  # 2026-05-24 Codex: deploy through the currently Ready control-plane node.
  # 120 is NotReady/SchedulingDisabled and its SSH/API endpoints are currently
  # unreachable; pinning CD to it blocks secret injection before GitOps deploy.
  K8S_SSH_HOST: 192.168.0.121
  K8S_API_SERVER: https://192.168.0.121:6443
  # 2026-06-01 Codex: post-deploy health/smoke probes use the production
  # public API. The old 192.168.0.125 NodePort VIP can be absent while the
  # public route and in-cluster service are healthy, causing false failures.
  API_HEALTH_URL: https://awoooi.wooo.work/api/v1/health
  ALERT_CHAIN_API_URL: https://awoooi.wooo.work

jobs:
  tests:
    # 2026-04-30 Codex: run the tests job on the host runner and launch the
    # CI image explicitly. The act-managed job container can disappear mid-test
    # with Docker RWLayer=nil on the shared 110 daemon.
    timeout-minutes: 30
    runs-on: awoooi-host
    # 2026-04-10 ogt: B5 改用 docker run 本地啟動，移除 services: 宣告
    # Gitea act runner 的 services: container name 為空，導致 CI 失敗
    steps:
      - name: Bootstrap Host Runner Tools
        # 2026-05-05 Codex: awoooi-host maps to the long-lived act-runner
        # container. After dirty reboots it may not contain node/curl/git, and
        # actions/checkout@v4 fails before tests can start.
        run: |
          if command -v apk >/dev/null 2>&1; then
            apk add --no-cache nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
          fi

      - uses: actions/checkout@v4

      - name: Guard Workflow Secret Surfaces
        run: node scripts/ci/check-gitea-step-env-secrets.js

      # 2026-03-31 ogt: 優化告警格式 - 提高可讀性
      - name: Get Commit Info
        id: commit
        run: |
          echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
          echo "message=$(git log -1 --pretty=%s | head -c 50)" >> $GITHUB_OUTPUT
          echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT

      - name: Notify Pipeline Start
        # 2026-04-16 ogt + Claude Sonnet 4.6: 改用 HTML 結構化格式，提升可讀性
        run: |
          COMMIT_MSG="${{ steps.commit.outputs.message }}"
          SHORT_SHA="${{ steps.commit.outputs.short_sha }}"
          ACTOR="${{ github.actor }}"
          # HTML escape commit message（防特殊字元破壞 HTML）
          COMMIT_ESC=$(echo "$COMMIT_MSG" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g')
          MSG=$(printf '🚀 <b>AWOOOI 部署開始</b>\n├ 📝 <code>%s</code>\n├ 🔖 <code>%s</code>\n└ 👤 %s' "${COMMIT_ESC}" "${SHORT_SHA}" "${ACTOR}")
          # 2026-05-02 Claude Opus 4.7 + 統帥 ogt: notify 失敗不該擋整條 CI（鐵證:
          # curl 400 從 5/1 起連續炸 14 個 commit 的 build-and-deploy）— 對齊 line 922 既有 pattern
          if AWOOI_CICD_STATUS=running \
            AWOOI_CICD_STAGE=tests \
            AWOOI_CICD_JOB_NAME="AWOOOI 部署開始" \
            AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
            AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
            AWOOI_CICD_SUMMARY="${COMMIT_MSG}" \
            scripts/ci/notify-awoooi-cicd.sh; then
            echo "✅ CI/CD start notification mirrored through AWOOI API"
          else
            curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \
              -d "chat_id=${{ env.SRE_GROUP_CHAT_ID }}" \
              -d "parse_mode=HTML" \
              --data-urlencode "text=${MSG}" || echo "TG notify failed (non-fatal): exit=$?"
          fi



      # 2026-03-31 ogt: Phase 22.0 CI 測試 (禁止 Mock - feedback_no_mock_testing.md)
      # 2026-04-01 ogt: 持久化 venv 加速 - /opt/api-venv 跨 run 保留
      # pyproject.toml hash 變才重裝，其餘直接 activate (節省 ~6-7 min)
      - name: Run API Tests
        run: |
          cat > /tmp/awoooi-api-tests.sh <<'CI_SCRIPT'
          VENV=/opt/api-venv
          HASH_FILE=/opt/api-venv/.deps_hash
          CURRENT_HASH=$(md5sum apps/api/pyproject.toml | awk '{print $1}')

          # python3.11 是 runner 層級持久安裝，只在首次或版本消失時才 apt-get
          # 2026-04-05 Claude Code: 分離 apt-get 與 venv hash-guard，避免每次 deps 變更都重跑 apt
          # 2026-04-16 ogt + Claude Sonnet 4.6: 修復 apt index 失敗 → 改用 --fix-missing + retry
          if ! command -v python3.11 &>/dev/null; then
            echo "📦 安裝 python3.11..."
            apt-get clean && rm -rf /var/lib/apt/lists/*
            apt-get update -q --fix-missing || apt-get update -q || true
            apt-get install -y -q python3.11-venv python3.11 || \
              (add-apt-repository ppa:deadsnakes/python -y 2>/dev/null && apt-get update -q && apt-get install -y -q python3.11-venv python3.11) || true
          else
            echo "⚡ python3.11 已安裝，跳過 apt-get"
          fi
          # 確保 python3.11 存在，否則 fallback 到系統 python3
          if ! command -v python3.11 &>/dev/null; then
            echo "⚠️ python3.11 安裝失敗，使用 python3 fallback"
            ln -sf "$(which python3)" /usr/local/bin/python3.11 || true
          fi

          if [ ! -d "$VENV/bin" ] || [ "$(cat $HASH_FILE 2>/dev/null)" != "$CURRENT_HASH" ]; then
            echo "📦 deps 已變更，重建 venv..."
            # 2026-04-17 ogt: /opt/api-venv 是 volume mount，不能 rm -rf 目錄本身
            # 改用 find 清空內容，保留 mount point 目錄
            find "$VENV" -mindepth 1 -delete 2>/dev/null || true
            python3.11 -m venv $VENV
            source $VENV/bin/activate
            pip install -q uv
            cd apps/api && uv pip install -q -e ".[dev]" && cd -
            echo "$CURRENT_HASH" > $HASH_FILE
          else
            echo "⚡ 使用快取 venv (deps 未變更)"
            source $VENV/bin/activate
          fi

          cd apps/api
          cleanup_pytest_workspace_cache() {
            # 2026-05-19 Codex: CI image runs as root against a bind-mounted
            # checkout. Remove Python cache artifacts before act-runner cleanup
            # so successful jobs do not end with root-owned __pycache__ noise.
            find tests src -type d -name __pycache__ -prune -exec rm -rf {} + 2>/dev/null || true
            rm -rf .pytest_cache 2>/dev/null || true
          }
          # CI 排除需外部服務的測試 (Redis pool / Ollama — 2026-04-01 Claude Code)
          # 2026-04-05 Claude Code: 修正 exit code — | tail 會吃掉 segfault (exit 139)
          #   改用 tee + PIPESTATUS[0] 正確捕捉 pytest 本身的 exit code
          # 2026-04-05 Claude Code: 加 --ignore=tests/integration 排除需 asyncpg 連線的 DB 測試
          #   integration tests 在 prod K8s 部署後由 E2E Smoke Test 覆蓋
          # PYTHONFAULTHANDLER=1: 若 C extension segfault，輸出完整 Python stacktrace
          # 2026-04-05 Claude Code: test_github_webhook.py 已根治
          #   原問題: import src.main → asyncpg C ext segfault (exit 139)
          #   修復: 改用最小化 app，只掛載 github_webhook router，不走 DB import chain
          #   現在可安全加入 CI 測試
          # 2026-04-22 ogt: DATABASE_URL 改為必填後，單元測試需要此 env var 讓 Settings 通過驗證
          # 單元測試不連 DB，此 CI placeholder 僅供 Pydantic 驗證，不產生真實連線
          DATABASE_URL="${DATABASE_URL:-postgresql+asyncpg://ci:ci@localhost/ci}" \
          PYTHONFAULTHANDLER=1 python3.11 -m pytest tests/ -v --tb=short -x -p no:cacheprovider \
            --ignore=tests/integration \
            --ignore=tests/test_anomaly_counter.py \
            --ignore=tests/test_global_repair_cooldown.py \
            --ignore=tests/test_redis_multisig.py \
            --ignore=tests/test_model_regression.py \
            --ignore=tests/test_prompt_validation.py \
            --ignore=tests/e2e_network_test.py \
            2>&1 | tee /tmp/pytest-output.txt; PYTEST_EXIT=${PIPESTATUS[0]}
          tail -60 /tmp/pytest-output.txt
          cleanup_pytest_workspace_cache
          exit $PYTEST_EXIT
          CI_SCRIPT
          docker run --rm \
            --name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-api-tests" \
            --cpus "2.0" \
            --memory "2g" \
            -v "$PWD:/workspace" \
            -v /tmp/awoooi-api-tests.sh:/tmp/awoooi-api-tests.sh:ro \
            -v awoooi-api-venv-cache:/opt/api-venv \
            -w /workspace \
            "${{ env.CI_IMAGE }}" \
            bash /tmp/awoooi-api-tests.sh

      # ── 整合測試 B5 (2026-04-10) ──────────────────────────────────────────
      # B5 整合測試 — postgres-test 由 services: 提供，localhost:15432 直連
      # 2026-04-10 Claude Sonnet 4.6: 用 psql 直連 localhost:15432 初始化 schema
      #   (docker exec 在 act runner 內無法取得 service container name)
      # B5: Gitea act runner 的 services: 實作與 GitHub Actions 不同
      # service container 啟動後需直連，但 act 的 container name 可能為空
      # 2026-04-10 ogt: 改用 docker run 本地啟動取代 services: 宣告
      # 2026-04-19 ogt + Claude Opus 4.7: cd 連續 2 次 fail (run 984/985)
      #   真因: act runner 把 ci-runner 跑在獨立 user-defined network,
      #         pg-test-b5 預設用 host bridge → 兩邊隔離無法連 (172.17.0.2 timeout)
      #   修法: 把 pg-test-b5 加入 act task 的 network,用 container name 連線
      - name: Integration Tests (B5 — 真實 DB)
        run: |
          cat > /tmp/awoooi-b5-tests.sh <<'CI_SCRIPT'
          cd apps/api
          # 安裝 psql client
          if ! command -v psql &>/dev/null; then
            apt-get install -y -q postgresql-client
          fi
          # 2026-04-19 ogt + Claude Opus 4.7 v3: 主動創 shared network
          # 之前 grep ACT_NET 在 c0f3509 run 沒 match → fallback bridge → container name DNS 失效
          # 真因: default bridge 不支援 container name DNS,必須 user-defined network
          # 修法: 主動建 'b5-test-net' (idempotent),ci-runner + pg-test-b5 都加入
          B5_NET="b5-test-net"
          docker network create "$B5_NET" 2>/dev/null || true
          # 當前 ci-runner container (hostname == short container id) 連上此 network
          # 若已連 → docker network connect 回 error 1,用 || true 吞掉
          docker network connect "$B5_NET" "$HOSTNAME" 2>/dev/null || true
          echo "B5 shared network: $B5_NET (ci-runner hostname: $HOSTNAME)"
          # 啟動測試 DB 於 shared network,用 container name 'pg-test-b5' 連線
          docker rm -f pg-test-b5 2>/dev/null || true
          docker run -d --name pg-test-b5 \
            --network="$B5_NET" \
            -e POSTGRES_DB=awoooi_test \
            -e POSTGRES_USER=awoooi \
            -e POSTGRES_PASSWORD=awoooi_test_2026 \
            pgvector/pgvector:pg16
          # 等待就緒（用 container name,最多 60 秒）
          for i in $(seq 1 30); do
            PGPASSWORD=awoooi_test_2026 pg_isready -h pg-test-b5 -p 5432 -U awoooi && break || sleep 2
          done
          # 初始化 schema
          PGPASSWORD=awoooi_test_2026 psql \
            -h pg-test-b5 -p 5432 -U awoooi -d awoooi_test \
            -f tests/integration/setup_test_schema.sql
          # 跑測試
          # B5 整合測試嚴格模式 (2026-04-13 ogt: 恢復 Break-Glass 移除)
          # -m integration: override pyproject.toml addopts "-m 'not integration'"，讓標記測試可執行
          # 2026-04-22 ogt: DATABASE_URL 改為必填後，import chain 需要此 env var 讓 Settings 通過驗證
          DATABASE_URL="postgresql+asyncpg://awoooi:awoooi_test_2026@pg-test-b5:5432/awoooi_test?ssl=disable" \
          TEST_DATABASE_URL="postgresql+asyncpg://awoooi:awoooi_test_2026@pg-test-b5:5432/awoooi_test?ssl=disable" \
            /opt/api-venv/bin/pytest tests/integration/test_b5_core_flows.py -v --tb=short -m integration -p no:cacheprovider || PYTEST_EXIT=$?
          # 清理
          docker rm -f pg-test-b5 || true
          # 2026-05-20 Codex: B5 imports shared tests helpers, so cleanup the
          # whole tests tree to avoid root-owned __pycache__ act-runner noise.
          find tests src -type d -name __pycache__ -prune -exec rm -rf {} + 2>/dev/null || true
          rm -rf .pytest_cache 2>/dev/null || true
          exit "${PYTEST_EXIT:-0}"
          CI_SCRIPT
          docker run --rm \
            --name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-b5-tests" \
            --cpus "2.0" \
            --memory "2g" \
            -v "$PWD:/workspace" \
            -v /tmp/awoooi-b5-tests.sh:/tmp/awoooi-b5-tests.sh:ro \
            -v /var/run/docker.sock:/var/run/docker.sock \
            -v awoooi-api-venv-cache:/opt/api-venv \
            -w /workspace \
            "${{ env.CI_IMAGE }}" \
            bash /tmp/awoooi-b5-tests.sh

      - name: Clean Test Workspace Artifacts
        if: always()
        env:
          HOST_RUNNER_CLEANUP_IMAGE: ${{ env.CI_IMAGE }}
        run: bash scripts/ci/cleanup-host-runner-workspace.sh

      - name: Notify Pipeline Failure
        # 2026-04-30 Codex: tests job failure notifier; no jq dependency for host parity.
        if: failure()
        run: |
          COMMIT_MSG="${{ steps.commit.outputs.message }}"
          SHORT_SHA="${{ steps.commit.outputs.short_sha }}"
          ACTOR="${{ github.actor }}"
          COMMIT_ESC=$(echo "$COMMIT_MSG" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g')
          MSG=$(printf '❌ <b>AWOOOI 部署失敗</b>\n├ 📝 <code>%s</code>\n├ 🔖 <code>%s</code>\n├ 👤 %s\n├ 🧪 Stage: tests\n└ 🔗 http://192.168.0.110:3001/wooo/awoooi/actions' "${COMMIT_ESC}" "${SHORT_SHA}" "${ACTOR}")
          if AWOOI_CICD_STATUS=failed \
            AWOOI_CICD_STAGE=tests \
            AWOOI_CICD_JOB_NAME="AWOOOI 部署失敗" \
            AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
            AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
            AWOOI_CICD_SUMMARY="${COMMIT_MSG}" \
            scripts/ci/notify-awoooi-cicd.sh; then
            echo "✅ CI/CD tests failure notification mirrored through AWOOI API"
          else
            curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \
              -d "chat_id=${{ env.SRE_GROUP_CHAT_ID }}" \
              -d "parse_mode=HTML" \
              --data-urlencode "text=${MSG}" || echo "TG notify failed (non-fatal): exit=$?"
          fi

  build-and-deploy:
    # 2026-04-30 Codex: Docker builds run on the host runner. Long docker build
    # steps were killing the transient act job container with RWLayer=nil.
    needs: tests
    timeout-minutes: 60
    runs-on: awoooi-host
    steps:
      - name: Bootstrap Host Runner Tools
        # 2026-05-05 Codex: keep the host-mode runner self-healing before
        # actions/checkout@v4 and Telegram failure notifications run.
        run: |
          if command -v apk >/dev/null 2>&1; then
            apk add --no-cache nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
          fi

      - uses: actions/checkout@v4

      - name: Get Commit Info
        id: commit
        run: |
          echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
          echo "message=$(git log -1 --pretty=%s | head -c 50)" >> $GITHUB_OUTPUT
          echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT

      - name: Notify Build Deploy Start
        run: |
          ACTOR="${{ github.actor }}"
          if AWOOI_CICD_STATUS=running \
            AWOOI_CICD_STAGE=build-and-deploy \
            AWOOI_CICD_JOB_NAME="AWOOOI 建置部署開始" \
            AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
            AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
            AWOOI_CICD_SUMMARY="${{ steps.commit.outputs.message }}" \
            scripts/ci/notify-awoooi-cicd.sh; then
            echo "✅ CI/CD build-deploy start notification mirrored through AWOOI API"
          else
            echo "⚠️ CI/CD build-deploy start notification failed (non-fatal)"
          fi

      - name: Login to Harbor
        run: |
          echo "${{ secrets.HARBOR_PASSWORD }}" | \
            docker login "${{ env.HARBOR }}" \
              -u "${{ secrets.HARBOR_USERNAME }}" \
              --password-stdin

      # 2026-05-21 Codex: AWOOI workflow concurrency and the Docker network
      # lock only protect AWOOI/Docker work. Other repos can still run
      # host-side Next/Turbo builds on the same 110 runner and starve this
      # deploy. Wait for those foreign web builds before starting our image
      # build; the gate is read-only and never kills another process.
      - name: Wait for Host Web Build Pressure
        run: bash scripts/ci/wait-host-web-build-pressure.sh

      # 2026-04-30 Codex: Gitea act-runner shares one Docker daemon across repos.
      # When another repo starts a heavy docker build while AWOOOI Web is still
      # building, the job container can disappear and Docker reports RWLayer=nil.
      # A Docker-network lock is global to the host daemon and survives container
      # namespaces, unlike /tmp/flock inside the transient job container.
      - name: Acquire Docker Build Lock
        run: |
          LOCK_NAME="awoooi-cd-docker-build-lock"
          STALE_SECONDS=7200
          EMPTY_LOCK_SECONDS=300
          WAIT_ATTEMPTS=180

          for attempt in $(seq 1 "$WAIT_ATTEMPTS"); do
            if docker network create \
              --label awoooi.ci-lock=docker-build \
              --label awoooi.owner=cd-pipeline \
              "$LOCK_NAME" >/dev/null 2>&1; then
              echo "DOCKER_BUILD_LOCK=${LOCK_NAME}" >> "$GITHUB_ENV"
              echo "✅ Docker build lock acquired: ${LOCK_NAME}"
              exit 0
            fi

            CREATED_AT=$(docker network inspect "$LOCK_NAME" \
              --format '{{.Created}}' 2>/dev/null || true)
            if [ -n "$CREATED_AT" ]; then
              # 2026-05-03 ogt: 修復 stale 偵測 — Docker 回傳 "2006-01-02 15:04:05.999999999 -0700 MST"
              # date -d 不接受奈秒小數點與末尾時區縮寫（CST/MST 等），導致 CREATED_EPOCH=0 → stale 永不觸發
              # 2026-06-18 Codex: act-runner 容器可能沒有 GNU date / python3；
              # node 由 bootstrap 安裝，作為 Docker CreatedAt 的穩定解析 fallback。
              # 2026-06-19 Codex: Docker / Gitea runner 可能回傳 ISO
              # `2026-06-18T16:20:00.123456789Z`；若 CREATED_EPOCH=0，
              # empty lock 永遠不會自清，下一輪 deploy 會卡滿 30 分鐘。
              CREATED_CLEAN=$(echo "$CREATED_AT" | sed 's/\.[0-9]*//' | sed 's/ [A-Z][A-Z]*$//')
              CREATED_EPOCH=$(date -d "$CREATED_CLEAN" +%s 2>/dev/null || \
                node -e 'const raw = process.argv[1] || ""; const base = raw.replace(/\.\d+/, "").replace(/\s+[A-Z]{2,4}$/, ""); const spaced = base.replace(/^(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2}:\d{2})\s+([+-]\d{2})(\d{2})$/, "$1T$2$3:$4"); const iso = base.replace(/^(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2}:\d{2})(Z|[+-]\d{2}:?\d{2})$/, "$1T$2$3"); const candidates = [raw, base, spaced, iso]; for (const candidate of candidates) { const ms = Date.parse(candidate); if (Number.isFinite(ms)) { console.log(Math.floor(ms / 1000)); process.exit(0); } } process.exit(1);' \
                "$CREATED_AT" 2>/dev/null || \
                python3 -c "import sys, datetime, re; ts = re.sub(r'\\.\d+', '', sys.argv[1]); ts = re.sub(r'\\s+[A-Z]{2,4}$', '', ts.strip()); print(int(datetime.datetime.strptime(ts, '%Y-%m-%d %H:%M:%S %z').timestamp()))" \
                "$CREATED_AT" 2>/dev/null || echo 0)
              NOW_EPOCH=$(date +%s)
              LOCK_AGE=$((NOW_EPOCH - CREATED_EPOCH))
              # 2026-05-05 Codex: dirty reboot / cancelled Actions can leave
              # the Docker-network lock behind with no active build or push.
              # Waiting the full 30m CD timeout keeps deploys queued even
              # though no job is protected, so clear empty locks after 5m.
              # 2026-06-18 Codex: 只靠 bracket pattern 仍會命中 lock-check
              # bash/awk 自己的指令列；必須排除檢查器本身，取消後留下的
              # empty lock network 才能在 5 分鐘後自清。
              ACTIVE_DOCKER_WORK=$(ps -eo pid,args | awk '
                $0 ~ /[d]ocker (build|push)|[b]uildx build/ &&
                $0 !~ /ACTIVE_DOCKER_WORK/ &&
                $0 !~ /awk/ &&
                $0 !~ /ps -eo pid,args/ {print}
              ' || true)
              if [ "$CREATED_EPOCH" -eq 0 ] && \
                 [ $((attempt * 10)) -gt $((EMPTY_LOCK_SECONDS * 2)) ] && \
                 [ -z "$ACTIVE_DOCKER_WORK" ]; then
                echo "⚠️ Docker build lock has unparsable CreatedAt (${CREATED_AT}) and no active docker build/push after $((attempt * 10))s, removing ${LOCK_NAME}"
                docker network rm "$LOCK_NAME" >/dev/null 2>&1 || true
                continue
              fi
              if [ "$CREATED_EPOCH" -gt 0 ] && \
                 [ "$LOCK_AGE" -gt "$EMPTY_LOCK_SECONDS" ] && \
                 [ -z "$ACTIVE_DOCKER_WORK" ]; then
                echo "⚠️ empty Docker build lock detected (age=${LOCK_AGE}s > ${EMPTY_LOCK_SECONDS}s, no active docker build/push), removing ${LOCK_NAME}"
                docker network rm "$LOCK_NAME" >/dev/null 2>&1 || true
                continue
              fi
              if [ "$CREATED_EPOCH" -gt 0 ] && \
                 [ "$LOCK_AGE" -gt "$STALE_SECONDS" ]; then
                echo "⚠️ stale Docker build lock detected (age=${LOCK_AGE}s > ${STALE_SECONDS}s), removing ${LOCK_NAME}"
                docker network rm "$LOCK_NAME" >/dev/null 2>&1 || true
                continue
              fi
            fi

            echo "⏳ Docker build lock busy (attempt ${attempt}/${WAIT_ATTEMPTS}); waiting..."
            sleep 10
          done

          echo "❌ timed out waiting for Docker build lock"
          exit 1

      # ── API 鏡像建置（含 Layer Cache 加速）──────────────────────────────
      # 2026-04-01 ogt: CACHE_BUST=git_sha 確保 src/ 和 models.json 層每次重建
      # deps 層 (pip install) 仍可 cache → 加速；代碼/配置層強制失效
      # 2026-05-05 Codex: host runner bootstrap installs docker-cli-buildx;
      # keep BuildKit enabled because the web Dockerfile uses RUN --mount.
      - name: Build and Push API
        env:
          DOCKER_BUILDKIT: "1"
        run: |
          docker build -f apps/api/Dockerfile \
            --build-arg BUILDKIT_INLINE_CACHE=1 \
            --cache-from ${{ env.HARBOR }}/awoooi/api:latest \
            --build-arg CACHE_BUST=${{ github.sha }} \
            -t ${{ env.HARBOR }}/awoooi/api:${{ github.sha }} \
            -t ${{ env.HARBOR }}/awoooi/api:latest \
            .
          docker push ${{ env.HARBOR }}/awoooi/api:${{ github.sha }}
          docker push ${{ env.HARBOR }}/awoooi/api:latest

      # 2026-03-31 ogt: 移除中間通知，減少訊息雜訊

      # ── Web 鏡像建置（精準快取失效）──────────────────────────────
      # 2026-03-30 ogt: NEXT_PUBLIC_* 必須用公網域名 (build-time 寫死)
      # 2026-04-01 Claude Code: CACHE_BUST=git_sha 取代 --no-cache
      # - deps 層 (pnpm install) 仍可 cache → 節省 ~2-3 min
      # - COPY . . 以下由 CACHE_BUST 強制失效 → 業務邏輯/CSRF 等變更正確進入 bundle
      # 2026-05-05 Codex: mirror API build mode; BuildKit required for cache mounts.
      - name: Build and Push Web
        env:
          DOCKER_BUILDKIT: "1"
        run: |
          docker build -f apps/web/Dockerfile \
            --build-arg NEXT_PUBLIC_API_URL=https://awoooi.wooo.work \
            --build-arg CACHE_BUST=${{ github.sha }} \
            --build-arg BUILDKIT_INLINE_CACHE=1 \
            --cache-from ${{ env.HARBOR }}/awoooi/web:latest \
            -t ${{ env.HARBOR }}/awoooi/web:${{ github.sha }} \
            -t ${{ env.HARBOR }}/awoooi/web:latest \
            .
          docker push ${{ env.HARBOR }}/awoooi/web:${{ github.sha }}
          docker push ${{ env.HARBOR }}/awoooi/web:latest

      - name: Release Docker Build Lock
        if: always()
        run: |
          if [ -n "${DOCKER_BUILD_LOCK:-}" ]; then
            docker network rm "$DOCKER_BUILD_LOCK" >/dev/null 2>&1 || true
            echo "✅ Docker build lock released: ${DOCKER_BUILD_LOCK}"
          else
            echo "⚡ no Docker build lock to release"
          fi

      # 2026-03-31 ogt: 移除中間通知

      # 2026-03-31 ogt: P0-1 Secrets 自動注入 (ADR-035 強制)
      # 2026-03-31 ogt: 加入 AI API Keys (修復 mock_fallback 問題)
      - name: Inject K8s Secrets
        run: |
          # 2026-05-18 Codex: 不把 secrets 放進 step-level env。
          # Gitea/act_runner 的 job log 可能展開 env；這裡只在 shell 內短暫轉
          # base64，並避免輸出原值。
          secret_b64() {
            if command -v python3.11 >/dev/null 2>&1; then
              python3.11 -c 'import base64, sys; data=sys.stdin.buffer.read(); data=data[:-1] if data.endswith(b"\n") else data; sys.stdout.write(base64.b64encode(data).decode())'
            elif command -v python3 >/dev/null 2>&1; then
              python3 -c 'import base64, sys; data=sys.stdin.buffer.read(); data=data[:-1] if data.endswith(b"\n") else data; sys.stdout.write(base64.b64encode(data).decode())'
            else
              secret_value="$(cat)"
              printf '%s' "${secret_value}" | base64 | tr -d '\n'
            fi
          }
          write_deploy_key() {
            mkdir -p "${HOME}/.ssh"
            umask 077
            cat > "${HOME}/.ssh/deploy_key" <<'AWOOOI_DEPLOY_KEY'
          ${{ secrets.DEPLOY_SSH_KEY }}
          AWOOOI_DEPLOY_KEY
            chmod 600 "${HOME}/.ssh/deploy_key"
          }

          TG_BOT_TOKEN_B64="$(secret_b64 <<'AWOOOI_SECRET_TG_BOT_TOKEN'
          ${{ secrets.TELEGRAM_BOT_TOKEN }}
          AWOOOI_SECRET_TG_BOT_TOKEN
          )"
          TG_CHAT_ID_B64="$(secret_b64 <<'AWOOOI_SECRET_SRE_GROUP_CHAT_ID_COMPAT'
          ${{ secrets.SRE_GROUP_CHAT_ID }}
          AWOOOI_SECRET_SRE_GROUP_CHAT_ID_COMPAT
          )"
          NVIDIA_API_KEY_B64="$(secret_b64 <<'AWOOOI_SECRET_NVIDIA_API_KEY'
          ${{ secrets.NVIDIA_API_KEY }}
          AWOOOI_SECRET_NVIDIA_API_KEY
          )"
          GEMINI_API_KEY_B64="$(secret_b64 <<'AWOOOI_SECRET_GEMINI_API_KEY'
          ${{ secrets.GEMINI_API_KEY }}
          AWOOOI_SECRET_GEMINI_API_KEY
          )"
          LANGFUSE_PUBLIC_KEY_B64="$(secret_b64 <<'AWOOOI_SECRET_LANGFUSE_PUBLIC_KEY'
          ${{ secrets.LANGFUSE_PUBLIC_KEY }}
          AWOOOI_SECRET_LANGFUSE_PUBLIC_KEY
          )"
          LANGFUSE_SECRET_KEY_B64="$(secret_b64 <<'AWOOOI_SECRET_LANGFUSE_SECRET_KEY'
          ${{ secrets.LANGFUSE_SECRET_KEY }}
          AWOOOI_SECRET_LANGFUSE_SECRET_KEY
          )"
          TG_USER_WHITELIST_B64="$(secret_b64 <<'AWOOOI_SECRET_TG_USER_WHITELIST'
          ${{ secrets.OPENCLAW_TG_USER_WHITELIST }}
          AWOOOI_SECRET_TG_USER_WHITELIST
          )"
          SENTRY_AUTH_TOKEN_B64="$(secret_b64 <<'AWOOOI_SECRET_SENTRY_AUTH_TOKEN'
          ${{ secrets.SENTRY_AUTH_TOKEN }}
          AWOOOI_SECRET_SENTRY_AUTH_TOKEN
          )"
          GITEA_WEBHOOK_SECRET_B64="$(secret_b64 <<'AWOOOI_SECRET_GITEA_WEBHOOK_SECRET'
          ${{ secrets.AWOOOI_GITEA_WEBHOOK_SECRET }}
          AWOOOI_SECRET_GITEA_WEBHOOK_SECRET
          )"
          ARGOCD_API_TOKEN_B64="$(secret_b64 <<'AWOOOI_SECRET_ARGOCD_API_TOKEN'
          ${{ secrets.ARGOCD_API_TOKEN }}
          AWOOOI_SECRET_ARGOCD_API_TOKEN
          )"
          DATABASE_URL_B64="$(secret_b64 <<'AWOOOI_SECRET_DATABASE_URL'
          ${{ secrets.DATABASE_URL }}
          AWOOOI_SECRET_DATABASE_URL
          )"
          MIGRATION_DATABASE_URL_B64="$(secret_b64 <<'AWOOOI_SECRET_MIGRATION_DATABASE_URL'
          ${{ secrets.MIGRATION_DATABASE_URL }}
          AWOOOI_SECRET_MIGRATION_DATABASE_URL
          )"
          REDIS_URL_B64="$(secret_b64 <<'AWOOOI_SECRET_REDIS_URL'
          ${{ secrets.REDIS_URL }}
          AWOOOI_SECRET_REDIS_URL
          )"
          JWT_SECRET_B64="$(secret_b64 <<'AWOOOI_SECRET_JWT_SECRET'
          ${{ secrets.JWT_SECRET }}
          AWOOOI_SECRET_JWT_SECRET
          )"
          JWT_ALGORITHM_B64="$(secret_b64 <<'AWOOOI_SECRET_JWT_ALGORITHM'
          ${{ secrets.JWT_ALGORITHM }}
          AWOOOI_SECRET_JWT_ALGORITHM
          )"
          WEBHOOK_HMAC_SECRET_B64="$(secret_b64 <<'AWOOOI_SECRET_WEBHOOK_HMAC_SECRET'
          ${{ secrets.WEBHOOK_HMAC_SECRET }}
          AWOOOI_SECRET_WEBHOOK_HMAC_SECRET
          )"
          AWOOOP_OPERATOR_API_KEY_B64="$(secret_b64 <<'AWOOOI_SECRET_AWOOOP_OPERATOR_API_KEY'
          ${{ secrets.AWOOOP_OPERATOR_API_KEY }}
          AWOOOI_SECRET_AWOOOP_OPERATOR_API_KEY
          )"
          SENTRY_DSN_B64="$(secret_b64 <<'AWOOOI_SECRET_SENTRY_DSN'
          ${{ secrets.SENTRY_DSN }}
          AWOOOI_SECRET_SENTRY_DSN
          )"
          CLAUDE_API_KEY_B64="$(secret_b64 <<'AWOOOI_SECRET_CLAUDE_API_KEY'
          ${{ secrets.CLAUDE_API_KEY }}
          AWOOOI_SECRET_CLAUDE_API_KEY
          )"
          GITEA_API_TOKEN_B64="$(secret_b64 <<'AWOOOI_SECRET_GITEA_API_TOKEN'
          ${{ secrets.AWOOOI_GITEA_API_TOKEN }}
          AWOOOI_SECRET_GITEA_API_TOKEN
          )"
          NEMOTRON_BOT_TOKEN_B64="$(secret_b64 <<'AWOOOI_SECRET_NEMOTRON_BOT_TOKEN'
          ${{ secrets.NEMOTRON_BOT_TOKEN }}
          AWOOOI_SECRET_NEMOTRON_BOT_TOKEN
          )"
          OPENCLAW_BOT_TOKEN_B64="$(secret_b64 <<'AWOOOI_SECRET_OPENCLAW_BOT_TOKEN'
          ${{ secrets.OPENCLAW_BOT_TOKEN }}
          AWOOOI_SECRET_OPENCLAW_BOT_TOKEN
          )"
          SMTP_HOST_B64="$(secret_b64 <<'AWOOOI_SECRET_SMTP_HOST'
          ${{ secrets.SMTP_HOST }}
          AWOOOI_SECRET_SMTP_HOST
          )"
          SRE_GROUP_CHAT_ID_B64="$(secret_b64 <<'AWOOOI_SECRET_SRE_GROUP_CHAT_ID'
          ${{ secrets.SRE_GROUP_CHAT_ID }}
          AWOOOI_SECRET_SRE_GROUP_CHAT_ID
          )"

          # S1/S2: 統一命名 deploy_key，改用 ssh-keyscan 與強制 host key 驗證。
          write_deploy_key
          # 2026-05-13 Codex: keyscan must include ED25519 explicitly. Some
          # OpenSSH builds otherwise record only RSA/ECDSA, then strict deploy
          # SSH fails with "No ED25519 host key is known" after image push.
          # 2026-06-13 Codex: keep deploy-time host keys in a dedicated file.
          # The runner user's global known_hosts is shared by cold-start and
          # backup checks for 120/188; overwriting it here caused strict SSH
          # recovery gates to flap after every CD run.
          DEPLOY_KNOWN_HOSTS="${HOME}/.ssh/deploy_known_hosts"
          ssh-keyscan -T 5 -t ed25519,rsa,ecdsa "${K8S_SSH_HOST}" > "${DEPLOY_KNOWN_HOSTS}" 2>/dev/null
          test -s "${DEPLOY_KNOWN_HOSTS}" || { echo "❌ K8S host keyscan failed: ${K8S_SSH_HOST}"; exit 1; }
          SSH_OPTS="-i ${HOME}/.ssh/deploy_key -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${DEPLOY_KNOWN_HOSTS} -o ConnectTimeout=10"
          ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" << SECRETS
          set -e
          K8S_API_SERVER="${{ env.K8S_API_SERVER }}"
          KUBECTL="sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=\${K8S_API_SERVER}"

          # 注入 Telegram Secrets (ADR-035 鐵律)
          # 2026-06-12 Codex: OPENCLAW_TG_CHAT_ID 僅作舊欄位相容，
          # 實際值必須與 SRE_GROUP_CHAT_ID 一致，避免正式告警旁路到其他群組。
          \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
            {"op":"add","path":"/data/OPENCLAW_TG_BOT_TOKEN","value":"${TG_BOT_TOKEN_B64}"},
            {"op":"add","path":"/data/OPENCLAW_TG_CHAT_ID","value":"${TG_CHAT_ID_B64}"}
          ]' || { echo "❌ Telegram Secrets patch 失敗 — ADR-035 鐵律"; exit 1; }

          # 2026-03-31 ogt: 注入 AI API Keys (修復 NVIDIA/Gemini mock_fallback)
          # 2026-04-01 Claude Code: base64 -w 0 防止長 key 換行破壞 JSON
          # NVIDIA NIM (免費 tier)
          if [ -n "${NVIDIA_API_KEY_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/NVIDIA_API_KEY","value":"${NVIDIA_API_KEY_B64}"}
            ]' && echo "✅ NVIDIA_API_KEY 已注入" || echo "⚠️ NVIDIA_API_KEY patch 失敗"
          else
            echo "⚠️ NVIDIA_API_KEY 未設定，跳過"
          fi

          # Gemini (備援)
          if [ -n "${GEMINI_API_KEY_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/GEMINI_API_KEY","value":"${GEMINI_API_KEY_B64}"}
            ]' && echo "✅ GEMINI_API_KEY 已注入" || echo "⚠️ GEMINI_API_KEY patch 失敗"
          else
            echo "⚠️ GEMINI_API_KEY 未設定，跳過"
          fi

          # 2026-04-01 Claude Code: Langfuse LLMOps keys (補齊 CD 注入，之前只有手動設定)
          if [ -n "${LANGFUSE_PUBLIC_KEY_B64}" ] && [ -n "${LANGFUSE_SECRET_KEY_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/LANGFUSE_PUBLIC_KEY","value":"${LANGFUSE_PUBLIC_KEY_B64}"},
              {"op":"add","path":"/data/LANGFUSE_SECRET_KEY","value":"${LANGFUSE_SECRET_KEY_B64}"}
            ]' && echo "✅ LANGFUSE keys 已注入" || echo "⚠️ LANGFUSE keys patch 失敗"
          else
            echo "⚠️ LANGFUSE_PUBLIC_KEY/SECRET_KEY 未設定，跳過 (現有 K8s secret 值維持不變)"
          fi

          # 2026-04-02 Claude Code: Telegram Whitelist (授權簽核用戶 ID)
          if [ -n "${TG_USER_WHITELIST_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/OPENCLAW_TG_USER_WHITELIST","value":"${TG_USER_WHITELIST_B64}"}
            ]' && echo "✅ TG_USER_WHITELIST 已注入" || echo "⚠️ TG_USER_WHITELIST patch 失敗"
          fi

          # Phase O-4.1 2026-04-02: Sentry Auth Token (Wave A.1 ADR-037)
          if [ -n "${SENTRY_AUTH_TOKEN_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/SENTRY_AUTH_TOKEN","value":"${SENTRY_AUTH_TOKEN_B64}"}
            ]' && echo "✅ SENTRY_AUTH_TOKEN 已注入" || echo "⚠️ SENTRY_AUTH_TOKEN patch 失敗"
          else
            echo "⚠️ SENTRY_AUTH_TOKEN 未設定，Sentry Comment API 將跳過"
          fi

          # ADR-059 2026-04-05 Claude Code: Gitea Webhook Secret
          if [ -n "${GITEA_WEBHOOK_SECRET_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/GITEA_WEBHOOK_SECRET","value":"${GITEA_WEBHOOK_SECRET_B64}"}
            ]' && echo "✅ GITEA_WEBHOOK_SECRET 已注入" || echo "⚠️ GITEA_WEBHOOK_SECRET patch 失敗"
          else
            echo "⚠️ GITEA_WEBHOOK_SECRET 未設定，Gitea Webhook 簽章驗證將在 prod 失效"
          fi

          # MCP Phase 3: ArgoCD API Token (2026-04-11 Claude Sonnet 4.6)
          if [ -n "${ARGOCD_API_TOKEN_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/ARGOCD_API_TOKEN","value":"${ARGOCD_API_TOKEN_B64}"}
            ]' && echo "✅ ARGOCD_API_TOKEN 已注入" || echo "⚠️ ARGOCD_API_TOKEN patch 失敗"
          else
            echo "⚠️ ARGOCD_API_TOKEN 未設定，ArgoCD MCP 將使用空 token"
          fi

          # ============================================================================
          # ADR-090-B 2026-04-18 ogt + Claude Opus 4.7: L3-only 升級 L2（13 個 key）
          # ============================================================================
          # 目的: 消滅「只存 K8s etcd 單點」的災難盲區，Gitea Secret 成為正式真相來源
          # 注意: 每個 block 與上方維持相同結構（if guard + base64 -w 0 + json patch）

          # DATABASE_URL — PG 應用連線串（2026-04-18 輪替）
          if [ -n "${DATABASE_URL_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/DATABASE_URL","value":"${DATABASE_URL_B64}"}
            ]' && echo "✅ DATABASE_URL 已注入" || echo "⚠️ DATABASE_URL patch 失敗"
          else
            echo "⚠️ DATABASE_URL 未設定，awoooi-api 將無法連 PG"
          fi

          # MIGRATION_DATABASE_URL — CI migration 用 awoooi_migrator 限權帳號（ADR-090-B）
          if [ -n "${MIGRATION_DATABASE_URL_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/MIGRATION_DATABASE_URL","value":"${MIGRATION_DATABASE_URL_B64}"}
            ]' && echo "✅ MIGRATION_DATABASE_URL 已注入" || echo "⚠️ MIGRATION_DATABASE_URL patch 失敗"
          fi

          # REDIS_URL — Redis 連線（6380 on 188）
          if [ -n "${REDIS_URL_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/REDIS_URL","value":"${REDIS_URL_B64}"}
            ]' && echo "✅ REDIS_URL 已注入" || echo "⚠️ REDIS_URL patch 失敗"
          else
            echo "⚠️ REDIS_URL 未設定"
          fi

          # JWT_SECRET / JWT_ALGORITHM — API 認證
          if [ -n "${JWT_SECRET_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/JWT_SECRET","value":"${JWT_SECRET_B64}"}
            ]' && echo "✅ JWT_SECRET 已注入" || echo "⚠️ JWT_SECRET patch 失敗"
          fi
          if [ -n "${JWT_ALGORITHM_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/JWT_ALGORITHM","value":"${JWT_ALGORITHM_B64}"}
            ]' && echo "✅ JWT_ALGORITHM 已注入" || echo "⚠️ JWT_ALGORITHM patch 失敗"
          fi

          # WEBHOOK_HMAC_SECRET — Alertmanager webhook HMAC 簽章
          if [ -n "${WEBHOOK_HMAC_SECRET_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/WEBHOOK_HMAC_SECRET","value":"${WEBHOOK_HMAC_SECRET_B64}"}
            ]' && echo "✅ WEBHOOK_HMAC_SECRET 已注入" || echo "⚠️ WEBHOOK_HMAC_SECRET patch 失敗"
          fi

          # AWOOOP_OPERATOR_API_KEY — AwoooP Operator mutation endpoints
          if [ -n "${AWOOOP_OPERATOR_API_KEY_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/AWOOOP_OPERATOR_API_KEY","value":"${AWOOOP_OPERATOR_API_KEY_B64}"}
            ]' && echo "✅ AWOOOP_OPERATOR_API_KEY 已注入" || echo "⚠️ AWOOOP_OPERATOR_API_KEY patch 失敗"
          fi

          # SENTRY_DSN — Sentry 錯誤追蹤（不是 auth token）
          if [ -n "${SENTRY_DSN_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/SENTRY_DSN","value":"${SENTRY_DSN_B64}"}
            ]' && echo "✅ SENTRY_DSN 已注入" || echo "⚠️ SENTRY_DSN patch 失敗"
          fi

          # CLAUDE_API_KEY — Claude 備援 LLM
          if [ -n "${CLAUDE_API_KEY_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/CLAUDE_API_KEY","value":"${CLAUDE_API_KEY_B64}"}
            ]' && echo "✅ CLAUDE_API_KEY 已注入" || echo "⚠️ CLAUDE_API_KEY patch 失敗"
          fi

          # GITEA_API_TOKEN — Gitea API Token（從 AWOOOI_GITEA_API_TOKEN 映射）
          if [ -n "${GITEA_API_TOKEN_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/GITEA_API_TOKEN","value":"${GITEA_API_TOKEN_B64}"}
            ]' && echo "✅ GITEA_API_TOKEN 已注入" || echo "⚠️ GITEA_API_TOKEN patch 失敗"
          fi

          # NEMOTRON_BOT_TOKEN / OPENCLAW_BOT_TOKEN — 多 Bot 架構
          if [ -n "${NEMOTRON_BOT_TOKEN_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/NEMOTRON_BOT_TOKEN","value":"${NEMOTRON_BOT_TOKEN_B64}"}
            ]' && echo "✅ NEMOTRON_BOT_TOKEN 已注入" || echo "⚠️ NEMOTRON_BOT_TOKEN patch 失敗"
          fi
          if [ -n "${OPENCLAW_BOT_TOKEN_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/OPENCLAW_BOT_TOKEN","value":"${OPENCLAW_BOT_TOKEN_B64}"}
            ]' && echo "✅ OPENCLAW_BOT_TOKEN 已注入" || echo "⚠️ OPENCLAW_BOT_TOKEN patch 失敗"
          fi

          # SMTP_HOST / SRE_GROUP_CHAT_ID
          if [ -n "${SMTP_HOST_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/SMTP_HOST","value":"${SMTP_HOST_B64}"}
            ]' && echo "✅ SMTP_HOST 已注入" || echo "⚠️ SMTP_HOST patch 失敗"
          fi
          if [ -n "${SRE_GROUP_CHAT_ID_B64}" ]; then
            \$KUBECTL patch secret awoooi-secrets -n awoooi-prod --type='json' -p='[
              {"op":"add","path":"/data/SRE_GROUP_CHAT_ID","value":"${SRE_GROUP_CHAT_ID_B64}"}
            ]' && echo "✅ SRE_GROUP_CHAT_ID 已注入" || echo "⚠️ SRE_GROUP_CHAT_ID patch 失敗"
          fi

          # 2026-04-06 Claude Code: Sprint 3 T2 — known_hosts Secret (Security Fix A1)
          # 替換關閉 host key 驗證的舊做法，讓 SSH 修復路徑使用已知主機指紋。
          # asyncssh reads /etc/ssh-mcp/known_hosts and requires a non-empty
          # OpenSSH known_hosts file. Keep hosts unhashed so both asyncssh and
          # CLI diagnostics can trust the same secret.
          # 2026-05-02 ogt + Claude Sonnet 4.6: 加 4 台主機完整性檢查
          # 根因：partial scan（如 110 timeout、其他成功）會讓 [-s file] 通過、
          #       後續 patch 推進缺漏的 known_hosts → asyncssh 拒所有 SSH。
          # 修法：scan 完用 grep -c 驗證 4 台主機都在；缺任何一台就 abort，
          #       不能覆蓋現有 secret，防止 production SSH 自動修復路徑癱瘓。
          ssh-keyscan 192.168.0.110 192.168.0.120 192.168.0.121 192.168.0.188 > /tmp/known_hosts_repair 2>/tmp/known_hosts_scan_err || true
          EXPECTED_HOSTS=4
          PRESENT=0
          for ip in 192.168.0.110 192.168.0.120 192.168.0.121 192.168.0.188; do
            if grep -qE "^\${ip}[[:space:]]" /tmp/known_hosts_repair 2>/dev/null; then
              PRESENT=\$((PRESENT + 1))
            else
              echo "⚠️ ssh-keyscan 缺主機 \${ip}"
            fi
          done
          if [ "\$PRESENT" -eq "\$EXPECTED_HOSTS" ]; then
            \$KUBECTL create secret generic awoooi-repair-known-hosts \
              -n awoooi-prod \
              --from-file=known_hosts=/tmp/known_hosts_repair \
              --dry-run=client -o yaml | \$KUBECTL apply -f - \
              && echo "✅ awoooi-repair-known-hosts Secret 已建立/更新" \
              || echo "⚠️ awoooi-repair-known-hosts Secret 建立失敗 (非致命)"
            KNOWN_HOSTS_B64=\$(base64 -w 0 /tmp/known_hosts_repair)
            \$KUBECTL patch secret ssh-mcp-key -n awoooi-prod --type=merge \
              -p="{\"data\":{\"known_hosts\":\"\${KNOWN_HOSTS_B64}\"}}" \
              && echo "✅ ssh-mcp-key known_hosts 已更新（4 台主機完整）" \
              || echo "⚠️ ssh-mcp-key known_hosts 更新失敗 (非致命)"
            rm -f /tmp/known_hosts_repair /tmp/known_hosts_scan_err
          else
            echo "❌ ssh-keyscan 只抓到 \${PRESENT}/\${EXPECTED_HOSTS} 台主機，跳過 patch（保留現有 secret）"
            cat /tmp/known_hosts_scan_err 2>/dev/null | head -10
            rm -f /tmp/known_hosts_repair /tmp/known_hosts_scan_err
          fi

          echo "✅ 所有 Secrets 注入完成"
          SECRETS

      # 2026-04-11 Claude Sonnet 4.6 (Sprint B-3 ADR-069):
      # Deploy 改為 ArgoCD GitOps 模式：更新 kustomization.yaml → git push [skip ci] → ArgoCD sync
      # 舊做法 (kubectl set image) 與 ArgoCD selfHeal 衝突 — ArgoCD 會 revert 任何直接 kubectl 操作
      # 新做法流程：
      #   1. 更新 kustomization.yaml image tag（用 kustomize edit set image）
      #   2. Apply ConfigMap/ServiceRegistry（不含 Deployment，由 ArgoCD 管）
      #   3. git commit [skip ci] + push → 觸發 ArgoCD automated sync
      #   4. 等待 ArgoCD sync + rollout 完成
      #   5. Health Check
      - name: Deploy to K8s (ArgoCD GitOps)
        run: |
          write_deploy_key() {
            mkdir -p "${HOME}/.ssh"
            umask 077
            cat > "${HOME}/.ssh/deploy_key" <<'AWOOOI_DEPLOY_KEY'
          ${{ secrets.DEPLOY_SSH_KEY }}
          AWOOOI_DEPLOY_KEY
            chmod 600 "${HOME}/.ssh/deploy_key"
          }

          mkdir -p ~/.ssh
          write_deploy_key
          # 2026-05-13 Codex: mirror Inject K8s Secrets host-key handling so the
          # deploy job never reaches SSH with a known_hosts file missing ED25519.
          # 2026-06-13 Codex: use the deploy-only known_hosts file so this
          # stage cannot wipe cold-start/backup host trust for 120/188.
          DEPLOY_KNOWN_HOSTS="${HOME}/.ssh/deploy_known_hosts"
          ssh-keyscan -T 5 -t ed25519,rsa,ecdsa "${K8S_SSH_HOST}" > "${DEPLOY_KNOWN_HOSTS}" 2>/dev/null
          test -s "${DEPLOY_KNOWN_HOSTS}" || { echo "❌ K8S host keyscan failed: ${K8S_SSH_HOST}"; exit 1; }
          SSH_OPTS="-i ${HOME}/.ssh/deploy_key -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${DEPLOY_KNOWN_HOSTS} -o ConnectTimeout=10"

          IMAGE_TAG="${{ github.sha }}"
          HARBOR=192.168.0.110:5000

          # ─── Step 1: Apply ConfigMap + ServiceRegistry (ArgoCD 管的是 Deployment，ConfigMap 仍直接 apply) ───
          cat k8s/awoooi-prod/04-configmap.yaml | \
            ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
            "KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"
          echo "✅ ConfigMap 已更新"

          cat k8s/awoooi-prod/15-service-registry-configmap.yaml | \
            ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
            "KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"
          echo "✅ Service Registry ConfigMap 已更新"

          # ─── Step 2: 更新 kustomization.yaml image tag ───
          # host runner 不保證有 root 權限，kustomize 安裝在使用者目錄。
          export PATH="${HOME}/.local/bin:${PATH}"
          if ! command -v kustomize &>/dev/null; then
            mkdir -p "${HOME}/.local/bin"
            curl -sL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.3.0/kustomize_v5.3.0_linux_amd64.tar.gz \
              | tar xz -C "${HOME}/.local/bin"
            chmod +x "${HOME}/.local/bin/kustomize"
          fi

          cd k8s/awoooi-prod
          # kustomize edit set image 更新 tag
          kustomize edit set image \
            192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER=${HARBOR}/awoooi/api:${IMAGE_TAG}
          kustomize edit set image \
            192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER=${HARBOR}/awoooi/web:${IMAGE_TAG}
          cd ../..

          # ─── Step 3: git commit [skip ci] + push → 觸發 ArgoCD sync ───
          git config user.email "cd@awoooi.internal"
          git config user.name "AWOOOI CD"
          git add k8s/awoooi-prod/kustomization.yaml
          DEPLOY_REVISION=""
          git diff --cached --quiet && echo "⚡ kustomization.yaml 無變化，跳過 push" || {
            git commit -m "chore(cd): deploy ${IMAGE_TAG::7} [skip ci]"
            # 用 token 推送（避免 SSH key 需要額外設定 push 權限）
            git remote remove gitea 2>/dev/null || true
            git remote add gitea "http://wooo:${{ secrets.CD_PUSH_TOKEN }}@192.168.0.110:3001/wooo/awoooi.git"
            # 先 rebase 避免 non-fast-forward (其他 commit 在 CI 期間已推入)
            # 2026-04-17 ogt: -X theirs — kustomization.yaml 衝突時採用當次部署的 image tag
            git fetch gitea main
            git rebase -X theirs gitea/main
            DEPLOY_REVISION=$(git rev-parse HEAD)
            git push gitea main
            echo "✅ kustomization.yaml 已 push，等待 ArgoCD sync 到 ${DEPLOY_REVISION:0:8}..."
          }

          # ─── Step 4: 等待 ArgoCD sync + rollout ───
          ROLLOUT_LOG="$(mktemp)"
          set +e
          ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
            "EXPECTED_REVISION='${DEPLOY_REVISION}' bash -s" 2>&1 << 'ARGOCD_WAIT' | tee "$ROLLOUT_LOG"
          set -e
          K8S_API_SERVER="${{ env.K8S_API_SERVER }}"
          KUBECTL="sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${K8S_API_SERVER}"
          RISK_FILE="$(mktemp)"
          UNKNOWN_STATUS_COUNT=0
          HEALTH_FAILURE_COUNT=0

          record_rollout_risk() {
            local message="$1"
            printf '%s\n' "$message" >> "$RISK_FILE"
            echo "⚠️ Rollout risk observed: $message" >&2
          }

          emit_rollout_evidence() {
            if [ -s "$RISK_FILE" ]; then
              local summary
              local kubectl_count
              kubectl_count=$(grep -c '^argocd_.*_query_failed=' "$RISK_FILE" 2>/dev/null || true)
              summary=$(tr '\n' '; ' < "$RISK_FILE" | sed 's/[[:cntrl:]]//g' | cut -c1-700)
              echo "AWOOOI_ROLLOUT_RISK=1"
              echo "AWOOOI_ROLLOUT_SUMMARY=unknown_status_count=${UNKNOWN_STATUS_COUNT}; health_failure_count=${HEALTH_FAILURE_COUNT}; kubectl_failure_count=${kubectl_count}; ${summary}"
            else
              echo "AWOOOI_ROLLOUT_RISK=0"
            fi
            rm -f "$RISK_FILE"
          }
          trap emit_rollout_evidence EXIT

          app_field() {
            local jsonpath="$1"
            local label="$2"
            local output
            local status
            local kubectl_seen
            set +e
            output=$($KUBECTL get application awoooi-prod -n argocd -o jsonpath="$jsonpath" 2>&1)
            status=$?
            set -e
            if [ "$status" -ne 0 ]; then
              kubectl_seen=$(grep -c '^argocd_.*_query_failed=' "$RISK_FILE" 2>/dev/null || true)
              if [ "$kubectl_seen" -lt 3 ]; then
                record_rollout_risk "argocd_${label}_query_failed=$(echo "$output" | head -c 180)"
              fi
              printf 'Unknown'
              return 0
            fi
            printf '%s' "$output"
          }

          probe_public_health() {
            local phase="$1"
            local http_code
            local status
            set +e
            http_code=$(curl -sS -w "%{http_code}" -o /dev/null --connect-timeout 3 --max-time 8 "${{ env.API_HEALTH_URL }}" 2>/dev/null)
            status=$?
            set -e
            if [ "$status" -ne 0 ]; then
              http_code="curl_error_${status}"
            fi
            if [ "$http_code" != "200" ]; then
              HEALTH_FAILURE_COUNT=$((HEALTH_FAILURE_COUNT + 1))
              if [ "$HEALTH_FAILURE_COUNT" -le 3 ]; then
                record_rollout_risk "public_health_${phase}_http=${http_code}"
              fi
            fi
          }

          collect_argocd_resource_evidence() {
            local template
            local output
            local status
            template='{{range .status.resources}}{{if ne .status "Synced"}}{{.kind}}/{{.name}}{{if .namespace}} ns={{.namespace}}{{end}} sync={{.status}}{{if .health.status}} health={{.health.status}}{{end}}{{"\n"}}{{end}}{{if .health.status}}{{if ne .health.status "Healthy"}}{{.kind}}/{{.name}}{{if .namespace}} ns={{.namespace}}{{end}} sync={{.status}} health={{.health.status}}{{if .health.message}} msg={{.health.message}}{{end}}{{"\n"}}{{end}}{{end}}{{end}}'
            set +e
            output=$($KUBECTL get application awoooi-prod -n argocd -o "go-template=${template}" 2>&1)
            status=$?
            set -e
            if [ "$status" -ne 0 ]; then
              local output_snippet
              output_snippet=$(printf '%s' "$output" | head -c 180)
              echo "resource_query_failed=${output_snippet}"
              return 0
            fi
            echo "$output" \
              | awk 'NF && !seen[$0]++ {print}' \
              | head -5 \
              | tr '\n' ';' \
              | sed 's/[[:cntrl:]]//g; s/;*$//'
          }

          validate_argocd_source_contract() {
            local target_revision
            local image_override

            target_revision=$(app_field '{.spec.source.targetRevision}' source_target_revision)
            image_override=$(app_field '{.spec.source.kustomize.images}' source_kustomize_images)

            if [ "$target_revision" != "main" ]; then
              record_rollout_risk "argocd_source_target_revision_not_main targetRevision=$target_revision"
              echo "❌ ArgoCD source targetRevision must be main, got: $target_revision" >&2
              exit 1
            fi

            if [ -n "$image_override" ]; then
              local image_override_snippet
              image_override_snippet=$(printf '%s' "$image_override" | head -c 180)
              record_rollout_risk "argocd_source_image_override_present images=${image_override_snippet}"
              echo "❌ ArgoCD source kustomize.images override must be empty; image truth belongs in k8s/awoooi-prod/kustomization.yaml" >&2
              exit 1
            fi
          }

          # 等待 ArgoCD Application 同步到目標 revision（最多 180s）。
          # 2026-05-24 Codex: top-level Application health can stay Degraded
          # without per-resource health detail. Treat that as rollout evidence,
          # then let kubectl rollout status and API health decide pass/fail.
          echo "⏳ 等待 ArgoCD sync..."
          validate_argocd_source_contract
          $KUBECTL annotate application awoooi-prod -n argocd \
            argocd.argoproj.io/refresh=hard --overwrite >/dev/null 2>&1 || true
          for i in $(seq 1 36); do
            SYNC=$(app_field '{.status.sync.status}' sync)
            HEALTH=$(app_field '{.status.health.status}' health)
            REVISION=$(app_field '{.status.sync.revision}' revision)
            SHORT_REVISION=$(echo "$REVISION" | cut -c1-8)
            SHORT_EXPECTED=$(echo "$EXPECTED_REVISION" | cut -c1-8)
            echo "  ArgoCD: sync=$SYNC health=$HEALTH revision=$SHORT_REVISION expected=${SHORT_EXPECTED:-any}"
            probe_public_health "argocd_wait"
            if [ "$SYNC" = "Unknown" ] || [ "$HEALTH" = "Unknown" ] || [ "$REVISION" = "Unknown" ]; then
              UNKNOWN_STATUS_COUNT=$((UNKNOWN_STATUS_COUNT + 1))
              if [ "$UNKNOWN_STATUS_COUNT" -le 3 ]; then
                record_rollout_risk "argocd_status_unknown sync=$SYNC health=$HEALTH revision=$SHORT_REVISION expected=${SHORT_EXPECTED:-any}"
              fi
            fi
            if [ "$SYNC" = "Synced" ]; then
              if [ -z "$EXPECTED_REVISION" ] || [ "$REVISION" = "$EXPECTED_REVISION" ]; then
                if [ "$HEALTH" != "Healthy" ]; then
                  RESOURCE_EVIDENCE=$(collect_argocd_resource_evidence)
                  if [ -n "$RESOURCE_EVIDENCE" ]; then
                    record_rollout_risk "argocd_health_not_healthy health=$HEALTH revision=$SHORT_REVISION resources=$RESOURCE_EVIDENCE"
                  else
                    record_rollout_risk "argocd_health_not_healthy health=$HEALTH revision=$SHORT_REVISION resources=none_visible"
                  fi
                fi
                echo "✅ ArgoCD Synced to target revision (health=$HEALTH)"
                break
              fi
            fi
            if [ "$i" = "36" ]; then
              echo "❌ ArgoCD 未在期限內同步到目標 revision"
              exit 1
            fi
            sleep 5
          done

          # 確認 rollout 完成
          $KUBECTL rollout status deployment/awoooi-api -n awoooi-prod --timeout=120s
          $KUBECTL rollout status deployment/awoooi-web -n awoooi-prod --timeout=120s
          $KUBECTL rollout status deployment/awoooi-worker -n awoooi-prod --timeout=120s
          echo "✅ 部署完成"

          # Health Check
          HEALTH_PASS=0
          for i in 1 2 3; do
            set +e
            HTTP_CODE=$(curl -sS -w "%{http_code}" -o /dev/null --connect-timeout 10 --max-time 20 "${{ env.API_HEALTH_URL }}" 2>/dev/null)
            CURL_STATUS=$?
            set -e
            if [ "$CURL_STATUS" -ne 0 ]; then
              HTTP_CODE="curl_error_${CURL_STATUS}"
            fi
            if [ "$HTTP_CODE" = "200" ]; then
              echo "✅ API 健康檢查通過"
              HEALTH_PASS=1
              break
            fi
            echo "⏳ 嘗試 #$i: HTTP $HTTP_CODE，等待 10s..."
            sleep 10
          done
          if [ "$HEALTH_PASS" = "0" ]; then
            record_rollout_risk "public_health_final_failed"
            echo "❌ API 健康檢查失敗"
            exit 1
          fi
          ARGOCD_WAIT
          ROLLOUT_EXIT=${PIPESTATUS[0]}
          set -e

          ROLLOUT_RISK="0"
          ROLLOUT_SUMMARY=""
          if grep -q '^AWOOOI_ROLLOUT_RISK=1$' "$ROLLOUT_LOG"; then
            ROLLOUT_RISK="1"
            ROLLOUT_SUMMARY=$(grep '^AWOOOI_ROLLOUT_SUMMARY=' "$ROLLOUT_LOG" | tail -1 | sed 's/^AWOOOI_ROLLOUT_SUMMARY=//' | cut -c1-700)
          fi
          if [ -n "${GITHUB_ENV:-}" ]; then
            {
              echo "AWOOI_ROLLOUT_RISK=${ROLLOUT_RISK}"
              echo "AWOOI_ROLLOUT_SUMMARY=${ROLLOUT_SUMMARY}"
            } >> "$GITHUB_ENV"
          fi
          rm -f "$ROLLOUT_LOG"

          if [ "$ROLLOUT_EXIT" -eq 0 ] && [ "$ROLLOUT_RISK" = "1" ]; then
            ACTOR="${GITHUB_ACTOR:-${{ github.actor }}}"
            if AWOOI_CICD_STATUS=pending \
              AWOOI_CICD_STAGE=rollout-risk \
              AWOOI_CICD_JOB_NAME="AWOOOI 部署完成但仍有風險證據" \
              AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
              AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
              AWOOI_CICD_SUMMARY="${ROLLOUT_SUMMARY}" \
              scripts/ci/notify-awoooi-cicd.sh; then
              echo "✅ CI/CD rollout risk notification mirrored through AWOOI API"
            else
              echo "⚠️ CI/CD rollout risk notification failed (non-fatal)"
            fi
          fi
          exit "$ROLLOUT_EXIT"

      - name: Notify Build Deploy Success
        run: |
          END_TIME=$(date +%s)
          DURATION=$((END_TIME - ${{ steps.commit.outputs.start_time }}))
          ACTOR="${{ github.actor }}"
          if AWOOI_CICD_STATUS=success \
            AWOOI_CICD_STAGE=build-and-deploy \
            AWOOI_CICD_JOB_NAME="AWOOOI 建置部署完成" \
            AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
            AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
            AWOOI_CICD_DURATION_SECONDS="${DURATION}" \
            AWOOI_CICD_SUMMARY="Image build/push + ArgoCD rollout + API health passed" \
            scripts/ci/notify-awoooi-cicd.sh; then
            echo "✅ CI/CD build-deploy success notification mirrored through AWOOI API"
          else
            echo "⚠️ CI/CD build-deploy success notification failed (non-fatal)"
          fi

      # 2026-04-09 Claude Sonnet 4.6: Sprint 5.2 — 同步 ops 腳本到 188 (ollama user)
      # 188 deploy key is rotated and must not be read by this disabled step.
      # 腳本: docker-health-monitor.sh + pg-backup.sh + notify-awoooi-ops.sh
      # 感知層與備份通知都先走 AWOOI API/AwoooP，Telegram 直發只保留 API 離線 fallback。
      - name: Sync Ops Scripts to 188
        # 2026-05-13 Codex T14e/P0:
        # Disabled until the 188 ops sync path is moved to a file-secret or
        # Ansible-controlled channel. Gitea Actions logs step env values, and
        # multiline SSH secrets must not be exposed through CD logs.
        if: ${{ false }}
        continue-on-error: true
        run: |
          echo "188 ops script sync disabled pending secure key rotation path"

      - name: Notify Pipeline Failure
        if: failure()
        run: |
          COMMIT_MSG="${{ steps.commit.outputs.message }}"
          SHORT_SHA="${{ steps.commit.outputs.short_sha }}"
          ACTOR="${{ github.actor }}"
          FAILURE_SUMMARY="${AWOOI_ROLLOUT_SUMMARY:-${COMMIT_MSG}}"
          COMMIT_ESC=$(echo "$COMMIT_MSG" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g')
          MSG=$(printf '❌ <b>AWOOOI 部署失敗</b>\n├ 📝 <code>%s</code>\n├ 🔖 <code>%s</code>\n├ 👤 %s\n├ 🏗️ Stage: build-and-deploy\n└ 🔗 http://192.168.0.110:3001/wooo/awoooi/actions' "${COMMIT_ESC}" "${SHORT_SHA}" "${ACTOR}")
          if AWOOI_CICD_STATUS=failed \
            AWOOI_CICD_STAGE=build-and-deploy \
            AWOOI_CICD_JOB_NAME="AWOOOI 部署失敗" \
            AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
            AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
            AWOOI_CICD_SUMMARY="${FAILURE_SUMMARY}" \
            scripts/ci/notify-awoooi-cicd.sh; then
            echo "✅ CI/CD build failure notification mirrored through AWOOI API"
          else
            curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \
              -d "chat_id=${{ env.SRE_GROUP_CHAT_ID }}" \
              -d "parse_mode=HTML" \
              --data-urlencode "text=${MSG}" || echo "TG notify failed (non-fatal): exit=$?"
          fi

  post-deploy-checks:
    needs: build-and-deploy
    timeout-minutes: 30
    # 2026-04-30 Codex: keep post-deploy on the host runner too. Playwright
    # install-deps can also kill the act-managed job container with RWLayer=nil.
    runs-on: awoooi-host
    steps:
      - name: Bootstrap Host Runner Tools
        # 2026-05-05 Codex: post-deploy also uses checkout and curl-based
        # notifications, so it needs the same runner bootstrap as earlier jobs.
        run: |
          if command -v apk >/dev/null 2>&1; then
            apk add --no-cache nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
          fi

      - uses: actions/checkout@v4

      - name: Get Commit Info
        id: commit
        run: |
          echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
          echo "message=$(git log -1 --pretty=%s | head -c 50)" >> $GITHUB_OUTPUT
          echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT

      - name: Notify Post Deploy Checks Start
        run: |
          ACTOR="${{ github.actor }}"
          if AWOOI_CICD_STATUS=running \
            AWOOI_CICD_STAGE=post-deploy-checks \
            AWOOI_CICD_JOB_NAME="AWOOOI 部署後驗證開始" \
            AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
            AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
            AWOOI_CICD_SUMMARY="Alert Chain / Source Link / Monitoring / Smoke gates started" \
            scripts/ci/notify-awoooi-cicd.sh; then
            echo "✅ CI/CD post-deploy start notification mirrored through AWOOI API"
          else
            echo "⚠️ CI/CD post-deploy start notification failed (non-fatal)"
          fi

      # Phase O-4.5 2026-04-02: Alert Chain Smoke Test (Wave A.6 + B.2 ADR-037)
      # 驗證告警鏈路 E2E: API Health + Webhook + OTEL + Event Exporter
      # 2026-04-05 Claude Code cache優化: 使用 /opt/api-venv (已有 requests)，移除 Setup Python Tools step
      # 2026-04-10 ogt: 移除 continue-on-error — 告警鏈路失敗必須阻塞部署
      - name: Alert Chain Smoke Test
        id: alert_chain_smoke
        run: |
          write_deploy_key() {
            mkdir -p "${HOME}/.ssh"
            umask 077
            cat > "${HOME}/.ssh/deploy_key" <<'AWOOOI_DEPLOY_KEY'
          ${{ secrets.DEPLOY_SSH_KEY }}
          AWOOOI_DEPLOY_KEY
            chmod 600 "${HOME}/.ssh/deploy_key"
          }
          collect_observability_statuses() {
            local component="$1"
            ssh $SSH_OPTS "wooo@${K8S_SSH_HOST}" \
              "sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${K8S_API_SERVER} get pods -n observability -l app.kubernetes.io/name=${component} --no-headers -o custom-columns=STATUS:.status.phase"
          }
          capture_observability_statuses() {
            local component="$1"
            local output
            if output="$(collect_observability_statuses "${component}" 2>&1)"; then
              printf '%s' "${output}"
              return 0
            fi
            printf '%s' "${output}"
            return 1
          }

          # 2026-05-19 Codex: the smoke test runs inside CI_IMAGE, but the
          # observability pod checks need the K3s host kubectl context. Capture
          # those read-only statuses on the host and pass them into the
          # container, instead of making the container own kube credentials.
          OBSERVABILITY_PREFLIGHT_ERROR=""
          OTEL_COLLECTOR_ERROR=""
          EVENT_EXPORTER_ERROR=""
          OTEL_COLLECTOR_STATUSES=""
          EVENT_EXPORTER_STATUSES=""

          write_deploy_key
          DEPLOY_KNOWN_HOSTS="${HOME}/.ssh/deploy_known_hosts"
          if ssh-keyscan -T 5 -t ed25519,rsa,ecdsa "${K8S_SSH_HOST}" > "${DEPLOY_KNOWN_HOSTS}" 2>/dev/null && test -s "${DEPLOY_KNOWN_HOSTS}"; then
            SSH_OPTS="-i ${HOME}/.ssh/deploy_key -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${DEPLOY_KNOWN_HOSTS} -o ConnectTimeout=10"
            if ! OTEL_COLLECTOR_STATUSES="$(capture_observability_statuses otel-collector)"; then
              OTEL_COLLECTOR_ERROR="$(printf '%s' "${OTEL_COLLECTOR_STATUSES}" | tail -1 | head -c 200)"
              OTEL_COLLECTOR_STATUSES=""
            fi
            if ! EVENT_EXPORTER_STATUSES="$(capture_observability_statuses event-exporter)"; then
              EVENT_EXPORTER_ERROR="$(printf '%s' "${EVENT_EXPORTER_STATUSES}" | tail -1 | head -c 200)"
              EVENT_EXPORTER_STATUSES=""
            fi
          else
            OBSERVABILITY_PREFLIGHT_ERROR="K8s host keyscan failed"
            OTEL_COLLECTOR_ERROR="${OBSERVABILITY_PREFLIGHT_ERROR}"
            EVENT_EXPORTER_ERROR="${OBSERVABILITY_PREFLIGHT_ERROR}"
          fi

          SOURCE_LINK_RUN_REF="gitea-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}"
          SOURCE_LINK_CANARY_WORK_ITEM_ID="source-evidence:sentry:upstream_canary:awoooi-source-link-canary-${SOURCE_LINK_RUN_REF}"
          SOURCE_LINK_CANARY_EVENT_ID="sentry:source_correlation_linked:awoooi-source-link-canary-${SOURCE_LINK_RUN_REF}"
          echo "source_link_canary_work_item_id=${SOURCE_LINK_CANARY_WORK_ITEM_ID}" >> "$GITHUB_OUTPUT"
          echo "source_link_canary_event_id=${SOURCE_LINK_CANARY_EVENT_ID}" >> "$GITHUB_OUTPUT"

          AWOOOP_OPERATOR_API_KEY="$(
            ssh $SSH_OPTS "wooo@${K8S_SSH_HOST}" \
              "sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${K8S_API_SERVER} get secret awoooi-secrets -n awoooi-prod -o jsonpath='{.data.AWOOOP_OPERATOR_API_KEY}' | base64 -d"
          )"
          if [ -z "${AWOOOP_OPERATOR_API_KEY}" ]; then
            echo "❌ AWOOOP_OPERATOR_API_KEY missing from production secret; source-link canary cannot run"
            exit 1
          fi
          export AWOOOP_OPERATOR_API_KEY

          # 2026-05-05 Codex: use the keepalived VIP instead of a fixed node.
          # Host runner launches the CI image explicitly to avoid act RWLayer=nil.
          if docker run --rm \
            --name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-alert-smoke" \
            --cpus "1.0" \
            --memory "1g" \
            -v "$PWD:/workspace" \
            -v awoooi-api-venv-cache:/opt/api-venv \
            -w /workspace \
            -e AWOOOI_OTEL_COLLECTOR_STATUSES="${OTEL_COLLECTOR_STATUSES}" \
            -e AWOOOI_OTEL_COLLECTOR_ERROR="${OTEL_COLLECTOR_ERROR}" \
            -e AWOOOI_EVENT_EXPORTER_STATUSES="${EVENT_EXPORTER_STATUSES}" \
            -e AWOOOI_EVENT_EXPORTER_ERROR="${EVENT_EXPORTER_ERROR}" \
            -e AWOOOP_OPERATOR_API_KEY \
            -e AWOOOP_OPERATOR_ID="gitea-cd-post-deploy" \
            -e SOURCE_LINK_RUN_REF="${SOURCE_LINK_RUN_REF}" \
            "${{ env.CI_IMAGE }}" \
            bash -lc 'set -o pipefail; source /opt/api-venv/bin/activate && python3 scripts/alert_chain_smoke_test.py --api-url ${{ env.ALERT_CHAIN_API_URL }} --source-link-canary-target-incident-id INC-20260505-25E744 --run-ref "${SOURCE_LINK_RUN_REF}" --json | tee /tmp/alert_chain_result.json'; then
            echo "alert_chain_status=pass" >> $GITHUB_OUTPUT
          else
            echo "alert_chain_status=fail" >> $GITHUB_OUTPUT
            exit 1
          fi

      # Phase O-5 Wave C.2 2026-04-02 ogt: 監控覆蓋率驗證 (generate_monitoring.py --check)
      # 2026-04-10 ogt: 移除 continue-on-error — 覆蓋率不足必須阻塞部署
      - name: Monitoring Coverage Check
        id: monitoring_coverage
        run: |
          if docker run --rm \
            --name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-coverage" \
            --cpus "1.0" \
            --memory "1g" \
            -v "$PWD:/workspace" \
            -v awoooi-api-venv-cache:/opt/api-venv \
            -w /workspace \
            "${{ env.CI_IMAGE }}" \
            bash -lc 'source /opt/api-venv/bin/activate && python3 scripts/generate_monitoring.py --check'; then
            echo "coverage_status=pass" >> $GITHUB_OUTPUT
          else
            echo "coverage_status=fail" >> $GITHUB_OUTPUT
            exit 1
          fi

      - name: AwoooP Source Correlation Applied-Link Smoke
        id: source_correlation_apply_smoke
        run: |
          SOURCE_LINK_CANARY_WORK_ITEM_ID="${{ steps.alert_chain_smoke.outputs.source_link_canary_work_item_id }}"
          SOURCE_LINK_CANARY_EVENT_ID="${{ steps.alert_chain_smoke.outputs.source_link_canary_event_id }}"
          export SOURCE_LINK_CANARY_WORK_ITEM_ID SOURCE_LINK_CANARY_EVENT_ID
          if docker run --rm \
            --name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-source-link-smoke" \
            --cpus "0.5" \
            --memory "512m" \
            -v "$PWD:/workspace" \
            -v awoooi-api-venv-cache:/opt/api-venv \
            -w /workspace \
            -e SOURCE_LINK_CANARY_WORK_ITEM_ID \
            -e SOURCE_LINK_CANARY_EVENT_ID \
            "${{ env.CI_IMAGE }}" \
            bash -lc 'set -o pipefail; source /opt/api-venv/bin/activate && python3 scripts/awooop_source_correlation_apply_smoke.py \
              --api-url ${{ env.ALERT_CHAIN_API_URL }} \
              --target-incident-id INC-20260505-25E744 \
              --work-item-id "${SOURCE_LINK_CANARY_WORK_ITEM_ID}" \
              --expected-source-event-provider-event-id "${SOURCE_LINK_CANARY_EVENT_ID}" \
              --allow-existing-apply \
              --refresh-if-stale-days 6 \
              --refresh-work-item-id "${SOURCE_LINK_CANARY_WORK_ITEM_ID}" \
              --verify-refresh-candidate \
              --reviewer-id gitea_cd_source_link_canary \
              --operator-note "CD dedicated source-link canary; append-only status-chain proof" \
              | tee /tmp/source_correlation_apply_smoke.json'; then
            echo "source_correlation_apply_status=pass" >> $GITHUB_OUTPUT
          else
            echo "source_correlation_apply_status=fail" >> $GITHUB_OUTPUT
            exit 1
          fi

      # [首席架構師] 新增 Playwright E2E Smoke Test 步驟 v1.0.0 2026-04-01 (台北時間)
      # continue-on-error: true — smoke 失敗不阻塞部署，但結果會反映在 TG 通知
      - name: E2E Smoke Test
        id: smoke
        continue-on-error: true
        run: |
          cat > /tmp/awoooi-smoke.sh <<'CI_SCRIPT'
          # 首席架構師 Review I4 + 2026-04-05 Claude Code cache優化:
          # playwright.config.ts import @playwright/test — 必須先安裝 pnpm node_modules
          # pnpm store 持久化到 /opt/pnpm-store，pnpm-lock.yaml hash 未變則 --prefer-offline
          cleanup_smoke_workspace_artifacts() {
            # 2026-05-19 Codex: pnpm creates a symlink-heavy node_modules tree
            # inside the bind-mounted checkout. Remove it before act-runner's
            # post-job cleanup so successful smoke jobs do not end with
            # errSymlink cleanup noise.
            rm -rf /workspace/node_modules \
              /workspace/apps/web/node_modules \
              /workspace/apps/web/tests/e2e/.auth \
              /workspace/apps/web/test-results \
              /workspace/apps/web/playwright-report \
              2>/dev/null || true
            find /workspace/apps /workspace/packages \
              -mindepth 2 -maxdepth 2 -type d -name node_modules -prune -exec rm -rf {} + \
              2>/dev/null || true
          }
          trap cleanup_smoke_workspace_artifacts EXIT

          PNPM_STORE=/opt/pnpm-store
          PNPM_HASH_FILE=/opt/pnpm-store/.lock_hash
          CURRENT_PNPM_HASH=$(md5sum pnpm-lock.yaml | awk '{print $1}')

          corepack enable 2>/dev/null || npm install -g pnpm@9 -q
          pnpm config set store-dir $PNPM_STORE

          if [ "$(cat $PNPM_HASH_FILE 2>/dev/null)" != "$CURRENT_PNPM_HASH" ]; then
            echo "📦 pnpm lock 已變更，重裝 node_modules..."
            pnpm install --frozen-lockfile 2>&1 | tail -5
            echo "$CURRENT_PNPM_HASH" > $PNPM_HASH_FILE
          else
            echo "⚡ 使用快取 pnpm store (lock 未變更)，prefer-offline..."
            pnpm install --frozen-lockfile --prefer-offline 2>&1 | tail -5
          fi

          cd apps/web
          # Playwright Chromium 持久化到 /opt/playwright-browsers，版本 hash guard
          export PLAYWRIGHT_BROWSERS_PATH=/opt/playwright-browsers
          PLAYWRIGHT_VER=$(node -e "console.log(require('./package.json').devDependencies['@playwright/test'] || '')" 2>/dev/null || echo "unknown")
          PLAYWRIGHT_HASH_FILE=/opt/playwright-browsers/.version_hash
          if [ "$(cat $PLAYWRIGHT_HASH_FILE 2>/dev/null)" != "$PLAYWRIGHT_VER" ]; then
            echo "📦 Playwright 版本變更 ($PLAYWRIGHT_VER)，重裝 Chromium..."
            npx playwright install chromium --with-deps 2>&1 | tail -5
            echo "$PLAYWRIGHT_VER" > $PLAYWRIGHT_HASH_FILE
          else
            echo "⚡ 使用快取 Playwright Chromium ($PLAYWRIGHT_VER)"
          fi
          # Browser cache 命中時也要確認 OS shared libs 存在；否則 smoke 會只測到
          # chromium launch failure（例如 libnspr4.so missing）。
          if ! ldconfig -p 2>/dev/null | grep -q 'libnspr4'; then
            echo "📦 Playwright system deps missing，補安裝 Chromium deps..."
            npx playwright install-deps chromium > /tmp/playwright-install-deps.log 2>&1 || {
              tail -40 /tmp/playwright-install-deps.log
              exit 1
            }
            tail -20 /tmp/playwright-install-deps.log
          fi
          # 對已部署的生產環境跑 smoke test
          SMOKE_STATUS=pass
          npx playwright test tests/e2e/smoke.spec.ts --reporter=line || SMOKE_STATUS=fail
          echo "smoke_status=${SMOKE_STATUS}" >> $GITHUB_OUTPUT
          CI_SCRIPT
          SMOKE_OUTPUT="$PWD/.awoooi-smoke-output"
          rm -f "$SMOKE_OUTPUT"
          touch "$SMOKE_OUTPUT"
          chmod 666 "$SMOKE_OUTPUT"
          SMOKE_DOCKER_STATUS=0
          # 2026-06-01 Codex: post-deploy smoke can pass, then hang in
          # runner cleanup and incorrectly mark the deploy failed. Bound only
          # the smoke container; preserve pass evidence if it was written.
          if command -v timeout >/dev/null 2>&1; then
            # 2026-06-14 Codex: act-runner host may provide BusyBox timeout,
            # which rejects GNU-only --kill-after. The short -k form works
            # with BusyBox and GNU timeout.
            timeout -k 20s 300s docker run --rm \
              --name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-e2e-smoke" \
              --cpus "1.5" \
              --memory "2g" \
              -v "$PWD:/workspace" \
              -v /tmp/awoooi-smoke.sh:/tmp/awoooi-smoke.sh:ro \
              -v awoooi-pnpm-store:/opt/pnpm-store \
              -v awoooi-playwright-browsers:/opt/playwright-browsers \
              -w /workspace \
              -e GITHUB_OUTPUT=/workspace/.awoooi-smoke-output \
              -e CI=true \
              -e PLAYWRIGHT_BASE_URL=https://awoooi.wooo.work \
              "${{ env.CI_IMAGE }}" \
              bash /tmp/awoooi-smoke.sh || SMOKE_DOCKER_STATUS=$?
          else
            docker run --rm \
              --name "awoooi-cd-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}-e2e-smoke" \
              --cpus "1.5" \
              --memory "2g" \
              -v "$PWD:/workspace" \
              -v /tmp/awoooi-smoke.sh:/tmp/awoooi-smoke.sh:ro \
              -v awoooi-pnpm-store:/opt/pnpm-store \
              -v awoooi-playwright-browsers:/opt/playwright-browsers \
              -w /workspace \
              -e GITHUB_OUTPUT=/workspace/.awoooi-smoke-output \
              -e CI=true \
              -e PLAYWRIGHT_BASE_URL=https://awoooi.wooo.work \
              "${{ env.CI_IMAGE }}" \
              bash /tmp/awoooi-smoke.sh || SMOKE_DOCKER_STATUS=$?
          fi
          if [ "$SMOKE_DOCKER_STATUS" != "0" ] && ! grep -q '^smoke_status=pass$' "$SMOKE_OUTPUT"; then
            echo "smoke_status=fail" > "$SMOKE_OUTPUT"
            echo "E2E smoke container failed before pass evidence: ${SMOKE_DOCKER_STATUS}"
            exit "$SMOKE_DOCKER_STATUS"
          fi
          if [ "$SMOKE_DOCKER_STATUS" != "0" ]; then
            echo "E2E smoke pass evidence was written; treating container exit ${SMOKE_DOCKER_STATUS} as cleanup timeout"
          fi
          cat "$SMOKE_OUTPUT" >> "$GITHUB_OUTPUT"
        env:
          CI: "true"
          # 直接測試已部署的生產環境，不啟動本地 dev server
          PLAYWRIGHT_BASE_URL: "https://awoooi.wooo.work"

      - name: Notify Health Check Success
        env:
          SMOKE_RESULT: ${{ steps.smoke.outputs.smoke_status == 'pass' && '✅' || '⚠️' }}
          ALERT_CHAIN_RESULT: ${{ steps.alert_chain_smoke.outputs.alert_chain_status == 'pass' && '✅' || '⚠️' }}
          MONITORING_RESULT: ${{ steps.monitoring_coverage.outputs.coverage_status == 'pass' && '✅' || '⚠️' }}
          SOURCE_LINK_RESULT: ${{ steps.source_correlation_apply_smoke.outputs.source_correlation_apply_status == 'pass' && '✅' || '⚠️' }}
        run: |
          END_TIME=$(date +%s)
          DURATION=$((END_TIME - ${{ steps.commit.outputs.start_time }}))
          MINUTES=$((DURATION / 60))
          SECONDS=$((DURATION % 60))
          # 2026-04-05 ogt: TG_MSG 必須在 shell 中組裝，才能展開 ${MINUTES}/${SECONDS} 等 shell 變數
          # 2026-04-05 ogt: 移除 parse_mode=HTML，避免 commit message 含特殊字元導致 400
          COMMIT_MSG="${{ steps.commit.outputs.message }}"
          SHORT_SHA="${{ steps.commit.outputs.short_sha }}"
          TG_MSG="✅ AWOOOI 部署完成\n├ 📝 ${COMMIT_MSG}\n├ 🔖 ${SHORT_SHA}\n├ ⏱️ 耗時: ${MINUTES}m ${SECONDS}s\n├ 📦 API: ✅ Web: ✅\n├ 🩺 Health: ✅\n├ 🔗 Alert Chain: ${ALERT_CHAIN_RESULT}\n├ 🧷 Source Link: ${SOURCE_LINK_RESULT}\n├ 📊 Monitoring: ${MONITORING_RESULT}\n└ 🎭 Smoke: ${SMOKE_RESULT}"
          if AWOOI_CICD_STATUS=success \
            AWOOI_CICD_STAGE=post-deploy \
            AWOOI_CICD_JOB_NAME="AWOOOI 部署完成" \
            AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
            AWOOI_CICD_DURATION_SECONDS="${DURATION}" \
            AWOOI_CICD_SUMMARY="API=✅; Web=✅; AlertChain=${ALERT_CHAIN_RESULT}; SourceLink=${SOURCE_LINK_RESULT}; Monitoring=${MONITORING_RESULT}; Smoke=${SMOKE_RESULT}" \
            scripts/ci/notify-awoooi-cicd.sh; then
            echo "✅ CI/CD success notification mirrored through AWOOI API"
          else
            printf '%b' "$TG_MSG" | curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \
              -d "chat_id=${{ env.SRE_GROUP_CHAT_ID }}" \
              --data-urlencode "text@-" || echo "TG notify warning (non-fatal)"
          fi

      - name: Notify Pipeline Failure
        # 2026-04-16 ogt + Claude Sonnet 4.6: 改用 HTML 結構化格式
        if: failure()
        run: |
          COMMIT_MSG="${{ steps.commit.outputs.message }}"
          SHORT_SHA="${{ steps.commit.outputs.short_sha }}"
          ACTOR="${{ github.actor }}"
          COMMIT_ESC=$(echo "$COMMIT_MSG" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g')
          MSG=$(printf '❌ <b>AWOOOI 部署失敗</b>\n├ 📝 <code>%s</code>\n├ 🔖 <code>%s</code>\n├ 👤 %s\n├ 🩺 Stage: post-deploy-checks\n└ 🔗 http://192.168.0.110:3001/wooo/awoooi/actions' "${COMMIT_ESC}" "${SHORT_SHA}" "${ACTOR}")
          if AWOOI_CICD_STATUS=failed \
            AWOOI_CICD_STAGE=post-deploy-checks \
            AWOOI_CICD_JOB_NAME="AWOOOI 部署失敗" \
            AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \
            AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \
            AWOOI_CICD_SUMMARY="${COMMIT_MSG}" \
            scripts/ci/notify-awoooi-cicd.sh; then
            echo "✅ CI/CD post-deploy failure notification mirrored through AWOOI API"
          else
            curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \
              -d "chat_id=${{ env.SRE_GROUP_CHAT_ID }}" \
              -d "parse_mode=HTML" \
              --data-urlencode "text=${MSG}" || echo "TG notify failed (non-fatal): exit=$?"
          fi

      - name: Clean Post-Deploy Workspace Artifacts
        if: always()
        env:
          HOST_RUNNER_CLEANUP_IMAGE: ${{ env.CI_IMAGE }}
        run: bash scripts/ci/cleanup-host-runner-workspace.sh
