fix(cd): coalesce stale carriers and harden verifiers
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 48s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m34s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 26s
CD Pipeline / revalidate-post-deploy-carrier (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 48s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m34s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 26s
CD Pipeline / revalidate-post-deploy-carrier (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
This commit is contained in:
@@ -24,11 +24,9 @@ on:
|
||||
workflow_dispatch:
|
||||
# 手動觸發永遠可用(用於補跑、緊急部署)
|
||||
|
||||
# Main deployments run serially so concurrent writers cannot starve production closure.
|
||||
# Job timeouts, rollout verification, and deploy markers terminate or expose failed runs.
|
||||
concurrency:
|
||||
group: cd-deploy-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
# Gitea 1.25 ignores the workflow concurrency key. The dedicated runner's verified
|
||||
# capacity=1 is the executable queue, while the selectors below revalidate linear
|
||||
# main ancestry before tests, immediately before deploy, and before post-deploy readback.
|
||||
|
||||
env:
|
||||
HARBOR: registry.wooo.work
|
||||
@@ -60,6 +58,40 @@ env:
|
||||
ALERT_CHAIN_API_URL: https://awoooi.wooo.work
|
||||
|
||||
jobs:
|
||||
select-latest-carrier:
|
||||
if: ${{ github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd') && !contains(github.event.head_commit.message, '[metadata-only]')) }}
|
||||
runs-on: awoooi-non110-host
|
||||
timeout-minutes: 3
|
||||
outputs:
|
||||
selected: ${{ steps.carrier.outputs.selected }}
|
||||
status: ${{ steps.carrier.outputs.status }}
|
||||
source_sha: ${{ steps.carrier.outputs.source_sha }}
|
||||
latest_sha: ${{ steps.carrier.outputs.latest_sha }}
|
||||
main_tip_sha: ${{ steps.carrier.outputs.main_tip_sha }}
|
||||
steps:
|
||||
- name: Checkout exact source from Gitea
|
||||
env:
|
||||
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git init .
|
||||
git remote remove origin 2>/dev/null || true
|
||||
git remote add origin "$GITEA_SOURCE_URL"
|
||||
git fetch --no-tags --prune --depth=1 origin "$GITHUB_SHA"
|
||||
git checkout --force --detach FETCH_HEAD
|
||||
|
||||
- name: Select latest inclusive main carrier
|
||||
id: carrier
|
||||
env:
|
||||
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 ops/runner/select-latest-inclusive-cd-carrier.py \
|
||||
--source-sha "$GITHUB_SHA" \
|
||||
--remote-url "$GITEA_SOURCE_URL" \
|
||||
--debounce-seconds 20 \
|
||||
--github-output "$GITHUB_OUTPUT"
|
||||
|
||||
workflow-shape:
|
||||
# 2026-06-28 Codex: Gitea 1.25 may mark a workflow invalid when every
|
||||
# root job has a job-level `if`. Keep one no-op root job without `if` so
|
||||
@@ -87,7 +119,8 @@ jobs:
|
||||
# on CD-generated deploy commits. Skip jobs explicitly so marker commits
|
||||
# do not trigger a self-feeding CD loop; `cancel-stale-cd` is a
|
||||
# controlled no-op trigger used only to cancel stale pre-guard runs.
|
||||
if: ${{ github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd')) }}
|
||||
if: ${{ needs.select-latest-carrier.outputs.selected == 'true' && (github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd'))) }}
|
||||
needs: [select-latest-carrier]
|
||||
# 2026-04-30 Codex: run the tests job on the host runner and launch the
|
||||
# CI image explicitly. The act-managed job container can disappear mid-test
|
||||
# with Docker RWLayer=nil on the shared 110 daemon.
|
||||
@@ -1370,18 +1403,53 @@ jobs:
|
||||
echo "AWOOI API notify failed; direct Telegram fallback disabled to preserve AwoooP receipt chain"
|
||||
fi
|
||||
|
||||
revalidate-deploy-carrier:
|
||||
if: ${{ needs.select-latest-carrier.outputs.selected == 'true' }}
|
||||
needs: [select-latest-carrier, tests]
|
||||
runs-on: awoooi-non110-host
|
||||
timeout-minutes: 3
|
||||
outputs:
|
||||
selected: ${{ steps.carrier.outputs.selected }}
|
||||
status: ${{ steps.carrier.outputs.status }}
|
||||
latest_sha: ${{ steps.carrier.outputs.latest_sha }}
|
||||
main_tip_sha: ${{ steps.carrier.outputs.main_tip_sha }}
|
||||
steps:
|
||||
- name: Checkout exact source from Gitea
|
||||
env:
|
||||
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git init .
|
||||
git remote remove origin 2>/dev/null || true
|
||||
git remote add origin "$GITEA_SOURCE_URL"
|
||||
git fetch --no-tags --prune --depth=1 origin "$GITHUB_SHA"
|
||||
git checkout --force --detach FETCH_HEAD
|
||||
|
||||
- name: Revalidate latest inclusive deploy carrier
|
||||
id: carrier
|
||||
env:
|
||||
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 ops/runner/select-latest-inclusive-cd-carrier.py \
|
||||
--source-sha "$GITHUB_SHA" \
|
||||
--remote-url "$GITEA_SOURCE_URL" \
|
||||
--github-output "$GITHUB_OUTPUT"
|
||||
|
||||
build-and-deploy:
|
||||
# 2026-06-28 Codex: keep CD-generated `[skip ci]` deploy commits and
|
||||
# `cancel-stale-cd` queue-cleaning commits from re-entering build/deploy.
|
||||
# 2026-07-01 Codex: metadata-only controlled-runtime fixes already run the
|
||||
# focused tests above; do not spend the Docker build lock or deploy marker
|
||||
# when no app image can change.
|
||||
if: ${{ github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd') && !contains(github.event.head_commit.message, '[metadata-only]')) }}
|
||||
if: ${{ needs.revalidate-deploy-carrier.outputs.selected == 'true' && (github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd') && !contains(github.event.head_commit.message, '[metadata-only]'))) }}
|
||||
# 2026-04-30 Codex: Docker builds run on the host runner. Long docker build
|
||||
# steps were killing the transient act job container with RWLayer=nil.
|
||||
needs: [tests]
|
||||
needs: [select-latest-carrier, tests, revalidate-deploy-carrier]
|
||||
timeout-minutes: 60
|
||||
runs-on: awoooi-non110-host
|
||||
outputs:
|
||||
deploy_applied: ${{ steps.deploy.outputs.applied }}
|
||||
steps:
|
||||
- name: Bootstrap Host Runner Tools
|
||||
# 2026-05-05 Codex: keep the host-mode runner self-healing before
|
||||
@@ -1977,11 +2045,30 @@ jobs:
|
||||
echo "⚡ no Docker build lock to release"
|
||||
fi
|
||||
|
||||
- name: Revalidate carrier before production mutations
|
||||
id: preproduction-carrier
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "selected=false" >> "$GITHUB_OUTPUT"
|
||||
CARRIER_JSON=$(python3 ops/runner/select-latest-inclusive-cd-carrier.py \
|
||||
--source-sha "$GITHUB_SHA" \
|
||||
--remote-url http://192.168.0.110:3001/wooo/awoooi.git)
|
||||
CARRIER_SELECTED=$(printf '%s' "$CARRIER_JSON" | \
|
||||
python3 -c 'import json,sys; print(str(json.load(sys.stdin)["selected"]).lower())')
|
||||
if [ "$CARRIER_SELECTED" != "true" ]; then
|
||||
CARRIER_LATEST=$(printf '%s' "$CARRIER_JSON" | \
|
||||
python3 -c 'import json,sys; print(json.load(sys.stdin)["latest_sha"])')
|
||||
echo "AWOOOI_BUILD_COALESCED=source=${GITHUB_SHA};latest=${CARRIER_LATEST};production_write_performed=false"
|
||||
exit 0
|
||||
fi
|
||||
echo "selected=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Production drift proved the original CREATE TABLE IF NOT EXISTS
|
||||
# migration did not update an older CHECK constraint. Reconcile this
|
||||
# one additive allowlist before rolling out the scanner and verify it
|
||||
# independently without exposing connection details.
|
||||
- name: Reconcile Asset Inventory Type Constraint
|
||||
if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}
|
||||
env:
|
||||
DATABASE_URL: ${{ secrets.DATABASE_URL }}
|
||||
MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}
|
||||
@@ -2122,6 +2209,7 @@ jobs:
|
||||
# the index transactionally, and prove the conflict-update path before
|
||||
# rolling out the scanner.
|
||||
- name: Repair Asset Inventory Canonical Integrity
|
||||
if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}
|
||||
env:
|
||||
DATABASE_URL: ${{ secrets.DATABASE_URL }}
|
||||
MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}
|
||||
@@ -2319,6 +2407,7 @@ jobs:
|
||||
# additive RLS schema, then verify both schema policy and runtime-role
|
||||
# read access without printing connection details or row content.
|
||||
- name: Apply MCP Version Lifecycle Receipt Schema
|
||||
if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}
|
||||
env:
|
||||
DATABASE_URL: ${{ secrets.DATABASE_URL }}
|
||||
MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}
|
||||
@@ -2463,6 +2552,7 @@ jobs:
|
||||
# independently verified. No source payload or connection detail is
|
||||
# printed by this controlled migration receipt.
|
||||
- name: Apply MCP Federation Receipt Schema
|
||||
if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}
|
||||
env:
|
||||
DATABASE_URL: ${{ secrets.DATABASE_URL }}
|
||||
MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}
|
||||
@@ -2612,6 +2702,7 @@ jobs:
|
||||
# 2026-03-31 ogt: P0-1 Secrets 自動注入 (ADR-035 強制)
|
||||
# 2026-03-31 ogt: 加入 AI API Keys (修復 mock_fallback 問題)
|
||||
- name: Inject K8s Secrets
|
||||
if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}
|
||||
env:
|
||||
ARGOCD_API_TOKEN: ${{ secrets.ARGOCD_API_TOKEN }}
|
||||
AWOOOI_GITEA_API_TOKEN: ${{ secrets.AWOOOI_GITEA_API_TOKEN }}
|
||||
@@ -3012,9 +3103,12 @@ jobs:
|
||||
# 4. 等待 ArgoCD sync + rollout 完成
|
||||
# 5. Health Check
|
||||
- name: Deploy to K8s (ArgoCD GitOps)
|
||||
if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}
|
||||
id: deploy
|
||||
env:
|
||||
CD_PUSH_TOKEN: ${{ secrets.CD_PUSH_TOKEN }}
|
||||
run: |
|
||||
echo "applied=false" >> "$GITHUB_OUTPUT"
|
||||
prepare_deploy_key() {
|
||||
mkdir -p "${HOME}/.ssh"
|
||||
umask 077
|
||||
@@ -3118,6 +3212,45 @@ jobs:
|
||||
fi
|
||||
}
|
||||
|
||||
# The two directly applied ConfigMaps must be part of the same rollback
|
||||
# boundary as NetworkPolicy. Otherwise a superseded carrier could leave
|
||||
# production configuration mutated without a deploy marker.
|
||||
CONFIGMAP_ROLLBACK_FILE="/tmp/awoooi-deploy-configmap-rollback.json"
|
||||
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
|
||||
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL get configmap awoooi-config service-registry -n awoooi-prod -o json" \
|
||||
| python3 -c '
|
||||
import json
|
||||
import sys
|
||||
|
||||
source = json.load(sys.stdin)
|
||||
items = []
|
||||
for item in source["items"]:
|
||||
clean = {
|
||||
"apiVersion": item["apiVersion"],
|
||||
"kind": item["kind"],
|
||||
"metadata": {
|
||||
"name": item["metadata"]["name"],
|
||||
"namespace": item["metadata"]["namespace"],
|
||||
"labels": item["metadata"].get("labels", {}),
|
||||
"annotations": item["metadata"].get("annotations", {}),
|
||||
},
|
||||
"data": item.get("data", {}),
|
||||
}
|
||||
if "binaryData" in item:
|
||||
clean["binaryData"] = item["binaryData"]
|
||||
if "immutable" in item:
|
||||
clean["immutable"] = item["immutable"]
|
||||
items.append(clean)
|
||||
json.dump({"apiVersion": "v1", "kind": "List", "items": items}, sys.stdout)
|
||||
' > "$CONFIGMAP_ROLLBACK_FILE"
|
||||
test -s "$CONFIGMAP_ROLLBACK_FILE"
|
||||
restore_deploy_configmaps() {
|
||||
echo "⏪ restoring previous AWOOOI deploy ConfigMaps"
|
||||
cat "$CONFIGMAP_ROLLBACK_FILE" | \
|
||||
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
|
||||
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"
|
||||
}
|
||||
|
||||
# NetworkPolicy is intentionally outside kustomize because commonLabels
|
||||
# mutates nested selectors. The checkout is depth=1, so snapshot live
|
||||
# policy truth and remove server-only metadata before any mutation.
|
||||
@@ -3182,12 +3315,15 @@ jobs:
|
||||
set +e
|
||||
restore_workload_database_projection
|
||||
workload_restore_exit=$?
|
||||
restore_deploy_configmaps
|
||||
configmap_restore_exit=$?
|
||||
restore_network_policy
|
||||
network_restore_exit=$?
|
||||
set -e
|
||||
if [ "$workload_restore_exit" -ne 0 ] \
|
||||
|| [ "$configmap_restore_exit" -ne 0 ] \
|
||||
|| [ "$network_restore_exit" -ne 0 ]; then
|
||||
echo "❌ deploy rollback incomplete: workload=${workload_restore_exit} network=${network_restore_exit}" >&2
|
||||
echo "❌ deploy rollback incomplete: workload=${workload_restore_exit} configmap=${configmap_restore_exit} network=${network_restore_exit}" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
@@ -3207,6 +3343,22 @@ jobs:
|
||||
fi
|
||||
exit "$exit_code"
|
||||
}
|
||||
|
||||
# Last read-only gate before the first production mutation. Builds may
|
||||
# finish out of order on Gitea 1.25; only the latest inclusive source
|
||||
# is allowed to cross this boundary.
|
||||
PREWRITE_CARRIER_JSON=$(python3 ops/runner/select-latest-inclusive-cd-carrier.py \
|
||||
--source-sha "$IMAGE_TAG" \
|
||||
--remote-url http://192.168.0.110:3001/wooo/awoooi.git)
|
||||
PREWRITE_CARRIER_SELECTED=$(printf '%s' "$PREWRITE_CARRIER_JSON" | \
|
||||
python3 -c 'import json,sys; print(str(json.load(sys.stdin)["selected"]).lower())')
|
||||
if [ "$PREWRITE_CARRIER_SELECTED" != "true" ]; then
|
||||
PREWRITE_CARRIER_LATEST=$(printf '%s' "$PREWRITE_CARRIER_JSON" | \
|
||||
python3 -c 'import json,sys; print(json.load(sys.stdin)["latest_sha"])')
|
||||
echo "AWOOOI_DEPLOY_COALESCED_AFTER_PREREQUISITES=source=${IMAGE_TAG};latest=${PREWRITE_CARRIER_LATEST};runtime_deploy_write_performed=false;monotonic_prerequisites_completed=true"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
trap rollback_deploy_controls_on_exit EXIT
|
||||
cat k8s/awoooi-prod/02-network-policy.yaml | \
|
||||
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
|
||||
@@ -3303,13 +3455,26 @@ jobs:
|
||||
k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml
|
||||
DEPLOY_REVISION=""
|
||||
git diff --cached --quiet && echo "⚡ kustomization.yaml 無變化,跳過 push" || {
|
||||
git commit -m "chore(cd): deploy ${IMAGE_TAG::7} [skip ci]"
|
||||
# 用 token 推送(避免 SSH key 需要額外設定 push 權限)
|
||||
git remote remove gitea 2>/dev/null || true
|
||||
git remote add gitea "http://wooo:${CD_PUSH_TOKEN}@192.168.0.110:3001/wooo/awoooi.git"
|
||||
# 先 rebase 避免 non-fast-forward (其他 commit 在 CI 期間已推入)
|
||||
# 2026-04-17 ogt: -X theirs — kustomization.yaml 衝突時採用當次部署的 image tag
|
||||
git fetch gitea main
|
||||
CARRIER_JSON=$(python3 ops/runner/select-latest-inclusive-cd-carrier.py \
|
||||
--source-sha "$IMAGE_TAG" \
|
||||
--remote-url http://192.168.0.110:3001/wooo/awoooi.git)
|
||||
CARRIER_SELECTED=$(printf '%s' "$CARRIER_JSON" | \
|
||||
python3 -c 'import json,sys; print(str(json.load(sys.stdin)["selected"]).lower())')
|
||||
if [ "$CARRIER_SELECTED" != "true" ]; then
|
||||
CARRIER_LATEST=$(printf '%s' "$CARRIER_JSON" | \
|
||||
python3 -c 'import json,sys; print(json.load(sys.stdin)["latest_sha"])')
|
||||
echo "AWOOOI_DEPLOY_SUPERSEDED_AFTER_WRITE=source=${IMAGE_TAG};latest=${CARRIER_LATEST};rollback=required"
|
||||
restore_deploy_controls
|
||||
echo "AWOOOI_DEPLOY_SUPERSEDED_ROLLBACK=source=${IMAGE_TAG};latest=${CARRIER_LATEST};rollback_verified=true"
|
||||
exit 1
|
||||
fi
|
||||
git commit -m "chore(cd): deploy ${IMAGE_TAG::7} [skip ci]"
|
||||
# Rebase only onto the exact main revision validated above. If main
|
||||
# advances after this fetch, the normal push rejects non-fast-forward.
|
||||
git rebase -X theirs gitea/main
|
||||
DEPLOY_REVISION=$(git rev-parse HEAD)
|
||||
git push gitea main
|
||||
@@ -3723,16 +3888,24 @@ jobs:
|
||||
container="$2"
|
||||
last_failure="workload_not_ready"
|
||||
for attempt in 1 2 3; do
|
||||
ready_pod_ref=$(
|
||||
candidate_pod_refs=$(
|
||||
$KUBECTL get pods -n awoooi-prod -l "app=$workload" \
|
||||
--field-selector=status.phase=Running \
|
||||
--sort-by=.metadata.creationTimestamp \
|
||||
-o name 2>/dev/null | tail -n 1
|
||||
) || ready_pod_ref=""
|
||||
if [ -z "$ready_pod_ref" ]; then
|
||||
-o name 2>/dev/null
|
||||
) || candidate_pod_refs=""
|
||||
ready_pod_ref=""
|
||||
while IFS= read -r candidate_pod_ref; do
|
||||
[ -n "$candidate_pod_ref" ] || continue
|
||||
if $KUBECTL wait --for=condition=Ready "$candidate_pod_ref" \
|
||||
-n awoooi-prod --timeout=1s >/dev/null 2>&1; then
|
||||
ready_pod_ref="$candidate_pod_ref"
|
||||
break
|
||||
fi
|
||||
done < <(printf '%s\n' "$candidate_pod_refs" | sed '/^$/d' | tac)
|
||||
if [ -z "$candidate_pod_refs" ]; then
|
||||
last_failure="running_pod_unavailable"
|
||||
elif ! $KUBECTL wait --for=condition=Ready "$ready_pod_ref" \
|
||||
-n awoooi-prod --timeout=10s >/dev/null 2>&1; then
|
||||
elif [ -z "$ready_pod_ref" ]; then
|
||||
last_failure="ready_pod_unavailable"
|
||||
else
|
||||
container_ready=$(
|
||||
@@ -4892,9 +5065,11 @@ jobs:
|
||||
echo "⚠️ CI/CD rollout risk notification failed (non-fatal)"
|
||||
fi
|
||||
fi
|
||||
echo "applied=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
|
||||
- name: Apply Runtime Image Mirror Policy
|
||||
if: ${{ steps.deploy.outputs.applied == 'true' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
prepare_deploy_key() {
|
||||
@@ -4944,6 +5119,7 @@ jobs:
|
||||
echo "runtime_image_policy_stage=verified"
|
||||
|
||||
- name: Notify Build Deploy Success
|
||||
if: ${{ steps.deploy.outputs.applied == 'true' }}
|
||||
run: |
|
||||
END_TIME=$(date +%s)
|
||||
DURATION=$((END_TIME - ${{ steps.commit.outputs.start_time }}))
|
||||
@@ -4998,13 +5174,46 @@ jobs:
|
||||
echo "AWOOI API notify failed; direct Telegram fallback disabled to preserve AwoooP receipt chain"
|
||||
fi
|
||||
|
||||
revalidate-post-deploy-carrier:
|
||||
if: ${{ needs.build-and-deploy.outputs.deploy_applied == 'true' }}
|
||||
needs: [build-and-deploy]
|
||||
runs-on: awoooi-non110-host
|
||||
timeout-minutes: 3
|
||||
outputs:
|
||||
selected: ${{ steps.carrier.outputs.selected }}
|
||||
status: ${{ steps.carrier.outputs.status }}
|
||||
latest_sha: ${{ steps.carrier.outputs.latest_sha }}
|
||||
main_tip_sha: ${{ steps.carrier.outputs.main_tip_sha }}
|
||||
steps:
|
||||
- name: Checkout exact source from Gitea
|
||||
env:
|
||||
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git init .
|
||||
git remote remove origin 2>/dev/null || true
|
||||
git remote add origin "$GITEA_SOURCE_URL"
|
||||
git fetch --no-tags --prune --depth=1 origin "$GITHUB_SHA"
|
||||
git checkout --force --detach FETCH_HEAD
|
||||
|
||||
- name: Revalidate latest inclusive post-deploy carrier
|
||||
id: carrier
|
||||
env:
|
||||
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 ops/runner/select-latest-inclusive-cd-carrier.py \
|
||||
--source-sha "$GITHUB_SHA" \
|
||||
--remote-url "$GITEA_SOURCE_URL" \
|
||||
--github-output "$GITHUB_OUTPUT"
|
||||
|
||||
post-deploy-checks:
|
||||
# 2026-06-28 Codex: post-deploy checks belong to real deploy runs; skip
|
||||
# marker/no-op commits already accounted for by the previous deploy run.
|
||||
# 2026-07-01 Codex: `[metadata-only]` commits do not roll a new image, so
|
||||
# post-deploy smokes would only retest the previous production artifact.
|
||||
if: ${{ github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd') && !contains(github.event.head_commit.message, '[metadata-only]')) }}
|
||||
needs: [build-and-deploy]
|
||||
if: ${{ needs.revalidate-post-deploy-carrier.outputs.selected == 'true' && needs.build-and-deploy.outputs.deploy_applied == 'true' && (github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd') && !contains(github.event.head_commit.message, '[metadata-only]'))) }}
|
||||
needs: [build-and-deploy, revalidate-post-deploy-carrier]
|
||||
# 2026-07-16 Codex: the fail-hard pressure gate below may legitimately
|
||||
# consume 20 minutes while a bounded backup and its metric freshness
|
||||
# window drain. Preserve enough time for the independent post-deploy
|
||||
|
||||
@@ -42,10 +42,67 @@ def test_cd_source_link_gate_uses_current_deploy_canary() -> None:
|
||||
assert "codex-sentry-20260513-t15b-v3" not in text
|
||||
|
||||
|
||||
def test_cd_main_pushes_queue_without_canceling_active_deploys() -> None:
|
||||
def test_cd_main_pushes_use_gitea_125_executable_inclusive_carrier() -> None:
|
||||
text = _workflow_text()
|
||||
concurrency = text.split("concurrency:", 1)[1].split("env:", 1)[0]
|
||||
|
||||
assert "group: cd-deploy-${{ github.ref }}" in concurrency
|
||||
assert "cancel-in-progress: false" in concurrency
|
||||
assert "cancel-in-progress: true" not in concurrency
|
||||
assert "\nconcurrency:" not in text
|
||||
assert "Gitea 1.25 ignores the workflow concurrency key" in text
|
||||
assert "select-latest-carrier:" in text
|
||||
assert "needs: [select-latest-carrier]" in text
|
||||
assert "revalidate-deploy-carrier:" in text
|
||||
assert "needs: [select-latest-carrier, tests]" in text
|
||||
assert "needs: [select-latest-carrier, tests, revalidate-deploy-carrier]" in text
|
||||
assert "revalidate-post-deploy-carrier:" in text
|
||||
assert "needs: [build-and-deploy, revalidate-post-deploy-carrier]" in text
|
||||
assert "needs.revalidate-deploy-carrier.outputs.selected == 'true'" in text
|
||||
assert "needs.revalidate-post-deploy-carrier.outputs.selected == 'true'" in text
|
||||
assert "deploy_applied: ${{ steps.deploy.outputs.applied }}" in text
|
||||
assert "AWOOOI_BUILD_COALESCED=" in text
|
||||
assert "AWOOOI_DEPLOY_COALESCED_AFTER_PREREQUISITES=" in text
|
||||
assert "select-latest-inclusive-cd-carrier.py" in text
|
||||
assert "--debounce-seconds 20" in text
|
||||
|
||||
|
||||
def test_deploy_revalidates_lineage_after_final_main_fetch() -> None:
|
||||
text = _workflow_text()
|
||||
build_job = text.split(" build-and-deploy:", 1)[1].split(
|
||||
" revalidate-post-deploy-carrier:", 1
|
||||
)[0]
|
||||
deploy = text.split("- name: Deploy to K8s (ArgoCD GitOps)", 1)[1].split(
|
||||
"- name: Apply Runtime Image Mirror Policy", 1
|
||||
)[0]
|
||||
|
||||
production_gate = build_job.index(
|
||||
"- name: Revalidate carrier before production mutations"
|
||||
)
|
||||
first_database_mutation = build_job.index(
|
||||
"- name: Reconcile Asset Inventory Type Constraint"
|
||||
)
|
||||
secret_mutation = build_job.index("- name: Inject K8s Secrets")
|
||||
deploy_step = build_job.index("- name: Deploy to K8s (ArgoCD GitOps)")
|
||||
prewrite_revalidate = deploy.index("PREWRITE_CARRIER_JSON=")
|
||||
first_production_write = deploy.index(
|
||||
"if ! cat k8s/awoooi-prod/02-network-policy.yaml"
|
||||
)
|
||||
final_fetch = deploy.index("git fetch gitea main", first_production_write)
|
||||
revalidate = deploy.index("CARRIER_JSON=$(python3", final_fetch)
|
||||
marker_commit = deploy.index('git commit -m "chore(cd): deploy', revalidate)
|
||||
normal_push = deploy.index("git push gitea main", marker_commit)
|
||||
assert production_gate < first_database_mutation < secret_mutation < deploy_step
|
||||
assert build_job.count(
|
||||
"if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}"
|
||||
) >= 6
|
||||
assert prewrite_revalidate < first_production_write
|
||||
assert final_fetch < revalidate < marker_commit < normal_push
|
||||
assert 'echo "applied=false" >> "$GITHUB_OUTPUT"' in deploy
|
||||
assert "AWOOOI_DEPLOY_COALESCED_AFTER_PREREQUISITES=" in deploy
|
||||
late_supersession = deploy.split(
|
||||
"AWOOOI_DEPLOY_SUPERSEDED_AFTER_WRITE=", 1
|
||||
)[1].split('git commit -m "chore(cd): deploy', 1)[0]
|
||||
assert "restore_deploy_controls" in late_supersession
|
||||
assert "AWOOOI_DEPLOY_SUPERSEDED_ROLLBACK=" in late_supersession
|
||||
assert "exit 1" in late_supersession
|
||||
assert "restore_deploy_configmaps" in deploy
|
||||
assert "awoooi-config service-registry" in deploy
|
||||
assert 'echo "applied=true" >> "$GITHUB_OUTPUT"' in deploy
|
||||
assert "git push --force" not in deploy
|
||||
|
||||
@@ -208,7 +208,11 @@ def test_cd_ssh_denial_probe_targets_ready_pod_and_classifies_transport() -> Non
|
||||
assert "for attempt in 1 2 3; do" in probe
|
||||
assert "--field-selector=status.phase=Running" in probe
|
||||
assert "--sort-by=.metadata.creationTimestamp" in probe
|
||||
assert '--for=condition=Ready "$ready_pod_ref"' in probe
|
||||
assert "candidate_pod_refs=" in probe
|
||||
assert "while IFS= read -r candidate_pod_ref; do" in probe
|
||||
assert '--for=condition=Ready "$candidate_pod_ref"' in probe
|
||||
assert 'ready_pod_ref="$candidate_pod_ref"' in probe
|
||||
assert "| tac)" in probe
|
||||
assert "status.containerStatuses" in probe
|
||||
assert '$KUBECTL exec -i "$ready_pod_ref"' in probe
|
||||
assert '$KUBECTL exec -i "deployment/$workload"' not in probe
|
||||
|
||||
@@ -297,6 +297,9 @@ def test_workload_database_bootstrap_keeps_secret_values_off_output() -> None:
|
||||
def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None:
|
||||
workflow = (REPO_ROOT / ".gitea/workflows/cd.yaml").read_text()
|
||||
normalized_workflow = " ".join(workflow.split())
|
||||
workload_rollback = workflow.split("WORKLOAD_DB_ROLLBACK_FILE", 1)[1].split(
|
||||
"# The two directly applied ConfigMaps", 1
|
||||
)[0]
|
||||
|
||||
assert "WORKLOAD_DB_ROLLBACK_FILE" in workflow
|
||||
assert "WORKLOAD_DB_SECRET_PREEXISTENCE_FILE" in workflow
|
||||
@@ -305,7 +308,7 @@ def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None:
|
||||
assert "bash -s -- verify" in workflow
|
||||
assert workflow.index("bash -s -- apply") < workflow.index("bash -s -- verify")
|
||||
assert '"spec": item["spec"]' in workflow
|
||||
assert '"data": item' not in workflow
|
||||
assert '"data": item' not in workload_rollback
|
||||
assert "get secret '${secret_name}' -n awoooi-prod -o jsonpath" not in workflow
|
||||
assert "get secret awoooi-api-db -n awoooi-prod -o jsonpath" not in workflow
|
||||
assert "HEAD^" not in workflow
|
||||
|
||||
@@ -136,6 +136,20 @@ _CD_BUILD_DEPLOY_SUCCESS_NOTIFICATION_RE = re.compile(
|
||||
r"Z\s+(?:[^\w\"']+\s+)?CI/CD build-deploy success notification mirrored"
|
||||
)
|
||||
_JOB_SUCCEEDED_RE = re.compile(r"Z\s+(?:[^\w\"']+\s+)?Job succeeded")
|
||||
_CD_EXECUTOR_BOUNDARY_FAILURES = (
|
||||
(
|
||||
"signal worker SSH-denial verifier did not prove the boundary",
|
||||
"signal_worker_ssh_denial_verifier_unproven",
|
||||
),
|
||||
(
|
||||
"API SSH-denial verifier did not prove the boundary",
|
||||
"api_ssh_denial_verifier_unproven",
|
||||
),
|
||||
(
|
||||
"execution broker has no live route evidence for allowlisted SSH endpoints",
|
||||
"execution_broker_allowlisted_ssh_route_unproven",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -694,6 +708,7 @@ def build_readback(
|
||||
build_docker_step_timeout_label=build_log_classifier[
|
||||
"docker_step_timeout_label"
|
||||
],
|
||||
build_failure_classifier=build_log_classifier["failure_classifier"],
|
||||
harbor_110_repair_no_matching_runner_label=(
|
||||
harbor_110_repair_no_matching_runner_label
|
||||
if not harbor_110_repair_ignored_for_current_cd_context
|
||||
@@ -1133,6 +1148,8 @@ def build_readback(
|
||||
if build_log_classifier["docker_step_timeout"]
|
||||
else "blocked_host_web_build_pressure"
|
||||
if effective_tests_log_classifier["host_pressure_blocked_or_waiting"]
|
||||
else "blocked_cd_build_verifier"
|
||||
if build_log_classifier["failure_classifier"]
|
||||
else "blocked_harbor_110_repair_no_matching_runner"
|
||||
if harbor_110_repair_no_matching_runner_label
|
||||
else "blocked_no_matching_online_runner"
|
||||
@@ -1413,6 +1430,7 @@ def _queue_safe_next_action(
|
||||
build_harbor_public_route_retrying_unavailable: bool,
|
||||
build_docker_step_timeout: bool,
|
||||
build_docker_step_timeout_label: str,
|
||||
build_failure_classifier: str,
|
||||
harbor_110_repair_no_matching_runner_label: str,
|
||||
harbor_110_repair_waiting: bool,
|
||||
harbor_110_repair_running: bool,
|
||||
@@ -1664,6 +1682,20 @@ def _queue_safe_next_action(
|
||||
blocker_fields=["latest_visible_cd_host_pressure_classifier"],
|
||||
)
|
||||
|
||||
if build_failure_classifier:
|
||||
return action(
|
||||
action_id="fix_exact_cd_build_verifier_then_single_replay",
|
||||
stage="cd_build_log_exact_failure",
|
||||
text=(
|
||||
"Reproduce and fix the exact build/deploy verifier locally, then "
|
||||
"perform one normal integration and terminal production readback."
|
||||
),
|
||||
reason=build_failure_classifier,
|
||||
command="focused local verifier replay before the next normal push",
|
||||
post_verifier="single exact CD terminal plus production deploy-marker readback",
|
||||
blocker_fields=["latest_visible_cd_failure_classifier"],
|
||||
)
|
||||
|
||||
if cd_jobs_stale_or_mismatched:
|
||||
return action(
|
||||
action_id="ignore_stale_cd_jobs_api_payload_and_poll_visible_cd_or_marker",
|
||||
@@ -1733,6 +1765,14 @@ def classify_cd_build_log(text: str) -> dict[str, Any]:
|
||||
)
|
||||
deploy_marker_matches = list(_CD_DEPLOY_MARKER_RE.finditer(text))
|
||||
latest_deploy_marker = deploy_marker_matches[-1] if deploy_marker_matches else None
|
||||
executor_boundary_failure = next(
|
||||
(
|
||||
classifier
|
||||
for marker, classifier in _CD_EXECUTOR_BOUNDARY_FAILURES
|
||||
if marker in text
|
||||
),
|
||||
"",
|
||||
)
|
||||
return {
|
||||
"job_succeeded": _JOB_SUCCEEDED_RE.search(text) is not None,
|
||||
"deploy_marker_pushed": latest_deploy_marker is not None,
|
||||
@@ -1749,7 +1789,9 @@ def classify_cd_build_log(text: str) -> dict[str, Any]:
|
||||
_CD_BUILD_DEPLOY_SUCCESS_NOTIFICATION_RE.search(text) is not None
|
||||
),
|
||||
"failure_classifier": (
|
||||
"harbor_registry_public_route_unavailable"
|
||||
executor_boundary_failure
|
||||
if executor_boundary_failure
|
||||
else "harbor_registry_public_route_unavailable"
|
||||
if harbor_public_route_blocked
|
||||
else "cd_docker_step_timeout"
|
||||
if latest_docker_step_timeout
|
||||
|
||||
191
ops/runner/select-latest-inclusive-cd-carrier.py
Normal file
191
ops/runner/select-latest-inclusive-cd-carrier.py
Normal file
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Select one exact main revision as the inclusive AWOOOI CD carrier."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
|
||||
SHA_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
EXCLUDED_MESSAGE_MARKERS = ("[skip ci]", "cancel-stale-cd", "[metadata-only]")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CarrierDecision:
|
||||
selected: bool
|
||||
status: str
|
||||
source_sha: str
|
||||
latest_sha: str
|
||||
main_tip_sha: str
|
||||
|
||||
|
||||
def _git(args: Sequence[str], *, cwd: Path, check: bool = True) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["git", *args],
|
||||
cwd=cwd,
|
||||
check=check,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def _validate_sha(value: str, *, label: str) -> str:
|
||||
normalized = value.strip().lower()
|
||||
if not SHA_RE.fullmatch(normalized):
|
||||
raise RuntimeError(f"{label}_invalid")
|
||||
return normalized
|
||||
|
||||
|
||||
def _latest_remote_sha(*, remote_url: str, branch: str, cwd: Path) -> str:
|
||||
result = _git(
|
||||
["ls-remote", "--heads", remote_url, f"refs/heads/{branch}"],
|
||||
cwd=cwd,
|
||||
)
|
||||
rows = [row.split() for row in result.stdout.splitlines() if row.strip()]
|
||||
if len(rows) != 1 or len(rows[0]) != 2:
|
||||
raise RuntimeError("latest_main_ref_unresolved")
|
||||
return _validate_sha(rows[0][0], label="latest_sha")
|
||||
|
||||
|
||||
def _fetch_latest_history(*, remote_url: str, branch: str, cwd: Path, depth: int) -> None:
|
||||
_git(
|
||||
[
|
||||
"fetch",
|
||||
"--no-tags",
|
||||
"--prune",
|
||||
f"--depth={depth}",
|
||||
remote_url,
|
||||
f"+refs/heads/{branch}:refs/remotes/awoooi-carrier/{branch}",
|
||||
],
|
||||
cwd=cwd,
|
||||
)
|
||||
|
||||
|
||||
def _latest_deploy_eligible_sha(*, branch: str, cwd: Path) -> str | None:
|
||||
result = _git(
|
||||
[
|
||||
"log",
|
||||
"--format=%H%x1f%B%x1e",
|
||||
f"refs/remotes/awoooi-carrier/{branch}",
|
||||
],
|
||||
cwd=cwd,
|
||||
)
|
||||
for record in result.stdout.split("\x1e"):
|
||||
sha, separator, message = record.strip().partition("\x1f")
|
||||
if not separator:
|
||||
continue
|
||||
normalized_message = message.lower()
|
||||
if any(marker in normalized_message for marker in EXCLUDED_MESSAGE_MARKERS):
|
||||
continue
|
||||
return _validate_sha(sha, label="latest_deploy_eligible_sha")
|
||||
return None
|
||||
|
||||
|
||||
def _source_is_ancestor(*, source_sha: str, latest_sha: str, cwd: Path) -> bool:
|
||||
result = _git(
|
||||
["merge-base", "--is-ancestor", source_sha, latest_sha],
|
||||
cwd=cwd,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode not in {0, 1}:
|
||||
raise RuntimeError("carrier_lineage_check_failed")
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def select_latest_inclusive_carrier(
|
||||
*,
|
||||
source_sha: str,
|
||||
remote_url: str,
|
||||
branch: str = "main",
|
||||
cwd: Path,
|
||||
debounce_seconds: float = 0,
|
||||
) -> CarrierDecision:
|
||||
source_sha = _validate_sha(source_sha, label="source_sha")
|
||||
if debounce_seconds < 0 or debounce_seconds > 60:
|
||||
raise RuntimeError("debounce_seconds_out_of_range")
|
||||
if debounce_seconds:
|
||||
time.sleep(debounce_seconds)
|
||||
|
||||
main_tip_sha = _latest_remote_sha(remote_url=remote_url, branch=branch, cwd=cwd)
|
||||
_fetch_latest_history(remote_url=remote_url, branch=branch, cwd=cwd, depth=256)
|
||||
latest_sha = _latest_deploy_eligible_sha(branch=branch, cwd=cwd)
|
||||
if latest_sha is None:
|
||||
_fetch_latest_history(remote_url=remote_url, branch=branch, cwd=cwd, depth=2048)
|
||||
latest_sha = _latest_deploy_eligible_sha(branch=branch, cwd=cwd)
|
||||
if latest_sha is None:
|
||||
raise RuntimeError("latest_deploy_eligible_main_unresolved")
|
||||
|
||||
if source_sha == latest_sha:
|
||||
return CarrierDecision(
|
||||
selected=True,
|
||||
status="selected_latest_main_carrier",
|
||||
source_sha=source_sha,
|
||||
latest_sha=latest_sha,
|
||||
main_tip_sha=main_tip_sha,
|
||||
)
|
||||
|
||||
if not _source_is_ancestor(source_sha=source_sha, latest_sha=latest_sha, cwd=cwd):
|
||||
shallow = _git(
|
||||
["rev-parse", "--is-shallow-repository"],
|
||||
cwd=cwd,
|
||||
).stdout.strip()
|
||||
if shallow == "true":
|
||||
_fetch_latest_history(remote_url=remote_url, branch=branch, cwd=cwd, depth=2048)
|
||||
|
||||
if not _source_is_ancestor(source_sha=source_sha, latest_sha=latest_sha, cwd=cwd):
|
||||
raise RuntimeError("latest_main_does_not_include_source")
|
||||
|
||||
return CarrierDecision(
|
||||
selected=False,
|
||||
status="superseded_by_latest_inclusive_carrier",
|
||||
source_sha=source_sha,
|
||||
latest_sha=latest_sha,
|
||||
main_tip_sha=main_tip_sha,
|
||||
)
|
||||
|
||||
|
||||
def _write_github_output(path: Path, decision: CarrierDecision) -> None:
|
||||
fields = {
|
||||
"selected": str(decision.selected).lower(),
|
||||
"status": decision.status,
|
||||
"source_sha": decision.source_sha,
|
||||
"latest_sha": decision.latest_sha,
|
||||
"main_tip_sha": decision.main_tip_sha,
|
||||
}
|
||||
with path.open("a", encoding="utf-8") as stream:
|
||||
for key, value in fields.items():
|
||||
stream.write(f"{key}={value}\n")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source-sha", required=True)
|
||||
parser.add_argument("--remote-url", required=True)
|
||||
parser.add_argument("--branch", default="main")
|
||||
parser.add_argument("--repo", type=Path, default=Path.cwd())
|
||||
parser.add_argument("--debounce-seconds", type=float, default=0)
|
||||
parser.add_argument("--github-output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
decision = select_latest_inclusive_carrier(
|
||||
source_sha=args.source_sha,
|
||||
remote_url=args.remote_url,
|
||||
branch=args.branch,
|
||||
cwd=args.repo.resolve(),
|
||||
debounce_seconds=args.debounce_seconds,
|
||||
)
|
||||
if args.github_output:
|
||||
_write_github_output(args.github_output, decision)
|
||||
print(json.dumps(asdict(decision), sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1904,6 +1904,43 @@ def test_failed_cd_with_stale_jobs_payload_gets_failure_classifier() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_exact_executor_boundary_failure_supersedes_stale_jobs_classifier() -> None:
|
||||
module = _load_module()
|
||||
payload = module.build_readback(
|
||||
actions_html=_actions_html_failed_cd_run(),
|
||||
actions_list_http_status=401,
|
||||
actions_list_payload={"message": "token is required"},
|
||||
cd_jobs_http_status=200,
|
||||
cd_jobs_payload={
|
||||
"jobs": [
|
||||
{
|
||||
"run_id": 4043,
|
||||
"name": "tests",
|
||||
"head_sha": "9c4e754d3369fa2de7606e7092712cf6d85e18e2",
|
||||
"conclusion": "success",
|
||||
}
|
||||
],
|
||||
"total_count": 1,
|
||||
},
|
||||
latest_cd_build_log_http_status=200,
|
||||
latest_cd_build_log_text=(
|
||||
"ssh_denial_probe_transport_unavailable workload=awoooi-worker "
|
||||
"reason=ready_pod_unavailable\n"
|
||||
"signal worker SSH-denial verifier did not prove the boundary\n"
|
||||
),
|
||||
)
|
||||
|
||||
assert payload["status"] == "blocked_cd_build_verifier"
|
||||
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
|
||||
assert payload["readback"]["latest_visible_cd_failure_classifier"] == (
|
||||
"signal_worker_ssh_denial_verifier_unproven"
|
||||
)
|
||||
assert (
|
||||
payload["readback"]["safe_next_action_id"]
|
||||
== "fix_exact_cd_build_verifier_then_single_replay"
|
||||
)
|
||||
|
||||
|
||||
def test_interrupted_host_pressure_reports_postgres_recovery_cpu() -> None:
|
||||
module = _load_module()
|
||||
payload = module.build_readback(
|
||||
|
||||
226
ops/runner/test_select_latest_inclusive_cd_carrier.py
Normal file
226
ops/runner/test_select_latest_inclusive_cd_carrier.py
Normal file
@@ -0,0 +1,226 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "ops/runner/select-latest-inclusive-cd-carrier.py"
|
||||
|
||||
|
||||
def _load_module():
|
||||
spec = importlib.util.spec_from_file_location("select_latest_inclusive_cd_carrier", SCRIPT)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _git(cwd: Path, *args: str) -> str:
|
||||
return subprocess.run(
|
||||
["git", *args],
|
||||
cwd=cwd,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def linear_remote(tmp_path: Path) -> tuple[Path, Path, str, str]:
|
||||
source = tmp_path / "source"
|
||||
remote = tmp_path / "remote.git"
|
||||
checkout = tmp_path / "checkout"
|
||||
source.mkdir()
|
||||
_git(source, "init", "-b", "main")
|
||||
_git(source, "config", "user.name", "Carrier Test")
|
||||
_git(source, "config", "user.email", "carrier@example.invalid")
|
||||
(source / "value.txt").write_text("one\n", encoding="utf-8")
|
||||
_git(source, "add", "value.txt")
|
||||
_git(source, "commit", "-m", "first")
|
||||
first = _git(source, "rev-parse", "HEAD")
|
||||
_git(tmp_path, "clone", "--bare", str(source), str(remote))
|
||||
_git(tmp_path, "clone", "--depth=1", remote.as_uri(), str(checkout))
|
||||
|
||||
(source / "value.txt").write_text("two\n", encoding="utf-8")
|
||||
_git(source, "commit", "-am", "second")
|
||||
second = _git(source, "rev-parse", "HEAD")
|
||||
_git(source, "push", str(remote), "main")
|
||||
return checkout, remote, first, second
|
||||
|
||||
|
||||
def test_latest_source_is_selected(linear_remote: tuple[Path, Path, str, str]) -> None:
|
||||
module = _load_module()
|
||||
checkout, remote, _, second = linear_remote
|
||||
|
||||
decision = module.select_latest_inclusive_carrier(
|
||||
source_sha=second,
|
||||
remote_url=str(remote),
|
||||
cwd=checkout,
|
||||
)
|
||||
|
||||
assert decision.selected is True
|
||||
assert decision.status == "selected_latest_main_carrier"
|
||||
assert decision.latest_sha == second
|
||||
|
||||
|
||||
def test_older_source_is_coalesced_only_when_latest_includes_it(
|
||||
linear_remote: tuple[Path, Path, str, str],
|
||||
) -> None:
|
||||
module = _load_module()
|
||||
checkout, remote, first, second = linear_remote
|
||||
|
||||
decision = module.select_latest_inclusive_carrier(
|
||||
source_sha=first,
|
||||
remote_url=str(remote),
|
||||
cwd=checkout,
|
||||
)
|
||||
|
||||
assert decision.selected is False
|
||||
assert decision.status == "superseded_by_latest_inclusive_carrier"
|
||||
assert decision.source_sha == first
|
||||
assert decision.latest_sha == second
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"marker_subject",
|
||||
[
|
||||
"chore(cd): deploy abcdef0 [skip ci]",
|
||||
"chore(cd): cancel-stale-cd",
|
||||
"chore(cd): refresh receipt [metadata-only]",
|
||||
],
|
||||
)
|
||||
def test_excluded_main_tip_does_not_displace_deployable_carrier(
|
||||
linear_remote: tuple[Path, Path, str, str],
|
||||
tmp_path: Path,
|
||||
marker_subject: str,
|
||||
) -> None:
|
||||
module = _load_module()
|
||||
checkout, remote, _, second = linear_remote
|
||||
marker_work = tmp_path / "marker-work"
|
||||
_git(tmp_path, "clone", str(remote), str(marker_work))
|
||||
_git(marker_work, "config", "user.name", "Carrier Test")
|
||||
_git(marker_work, "config", "user.email", "carrier@example.invalid")
|
||||
_git(marker_work, "commit", "--allow-empty", "-m", marker_subject)
|
||||
marker_sha = _git(marker_work, "rev-parse", "HEAD")
|
||||
_git(marker_work, "push", "origin", "main")
|
||||
|
||||
decision = module.select_latest_inclusive_carrier(
|
||||
source_sha=second,
|
||||
remote_url=str(remote),
|
||||
cwd=checkout,
|
||||
)
|
||||
|
||||
assert decision.selected is True
|
||||
assert decision.latest_sha == second
|
||||
assert decision.main_tip_sha == marker_sha
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"marker_body",
|
||||
[
|
||||
"release receipt [skip ci]",
|
||||
"queue receipt cancel-stale-cd",
|
||||
"runtime receipt [metadata-only]",
|
||||
],
|
||||
)
|
||||
def test_excluded_marker_in_commit_body_does_not_displace_deployable_carrier(
|
||||
linear_remote: tuple[Path, Path, str, str],
|
||||
tmp_path: Path,
|
||||
marker_body: str,
|
||||
) -> None:
|
||||
module = _load_module()
|
||||
checkout, remote, _, second = linear_remote
|
||||
marker_work = tmp_path / "marker-body-work"
|
||||
_git(tmp_path, "clone", str(remote), str(marker_work))
|
||||
_git(marker_work, "config", "user.name", "Carrier Test")
|
||||
_git(marker_work, "config", "user.email", "carrier@example.invalid")
|
||||
_git(
|
||||
marker_work,
|
||||
"commit",
|
||||
"--allow-empty",
|
||||
"-m",
|
||||
"receipt marker",
|
||||
"-m",
|
||||
marker_body,
|
||||
)
|
||||
marker_sha = _git(marker_work, "rev-parse", "HEAD")
|
||||
_git(marker_work, "push", "origin", "main")
|
||||
|
||||
decision = module.select_latest_inclusive_carrier(
|
||||
source_sha=second,
|
||||
remote_url=str(remote),
|
||||
cwd=checkout,
|
||||
)
|
||||
|
||||
assert decision.selected is True
|
||||
assert decision.latest_sha == second
|
||||
assert decision.main_tip_sha == marker_sha
|
||||
|
||||
|
||||
def test_unrelated_latest_main_fails_closed(tmp_path: Path) -> None:
|
||||
module = _load_module()
|
||||
checkout = tmp_path / "checkout"
|
||||
unrelated = tmp_path / "unrelated"
|
||||
remote = tmp_path / "remote.git"
|
||||
for repo, value in ((checkout, "source"), (unrelated, "unrelated")):
|
||||
repo.mkdir()
|
||||
_git(repo, "init", "-b", "main")
|
||||
_git(repo, "config", "user.name", "Carrier Test")
|
||||
_git(repo, "config", "user.email", "carrier@example.invalid")
|
||||
(repo / "value.txt").write_text(f"{value}\n", encoding="utf-8")
|
||||
_git(repo, "add", "value.txt")
|
||||
_git(repo, "commit", "-m", value)
|
||||
source_sha = _git(checkout, "rev-parse", "HEAD")
|
||||
_git(tmp_path, "clone", "--bare", str(unrelated), str(remote))
|
||||
|
||||
with pytest.raises(RuntimeError, match="latest_main_does_not_include_source"):
|
||||
module.select_latest_inclusive_carrier(
|
||||
source_sha=source_sha,
|
||||
remote_url=str(remote),
|
||||
cwd=checkout,
|
||||
)
|
||||
|
||||
|
||||
def test_deep_linear_history_is_proven_after_bounded_deepen(tmp_path: Path) -> None:
|
||||
module = _load_module()
|
||||
source = tmp_path / "source"
|
||||
remote = tmp_path / "remote.git"
|
||||
checkout = tmp_path / "checkout"
|
||||
source.mkdir()
|
||||
_git(source, "init", "-b", "main")
|
||||
_git(source, "config", "user.name", "Carrier Test")
|
||||
_git(source, "config", "user.email", "carrier@example.invalid")
|
||||
_git(source, "commit", "--allow-empty", "-m", "first")
|
||||
first = _git(source, "rev-parse", "HEAD")
|
||||
_git(tmp_path, "clone", "--bare", str(source), str(remote))
|
||||
_git(tmp_path, "clone", "--depth=1", remote.as_uri(), str(checkout))
|
||||
for index in range(270):
|
||||
_git(source, "commit", "--allow-empty", "-m", f"linear-{index}")
|
||||
latest = _git(source, "rev-parse", "HEAD")
|
||||
_git(source, "push", str(remote), "main")
|
||||
|
||||
decision = module.select_latest_inclusive_carrier(
|
||||
source_sha=first,
|
||||
remote_url=str(remote),
|
||||
cwd=checkout,
|
||||
)
|
||||
|
||||
assert decision.selected is False
|
||||
assert decision.latest_sha == latest
|
||||
|
||||
|
||||
def test_invalid_source_sha_fails_closed(tmp_path: Path) -> None:
|
||||
module = _load_module()
|
||||
|
||||
with pytest.raises(RuntimeError, match="source_sha_invalid"):
|
||||
module.select_latest_inclusive_carrier(
|
||||
source_sha="not-a-sha",
|
||||
remote_url="unused",
|
||||
cwd=tmp_path,
|
||||
)
|
||||
Reference in New Issue
Block a user