fix(cd): coalesce stale carriers and harden verifiers
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 48s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m34s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 26s
CD Pipeline / revalidate-post-deploy-carrier (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled

This commit is contained in:
Your Name
2026-07-18 18:54:04 +08:00
parent dbabdeab7e
commit 0d3c3b483e
8 changed files with 797 additions and 28 deletions

View File

@@ -24,11 +24,9 @@ on:
workflow_dispatch:
# 手動觸發永遠可用(用於補跑、緊急部署)
# Main deployments run serially so concurrent writers cannot starve production closure.
# Job timeouts, rollout verification, and deploy markers terminate or expose failed runs.
concurrency:
group: cd-deploy-${{ github.ref }}
cancel-in-progress: false
# Gitea 1.25 ignores the workflow concurrency key. The dedicated runner's verified
# capacity=1 is the executable queue, while the selectors below revalidate linear
# main ancestry before tests, immediately before deploy, and before post-deploy readback.
env:
HARBOR: registry.wooo.work
@@ -60,6 +58,40 @@ env:
ALERT_CHAIN_API_URL: https://awoooi.wooo.work
jobs:
select-latest-carrier:
if: ${{ github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd') && !contains(github.event.head_commit.message, '[metadata-only]')) }}
runs-on: awoooi-non110-host
timeout-minutes: 3
outputs:
selected: ${{ steps.carrier.outputs.selected }}
status: ${{ steps.carrier.outputs.status }}
source_sha: ${{ steps.carrier.outputs.source_sha }}
latest_sha: ${{ steps.carrier.outputs.latest_sha }}
main_tip_sha: ${{ steps.carrier.outputs.main_tip_sha }}
steps:
- name: Checkout exact source from Gitea
env:
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
run: |
set -euo pipefail
git init .
git remote remove origin 2>/dev/null || true
git remote add origin "$GITEA_SOURCE_URL"
git fetch --no-tags --prune --depth=1 origin "$GITHUB_SHA"
git checkout --force --detach FETCH_HEAD
- name: Select latest inclusive main carrier
id: carrier
env:
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
run: |
set -euo pipefail
python3 ops/runner/select-latest-inclusive-cd-carrier.py \
--source-sha "$GITHUB_SHA" \
--remote-url "$GITEA_SOURCE_URL" \
--debounce-seconds 20 \
--github-output "$GITHUB_OUTPUT"
workflow-shape:
# 2026-06-28 Codex: Gitea 1.25 may mark a workflow invalid when every
# root job has a job-level `if`. Keep one no-op root job without `if` so
@@ -87,7 +119,8 @@ jobs:
# on CD-generated deploy commits. Skip jobs explicitly so marker commits
# do not trigger a self-feeding CD loop; `cancel-stale-cd` is a
# controlled no-op trigger used only to cancel stale pre-guard runs.
if: ${{ github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd')) }}
if: ${{ needs.select-latest-carrier.outputs.selected == 'true' && (github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd'))) }}
needs: [select-latest-carrier]
# 2026-04-30 Codex: run the tests job on the host runner and launch the
# CI image explicitly. The act-managed job container can disappear mid-test
# with Docker RWLayer=nil on the shared 110 daemon.
@@ -1370,18 +1403,53 @@ jobs:
echo "AWOOI API notify failed; direct Telegram fallback disabled to preserve AwoooP receipt chain"
fi
revalidate-deploy-carrier:
if: ${{ needs.select-latest-carrier.outputs.selected == 'true' }}
needs: [select-latest-carrier, tests]
runs-on: awoooi-non110-host
timeout-minutes: 3
outputs:
selected: ${{ steps.carrier.outputs.selected }}
status: ${{ steps.carrier.outputs.status }}
latest_sha: ${{ steps.carrier.outputs.latest_sha }}
main_tip_sha: ${{ steps.carrier.outputs.main_tip_sha }}
steps:
- name: Checkout exact source from Gitea
env:
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
run: |
set -euo pipefail
git init .
git remote remove origin 2>/dev/null || true
git remote add origin "$GITEA_SOURCE_URL"
git fetch --no-tags --prune --depth=1 origin "$GITHUB_SHA"
git checkout --force --detach FETCH_HEAD
- name: Revalidate latest inclusive deploy carrier
id: carrier
env:
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
run: |
set -euo pipefail
python3 ops/runner/select-latest-inclusive-cd-carrier.py \
--source-sha "$GITHUB_SHA" \
--remote-url "$GITEA_SOURCE_URL" \
--github-output "$GITHUB_OUTPUT"
build-and-deploy:
# 2026-06-28 Codex: keep CD-generated `[skip ci]` deploy commits and
# `cancel-stale-cd` queue-cleaning commits from re-entering build/deploy.
# 2026-07-01 Codex: metadata-only controlled-runtime fixes already run the
# focused tests above; do not spend the Docker build lock or deploy marker
# when no app image can change.
if: ${{ github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd') && !contains(github.event.head_commit.message, '[metadata-only]')) }}
if: ${{ needs.revalidate-deploy-carrier.outputs.selected == 'true' && (github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd') && !contains(github.event.head_commit.message, '[metadata-only]'))) }}
# 2026-04-30 Codex: Docker builds run on the host runner. Long docker build
# steps were killing the transient act job container with RWLayer=nil.
needs: [tests]
needs: [select-latest-carrier, tests, revalidate-deploy-carrier]
timeout-minutes: 60
runs-on: awoooi-non110-host
outputs:
deploy_applied: ${{ steps.deploy.outputs.applied }}
steps:
- name: Bootstrap Host Runner Tools
# 2026-05-05 Codex: keep the host-mode runner self-healing before
@@ -1977,11 +2045,30 @@ jobs:
echo "⚡ no Docker build lock to release"
fi
- name: Revalidate carrier before production mutations
id: preproduction-carrier
run: |
set -euo pipefail
echo "selected=false" >> "$GITHUB_OUTPUT"
CARRIER_JSON=$(python3 ops/runner/select-latest-inclusive-cd-carrier.py \
--source-sha "$GITHUB_SHA" \
--remote-url http://192.168.0.110:3001/wooo/awoooi.git)
CARRIER_SELECTED=$(printf '%s' "$CARRIER_JSON" | \
python3 -c 'import json,sys; print(str(json.load(sys.stdin)["selected"]).lower())')
if [ "$CARRIER_SELECTED" != "true" ]; then
CARRIER_LATEST=$(printf '%s' "$CARRIER_JSON" | \
python3 -c 'import json,sys; print(json.load(sys.stdin)["latest_sha"])')
echo "AWOOOI_BUILD_COALESCED=source=${GITHUB_SHA};latest=${CARRIER_LATEST};production_write_performed=false"
exit 0
fi
echo "selected=true" >> "$GITHUB_OUTPUT"
# Production drift proved the original CREATE TABLE IF NOT EXISTS
# migration did not update an older CHECK constraint. Reconcile this
# one additive allowlist before rolling out the scanner and verify it
# independently without exposing connection details.
- name: Reconcile Asset Inventory Type Constraint
if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}
env:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}
@@ -2122,6 +2209,7 @@ jobs:
# the index transactionally, and prove the conflict-update path before
# rolling out the scanner.
- name: Repair Asset Inventory Canonical Integrity
if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}
env:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}
@@ -2319,6 +2407,7 @@ jobs:
# additive RLS schema, then verify both schema policy and runtime-role
# read access without printing connection details or row content.
- name: Apply MCP Version Lifecycle Receipt Schema
if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}
env:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}
@@ -2463,6 +2552,7 @@ jobs:
# independently verified. No source payload or connection detail is
# printed by this controlled migration receipt.
- name: Apply MCP Federation Receipt Schema
if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}
env:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}
@@ -2612,6 +2702,7 @@ jobs:
# 2026-03-31 ogt: P0-1 Secrets 自動注入 (ADR-035 強制)
# 2026-03-31 ogt: 加入 AI API Keys (修復 mock_fallback 問題)
- name: Inject K8s Secrets
if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}
env:
ARGOCD_API_TOKEN: ${{ secrets.ARGOCD_API_TOKEN }}
AWOOOI_GITEA_API_TOKEN: ${{ secrets.AWOOOI_GITEA_API_TOKEN }}
@@ -3012,9 +3103,12 @@ jobs:
# 4. 等待 ArgoCD sync + rollout 完成
# 5. Health Check
- name: Deploy to K8s (ArgoCD GitOps)
if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}
id: deploy
env:
CD_PUSH_TOKEN: ${{ secrets.CD_PUSH_TOKEN }}
run: |
echo "applied=false" >> "$GITHUB_OUTPUT"
prepare_deploy_key() {
mkdir -p "${HOME}/.ssh"
umask 077
@@ -3118,6 +3212,45 @@ jobs:
fi
}
# The two directly applied ConfigMaps must be part of the same rollback
# boundary as NetworkPolicy. Otherwise a superseded carrier could leave
# production configuration mutated without a deploy marker.
CONFIGMAP_ROLLBACK_FILE="/tmp/awoooi-deploy-configmap-rollback.json"
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL get configmap awoooi-config service-registry -n awoooi-prod -o json" \
| python3 -c '
import json
import sys
source = json.load(sys.stdin)
items = []
for item in source["items"]:
clean = {
"apiVersion": item["apiVersion"],
"kind": item["kind"],
"metadata": {
"name": item["metadata"]["name"],
"namespace": item["metadata"]["namespace"],
"labels": item["metadata"].get("labels", {}),
"annotations": item["metadata"].get("annotations", {}),
},
"data": item.get("data", {}),
}
if "binaryData" in item:
clean["binaryData"] = item["binaryData"]
if "immutable" in item:
clean["immutable"] = item["immutable"]
items.append(clean)
json.dump({"apiVersion": "v1", "kind": "List", "items": items}, sys.stdout)
' > "$CONFIGMAP_ROLLBACK_FILE"
test -s "$CONFIGMAP_ROLLBACK_FILE"
restore_deploy_configmaps() {
echo "⏪ restoring previous AWOOOI deploy ConfigMaps"
cat "$CONFIGMAP_ROLLBACK_FILE" | \
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"
}
# NetworkPolicy is intentionally outside kustomize because commonLabels
# mutates nested selectors. The checkout is depth=1, so snapshot live
# policy truth and remove server-only metadata before any mutation.
@@ -3182,12 +3315,15 @@ jobs:
set +e
restore_workload_database_projection
workload_restore_exit=$?
restore_deploy_configmaps
configmap_restore_exit=$?
restore_network_policy
network_restore_exit=$?
set -e
if [ "$workload_restore_exit" -ne 0 ] \
|| [ "$configmap_restore_exit" -ne 0 ] \
|| [ "$network_restore_exit" -ne 0 ]; then
echo "❌ deploy rollback incomplete: workload=${workload_restore_exit} network=${network_restore_exit}" >&2
echo "❌ deploy rollback incomplete: workload=${workload_restore_exit} configmap=${configmap_restore_exit} network=${network_restore_exit}" >&2
return 1
fi
}
@@ -3207,6 +3343,22 @@ jobs:
fi
exit "$exit_code"
}
# Last read-only gate before the first production mutation. Builds may
# finish out of order on Gitea 1.25; only the latest inclusive source
# is allowed to cross this boundary.
PREWRITE_CARRIER_JSON=$(python3 ops/runner/select-latest-inclusive-cd-carrier.py \
--source-sha "$IMAGE_TAG" \
--remote-url http://192.168.0.110:3001/wooo/awoooi.git)
PREWRITE_CARRIER_SELECTED=$(printf '%s' "$PREWRITE_CARRIER_JSON" | \
python3 -c 'import json,sys; print(str(json.load(sys.stdin)["selected"]).lower())')
if [ "$PREWRITE_CARRIER_SELECTED" != "true" ]; then
PREWRITE_CARRIER_LATEST=$(printf '%s' "$PREWRITE_CARRIER_JSON" | \
python3 -c 'import json,sys; print(json.load(sys.stdin)["latest_sha"])')
echo "AWOOOI_DEPLOY_COALESCED_AFTER_PREREQUISITES=source=${IMAGE_TAG};latest=${PREWRITE_CARRIER_LATEST};runtime_deploy_write_performed=false;monotonic_prerequisites_completed=true"
exit 0
fi
trap rollback_deploy_controls_on_exit EXIT
cat k8s/awoooi-prod/02-network-policy.yaml | \
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
@@ -3303,13 +3455,26 @@ jobs:
k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml
DEPLOY_REVISION=""
git diff --cached --quiet && echo "⚡ kustomization.yaml 無變化,跳過 push" || {
git commit -m "chore(cd): deploy ${IMAGE_TAG::7} [skip ci]"
# 用 token 推送(避免 SSH key 需要額外設定 push 權限)
git remote remove gitea 2>/dev/null || true
git remote add gitea "http://wooo:${CD_PUSH_TOKEN}@192.168.0.110:3001/wooo/awoooi.git"
# 先 rebase 避免 non-fast-forward (其他 commit 在 CI 期間已推入)
# 2026-04-17 ogt: -X theirs — kustomization.yaml 衝突時採用當次部署的 image tag
git fetch gitea main
CARRIER_JSON=$(python3 ops/runner/select-latest-inclusive-cd-carrier.py \
--source-sha "$IMAGE_TAG" \
--remote-url http://192.168.0.110:3001/wooo/awoooi.git)
CARRIER_SELECTED=$(printf '%s' "$CARRIER_JSON" | \
python3 -c 'import json,sys; print(str(json.load(sys.stdin)["selected"]).lower())')
if [ "$CARRIER_SELECTED" != "true" ]; then
CARRIER_LATEST=$(printf '%s' "$CARRIER_JSON" | \
python3 -c 'import json,sys; print(json.load(sys.stdin)["latest_sha"])')
echo "AWOOOI_DEPLOY_SUPERSEDED_AFTER_WRITE=source=${IMAGE_TAG};latest=${CARRIER_LATEST};rollback=required"
restore_deploy_controls
echo "AWOOOI_DEPLOY_SUPERSEDED_ROLLBACK=source=${IMAGE_TAG};latest=${CARRIER_LATEST};rollback_verified=true"
exit 1
fi
git commit -m "chore(cd): deploy ${IMAGE_TAG::7} [skip ci]"
# Rebase only onto the exact main revision validated above. If main
# advances after this fetch, the normal push rejects non-fast-forward.
git rebase -X theirs gitea/main
DEPLOY_REVISION=$(git rev-parse HEAD)
git push gitea main
@@ -3723,16 +3888,24 @@ jobs:
container="$2"
last_failure="workload_not_ready"
for attempt in 1 2 3; do
ready_pod_ref=$(
candidate_pod_refs=$(
$KUBECTL get pods -n awoooi-prod -l "app=$workload" \
--field-selector=status.phase=Running \
--sort-by=.metadata.creationTimestamp \
-o name 2>/dev/null | tail -n 1
) || ready_pod_ref=""
if [ -z "$ready_pod_ref" ]; then
-o name 2>/dev/null
) || candidate_pod_refs=""
ready_pod_ref=""
while IFS= read -r candidate_pod_ref; do
[ -n "$candidate_pod_ref" ] || continue
if $KUBECTL wait --for=condition=Ready "$candidate_pod_ref" \
-n awoooi-prod --timeout=1s >/dev/null 2>&1; then
ready_pod_ref="$candidate_pod_ref"
break
fi
done < <(printf '%s\n' "$candidate_pod_refs" | sed '/^$/d' | tac)
if [ -z "$candidate_pod_refs" ]; then
last_failure="running_pod_unavailable"
elif ! $KUBECTL wait --for=condition=Ready "$ready_pod_ref" \
-n awoooi-prod --timeout=10s >/dev/null 2>&1; then
elif [ -z "$ready_pod_ref" ]; then
last_failure="ready_pod_unavailable"
else
container_ready=$(
@@ -4892,9 +5065,11 @@ jobs:
echo "⚠️ CI/CD rollout risk notification failed (non-fatal)"
fi
fi
echo "applied=true" >> "$GITHUB_OUTPUT"
exit 0
- name: Apply Runtime Image Mirror Policy
if: ${{ steps.deploy.outputs.applied == 'true' }}
run: |
set -euo pipefail
prepare_deploy_key() {
@@ -4944,6 +5119,7 @@ jobs:
echo "runtime_image_policy_stage=verified"
- name: Notify Build Deploy Success
if: ${{ steps.deploy.outputs.applied == 'true' }}
run: |
END_TIME=$(date +%s)
DURATION=$((END_TIME - ${{ steps.commit.outputs.start_time }}))
@@ -4998,13 +5174,46 @@ jobs:
echo "AWOOI API notify failed; direct Telegram fallback disabled to preserve AwoooP receipt chain"
fi
revalidate-post-deploy-carrier:
if: ${{ needs.build-and-deploy.outputs.deploy_applied == 'true' }}
needs: [build-and-deploy]
runs-on: awoooi-non110-host
timeout-minutes: 3
outputs:
selected: ${{ steps.carrier.outputs.selected }}
status: ${{ steps.carrier.outputs.status }}
latest_sha: ${{ steps.carrier.outputs.latest_sha }}
main_tip_sha: ${{ steps.carrier.outputs.main_tip_sha }}
steps:
- name: Checkout exact source from Gitea
env:
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
run: |
set -euo pipefail
git init .
git remote remove origin 2>/dev/null || true
git remote add origin "$GITEA_SOURCE_URL"
git fetch --no-tags --prune --depth=1 origin "$GITHUB_SHA"
git checkout --force --detach FETCH_HEAD
- name: Revalidate latest inclusive post-deploy carrier
id: carrier
env:
GITEA_SOURCE_URL: http://192.168.0.110:3001/wooo/awoooi.git
run: |
set -euo pipefail
python3 ops/runner/select-latest-inclusive-cd-carrier.py \
--source-sha "$GITHUB_SHA" \
--remote-url "$GITEA_SOURCE_URL" \
--github-output "$GITHUB_OUTPUT"
post-deploy-checks:
# 2026-06-28 Codex: post-deploy checks belong to real deploy runs; skip
# marker/no-op commits already accounted for by the previous deploy run.
# 2026-07-01 Codex: `[metadata-only]` commits do not roll a new image, so
# post-deploy smokes would only retest the previous production artifact.
if: ${{ github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd') && !contains(github.event.head_commit.message, '[metadata-only]')) }}
needs: [build-and-deploy]
if: ${{ needs.revalidate-post-deploy-carrier.outputs.selected == 'true' && needs.build-and-deploy.outputs.deploy_applied == 'true' && (github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd') && !contains(github.event.head_commit.message, '[metadata-only]'))) }}
needs: [build-and-deploy, revalidate-post-deploy-carrier]
# 2026-07-16 Codex: the fail-hard pressure gate below may legitimately
# consume 20 minutes while a bounded backup and its metric freshness
# window drain. Preserve enough time for the independent post-deploy

View File

@@ -42,10 +42,67 @@ def test_cd_source_link_gate_uses_current_deploy_canary() -> None:
assert "codex-sentry-20260513-t15b-v3" not in text
def test_cd_main_pushes_queue_without_canceling_active_deploys() -> None:
def test_cd_main_pushes_use_gitea_125_executable_inclusive_carrier() -> None:
text = _workflow_text()
concurrency = text.split("concurrency:", 1)[1].split("env:", 1)[0]
assert "group: cd-deploy-${{ github.ref }}" in concurrency
assert "cancel-in-progress: false" in concurrency
assert "cancel-in-progress: true" not in concurrency
assert "\nconcurrency:" not in text
assert "Gitea 1.25 ignores the workflow concurrency key" in text
assert "select-latest-carrier:" in text
assert "needs: [select-latest-carrier]" in text
assert "revalidate-deploy-carrier:" in text
assert "needs: [select-latest-carrier, tests]" in text
assert "needs: [select-latest-carrier, tests, revalidate-deploy-carrier]" in text
assert "revalidate-post-deploy-carrier:" in text
assert "needs: [build-and-deploy, revalidate-post-deploy-carrier]" in text
assert "needs.revalidate-deploy-carrier.outputs.selected == 'true'" in text
assert "needs.revalidate-post-deploy-carrier.outputs.selected == 'true'" in text
assert "deploy_applied: ${{ steps.deploy.outputs.applied }}" in text
assert "AWOOOI_BUILD_COALESCED=" in text
assert "AWOOOI_DEPLOY_COALESCED_AFTER_PREREQUISITES=" in text
assert "select-latest-inclusive-cd-carrier.py" in text
assert "--debounce-seconds 20" in text
def test_deploy_revalidates_lineage_after_final_main_fetch() -> None:
text = _workflow_text()
build_job = text.split(" build-and-deploy:", 1)[1].split(
" revalidate-post-deploy-carrier:", 1
)[0]
deploy = text.split("- name: Deploy to K8s (ArgoCD GitOps)", 1)[1].split(
"- name: Apply Runtime Image Mirror Policy", 1
)[0]
production_gate = build_job.index(
"- name: Revalidate carrier before production mutations"
)
first_database_mutation = build_job.index(
"- name: Reconcile Asset Inventory Type Constraint"
)
secret_mutation = build_job.index("- name: Inject K8s Secrets")
deploy_step = build_job.index("- name: Deploy to K8s (ArgoCD GitOps)")
prewrite_revalidate = deploy.index("PREWRITE_CARRIER_JSON=")
first_production_write = deploy.index(
"if ! cat k8s/awoooi-prod/02-network-policy.yaml"
)
final_fetch = deploy.index("git fetch gitea main", first_production_write)
revalidate = deploy.index("CARRIER_JSON=$(python3", final_fetch)
marker_commit = deploy.index('git commit -m "chore(cd): deploy', revalidate)
normal_push = deploy.index("git push gitea main", marker_commit)
assert production_gate < first_database_mutation < secret_mutation < deploy_step
assert build_job.count(
"if: ${{ steps.preproduction-carrier.outputs.selected == 'true' }}"
) >= 6
assert prewrite_revalidate < first_production_write
assert final_fetch < revalidate < marker_commit < normal_push
assert 'echo "applied=false" >> "$GITHUB_OUTPUT"' in deploy
assert "AWOOOI_DEPLOY_COALESCED_AFTER_PREREQUISITES=" in deploy
late_supersession = deploy.split(
"AWOOOI_DEPLOY_SUPERSEDED_AFTER_WRITE=", 1
)[1].split('git commit -m "chore(cd): deploy', 1)[0]
assert "restore_deploy_controls" in late_supersession
assert "AWOOOI_DEPLOY_SUPERSEDED_ROLLBACK=" in late_supersession
assert "exit 1" in late_supersession
assert "restore_deploy_configmaps" in deploy
assert "awoooi-config service-registry" in deploy
assert 'echo "applied=true" >> "$GITHUB_OUTPUT"' in deploy
assert "git push --force" not in deploy

View File

@@ -208,7 +208,11 @@ def test_cd_ssh_denial_probe_targets_ready_pod_and_classifies_transport() -> Non
assert "for attempt in 1 2 3; do" in probe
assert "--field-selector=status.phase=Running" in probe
assert "--sort-by=.metadata.creationTimestamp" in probe
assert '--for=condition=Ready "$ready_pod_ref"' in probe
assert "candidate_pod_refs=" in probe
assert "while IFS= read -r candidate_pod_ref; do" in probe
assert '--for=condition=Ready "$candidate_pod_ref"' in probe
assert 'ready_pod_ref="$candidate_pod_ref"' in probe
assert "| tac)" in probe
assert "status.containerStatuses" in probe
assert '$KUBECTL exec -i "$ready_pod_ref"' in probe
assert '$KUBECTL exec -i "deployment/$workload"' not in probe

View File

@@ -297,6 +297,9 @@ def test_workload_database_bootstrap_keeps_secret_values_off_output() -> None:
def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None:
workflow = (REPO_ROOT / ".gitea/workflows/cd.yaml").read_text()
normalized_workflow = " ".join(workflow.split())
workload_rollback = workflow.split("WORKLOAD_DB_ROLLBACK_FILE", 1)[1].split(
"# The two directly applied ConfigMaps", 1
)[0]
assert "WORKLOAD_DB_ROLLBACK_FILE" in workflow
assert "WORKLOAD_DB_SECRET_PREEXISTENCE_FILE" in workflow
@@ -305,7 +308,7 @@ def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None:
assert "bash -s -- verify" in workflow
assert workflow.index("bash -s -- apply") < workflow.index("bash -s -- verify")
assert '"spec": item["spec"]' in workflow
assert '"data": item' not in workflow
assert '"data": item' not in workload_rollback
assert "get secret '${secret_name}' -n awoooi-prod -o jsonpath" not in workflow
assert "get secret awoooi-api-db -n awoooi-prod -o jsonpath" not in workflow
assert "HEAD^" not in workflow

View File

@@ -136,6 +136,20 @@ _CD_BUILD_DEPLOY_SUCCESS_NOTIFICATION_RE = re.compile(
r"Z\s+(?:[^\w\"']+\s+)?CI/CD build-deploy success notification mirrored"
)
_JOB_SUCCEEDED_RE = re.compile(r"Z\s+(?:[^\w\"']+\s+)?Job succeeded")
_CD_EXECUTOR_BOUNDARY_FAILURES = (
(
"signal worker SSH-denial verifier did not prove the boundary",
"signal_worker_ssh_denial_verifier_unproven",
),
(
"API SSH-denial verifier did not prove the boundary",
"api_ssh_denial_verifier_unproven",
),
(
"execution broker has no live route evidence for allowlisted SSH endpoints",
"execution_broker_allowlisted_ssh_route_unproven",
),
)
@dataclass(frozen=True)
@@ -694,6 +708,7 @@ def build_readback(
build_docker_step_timeout_label=build_log_classifier[
"docker_step_timeout_label"
],
build_failure_classifier=build_log_classifier["failure_classifier"],
harbor_110_repair_no_matching_runner_label=(
harbor_110_repair_no_matching_runner_label
if not harbor_110_repair_ignored_for_current_cd_context
@@ -1133,6 +1148,8 @@ def build_readback(
if build_log_classifier["docker_step_timeout"]
else "blocked_host_web_build_pressure"
if effective_tests_log_classifier["host_pressure_blocked_or_waiting"]
else "blocked_cd_build_verifier"
if build_log_classifier["failure_classifier"]
else "blocked_harbor_110_repair_no_matching_runner"
if harbor_110_repair_no_matching_runner_label
else "blocked_no_matching_online_runner"
@@ -1413,6 +1430,7 @@ def _queue_safe_next_action(
build_harbor_public_route_retrying_unavailable: bool,
build_docker_step_timeout: bool,
build_docker_step_timeout_label: str,
build_failure_classifier: str,
harbor_110_repair_no_matching_runner_label: str,
harbor_110_repair_waiting: bool,
harbor_110_repair_running: bool,
@@ -1664,6 +1682,20 @@ def _queue_safe_next_action(
blocker_fields=["latest_visible_cd_host_pressure_classifier"],
)
if build_failure_classifier:
return action(
action_id="fix_exact_cd_build_verifier_then_single_replay",
stage="cd_build_log_exact_failure",
text=(
"Reproduce and fix the exact build/deploy verifier locally, then "
"perform one normal integration and terminal production readback."
),
reason=build_failure_classifier,
command="focused local verifier replay before the next normal push",
post_verifier="single exact CD terminal plus production deploy-marker readback",
blocker_fields=["latest_visible_cd_failure_classifier"],
)
if cd_jobs_stale_or_mismatched:
return action(
action_id="ignore_stale_cd_jobs_api_payload_and_poll_visible_cd_or_marker",
@@ -1733,6 +1765,14 @@ def classify_cd_build_log(text: str) -> dict[str, Any]:
)
deploy_marker_matches = list(_CD_DEPLOY_MARKER_RE.finditer(text))
latest_deploy_marker = deploy_marker_matches[-1] if deploy_marker_matches else None
executor_boundary_failure = next(
(
classifier
for marker, classifier in _CD_EXECUTOR_BOUNDARY_FAILURES
if marker in text
),
"",
)
return {
"job_succeeded": _JOB_SUCCEEDED_RE.search(text) is not None,
"deploy_marker_pushed": latest_deploy_marker is not None,
@@ -1749,7 +1789,9 @@ def classify_cd_build_log(text: str) -> dict[str, Any]:
_CD_BUILD_DEPLOY_SUCCESS_NOTIFICATION_RE.search(text) is not None
),
"failure_classifier": (
"harbor_registry_public_route_unavailable"
executor_boundary_failure
if executor_boundary_failure
else "harbor_registry_public_route_unavailable"
if harbor_public_route_blocked
else "cd_docker_step_timeout"
if latest_docker_step_timeout

View File

@@ -0,0 +1,191 @@
#!/usr/bin/env python3
"""Select one exact main revision as the inclusive AWOOOI CD carrier."""
from __future__ import annotations
import argparse
import json
import re
import subprocess
import time
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Sequence
SHA_RE = re.compile(r"^[0-9a-f]{40}$")
EXCLUDED_MESSAGE_MARKERS = ("[skip ci]", "cancel-stale-cd", "[metadata-only]")
@dataclass(frozen=True)
class CarrierDecision:
selected: bool
status: str
source_sha: str
latest_sha: str
main_tip_sha: str
def _git(args: Sequence[str], *, cwd: Path, check: bool = True) -> subprocess.CompletedProcess[str]:
return subprocess.run(
["git", *args],
cwd=cwd,
check=check,
capture_output=True,
text=True,
)
def _validate_sha(value: str, *, label: str) -> str:
normalized = value.strip().lower()
if not SHA_RE.fullmatch(normalized):
raise RuntimeError(f"{label}_invalid")
return normalized
def _latest_remote_sha(*, remote_url: str, branch: str, cwd: Path) -> str:
result = _git(
["ls-remote", "--heads", remote_url, f"refs/heads/{branch}"],
cwd=cwd,
)
rows = [row.split() for row in result.stdout.splitlines() if row.strip()]
if len(rows) != 1 or len(rows[0]) != 2:
raise RuntimeError("latest_main_ref_unresolved")
return _validate_sha(rows[0][0], label="latest_sha")
def _fetch_latest_history(*, remote_url: str, branch: str, cwd: Path, depth: int) -> None:
_git(
[
"fetch",
"--no-tags",
"--prune",
f"--depth={depth}",
remote_url,
f"+refs/heads/{branch}:refs/remotes/awoooi-carrier/{branch}",
],
cwd=cwd,
)
def _latest_deploy_eligible_sha(*, branch: str, cwd: Path) -> str | None:
result = _git(
[
"log",
"--format=%H%x1f%B%x1e",
f"refs/remotes/awoooi-carrier/{branch}",
],
cwd=cwd,
)
for record in result.stdout.split("\x1e"):
sha, separator, message = record.strip().partition("\x1f")
if not separator:
continue
normalized_message = message.lower()
if any(marker in normalized_message for marker in EXCLUDED_MESSAGE_MARKERS):
continue
return _validate_sha(sha, label="latest_deploy_eligible_sha")
return None
def _source_is_ancestor(*, source_sha: str, latest_sha: str, cwd: Path) -> bool:
result = _git(
["merge-base", "--is-ancestor", source_sha, latest_sha],
cwd=cwd,
check=False,
)
if result.returncode not in {0, 1}:
raise RuntimeError("carrier_lineage_check_failed")
return result.returncode == 0
def select_latest_inclusive_carrier(
*,
source_sha: str,
remote_url: str,
branch: str = "main",
cwd: Path,
debounce_seconds: float = 0,
) -> CarrierDecision:
source_sha = _validate_sha(source_sha, label="source_sha")
if debounce_seconds < 0 or debounce_seconds > 60:
raise RuntimeError("debounce_seconds_out_of_range")
if debounce_seconds:
time.sleep(debounce_seconds)
main_tip_sha = _latest_remote_sha(remote_url=remote_url, branch=branch, cwd=cwd)
_fetch_latest_history(remote_url=remote_url, branch=branch, cwd=cwd, depth=256)
latest_sha = _latest_deploy_eligible_sha(branch=branch, cwd=cwd)
if latest_sha is None:
_fetch_latest_history(remote_url=remote_url, branch=branch, cwd=cwd, depth=2048)
latest_sha = _latest_deploy_eligible_sha(branch=branch, cwd=cwd)
if latest_sha is None:
raise RuntimeError("latest_deploy_eligible_main_unresolved")
if source_sha == latest_sha:
return CarrierDecision(
selected=True,
status="selected_latest_main_carrier",
source_sha=source_sha,
latest_sha=latest_sha,
main_tip_sha=main_tip_sha,
)
if not _source_is_ancestor(source_sha=source_sha, latest_sha=latest_sha, cwd=cwd):
shallow = _git(
["rev-parse", "--is-shallow-repository"],
cwd=cwd,
).stdout.strip()
if shallow == "true":
_fetch_latest_history(remote_url=remote_url, branch=branch, cwd=cwd, depth=2048)
if not _source_is_ancestor(source_sha=source_sha, latest_sha=latest_sha, cwd=cwd):
raise RuntimeError("latest_main_does_not_include_source")
return CarrierDecision(
selected=False,
status="superseded_by_latest_inclusive_carrier",
source_sha=source_sha,
latest_sha=latest_sha,
main_tip_sha=main_tip_sha,
)
def _write_github_output(path: Path, decision: CarrierDecision) -> None:
fields = {
"selected": str(decision.selected).lower(),
"status": decision.status,
"source_sha": decision.source_sha,
"latest_sha": decision.latest_sha,
"main_tip_sha": decision.main_tip_sha,
}
with path.open("a", encoding="utf-8") as stream:
for key, value in fields.items():
stream.write(f"{key}={value}\n")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source-sha", required=True)
parser.add_argument("--remote-url", required=True)
parser.add_argument("--branch", default="main")
parser.add_argument("--repo", type=Path, default=Path.cwd())
parser.add_argument("--debounce-seconds", type=float, default=0)
parser.add_argument("--github-output", type=Path)
args = parser.parse_args()
decision = select_latest_inclusive_carrier(
source_sha=args.source_sha,
remote_url=args.remote_url,
branch=args.branch,
cwd=args.repo.resolve(),
debounce_seconds=args.debounce_seconds,
)
if args.github_output:
_write_github_output(args.github_output, decision)
print(json.dumps(asdict(decision), sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -1904,6 +1904,43 @@ def test_failed_cd_with_stale_jobs_payload_gets_failure_classifier() -> None:
)
def test_exact_executor_boundary_failure_supersedes_stale_jobs_classifier() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_failed_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={
"jobs": [
{
"run_id": 4043,
"name": "tests",
"head_sha": "9c4e754d3369fa2de7606e7092712cf6d85e18e2",
"conclusion": "success",
}
],
"total_count": 1,
},
latest_cd_build_log_http_status=200,
latest_cd_build_log_text=(
"ssh_denial_probe_transport_unavailable workload=awoooi-worker "
"reason=ready_pod_unavailable\n"
"signal worker SSH-denial verifier did not prove the boundary\n"
),
)
assert payload["status"] == "blocked_cd_build_verifier"
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
assert payload["readback"]["latest_visible_cd_failure_classifier"] == (
"signal_worker_ssh_denial_verifier_unproven"
)
assert (
payload["readback"]["safe_next_action_id"]
== "fix_exact_cd_build_verifier_then_single_replay"
)
def test_interrupted_host_pressure_reports_postgres_recovery_cpu() -> None:
module = _load_module()
payload = module.build_readback(

View File

@@ -0,0 +1,226 @@
from __future__ import annotations
import importlib.util
import subprocess
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "ops/runner/select-latest-inclusive-cd-carrier.py"
def _load_module():
spec = importlib.util.spec_from_file_location("select_latest_inclusive_cd_carrier", SCRIPT)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _git(cwd: Path, *args: str) -> str:
return subprocess.run(
["git", *args],
cwd=cwd,
check=True,
capture_output=True,
text=True,
).stdout.strip()
@pytest.fixture
def linear_remote(tmp_path: Path) -> tuple[Path, Path, str, str]:
source = tmp_path / "source"
remote = tmp_path / "remote.git"
checkout = tmp_path / "checkout"
source.mkdir()
_git(source, "init", "-b", "main")
_git(source, "config", "user.name", "Carrier Test")
_git(source, "config", "user.email", "carrier@example.invalid")
(source / "value.txt").write_text("one\n", encoding="utf-8")
_git(source, "add", "value.txt")
_git(source, "commit", "-m", "first")
first = _git(source, "rev-parse", "HEAD")
_git(tmp_path, "clone", "--bare", str(source), str(remote))
_git(tmp_path, "clone", "--depth=1", remote.as_uri(), str(checkout))
(source / "value.txt").write_text("two\n", encoding="utf-8")
_git(source, "commit", "-am", "second")
second = _git(source, "rev-parse", "HEAD")
_git(source, "push", str(remote), "main")
return checkout, remote, first, second
def test_latest_source_is_selected(linear_remote: tuple[Path, Path, str, str]) -> None:
module = _load_module()
checkout, remote, _, second = linear_remote
decision = module.select_latest_inclusive_carrier(
source_sha=second,
remote_url=str(remote),
cwd=checkout,
)
assert decision.selected is True
assert decision.status == "selected_latest_main_carrier"
assert decision.latest_sha == second
def test_older_source_is_coalesced_only_when_latest_includes_it(
linear_remote: tuple[Path, Path, str, str],
) -> None:
module = _load_module()
checkout, remote, first, second = linear_remote
decision = module.select_latest_inclusive_carrier(
source_sha=first,
remote_url=str(remote),
cwd=checkout,
)
assert decision.selected is False
assert decision.status == "superseded_by_latest_inclusive_carrier"
assert decision.source_sha == first
assert decision.latest_sha == second
@pytest.mark.parametrize(
"marker_subject",
[
"chore(cd): deploy abcdef0 [skip ci]",
"chore(cd): cancel-stale-cd",
"chore(cd): refresh receipt [metadata-only]",
],
)
def test_excluded_main_tip_does_not_displace_deployable_carrier(
linear_remote: tuple[Path, Path, str, str],
tmp_path: Path,
marker_subject: str,
) -> None:
module = _load_module()
checkout, remote, _, second = linear_remote
marker_work = tmp_path / "marker-work"
_git(tmp_path, "clone", str(remote), str(marker_work))
_git(marker_work, "config", "user.name", "Carrier Test")
_git(marker_work, "config", "user.email", "carrier@example.invalid")
_git(marker_work, "commit", "--allow-empty", "-m", marker_subject)
marker_sha = _git(marker_work, "rev-parse", "HEAD")
_git(marker_work, "push", "origin", "main")
decision = module.select_latest_inclusive_carrier(
source_sha=second,
remote_url=str(remote),
cwd=checkout,
)
assert decision.selected is True
assert decision.latest_sha == second
assert decision.main_tip_sha == marker_sha
@pytest.mark.parametrize(
"marker_body",
[
"release receipt [skip ci]",
"queue receipt cancel-stale-cd",
"runtime receipt [metadata-only]",
],
)
def test_excluded_marker_in_commit_body_does_not_displace_deployable_carrier(
linear_remote: tuple[Path, Path, str, str],
tmp_path: Path,
marker_body: str,
) -> None:
module = _load_module()
checkout, remote, _, second = linear_remote
marker_work = tmp_path / "marker-body-work"
_git(tmp_path, "clone", str(remote), str(marker_work))
_git(marker_work, "config", "user.name", "Carrier Test")
_git(marker_work, "config", "user.email", "carrier@example.invalid")
_git(
marker_work,
"commit",
"--allow-empty",
"-m",
"receipt marker",
"-m",
marker_body,
)
marker_sha = _git(marker_work, "rev-parse", "HEAD")
_git(marker_work, "push", "origin", "main")
decision = module.select_latest_inclusive_carrier(
source_sha=second,
remote_url=str(remote),
cwd=checkout,
)
assert decision.selected is True
assert decision.latest_sha == second
assert decision.main_tip_sha == marker_sha
def test_unrelated_latest_main_fails_closed(tmp_path: Path) -> None:
module = _load_module()
checkout = tmp_path / "checkout"
unrelated = tmp_path / "unrelated"
remote = tmp_path / "remote.git"
for repo, value in ((checkout, "source"), (unrelated, "unrelated")):
repo.mkdir()
_git(repo, "init", "-b", "main")
_git(repo, "config", "user.name", "Carrier Test")
_git(repo, "config", "user.email", "carrier@example.invalid")
(repo / "value.txt").write_text(f"{value}\n", encoding="utf-8")
_git(repo, "add", "value.txt")
_git(repo, "commit", "-m", value)
source_sha = _git(checkout, "rev-parse", "HEAD")
_git(tmp_path, "clone", "--bare", str(unrelated), str(remote))
with pytest.raises(RuntimeError, match="latest_main_does_not_include_source"):
module.select_latest_inclusive_carrier(
source_sha=source_sha,
remote_url=str(remote),
cwd=checkout,
)
def test_deep_linear_history_is_proven_after_bounded_deepen(tmp_path: Path) -> None:
module = _load_module()
source = tmp_path / "source"
remote = tmp_path / "remote.git"
checkout = tmp_path / "checkout"
source.mkdir()
_git(source, "init", "-b", "main")
_git(source, "config", "user.name", "Carrier Test")
_git(source, "config", "user.email", "carrier@example.invalid")
_git(source, "commit", "--allow-empty", "-m", "first")
first = _git(source, "rev-parse", "HEAD")
_git(tmp_path, "clone", "--bare", str(source), str(remote))
_git(tmp_path, "clone", "--depth=1", remote.as_uri(), str(checkout))
for index in range(270):
_git(source, "commit", "--allow-empty", "-m", f"linear-{index}")
latest = _git(source, "rev-parse", "HEAD")
_git(source, "push", str(remote), "main")
decision = module.select_latest_inclusive_carrier(
source_sha=first,
remote_url=str(remote),
cwd=checkout,
)
assert decision.selected is False
assert decision.latest_sha == latest
def test_invalid_source_sha_fails_closed(tmp_path: Path) -> None:
module = _load_module()
with pytest.raises(RuntimeError, match="source_sha_invalid"):
module.select_latest_inclusive_carrier(
source_sha="not-a-sha",
remote_url="unused",
cwd=tmp_path,
)