fix(ci): reopen hard-limited controlled cd lane
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m42s
Code Review / ai-code-review (push) Successful in 15s
CD Pipeline / build-and-deploy (push) Successful in 6m33s
CD Pipeline / post-deploy-checks (push) Successful in 3m10s

This commit is contained in:
Your Name
2026-06-28 11:52:33 +08:00
parent 18b4f53e26
commit 4414ec991f
11 changed files with 342 additions and 59 deletions

View File

@@ -193,12 +193,19 @@ RUNNER_DIR="/home/wooo/act-runner"
RUNNER_SERVICE="gitea-act-runner-host.service"
RUNNER_ENABLE_SENTINEL="/run/awoooi-runner-host-enabled"
CD_LANE_DIR="/home/wooo/awoooi-cd-lane"
CD_LANE_SERVICE="awoooi-cd-lane.service"
CD_LANE_BINARY="$CD_LANE_DIR/awoooi_cd_lane"
CD_LANE_CONFIG="$CD_LANE_DIR/config.yaml"
CD_LANE_DRAIN_DIR="/home/wooo/awoooi-cd-lane-drain"
CD_LANE_DRAIN_SERVICE="awoooi-cd-lane-drain.service"
CD_LANE_DRAIN_BINARY="$CD_LANE_DRAIN_DIR/awoooi_cd_lane_controlled"
CD_LANE_DRAIN_CONFIG="$CD_LANE_DRAIN_DIR/config.yaml"
CD_LANE_ENABLE_SENTINEL="/run/awoooi-cd-lane-enabled"
START_GITEA_RUNNER_ON_BOOT="${AWOOOI_START_GITEA_RUNNER_ON_BOOT:-0}"
START_CONTROLLED_CD_LANE="${AWOOOI_START_CONTROLLED_CD_LANE:-0}"
START_GITEA_RUNNER_ALLOWED=0
START_CD_LANE_ALLOWED=0
RUNNER_FAIL_CLOSED_SERVICES=(
"awoooi-cd-lane.service"
"awoooi-cd-lane-drain.service"
"awoooi-direct-runner-open.service"
"awoooi-direct-runner.service"
"gitea-act-runner-host.service"
@@ -207,18 +214,19 @@ RUNNER_FAIL_CLOSED_SERVICES=(
"gitea-act-runner-awoooi-open.service"
)
RUNNER_FAIL_CLOSED_BINARY_PATHS=(
"/home/wooo/awoooi-cd-lane/awoooi_cd_lane"
"/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled"
"/home/wooo/act-runner/act_runner"
"/home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard"
"/home/wooo/act-runner-controlled/act_runner"
"/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner"
)
# Host runner still needs both keys. The direct cd-lane stays fail-closed until
# it is migrated or hard-limited outside this production host pressure lane.
# Legacy host runner still needs both keys. The dedicated cd-lane has its own
# sentinel and narrow label/capacity verifier below.
if [ "$START_GITEA_RUNNER_ON_BOOT" = "1" ] && [ -e "$RUNNER_ENABLE_SENTINEL" ]; then
START_GITEA_RUNNER_ALLOWED=1
fi
if [ -e "$CD_LANE_ENABLE_SENTINEL" ] || [ "$START_CONTROLLED_CD_LANE" = "1" ]; then
START_CD_LANE_ALLOWED=1
fi
mask_runner_unit_file() {
local unit="$1"
@@ -271,17 +279,153 @@ EOF
install_cd_lane_fail_closed_unit() {
local unit_file="/etc/systemd/system/awoooi-cd-lane.service"
local tmp
local quarantine_stamp
quarantine_stamp="$(date +%Y%m%d%H%M%S)"
systemctl mask awoooi-cd-lane.service >/dev/null 2>&1 || true
if [ -e "$unit_file" ] || [ -L "$unit_file" ]; then
chattr -i "$unit_file" >/dev/null 2>&1 || true
if ! { [ -L "$unit_file" ] && [ "$(readlink "$unit_file" 2>/dev/null || true)" = "/dev/null" ]; }; then
if ! grep -q "AWOOOI direct CD lane fail-closed" "$unit_file" 2>/dev/null; then
mv "$unit_file" "${unit_file}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true
fi
fi
ln -sfn /dev/null "$unit_file" >/dev/null 2>&1 || true
tmp="$(mktemp)"
cat >"$tmp" <<'EOF'
[Unit]
Description=AWOOOI direct CD lane fail-closed after 2026-06-28 pressure incident
ConditionPathExists=/run/awoooi-cd-lane-enabled
[Service]
Type=oneshot
ExecStart=/bin/false
EOF
install -o root -g root -m 0444 "$tmp" "$unit_file" >/dev/null 2>&1 || true
rm -f "$tmp"
chattr +i "$unit_file" >/dev/null 2>&1 || true
}
install_controlled_cd_lane_unit() {
local unit_file="/etc/systemd/system/$CD_LANE_SERVICE"
local tmp
chattr -i "$unit_file" "$CD_LANE_BINARY" >/dev/null 2>&1 || true
tmp="$(mktemp)"
cat >"$tmp" <<EOF
[Unit]
Description=AWOOOI controlled CD lane
After=network-online.target docker.service
Wants=network-online.target
Requires=docker.service
[Service]
Type=simple
User=wooo
WorkingDirectory=${CD_LANE_DIR}/data
Environment=HOME=/home/wooo
Environment=AWOOOI_CONTROLLED_RUNNER_OPEN=1
Environment=HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1
Environment=HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=1
ExecStart=${CD_LANE_BINARY} daemon --config ${CD_LANE_CONFIG}
Restart=always
RestartSec=10
KillSignal=SIGINT
TimeoutStopSec=3700
SuccessExitStatus=0 130 143
CPUAccounting=true
CPUQuota=250%
MemoryAccounting=true
MemoryHigh=8G
MemoryMax=12G
TasksAccounting=true
TasksMax=512
IOAccounting=true
IOWeight=100
[Install]
WantedBy=multi-user.target
EOF
install -o root -g root -m 0644 "$tmp" "$unit_file" >/dev/null 2>&1 || true
rm -f "$tmp"
}
install_controlled_cd_lane_drain_unit() {
local unit_file="/etc/systemd/system/$CD_LANE_DRAIN_SERVICE"
local tmp
chattr -i "$unit_file" "$CD_LANE_DRAIN_BINARY" >/dev/null 2>&1 || true
if [ -L "$unit_file" ] && [ "$(readlink "$unit_file" 2>/dev/null || true)" = "/dev/null" ]; then
rm -f "$unit_file" >/dev/null 2>&1 || true
fi
tmp="$(mktemp)"
cat >"$tmp" <<EOF
[Unit]
Description=AWOOOI controlled CD lane drain bypass for old queued guards
After=network-online.target docker.service
Wants=network-online.target
Requires=docker.service
[Service]
Type=simple
User=wooo
WorkingDirectory=${CD_LANE_DRAIN_DIR}/data
Environment=HOME=/home/wooo
Environment=AWOOOI_CONTROLLED_RUNNER_OPEN=1
Environment=HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1
Environment=HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=1
ExecStart=${CD_LANE_DRAIN_BINARY} daemon --config ${CD_LANE_DRAIN_CONFIG}
Restart=always
RestartSec=10
KillSignal=SIGINT
TimeoutStopSec=3700
SuccessExitStatus=0 130 143
CPUAccounting=true
CPUQuota=250%
MemoryAccounting=true
MemoryHigh=8G
MemoryMax=12G
TasksAccounting=true
TasksMax=512
IOAccounting=true
IOWeight=100
[Install]
WantedBy=multi-user.target
EOF
install -o root -g root -m 0644 "$tmp" "$unit_file" >/dev/null 2>&1 || true
rm -f "$tmp"
}
cd_lane_config_path_is_controlled() {
local config_path="$1"
[ -f "$config_path" ] || return 1
grep -Eq '^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$' "$config_path" || return 1
grep -q 'awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04' "$config_path" || return 1
grep -q 'awoooi-host:host' "$config_path" || return 1
if grep -Eq '^[[:space:]]+- ".*(ubuntu-latest|stockplatform|headless|playwright)' "$config_path"; then
return 1
fi
return 0
}
cd_lane_config_is_controlled() {
cd_lane_config_path_is_controlled "$CD_LANE_CONFIG"
}
cd_lane_drain_config_is_controlled() {
cd_lane_config_path_is_controlled "$CD_LANE_DRAIN_CONFIG"
}
cd_lane_drain_is_controlled_open() {
local active
active="$(systemctl show "$CD_LANE_DRAIN_SERVICE" -p ActiveState --value 2>/dev/null || true)"
[ "$active" = "active" ] || return 1
cd_lane_drain_config_is_controlled || return 1
file "$CD_LANE_DRAIN_BINARY" 2>/dev/null | grep -qi "ELF" || return 1
return 0
}
cd_lane_drain_is_controlled_available() {
cd_lane_drain_config_is_controlled || return 1
file "$CD_LANE_DRAIN_BINARY" 2>/dev/null | grep -qi "ELF" || return 1
return 0
}
quarantine_cd_lane_registration_fail_closed() {
@@ -336,6 +480,33 @@ quarantine_cd_lane_root_restore_sources_fail_closed() {
apply_cd_lane_fail_closed_guard() {
local unit
if cd_lane_drain_is_controlled_available; then
if cd_lane_drain_is_controlled_open; then
log "✅ controlled cd-lane drain verifier passed; preserving drain lane and fail-closing regular lane only"
else
log "✅ controlled cd-lane drain assets verified; restoring drain unit and fail-closing regular lane only"
fi
systemctl kill --signal=SIGKILL "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
systemctl stop "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
systemctl disable "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
install_cd_lane_fail_closed_unit
pkill -KILL -f "^${CD_LANE_BINARY} daemon" >/dev/null 2>&1 || true
install_controlled_cd_lane_drain_unit
quarantine_cd_lane_root_restore_sources_fail_closed
systemctl daemon-reload >/dev/null 2>&1 || true
systemctl enable --now "$CD_LANE_DRAIN_SERVICE" >/dev/null 2>&1 || true
return 0
fi
if { [ -e "$CD_LANE_ENABLE_SENTINEL" ] || [ -e "/run/awoooi-cd-lane-controlled-open" ] || [ "$START_CONTROLLED_CD_LANE" = "1" ]; } \
&& cd_lane_config_is_controlled \
&& file "$CD_LANE_BINARY" 2>/dev/null | grep -qi "ELF"; then
log "✅ controlled cd-lane verifier passed; keeping dedicated lane open"
install_controlled_cd_lane_unit
quarantine_cd_lane_root_restore_sources_fail_closed
systemctl daemon-reload >/dev/null 2>&1 || true
systemctl enable --now "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
return 0
fi
for unit in awoooi-cd-lane.service awoooi-cd-lane-drain.service; do
systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true
systemctl stop "$unit" >/dev/null 2>&1 || true
@@ -361,6 +532,22 @@ ensure_cd_lane_fail_closed() {
apply_cd_lane_fail_closed_guard
}
ensure_controlled_cd_lane_open() {
if ! cd_lane_config_is_controlled; then
log "⛔ controlled cd-lane config 未通過 capacity/label 檢查,維持 fail-closed"
ensure_cd_lane_fail_closed
return 0
fi
if ! file "$CD_LANE_BINARY" 2>/dev/null | grep -qi "ELF"; then
log "⛔ controlled cd-lane binary 不是可執行 ELF維持 fail-closed"
ensure_cd_lane_fail_closed
return 0
fi
install_controlled_cd_lane_unit
systemctl daemon-reload >/dev/null 2>&1 || true
systemctl enable --now "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
}
ensure_host_runner_fail_closed() {
local unit
local binary
@@ -390,9 +577,6 @@ ensure_host_runner_fail_closed() {
fi
pkill -KILL -f "^${RUNNER_DIR}/act_runner(\\.real-[^ ]*)? daemon" >/dev/null 2>&1 || true
pkill -KILL -f "^${CD_LANE_DIR}/awoooi_cd_lane daemon" >/dev/null 2>&1 || true
pkill -KILL -f "^${CD_LANE_DRAIN_DIR}/awoooi_cd_lane_controlled daemon" >/dev/null 2>&1 || true
quarantine_cd_lane_registration_fail_closed
quarantine_cd_lane_root_restore_sources_fail_closed
for binary in "${RUNNER_FAIL_CLOSED_BINARY_PATHS[@]}"; do
guard_runner_binary_fail_closed "$binary"
@@ -499,8 +683,13 @@ else
log "⚠️ 找不到 act-runner binary/config: $RUNNER_DIR"
fi
log "⏸️ direct cd-lane / drain lane 維持 fail-closed需完成搬遷或硬限流後才可用獨立變更恢復"
ensure_cd_lane_fail_closed
if [ "$START_CD_LANE_ALLOWED" = "1" ]; then
log "✅ controlled cd-lane 具備 sentinel/env 授權,執行 capacity/label/binary verifier 後受控開啟"
ensure_controlled_cd_lane_open
else
log "⏸️ controlled cd-lane 未要求啟動;保留合格 drain laneregular lane 維持 fail-closed"
ensure_cd_lane_fail_closed
fi
# ──────────────────────────────────────────────
# STEP 7: SentryError Tracking

View File

@@ -327,12 +327,30 @@ if [ "$cd_lane_active" = "inactive" ] \
&& { { [ "$cd_lane_load" = "masked" ] && [ "$cd_lane_unitfile" = "masked" ]; } || echo "$cd_lane_execstart" | grep -q "/bin/false"; }; then
cd_lane_ok=1
cd_lane_mode=failclosed
elif [ "$cd_lane_sentinel" = "present" ] && [ "$cd_lane_active" = "active" ] && [ "$cd_lane_capacity_ok" = "1" ] && [ "$cd_lane_labels_ok" = "1" ] && [ "$cd_lane_binary_elf" = "1" ]; then
cd_lane_ok=1
cd_lane_mode=controlled_open
fi
echo "CD_LANE_CONTROLLED mode=$cd_lane_mode load=$cd_lane_load unitfile=$cd_lane_unitfile active=$cd_lane_active mainpid=$cd_lane_mainpid sentinel=$cd_lane_sentinel capacity=$cd_lane_capacity_ok labels=$cd_lane_labels_ok binary_elf=$cd_lane_binary_elf process_count=$cd_lane_process_count ok=$cd_lane_ok"
cd_lane_drain_load=$(systemctl show awoooi-cd-lane-drain.service -p LoadState --value 2>/dev/null || true)
cd_lane_drain_unitfile=$(systemctl show awoooi-cd-lane-drain.service -p UnitFileState --value 2>/dev/null || true)
cd_lane_drain_active=$(systemctl show awoooi-cd-lane-drain.service -p ActiveState --value 2>/dev/null || true)
cd_lane_drain_mainpid=$(systemctl show awoooi-cd-lane-drain.service -p MainPID --value 2>/dev/null || true)
cd_lane_drain_cpu_accounting=$(systemctl show awoooi-cd-lane-drain.service -p CPUAccounting --value 2>/dev/null || true)
cd_lane_drain_cpu_quota=$(systemctl show awoooi-cd-lane-drain.service -p CPUQuotaPerSecUSec --value 2>/dev/null || true)
cd_lane_drain_memory_accounting=$(systemctl show awoooi-cd-lane-drain.service -p MemoryAccounting --value 2>/dev/null || true)
cd_lane_drain_memory_max=$(systemctl show awoooi-cd-lane-drain.service -p MemoryMax --value 2>/dev/null || true)
cd_lane_drain_tasks_accounting=$(systemctl show awoooi-cd-lane-drain.service -p TasksAccounting --value 2>/dev/null || true)
cd_lane_drain_tasks_max=$(systemctl show awoooi-cd-lane-drain.service -p TasksMax --value 2>/dev/null || true)
cd_lane_drain_limits_ok=0
if [ "$cd_lane_drain_cpu_accounting" = "yes" ] \
&& [ -n "$cd_lane_drain_cpu_quota" ] && [ "$cd_lane_drain_cpu_quota" != "infinity" ] \
&& [ "$cd_lane_drain_memory_accounting" = "yes" ] \
&& [ -n "$cd_lane_drain_memory_max" ] && [ "$cd_lane_drain_memory_max" != "infinity" ] \
&& [ "$cd_lane_drain_tasks_accounting" = "yes" ] \
&& [ -n "$cd_lane_drain_tasks_max" ] && [ "$cd_lane_drain_tasks_max" != "infinity" ]; then
cd_lane_drain_limits_ok=1
fi
cd_lane_drain_capacity_ok=0
cd_lane_drain_labels_ok=0
if grep -Eq "^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$" /home/wooo/awoooi-cd-lane-drain/config.yaml 2>/dev/null; then
@@ -355,21 +373,28 @@ if [ "$cd_lane_drain_active" != "active" ] \
&& { [ "$cd_lane_drain_load" = "not-found" ] || { [ "$cd_lane_drain_load" = "masked" ] && [ "$cd_lane_drain_unitfile" = "masked" ]; }; }; then
cd_lane_drain_ok=1
cd_lane_drain_mode=failclosed
elif [ "$cd_lane_drain_active" = "active" ] \
&& [ "$cd_lane_drain_capacity_ok" = "1" ] \
&& [ "$cd_lane_drain_labels_ok" = "1" ] \
&& [ "$cd_lane_drain_binary_elf" = "1" ] \
&& [ "$cd_lane_drain_limits_ok" = "1" ]; then
cd_lane_drain_ok=1
cd_lane_drain_mode=controlled_open
fi
echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active mainpid=$cd_lane_drain_mainpid capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok"
echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active mainpid=$cd_lane_drain_mainpid capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf limits=$cd_lane_drain_limits_ok process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok"
cd_lane_root_restore_left=unknown
if sudo -n true >/dev/null 2>&1; then
cd_lane_root_restore_left=$(sudo -n find /root -maxdepth 1 -type d \( -name "awoooi-cd-lane-disabled-*" -o -name "awoooi-cd-lane-drain-disabled-*" \) -print 2>/dev/null | wc -l | tr -d " ")
fi
echo "CD_LANE_ROOT_RESTORE_SOURCES left=$cd_lane_root_restore_left"
cd_lane_guard_ok=0
if [ "$cd_lane_ok" = "1" ] && [ "$cd_lane_drain_ok" = "1" ] && [ "$cd_lane_root_restore_left" = "0" ]; then
if { [ "$cd_lane_ok" = "1" ] || [ "$cd_lane_drain_ok" = "1" ]; } && [ "$cd_lane_root_restore_left" = "0" ]; then
cd_lane_guard_ok=1
fi
echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok"
direct_runner_count=$(pgrep -f "^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" 2>/dev/null | wc -l | tr -d " ")
echo "RUNNER_DIRECT_PROCESS_COUNT $direct_runner_count"
for p in /home/wooo/awoooi-cd-lane/awoooi_cd_lane /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
for p in /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
kind=$(file -b "$p" 2>/dev/null || echo missing)
echo "RUNNER_FAILCLOSED_BINARY $p kind=$kind"
echo "$kind" | grep -qi "ELF" && echo "RUNNER_FAILCLOSED_BINARY_ELF $p"
@@ -402,7 +427,7 @@ docker ps --format "DOCKER {{.Names}}\t{{.Status}}" | head -120
else
fail "110 legacy direct/Gitea runner units are not fail-closed"
fi
grep -q "CD_LANE_GUARDRAILS_OK 1" <<<"$out" && ok "110 runner/CD lane fail-closed guardrails complete" || fail "110 runner/CD lane fail-closed guardrails incomplete"
grep -q "CD_LANE_GUARDRAILS_OK 1" <<<"$out" && ok "110 controlled cd-lane is safe, drained, or fail-closed" || fail "110 controlled cd-lane is neither safe-open/drained nor fail-closed"
grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" <<<"$out" && ok "110 legacy direct runner process count is zero" || fail "110 legacy direct runner process detected"
grep -q "RUNNER_FAILCLOSED_BINARY_ELF" <<<"$out" && fail "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing"
grep -q "sentry-self-hosted-clickhouse-1.*Restarting" <<<"$out" && warn "Sentry ClickHouse restarting" || ok "Sentry ClickHouse not visibly restarting"

View File

@@ -346,11 +346,29 @@ if [ "$cd_lane_active" = "inactive" ] \
&& { { [ "$cd_lane_load" = "masked" ] && [ "$cd_lane_unitfile" = "masked" ]; } || echo "$cd_lane_execstart" | grep -q "/bin/false"; }; then
cd_lane_ok=1
cd_lane_mode=failclosed
elif [ "$cd_lane_sentinel" = "present" ] && [ "$cd_lane_active" = "active" ] && [ "$cd_lane_capacity_ok" = "1" ] && [ "$cd_lane_labels_ok" = "1" ] && [ "$cd_lane_binary_elf" = "1" ]; then
cd_lane_ok=1
cd_lane_mode=controlled_open
fi
echo "CD_LANE_CONTROLLED mode=$cd_lane_mode load=$cd_lane_load unitfile=$cd_lane_unitfile active=$cd_lane_active sentinel=$cd_lane_sentinel capacity=$cd_lane_capacity_ok labels=$cd_lane_labels_ok binary_elf=$cd_lane_binary_elf process_count=$cd_lane_process_count ok=$cd_lane_ok"
cd_lane_drain_load=$(systemctl show awoooi-cd-lane-drain.service -p LoadState --value 2>/dev/null || true)
cd_lane_drain_unitfile=$(systemctl show awoooi-cd-lane-drain.service -p UnitFileState --value 2>/dev/null || true)
cd_lane_drain_active=$(systemctl show awoooi-cd-lane-drain.service -p ActiveState --value 2>/dev/null || true)
cd_lane_drain_cpu_accounting=$(systemctl show awoooi-cd-lane-drain.service -p CPUAccounting --value 2>/dev/null || true)
cd_lane_drain_cpu_quota=$(systemctl show awoooi-cd-lane-drain.service -p CPUQuotaPerSecUSec --value 2>/dev/null || true)
cd_lane_drain_memory_accounting=$(systemctl show awoooi-cd-lane-drain.service -p MemoryAccounting --value 2>/dev/null || true)
cd_lane_drain_memory_max=$(systemctl show awoooi-cd-lane-drain.service -p MemoryMax --value 2>/dev/null || true)
cd_lane_drain_tasks_accounting=$(systemctl show awoooi-cd-lane-drain.service -p TasksAccounting --value 2>/dev/null || true)
cd_lane_drain_tasks_max=$(systemctl show awoooi-cd-lane-drain.service -p TasksMax --value 2>/dev/null || true)
cd_lane_drain_limits_ok=0
if [ "$cd_lane_drain_cpu_accounting" = "yes" ] \
&& [ -n "$cd_lane_drain_cpu_quota" ] && [ "$cd_lane_drain_cpu_quota" != "infinity" ] \
&& [ "$cd_lane_drain_memory_accounting" = "yes" ] \
&& [ -n "$cd_lane_drain_memory_max" ] && [ "$cd_lane_drain_memory_max" != "infinity" ] \
&& [ "$cd_lane_drain_tasks_accounting" = "yes" ] \
&& [ -n "$cd_lane_drain_tasks_max" ] && [ "$cd_lane_drain_tasks_max" != "infinity" ]; then
cd_lane_drain_limits_ok=1
fi
cd_lane_drain_capacity_ok=0
cd_lane_drain_labels_ok=0
if grep -Eq "^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$" /home/wooo/awoooi-cd-lane-drain/config.yaml 2>/dev/null; then
@@ -373,15 +391,22 @@ if [ "$cd_lane_drain_active" != "active" ] \
&& { [ "$cd_lane_drain_load" = "not-found" ] || { [ "$cd_lane_drain_load" = "masked" ] && [ "$cd_lane_drain_unitfile" = "masked" ]; }; }; then
cd_lane_drain_ok=1
cd_lane_drain_mode=failclosed
elif [ "$cd_lane_drain_active" = "active" ] \
&& [ "$cd_lane_drain_capacity_ok" = "1" ] \
&& [ "$cd_lane_drain_labels_ok" = "1" ] \
&& [ "$cd_lane_drain_binary_elf" = "1" ] \
&& [ "$cd_lane_drain_limits_ok" = "1" ]; then
cd_lane_drain_ok=1
cd_lane_drain_mode=controlled_open
fi
echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok"
echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf limits=$cd_lane_drain_limits_ok process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok"
cd_lane_root_restore_left=unknown
if sudo -n true >/dev/null 2>&1; then
cd_lane_root_restore_left=$(sudo -n find /root -maxdepth 1 -type d \( -name "awoooi-cd-lane-disabled-*" -o -name "awoooi-cd-lane-drain-disabled-*" \) -print 2>/dev/null | wc -l | tr -d " ")
fi
echo "CD_LANE_ROOT_RESTORE_SOURCES left=$cd_lane_root_restore_left"
cd_lane_guard_ok=0
if [ "$cd_lane_ok" = "1" ] && [ "$cd_lane_drain_ok" = "1" ] && [ "$cd_lane_root_restore_left" = "0" ]; then
if { [ "$cd_lane_ok" = "1" ] || [ "$cd_lane_drain_ok" = "1" ]; } && [ "$cd_lane_root_restore_left" = "0" ]; then
cd_lane_guard_ok=1
fi
echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok"
@@ -389,7 +414,7 @@ echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok"
direct_runner_count=$(pgrep -f "^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" 2>/dev/null | wc -l | tr -d " ")
echo "RUNNER_DIRECT_PROCESS_COUNT $direct_runner_count"
[ "$direct_runner_count" = "0" ] || bad=1
for p in /home/wooo/awoooi-cd-lane/awoooi_cd_lane /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
for p in /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
kind=$(file -b "$p" 2>/dev/null || echo missing)
echo "RUNNER_FAILCLOSED_BINARY $p kind=$kind"
echo "$kind" | grep -qi "ELF" && bad=1

View File

@@ -585,6 +585,21 @@ cd_lane_drain_load=$(systemctl show awoooi-cd-lane-drain.service -p LoadState --
cd_lane_drain_unitfile=$(systemctl show awoooi-cd-lane-drain.service -p UnitFileState --value 2>/dev/null || true)
cd_lane_drain_active=$(systemctl show awoooi-cd-lane-drain.service -p ActiveState --value 2>/dev/null || true)
cd_lane_drain_mainpid=$(systemctl show awoooi-cd-lane-drain.service -p MainPID --value 2>/dev/null || true)
cd_lane_drain_cpu_accounting=$(systemctl show awoooi-cd-lane-drain.service -p CPUAccounting --value 2>/dev/null || true)
cd_lane_drain_cpu_quota=$(systemctl show awoooi-cd-lane-drain.service -p CPUQuotaPerSecUSec --value 2>/dev/null || true)
cd_lane_drain_memory_accounting=$(systemctl show awoooi-cd-lane-drain.service -p MemoryAccounting --value 2>/dev/null || true)
cd_lane_drain_memory_max=$(systemctl show awoooi-cd-lane-drain.service -p MemoryMax --value 2>/dev/null || true)
cd_lane_drain_tasks_accounting=$(systemctl show awoooi-cd-lane-drain.service -p TasksAccounting --value 2>/dev/null || true)
cd_lane_drain_tasks_max=$(systemctl show awoooi-cd-lane-drain.service -p TasksMax --value 2>/dev/null || true)
cd_lane_drain_limits_ok=0
if [ "$cd_lane_drain_cpu_accounting" = "yes" ] \
&& [ -n "$cd_lane_drain_cpu_quota" ] && [ "$cd_lane_drain_cpu_quota" != "infinity" ] \
&& [ "$cd_lane_drain_memory_accounting" = "yes" ] \
&& [ -n "$cd_lane_drain_memory_max" ] && [ "$cd_lane_drain_memory_max" != "infinity" ] \
&& [ "$cd_lane_drain_tasks_accounting" = "yes" ] \
&& [ -n "$cd_lane_drain_tasks_max" ] && [ "$cd_lane_drain_tasks_max" != "infinity" ]; then
cd_lane_drain_limits_ok=1
fi
cd_lane_drain_capacity_ok=0
cd_lane_drain_labels_ok=0
if grep -Eq "^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$" /home/wooo/awoooi-cd-lane-drain/config.yaml 2>/dev/null; then
@@ -607,21 +622,28 @@ if [ "$cd_lane_drain_active" != "active" ] \
&& { [ "$cd_lane_drain_load" = "not-found" ] || { [ "$cd_lane_drain_load" = "masked" ] && [ "$cd_lane_drain_unitfile" = "masked" ]; }; }; then
cd_lane_drain_ok=1
cd_lane_drain_mode=failclosed
elif [ "$cd_lane_drain_active" = "active" ] \
&& [ "$cd_lane_drain_capacity_ok" = "1" ] \
&& [ "$cd_lane_drain_labels_ok" = "1" ] \
&& [ "$cd_lane_drain_binary_elf" = "1" ] \
&& [ "$cd_lane_drain_limits_ok" = "1" ]; then
cd_lane_drain_ok=1
cd_lane_drain_mode=controlled_open
fi
echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active mainpid=$cd_lane_drain_mainpid capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok"
echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active mainpid=$cd_lane_drain_mainpid capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf limits=$cd_lane_drain_limits_ok process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok"
cd_lane_root_restore_left=unknown
if sudo -n true >/dev/null 2>&1; then
cd_lane_root_restore_left=$(sudo -n find /root -maxdepth 1 -type d \( -name "awoooi-cd-lane-disabled-*" -o -name "awoooi-cd-lane-drain-disabled-*" \) -print 2>/dev/null | wc -l | tr -d " ")
fi
echo "CD_LANE_ROOT_RESTORE_SOURCES left=$cd_lane_root_restore_left"
cd_lane_guard_ok=0
if [ "$cd_lane_ok" = "1" ] && [ "$cd_lane_drain_ok" = "1" ] && [ "$cd_lane_root_restore_left" = "0" ]; then
if { [ "$cd_lane_ok" = "1" ] || [ "$cd_lane_drain_ok" = "1" ]; } && [ "$cd_lane_root_restore_left" = "0" ]; then
cd_lane_guard_ok=1
fi
echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok"
direct_runner_count=$(pgrep -f "^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" 2>/dev/null | wc -l | tr -d " ")
echo "RUNNER_DIRECT_PROCESS_COUNT $direct_runner_count"
for p in /home/wooo/awoooi-cd-lane/awoooi_cd_lane /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
for p in /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
kind=$(file -b "$p" 2>/dev/null || echo missing)
echo "RUNNER_FAILCLOSED_BINARY $p kind=$kind"
echo "$kind" | grep -qi "ELF" && echo "RUNNER_FAILCLOSED_BINARY_ELF $p"
@@ -629,9 +651,9 @@ done
HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1 HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=0 /usr/local/bin/awoooi-wait-host-web-build-pressure.sh
echo "RUNNER_PRESSURE_GATE_RC $?"
' >"$runner_tmp" 2>&1; then
ok "110 runner fail-closed readback succeeded"
ok "110 controlled runner readback succeeded"
else
blocked "110 runner fail-closed readback failed"
blocked "110 controlled runner readback failed"
fi
cat "$runner_tmp"
if awk '$1 == "RUNNER_FAILCLOSED_UNIT" && $NF != "ok=1" {bad=1} END {exit bad}' "$runner_tmp"; then
@@ -639,7 +661,7 @@ if awk '$1 == "RUNNER_FAILCLOSED_UNIT" && $NF != "ok=1" {bad=1} END {exit bad}'
else
blocked "110 legacy direct/Gitea runner units are not fail-closed"
fi
grep -q "CD_LANE_GUARDRAILS_OK 1" "$runner_tmp" && ok "110 runner/CD lane fail-closed guardrails complete" || blocked "110 runner/CD lane fail-closed guardrails incomplete"
grep -q "CD_LANE_GUARDRAILS_OK 1" "$runner_tmp" && ok "110 controlled cd-lane is safe-open/drained or fail-closed" || blocked "110 controlled cd-lane guardrails incomplete"
grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" "$runner_tmp" && ok "110 legacy direct runner process count is zero" || blocked "110 legacy direct runner process detected"
grep -q "RUNNER_FAILCLOSED_BINARY_ELF" "$runner_tmp" && blocked "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing"
grep -q "RUNNER_PRESSURE_GATE_RC 0" "$runner_tmp" && ok "110 host pressure gate returned 0" || blocked "110 host pressure gate is blocking"