fix(ops): bound CD DB probes and restore SigNoz collector

2026-07-15 02:29:51 +08:00
parent 27736add14
commit bfb2d2d040
11 changed files with 442 additions and 19 deletions
--- a/.gitea/workflows/cd.yaml
+++ b/.gitea/workflows/cd.yaml
@@ -3868,6 +3868,25 @@ jobs:
          API_DESIRED_REPLICA_COUNT=$($KUBECTL get deployment awoooi-api \
            -n awoooi-prod -o jsonpath='{.spec.replicas}')

+          API_HTTP_PROBE_SAFETY_RESERVE=4
+          API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE=$((
+            API_DB_AVAILABLE_CONNECTION_HEADROOM - API_HTTP_PROBE_SAFETY_RESERVE
+          ))
+          if [ "$API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE" -lt 0 ]; then
+            API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE=0
+          fi
+          API_HTTP_CONCURRENCY_WORKERS=$API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE
+          if [ "$API_HTTP_CONCURRENCY_WORKERS" -gt 4 ]; then
+            API_HTTP_CONCURRENCY_WORKERS=4
+          fi
+          API_HTTP_CONCURRENCY_VERIFIED=false
+          API_HTTP_PROBE_SKIP_REASON=""
+          if [ "$CONNECTION_BUDGET_VERIFIED" != "true" ]; then
+            API_HTTP_PROBE_SKIP_REASON=connection_budget_not_verified
+          elif [ "$API_HTTP_CONCURRENCY_WORKERS" -lt 1 ]; then
+            API_HTTP_PROBE_SKIP_REASON=connection_headroom_reserved_for_runtime
+          else
+            export API_HTTP_CONCURRENCY_WORKERS
          API_HTTP_CONCURRENCY_VERIFIED=$(python3 - <<'PY'
          import concurrent.futures
          import os
@@ -3904,11 +3923,14 @@ jobs:
              "API_PUBLIC_READINESS_DELAY_SECONDS", 5, 0, 30
          )
          concurrency_attempts = bounded_int(
-              "API_HTTP_CONCURRENCY_ATTEMPTS", 4, 1, 8
+              "API_HTTP_CONCURRENCY_ATTEMPTS", 1, 1, 1
          )
          concurrency_delay_seconds = bounded_int(
              "API_HTTP_CONCURRENCY_DELAY_SECONDS", 5, 0, 30
          )
+          probe_worker_count = bounded_int(
+              "API_HTTP_CONCURRENCY_WORKERS", 4, 1, 4
+          )


          def fetch_status(url: str, timeout: int) -> tuple[int | None, str]:
@@ -3955,18 +3977,18 @@ jobs:


          def fetch(url: str) -> int:
-              status, error_type = fetch_status(url, timeout=20)
+              status, error_type = fetch_status(url, timeout=5)
              if status is None:
                  raise RuntimeError(error_type)
              return status


          last_result = "unavailable"
-          concurrency_urls = (db_probe_url,) * 8
+          concurrency_urls = (db_probe_url,) * probe_worker_count
          for attempt in range(1, concurrency_attempts + 1):
              try:
                  with concurrent.futures.ThreadPoolExecutor(
-                      max_workers=8
+                      max_workers=probe_worker_count
                  ) as executor:
                      statuses = list(executor.map(fetch, concurrency_urls))
              except Exception as exc:
@@ -4000,6 +4022,10 @@ jobs:
              print("false")
          PY
          )
+          fi
+          if [ -n "$API_HTTP_PROBE_SKIP_REASON" ]; then
+            echo "API database concurrency probe skipped: ${API_HTTP_PROBE_SKIP_REASON} headroom=${API_DB_AVAILABLE_CONNECTION_HEADROOM} reserve=${API_HTTP_PROBE_SAFETY_RESERVE}" >&2
+          fi
          if [ "$API_HTTP_CONCURRENCY_VERIFIED" != "true" ]; then
            CONNECTION_BUDGET_VERIFIED=false
          fi
@@ -4100,6 +4126,9 @@ jobs:
            --from-literal=broker_sequence_privilege_gap_count="$BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" \
            --from-literal=representative_db_preflights_verified="$REPRESENTATIVE_DB_PREFLIGHTS_VERIFIED" \
            --from-literal=api_http_concurrency_probe_passed="$API_HTTP_CONCURRENCY_VERIFIED" \
+            --from-literal=api_http_concurrency_probe_workers="$API_HTTP_CONCURRENCY_WORKERS" \
+            --from-literal=api_http_concurrency_probe_safety_reserve="$API_HTTP_PROBE_SAFETY_RESERVE" \
+            --from-literal=api_http_concurrency_probe_skip_reason="$API_HTTP_PROBE_SKIP_REASON" \
            --from-literal=api_http_probe_restart_free="$API_PROBE_RESTART_FREE" \
            --from-literal=api_restart_count_before="$API_RESTART_COUNT_BEFORE" \
            --from-literal=api_restart_count_after="$API_RESTART_COUNT_AFTER" \
--- a/apps/api/tests/test_executor_network_policy_boundary.py
+++ b/apps/api/tests/test_executor_network_policy_boundary.py
@@ -208,7 +208,10 @@ def test_cd_concurrency_probe_degrades_to_terminal_false_receipt() -> None:
    assert "import sys" in probe
    assert "concurrency_urls =" in probe
    assert "ThreadPoolExecutor(" in probe
-    assert "max_workers=8" in probe
+    assert "probe_worker_count" in probe
+    assert "max_workers=probe_worker_count" in probe
+    assert "max_workers=8" not in probe
+    assert "API_HTTP_CONCURRENCY_WORKERS" in probe
    assert "for attempt in range(1, concurrency_attempts + 1):" in probe
    assert "if attempt < concurrency_attempts:" in probe
    assert "transient_http_statuses" in probe
@@ -220,6 +223,9 @@ def test_cd_concurrency_probe_degrades_to_terminal_false_receipt() -> None:
        'api_http_concurrency_probe_passed="$API_HTTP_CONCURRENCY_VERIFIED"'
        in workflow
    )
+    assert "API_HTTP_PROBE_SAFETY_RESERVE=4" in workflow
+    assert "connection_budget_not_verified" in workflow
+    assert 'api_http_concurrency_probe_safety_reserve="$API_HTTP_PROBE_SAFETY_RESERVE"' in workflow
    assert 'connection_budget_verified="$CONNECTION_BUDGET_VERIFIED"' in workflow
    assert "workload DB identity/budget remains in progress" in workflow

--- a/apps/api/tests/test_signal_worker_ansible_executor_binding.py
+++ b/apps/api/tests/test_signal_worker_ansible_executor_binding.py
@@ -272,7 +272,9 @@ def test_cd_prunes_and_verifies_api_executor_boundary_in_production() -> None:
    assert "AWOOOI_DB_REQUIRED_CONNECTION_BUDGET" in workflow
    assert "api_http_concurrency_probe_passed" in workflow
    assert "ThreadPoolExecutor" in workflow
-    assert "max_workers=8" in workflow
+    assert "max_workers=probe_worker_count" in workflow
+    assert "max_workers=8" not in workflow
+    assert "api_http_concurrency_probe_safety_reserve" in workflow
    assert "WORKLOAD_DB_ROLLBACK_FILE" in workflow
    assert "restore_workload_database_projection" in workflow
    assert "awoooi-workload-db-identity-bootstrap.sh" in workflow
--- a/apps/api/tests/test_workload_database_identity_projection.py
+++ b/apps/api/tests/test_workload_database_identity_projection.py
@@ -280,13 +280,22 @@ def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None:
    assert "workload DB identity attempts exhausted before receipt write" in workflow
    assert "API_PUBLIC_READINESS_ATTEMPTS" in workflow
    assert "API_HTTP_CONCURRENCY_ATTEMPTS" in workflow
+    assert "API_HTTP_PROBE_SAFETY_RESERVE=4" in workflow
+    assert "API_HTTP_CONCURRENCY_WORKERS" in workflow
    assert "API sequential readiness permanent HTTP status" in workflow
    assert "api/v1/health/ready" in workflow
    assert "events/dossier/coverage?project_id=awoooi&limit=1" not in workflow
    assert "api/v1/platform/cicd/events" in workflow
    assert "?project_id=awoooi&limit=1" in workflow
-    assert "concurrency_urls = (db_probe_url,) * 8" in workflow
+    assert "concurrency_urls = (db_probe_url,) * probe_worker_count" in workflow
+    assert "max_workers=probe_worker_count" in workflow
+    assert "concurrency_urls = (db_probe_url,) * 8" not in workflow
+    assert "max_workers=8" not in workflow
    assert "concurrency_urls = urls * 6" not in workflow
+    assert "connection_budget_not_verified" in workflow
+    assert "connection_headroom_reserved_for_runtime" in workflow
+    assert "api_http_concurrency_probe_safety_reserve" in workflow
+    assert "api_http_concurrency_probe_skip_reason" in workflow
    assert "API_RESTART_COUNT_BEFORE=$(api_restart_total)" in workflow
    assert "API_RESTART_COUNT_AFTER=$(api_restart_total)" in workflow
    assert '"$API_PROBE_RESTART_FREE" != "true"' in workflow
@@ -353,7 +362,7 @@ def test_cd_api_readiness_retries_http_503_then_runs_all_200_concurrency(
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
 ) -> None:
-    responses = [503, 200, *([200] * 8)]
+    responses = [503, 200, *([200] * 4)]
    calls: list[str] = []

    def fake_urlopen(url: str, *, timeout: int) -> _FakeHTTPResponse:
@@ -367,8 +376,9 @@ def test_cd_api_readiness_retries_http_503_then_runs_all_200_concurrency(
    monkeypatch.setattr(time, "sleep", lambda _seconds: None)
    monkeypatch.setenv("API_PUBLIC_READINESS_ATTEMPTS", "3")
    monkeypatch.setenv("API_PUBLIC_READINESS_DELAY_SECONDS", "0")
-    monkeypatch.setenv("API_HTTP_CONCURRENCY_ATTEMPTS", "2")
+    monkeypatch.setenv("API_HTTP_CONCURRENCY_ATTEMPTS", "1")
    monkeypatch.setenv("API_HTTP_CONCURRENCY_DELAY_SECONDS", "0")
+    monkeypatch.setenv("API_HTTP_CONCURRENCY_WORKERS", "4")

    exec(
        compile(
@@ -382,13 +392,43 @@ def test_cd_api_readiness_retries_http_503_then_runs_all_200_concurrency(
    output = capsys.readouterr()
    assert output.out.strip() == "true"
    assert "api_sequential_readiness=ready;attempt=2" in output.err
-    assert len(calls) == 10
+    assert len(calls) == 6
    assert calls[0].startswith("https://awoooi.wooo.work/api/v1/health/ready|")
    assert calls[1].startswith("https://awoooi.wooo.work/api/v1/health/ready|")
    assert all("platform/cicd/events" in call for call in calls[2:])
    assert responses == []


+def test_cd_api_concurrency_workers_are_capped_at_four(
+    monkeypatch: pytest.MonkeyPatch,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    responses = [200, *([200] * 4)]
+    calls: list[str] = []
+
+    def fake_urlopen(url: str, *, timeout: int) -> _FakeHTTPResponse:
+        calls.append(f"{url}|{timeout}")
+        return _FakeHTTPResponse(responses.pop(0))
+
+    monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
+    monkeypatch.setenv("API_HTTP_CONCURRENCY_WORKERS", "8")
+
+    exec(
+        compile(
+            _api_http_readiness_and_concurrency_python(),
+            "<cd-api-http-verifier>",
+            "exec",
+        ),
+        {},
+    )
+
+    output = capsys.readouterr()
+    assert output.out.strip() == "true"
+    assert len(calls) == 5
+    assert all("platform/cicd/events" in call for call in calls[1:])
+    assert responses == []
+
+
 def test_cd_api_readiness_exhaustion_stops_before_green_receipt_write(
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
--- a/docs/LOGBOOK.md
+++ b/docs/LOGBOOK.md
@@ -1,3 +1,19 @@
+## 2026-07-15 — P0-OBS-002 post-closure runtime regression 與 guard 修復
+
+**runtime evidence**：
+- gRPC bridge run `P0-OBS-002-20260715T013800+0800` 先通過完整 600 秒 verifier；後續 revision `2004` 於 02:11:22 起發生 liveness/readiness timeout，最低 `0/2 Ready`，兩 pod 各累積 restart `4`。bridge 維持同 PID、NRestarts `0`，因此不執行 bridge rollback。
+- API log 捕獲 `too many connections for role awoooi_api_runtime`；既有 CD verifier 在 role limit `12`、最低 headroom `8` 的條件下仍固定發出 8 個 DB-backed requests，存在把 `4 + 8` 打滿 `12/12` 的結構性風險。已完成的 #5119 receipt 本身顯示 probe 當時 active=`0`、restart `0->0`，故只把此問題列為高信心 structural amplifier，不誤報為唯一直接觸發。
+- 每日 `backup-all` 02:12 執行 SignOz 備份時停止 collector；volume backup 立即失敗，總備份僅 `3/9` 成功且 collector 未恢復。`P0-OBS-002-static-recovery-20260715T021600+0800` 只恢復 stateless collector，4317/4318、OTLP HTTP `200`、`.120/.121` gRPC TCP 全數通過，stateful services touched=`0`。
+
+**source/test evidence**：
+- CD probe 改為先檢查 connection budget、保留 4 條安全餘量、最多 4 workers、單輪、每 request 5 秒；新增 workers/reserve/skip reason machine-readable receipt fields。
+- `backup-signoz.sh` 在 stop 前先 arm restore，加入 `EXIT/HUP/INT/TERM` trap、防重入 cleanup 與首次 restart 失敗後的有界 EXIT retry。
+- 新增 `ops/signoz/p0-obs-002-post-closure-regression.yaml`，分離 API probe starvation、collector backup interruption、controlled recovery、source candidate 與 scope boundary。
+
+**仍維持**：
+- API 最近一次讀回雖恢復 `2/2`，但 restart 已是 `4/4` 且較新 Gitea CD 仍連續進行，尚未完成 10 分鐘 cooldown；program 仍是 `partial_degraded`。
+- 兩個 source guard 尚未由 main/CD/runtime readback 證明；filelog normalization、organization、direct ingress policy、HTTP bridge supervision 與 producer promotion 仍未完成。
+
 ## 2026-07-15 — P0-OBS-002 production OTLP/gRPC runtime closure

 **完成內容**：
--- a/ops/signoz/organization-canonical-route-migration.yaml
+++ b/ops/signoz/organization-canonical-route-migration.yaml
@@ -152,6 +152,13 @@ runtime_observations:
    real_application_trace_freshness_inferred: false
    producer_queue_failure_gate_inferred: false
    promotion_authorized: false
+  post_closure_regression:
+    snapshot: ops/signoz/p0-obs-002-post-closure-regression.yaml
+    grpc_bridge_rollback_indicated: false
+    canonical_collector_recovered: true
+    api_runtime_cooldown_closed: false
+    cd_probe_safety_fix_deployed: false
+    backup_collector_restore_guard_deployed: false

 terminal:
  status: partial_degraded
@@ -160,4 +167,7 @@ terminal:
    - direct_ingress_policy_not_closed
    - http_bridge_not_supervised
    - filelog_normalization_degraded
+    - api_runtime_cooldown_pending
+    - cd_probe_safety_fix_not_deployed
+    - backup_collector_restore_guard_not_deployed
    - wave_b_not_authorized
--- a/ops/signoz/p0-obs-002-post-closure-regression.yaml
+++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml
@@ -0,0 +1,93 @@
+schema: awoooi_signoz_post_closure_regression_v1
+work_item_id: P0-OBS-002
+trace_id: P0-OBS-002
+observed_at: "2026-07-15T02:20:00+08:00"
+
+baseline:
+  grpc_runtime_run_id: P0-OBS-002-20260715T013800+0800
+  verifier_window_seconds: 600
+  verifier_terminal: pass
+  api_restart_delta: 0
+  trace_count: 1098
+  metric_sample_count: 60180
+
+regressions:
+  api_runtime:
+    first_probe_timeout_at: "2026-07-15T02:11:22+08:00"
+    deployed_source_sha: 31de3a709815f33a11ba6de6e78b771b7160a245
+    deploy_marker_sha: f331ae87c27f42ecc96d3cd4e86f30499b5fbfba
+    image_digest: sha256:a99b3b276def966c03be56f7e8e15feb69d5e2029c8d4ec11dd4b839709351b8
+    minimum_ready_replicas_observed: 0
+    expected_replicas: 2
+    restart_count_per_pod_at_recovery: 4
+    last_observed_ready_replicas: 2
+    last_observed_cooldown_closed: false
+    primary_classification: db_connection_budget_and_request_pressure
+    structural_risk_confidence: 0.98
+    direct_application_commit_regression_confidence: 0.12
+    evidence:
+      - too_many_connections_for_awoooi_api_runtime
+      - constant_response_liveness_and_readiness_timed_out
+      - no_oom_or_cgroup_cpu_throttle
+      - fixed_eight_worker_cd_db_probe_can_exhaust_role_limit_twelve
+  canonical_collector:
+    stopped_at: "2026-07-15T02:12:15+08:00"
+    exit_code: 0
+    oom_killed: false
+    trigger: daily_backup_signoz_collector_stop
+    backup_terminal: failed
+    backup_services_succeeded: 3
+    backup_services_total: 9
+    automatic_restore_observed: false
+
+controlled_recovery:
+  run_id: P0-OBS-002-static-recovery-20260715T021600+0800
+  collector_only: true
+  stateful_services_touched: 0
+  collector_running: true
+  grpc_4317_listening: true
+  http_4318_listening: true
+  empty_otlp_http_status: 200
+  grpc_bridge_enabled_active: true
+  grpc_bridge_restart_count: 0
+  source_nodes_tcp_verified:
+    - 192.168.0.120
+    - 192.168.0.121
+
+source_candidates:
+  cd_probe:
+    path: .gitea/workflows/cd.yaml
+    connection_budget_precheck_required: true
+    max_workers: 4
+    max_attempts: 1
+    request_timeout_seconds: 5
+    safety_reserve_connections: 4
+    receipt_fields_added: true
+    deployment_status: pending
+  backup_restore_guard:
+    path: scripts/backup/backup-signoz.sh
+    trap_exit_hup_int_term: true
+    restore_armed_before_stop: true
+    bounded_exit_retry: true
+    deployment_status: pending
+
+scope_boundaries:
+  grpc_bridge_rollback_indicated: false
+  api_manual_rollback_performed: false
+  firewall_changed: false
+  database_changed: false
+  stateful_volume_read: false
+  secret_read: false
+  github_used: false
+
+terminal:
+  status: partial_degraded
+  blockers:
+    - api_runtime_cooldown_pending
+    - cd_probe_safety_fix_not_deployed
+    - backup_collector_restore_guard_not_deployed
+    - newer_main_cd_nonterminal
+  safe_next_action: >-
+    Merge current Gitea main, validate and deploy the bounded CD probe and backup
+    restore guard, then require a fresh CD terminal plus ten-minute API and OTLP
+    runtime verifier with zero restart and exporter-error deltas.
--- a/scripts/backup/backup-signoz.sh
+++ b/scripts/backup/backup-signoz.sh
@@ -1,10 +1,11 @@
 #!/bin/bash
 # =============================================================================
 # WOOO AIOps - SignOz 備份腳本 (ClickHouse + SQLite)
-# 版本: 1.1.0
+# 版本: 1.2.0
 # 建立日期: 2026-04-05
 # 2026-04-05 Claude Code: 新增 SignOz 分散式追蹤備份 — 首席架構師備份審計
 # 2026-04-05 Claude Code: v1.1 修正 tar pipeline exit code 處理 + || true
+# 2026-07-15 Codex: v1.2 確保任何 exit/signal/failure 都會受控還原 OTEL Collector
 # =============================================================================

 set -euo pipefail
@@ -14,11 +15,42 @@ source "$(dirname "$0")/common.sh"
 SERVICE="signoz"
 LOCAL_REPO="${BACKUP_BASE}/signoz"
 DUMP_DIR="/tmp/signoz-backup-$$"
+COLLECTOR_RESTORE_PENDING=0
+CLEANUP_COMPLETE=0
+
+restore_collector() {
+    if [ "${COLLECTOR_RESTORE_PENDING}" -ne 1 ]; then
+        return 0
+    fi
+
+    log_info "重啟 signoz-otel-collector..."
+    if docker start signoz-otel-collector 2>/dev/null; then
+        COLLECTOR_RESTORE_PENDING=0
+        return 0
+    fi
+
+    log_warn "signoz-otel-collector 重啟失敗"
+    return 1
+}

 cleanup() {
-    # 確保 collector 已重啟
-    docker start signoz-otel-collector 2>/dev/null || true
+    local exit_code=$?
+
+    if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then
+        return "${exit_code}"
+    fi
+    CLEANUP_COMPLETE=1
+
+    # cleanup 期間忽略重複 signal，避免還原與刪除流程重入。
+    trap '' HUP INT TERM
+    restore_collector || true
    rm -rf "${DUMP_DIR}"
+    return "${exit_code}"
+}
+
+on_signal() {
+    local signal_number="$1"
+    exit "$((128 + signal_number))"
 }

 backup_volume() {
@@ -44,6 +76,12 @@ backup_volume() {

 main() {
    local start_time=$(date +%s)
+
+    trap cleanup EXIT
+    trap 'on_signal 1' HUP
+    trap 'on_signal 2' INT
+    trap 'on_signal 15' TERM
+
    log_info "========== 開始 SignOz 備份 =========="
    mkdir -p "${DUMP_DIR}"

@@ -51,13 +89,14 @@ main() {

    # Step 1: 停止 OTEL Collector 確保數據一致性
    log_info "暫停 signoz-otel-collector 以確保數據一致性..."
+    # 先 arm 還原再停止，消除 signal 發生在 stop 與狀態記錄之間的空窗。
+    COLLECTOR_RESTORE_PENDING=1
    docker stop signoz-otel-collector 2>/dev/null || log_warn "signoz-otel-collector 未在運行，繼續"
    docker stop signoz-telemetrystore-migrator 2>/dev/null || true

    # Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積)
    backup_volume "signoz-clickhouse" "${DUMP_DIR}/clickhouse_${timestamp}.tar.gz" "--exclude=/data/tmp" || {
        log_error "ClickHouse volume 備份失敗"
-        cleanup
        notify_clawbot "failed" "${SERVICE}" "SignOz ClickHouse 備份失敗"
        exit 1
    }
@@ -68,15 +107,14 @@ main() {
    }

    # Step 4: 重啟 Collector
-    log_info "重啟 signoz-otel-collector..."
-    docker start signoz-otel-collector 2>/dev/null || log_warn "signoz-otel-collector 重啟失敗"
+    # 若此處失敗，保留 pending 讓 EXIT cleanup 再做一次有界還原。
+    restore_collector || true

    # Step 5: 初始化 Restic 倉庫
    if [ ! -d "${LOCAL_REPO}/data" ]; then
        log_info "初始化 Restic 倉庫: ${LOCAL_REPO}"
        restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
            log_error "Restic 倉庫初始化失敗"
-            rm -rf "${DUMP_DIR}"
            exit 1
        }
    fi
@@ -92,8 +130,6 @@ main() {
    # Step 7: GFS 清理
    cleanup_old_backups "${LOCAL_REPO}"

-    rm -rf "${DUMP_DIR}"
-
    local end_time=$(date +%s)
    local duration=$((end_time - start_time))
    log_success "========== SignOz 備份完成 (${duration}s) =========="
--- a/scripts/backup/tests/test_backup_signoz_collector_restore.py
+++ b/scripts/backup/tests/test_backup_signoz_collector_restore.py
@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh"
+
+
+def _write_executable(path: Path, text: str) -> None:
+    path.write_text(text, encoding="utf-8")
+    path.chmod(0o755)
+
+
+def _run_backup(
+    tmp_path: Path,
+    *,
+    docker_run_mode: str = "success",
+    docker_start_fail_once: bool = False,
+    restic_backup_status: int = 0,
+) -> tuple[subprocess.CompletedProcess[str], list[str], Path]:
+    harness = tmp_path / "backup"
+    fake_bin = tmp_path / "bin"
+    backup_base = tmp_path / "repository"
+    docker_log = tmp_path / "docker.log"
+    harness.mkdir()
+    fake_bin.mkdir()
+    (backup_base / "signoz" / "data").mkdir(parents=True)
+    shutil.copy2(SCRIPT, harness / SCRIPT.name)
+
+    (harness / "common.sh").write_text(
+        """\
+BACKUP_BASE="${TEST_BACKUP_BASE:?}"
+RESTIC_PASSWORD_FILE="${TEST_BACKUP_BASE}/password"
+log_info() { :; }
+log_warn() { :; }
+log_error() { :; }
+log_success() { :; }
+notify_clawbot() { :; }
+build_tags() { :; }
+cleanup_old_backups() { :; }
+""",
+        encoding="utf-8",
+    )
+
+    _write_executable(
+        fake_bin / "docker",
+        """\
+#!/bin/bash
+set -eu
+printf '%s\n' "$*" >> "${TEST_DOCKER_LOG:?}"
+case "${1:-}" in
+    stop)
+        exit 0
+        ;;
+    start)
+        if [ "${FAKE_DOCKER_START_FAIL_ONCE:-0}" = "1" ] && \
+           [ ! -e "${TEST_DOCKER_START_MARKER:?}" ]; then
+            : > "${TEST_DOCKER_START_MARKER}"
+            exit 1
+        fi
+        exit 0
+        ;;
+    run)
+        if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "signal" ]; then
+            kill -TERM "${PPID}"
+            sleep 0.1
+            exit 0
+        fi
+        if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "empty" ]; then
+            exit 0
+        fi
+        printf 'archive-data\n'
+        ;;
+esac
+""",
+    )
+    _write_executable(
+        fake_bin / "restic",
+        """\
+#!/bin/bash
+set -eu
+case " $* " in
+    *" backup "*) exit "${FAKE_RESTIC_BACKUP_STATUS:-0}" ;;
+    *" snapshots "*) printf '[{"short_id":"test123"}]\n' ;;
+esac
+""",
+    )
+    _write_executable(
+        fake_bin / "du",
+        """\
+#!/bin/bash
+printf '4K\t%s\n' "${2:-unknown}"
+""",
+    )
+
+    env = {
+        **os.environ,
+        "PATH": f"{fake_bin}:{os.environ['PATH']}",
+        "TEST_BACKUP_BASE": str(backup_base),
+        "TEST_DOCKER_LOG": str(docker_log),
+        "TEST_DOCKER_START_MARKER": str(tmp_path / "docker-start-failed-once"),
+        "FAKE_DOCKER_RUN_MODE": docker_run_mode,
+        "FAKE_DOCKER_START_FAIL_ONCE": "1" if docker_start_fail_once else "0",
+        "FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status),
+    }
+    process = subprocess.Popen(
+        ["bash", str(harness / SCRIPT.name)],
+        env=env,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    stdout, stderr = process.communicate(timeout=10)
+    result = subprocess.CompletedProcess(
+        process.args,
+        process.returncode,
+        stdout,
+        stderr,
+    )
+    docker_calls = docker_log.read_text(encoding="utf-8").splitlines()
+    return result, docker_calls, Path(f"/tmp/signoz-backup-{process.pid}")
+
+
+def _collector_start_count(docker_calls: list[str]) -> int:
+    return docker_calls.count("start signoz-otel-collector")
+
+
+def test_collector_is_restored_once_when_backup_receives_term(tmp_path: Path) -> None:
+    result, docker_calls, dump_dir = _run_backup(
+        tmp_path,
+        docker_run_mode="signal",
+    )
+
+    assert result.returncode != 0
+    assert "stop signoz-otel-collector" in docker_calls
+    assert _collector_start_count(docker_calls) == 1
+    assert not dump_dir.exists()
+
+
+def test_collector_is_restored_once_when_clickhouse_backup_fails(tmp_path: Path) -> None:
+    result, docker_calls, dump_dir = _run_backup(
+        tmp_path,
+        docker_run_mode="empty",
+    )
+
+    assert result.returncode == 1
+    assert _collector_start_count(docker_calls) == 1
+    assert not dump_dir.exists()
+
+
+def test_exit_cleanup_does_not_restart_an_already_restored_collector(
+    tmp_path: Path,
+) -> None:
+    result, docker_calls, dump_dir = _run_backup(
+        tmp_path,
+        restic_backup_status=17,
+    )
+
+    assert result.returncode == 17
+    assert _collector_start_count(docker_calls) == 1
+    assert not dump_dir.exists()
+
+
+def test_exit_cleanup_retries_only_after_explicit_restore_fails(
+    tmp_path: Path,
+) -> None:
+    result, docker_calls, dump_dir = _run_backup(
+        tmp_path,
+        docker_start_fail_once=True,
+        restic_backup_status=17,
+    )
+
+    assert result.returncode == 17
+    assert _collector_start_count(docker_calls) == 2
+    assert not dump_dir.exists()
--- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py
+++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py
@@ -22,6 +22,9 @@ EXPECTED_BLOCKERS = [
    "direct_ingress_policy_not_closed",
    "http_bridge_not_supervised",
    "filelog_normalization_degraded",
+    "api_runtime_cooldown_pending",
+    "cd_probe_safety_fix_not_deployed",
+    "backup_collector_restore_guard_not_deployed",
    "wave_b_not_authorized",
 ]

--- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
+++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
@@ -23,6 +23,7 @@ GRPC_BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh"
 GRPC_UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service"
 GRPC_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"
 GRPC_RUNTIME_RECEIPT = ROOT / "ops/signoz/p0-obs-002-grpc-runtime-receipt.yaml"
+POST_CLOSURE_REGRESSION = ROOT / "ops/signoz/p0-obs-002-post-closure-regression.yaml"


 def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
@@ -266,6 +267,14 @@ def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate(
    assert receipt["completion"]["grpc_runtime_asset"] == "closed"
    assert receipt["completion"]["program"] == "partial_degraded"

+    regression = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+    assert regression["baseline"]["verifier_terminal"] == "pass"
+    assert regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is False
+    assert regression["controlled_recovery"]["stateful_services_touched"] == 0
+    assert regression["controlled_recovery"]["grpc_4317_listening"] is True
+    assert regression["scope_boundaries"]["grpc_bridge_rollback_indicated"] is False
+    assert regression["terminal"]["status"] == "partial_degraded"
+

 def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None:
    env = os.environ.copy()