From 867e0e73df7c58dee620c759a5dcaab6f3a2832e Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 21 May 2026 12:34:51 +0800 Subject: [PATCH] ci(awooop): add dedicated source link canary --- .gitea/workflows/e2e-health.yaml | 7 +- .../tests/test_alert_chain_smoke_metric.py | 56 +++++++++ scripts/alert_chain_smoke_test.py | 118 ++++++++++++++++++ 3 files changed, 178 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/e2e-health.yaml b/.gitea/workflows/e2e-health.yaml index ea800552..09e1ccf6 100644 --- a/.gitea/workflows/e2e-health.yaml +++ b/.gitea/workflows/e2e-health.yaml @@ -55,7 +55,7 @@ jobs: run: | SOURCE_CANARY_RUN_REF="gitea-e2e-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}" echo "SOURCE_CANARY_RUN_REF=${SOURCE_CANARY_RUN_REF}" >> "$GITHUB_ENV" - echo "SOURCE_CANARY_WORK_ITEM_ID=source-evidence:sentry:upstream_canary:awoooi-canary-${SOURCE_CANARY_RUN_REF}" >> "$GITHUB_ENV" + echo "SOURCE_LINK_CANARY_WORK_ITEM_ID=source-evidence:sentry:upstream_canary:awoooi-source-link-canary-${SOURCE_CANARY_RUN_REF}" >> "$GITHUB_ENV" OPERATOR_KEY="$(cat <<'AWOOOI_SECRET_AWOOOP_OPERATOR_API_KEY' ${{ secrets.AWOOOP_OPERATOR_API_KEY }} AWOOOI_SECRET_AWOOOP_OPERATOR_API_KEY @@ -68,6 +68,7 @@ jobs: --source-provider-heartbeat \ --source-provider-upstream-canary \ --run-ref "${SOURCE_CANARY_RUN_REF}" \ + --source-link-canary-target-incident-id INC-20260505-25E744 \ --json - name: Source Correlation Applied-Link Smoke @@ -77,10 +78,10 @@ jobs: --target-incident-id INC-20260505-25E744 \ --allow-existing-apply \ --refresh-if-stale-days 6 \ - --refresh-work-item-id "${SOURCE_CANARY_WORK_ITEM_ID}" \ + --refresh-work-item-id "${SOURCE_LINK_CANARY_WORK_ITEM_ID}" \ --verify-refresh-candidate \ --reviewer-id gitea_e2e_source_link_canary \ - --operator-note "T122 rolling source-correlation canary refresh; append-only status-chain proof" + --operator-note "T124 dedicated source-link canary refresh; append-only status-chain proof" - name: Notify Telegram on Failure if: failure() diff --git a/apps/api/tests/test_alert_chain_smoke_metric.py b/apps/api/tests/test_alert_chain_smoke_metric.py index 623f803e..39bf64c7 100644 --- a/apps/api/tests/test_alert_chain_smoke_metric.py +++ b/apps/api/tests/test_alert_chain_smoke_metric.py @@ -244,6 +244,62 @@ class AlertChainSmokeMetricTest(unittest.TestCase): self.assertEqual(calls[0]["headers"]["X-AwoooP-Operator-Id"], "gitea-e2e-health") self.assertEqual(calls[1]["headers"]["X-AwoooP-Operator-Key"], "secret") + def test_source_link_canary_requires_operator_key(self): + result = alert_chain_smoke_test.send_source_link_canary( + "https://awoooi.example", + target_incident_id="INC-20260505-25E744", + operator_key=None, + operator_id="gitea-e2e-health", + run_ref="run-123", + ) + + self.assertFalse(result.passed) + self.assertTrue(result.critical) + self.assertIn("AWOOOP_OPERATOR_API_KEY", result.message) + + def test_source_link_canary_posts_dedicated_sentry_payload(self): + calls = [] + + def fake_post(url, payload, *, headers=None, timeout=None): + calls.append( + { + "url": url, + "payload": payload, + "headers": headers, + "timeout": timeout, + } + ) + return alert_chain_smoke_test.HttpGetResult( + 200, + '{"status":"canary_recorded","provider":"sentry"}', + ) + + original_post = alert_chain_smoke_test.http_post_json + try: + alert_chain_smoke_test.http_post_json = fake_post + result = alert_chain_smoke_test.send_source_link_canary( + "https://awoooi.example", + target_incident_id="INC-20260505-25E744", + operator_key="secret", + operator_id="gitea-e2e-health", + run_ref="run/123", + ) + finally: + alert_chain_smoke_test.http_post_json = original_post + + self.assertTrue(result.passed) + self.assertEqual( + calls[0]["url"], + "https://awoooi.example/api/v1/webhooks/sentry/error", + ) + issue = calls[0]["payload"]["data"]["issue"] + tags = calls[0]["payload"]["data"]["event"]["tags"] + self.assertEqual(issue["id"], "awoooi-source-link-canary-run-123") + self.assertEqual(issue["title"], "AwoooPSourceLinkCanary") + self.assertIn(["source_link_canary", "true"], tags) + self.assertIn(["target_incident_id", "INC-20260505-25E744"], tags) + self.assertEqual(calls[0]["headers"]["X-AwoooP-Operator-Key"], "secret") + if __name__ == "__main__": unittest.main() diff --git a/scripts/alert_chain_smoke_test.py b/scripts/alert_chain_smoke_test.py index 4949c4b1..40a61e72 100644 --- a/scripts/alert_chain_smoke_test.py +++ b/scripts/alert_chain_smoke_test.py @@ -569,6 +569,39 @@ def _build_sentry_upstream_canary_payload(safe_ref: str) -> dict[str, Any]: } +def _build_sentry_source_link_canary_payload( + safe_ref: str, + *, + target_incident_id: str, +) -> dict[str, Any]: + issue_id = f"awoooi-source-link-canary-{safe_ref}" + return { + "action": "triggered", + "data": { + "issue": { + "id": issue_id, + "shortId": "AWOOOI-CANARY-SOURCE-LINK", + "title": "AwoooPSourceLinkCanary", + "culprit": "source-correlation-refresh", + "level": "info", + "project": {"slug": "awoooi"}, + "permalink": "https://awoooi.wooo.work/zh-TW/awooop/work-items", + }, + "event": { + "message": "AwoooP source correlation refresh canary", + "platform": "python", + "tags": [ + ["awoooi_canary", "true"], + ["source_link_canary", "true"], + ["run_ref", safe_ref], + ["target_incident_id", target_incident_id], + ], + }, + }, + "actor": {"type": "application", "name": "AwoooP E2E"}, + } + + def _build_signoz_upstream_canary_payload(safe_ref: str) -> dict[str, Any]: fingerprint = f"source-provider-canary:signoz:{safe_ref}" return { @@ -693,6 +726,68 @@ def send_source_provider_upstream_canary( ) +def send_source_link_canary( + api_url: str, + *, + target_incident_id: str, + operator_key: str | None, + operator_id: str, + run_ref: str | None = None, +) -> CheckResult: + """Send a Sentry canary meant specifically for source-link refresh proof.""" + target_id = str(target_incident_id or "").strip() + if not target_id: + return CheckResult( + "Source Link Canary", + False, + "target_incident_id 未設定；無法建立 source-link refresh canary", + ) + if not operator_key: + return CheckResult( + "Source Link Canary", + False, + "AWOOOP_OPERATOR_API_KEY 未設定；無法打入受保護 source-link canary", + ) + + safe_ref = _safe_run_ref(run_ref) + url = f"{api_url}/api/v1/webhooks/sentry/error" + payload = _build_sentry_source_link_canary_payload( + safe_ref, + target_incident_id=target_id, + ) + try: + resp = http_post_json( + url, + payload, + headers={ + "X-AwoooP-Operator-Id": operator_id, + "X-AwoooP-Operator-Key": operator_key, + }, + timeout=TIMEOUT, + ) + data = resp.json() + except (URLError, TimeoutError, OSError, json.JSONDecodeError) as e: + return CheckResult( + "Source Link Canary", + False, + f"sentry source-link canary failed: {_http_error_message(e)}", + ) + if resp.status_code >= 400: + return CheckResult( + "Source Link Canary", + False, + f"sentry HTTP {resp.status_code}: {data.get('detail', resp.text) if isinstance(data, dict) else resp.text}", + ) + validation_error = _validate_upstream_canary_response("sentry", data) + if validation_error: + return CheckResult("Source Link Canary", False, validation_error) + return CheckResult( + "Source Link Canary", + True, + f"recorded sentry source-link canary event for {target_id}", + ) + + def check_signoz_reachable(signoz_url: str) -> CheckResult: """Check 4: SigNoz UI 可達""" try: @@ -793,6 +888,7 @@ def run_smoke_test( operator_key: str | None = None, operator_id: str = "gitea-e2e-health", run_ref: str | None = None, + source_link_canary_target_incident_id: str | None = None, ) -> SmokeTestReport: report = SmokeTestReport() metrics_url = metrics_api_url or api_url @@ -864,6 +960,18 @@ def run_smoke_test( ) ) + if source_link_canary_target_incident_id: + source_link_result = send_source_link_canary( + api_url, + target_incident_id=source_link_canary_target_incident_id, + operator_key=operator_key, + operator_id=operator_id, + run_ref=run_ref, + ) + report.add(source_link_result) + if fail_fast and not source_link_result.passed and source_link_result.critical: + return report + # Check 4: SigNoz report.add(check_signoz_reachable(SIGNOZ_URL)) @@ -926,6 +1034,13 @@ def main() -> int: default=os.environ.get("GITHUB_RUN_ID") or os.environ.get("GITEA_RUN_ID"), help="CI run reference stored in heartbeat payload", ) + parser.add_argument( + "--source-link-canary-target-incident-id", + help=( + "Write a Sentry source-link canary for this target Incident so " + "source-correlation refresh can use semantically dedicated evidence." + ), + ) args = parser.parse_args() report = run_smoke_test( @@ -938,6 +1053,9 @@ def main() -> int: operator_key=os.environ.get(args.operator_key_env), operator_id=args.operator_id, run_ref=args.run_ref, + source_link_canary_target_incident_id=( + args.source_link_canary_target_incident_id + ), ) print("-" * 50)