ci(awooop): add dedicated source link canary
This commit is contained in:
@@ -55,7 +55,7 @@ jobs:
|
||||
run: |
|
||||
SOURCE_CANARY_RUN_REF="gitea-e2e-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-1}"
|
||||
echo "SOURCE_CANARY_RUN_REF=${SOURCE_CANARY_RUN_REF}" >> "$GITHUB_ENV"
|
||||
echo "SOURCE_CANARY_WORK_ITEM_ID=source-evidence:sentry:upstream_canary:awoooi-canary-${SOURCE_CANARY_RUN_REF}" >> "$GITHUB_ENV"
|
||||
echo "SOURCE_LINK_CANARY_WORK_ITEM_ID=source-evidence:sentry:upstream_canary:awoooi-source-link-canary-${SOURCE_CANARY_RUN_REF}" >> "$GITHUB_ENV"
|
||||
OPERATOR_KEY="$(cat <<'AWOOOI_SECRET_AWOOOP_OPERATOR_API_KEY'
|
||||
${{ secrets.AWOOOP_OPERATOR_API_KEY }}
|
||||
AWOOOI_SECRET_AWOOOP_OPERATOR_API_KEY
|
||||
@@ -68,6 +68,7 @@ jobs:
|
||||
--source-provider-heartbeat \
|
||||
--source-provider-upstream-canary \
|
||||
--run-ref "${SOURCE_CANARY_RUN_REF}" \
|
||||
--source-link-canary-target-incident-id INC-20260505-25E744 \
|
||||
--json
|
||||
|
||||
- name: Source Correlation Applied-Link Smoke
|
||||
@@ -77,10 +78,10 @@ jobs:
|
||||
--target-incident-id INC-20260505-25E744 \
|
||||
--allow-existing-apply \
|
||||
--refresh-if-stale-days 6 \
|
||||
--refresh-work-item-id "${SOURCE_CANARY_WORK_ITEM_ID}" \
|
||||
--refresh-work-item-id "${SOURCE_LINK_CANARY_WORK_ITEM_ID}" \
|
||||
--verify-refresh-candidate \
|
||||
--reviewer-id gitea_e2e_source_link_canary \
|
||||
--operator-note "T122 rolling source-correlation canary refresh; append-only status-chain proof"
|
||||
--operator-note "T124 dedicated source-link canary refresh; append-only status-chain proof"
|
||||
|
||||
- name: Notify Telegram on Failure
|
||||
if: failure()
|
||||
|
||||
@@ -244,6 +244,62 @@ class AlertChainSmokeMetricTest(unittest.TestCase):
|
||||
self.assertEqual(calls[0]["headers"]["X-AwoooP-Operator-Id"], "gitea-e2e-health")
|
||||
self.assertEqual(calls[1]["headers"]["X-AwoooP-Operator-Key"], "secret")
|
||||
|
||||
def test_source_link_canary_requires_operator_key(self):
|
||||
result = alert_chain_smoke_test.send_source_link_canary(
|
||||
"https://awoooi.example",
|
||||
target_incident_id="INC-20260505-25E744",
|
||||
operator_key=None,
|
||||
operator_id="gitea-e2e-health",
|
||||
run_ref="run-123",
|
||||
)
|
||||
|
||||
self.assertFalse(result.passed)
|
||||
self.assertTrue(result.critical)
|
||||
self.assertIn("AWOOOP_OPERATOR_API_KEY", result.message)
|
||||
|
||||
def test_source_link_canary_posts_dedicated_sentry_payload(self):
|
||||
calls = []
|
||||
|
||||
def fake_post(url, payload, *, headers=None, timeout=None):
|
||||
calls.append(
|
||||
{
|
||||
"url": url,
|
||||
"payload": payload,
|
||||
"headers": headers,
|
||||
"timeout": timeout,
|
||||
}
|
||||
)
|
||||
return alert_chain_smoke_test.HttpGetResult(
|
||||
200,
|
||||
'{"status":"canary_recorded","provider":"sentry"}',
|
||||
)
|
||||
|
||||
original_post = alert_chain_smoke_test.http_post_json
|
||||
try:
|
||||
alert_chain_smoke_test.http_post_json = fake_post
|
||||
result = alert_chain_smoke_test.send_source_link_canary(
|
||||
"https://awoooi.example",
|
||||
target_incident_id="INC-20260505-25E744",
|
||||
operator_key="secret",
|
||||
operator_id="gitea-e2e-health",
|
||||
run_ref="run/123",
|
||||
)
|
||||
finally:
|
||||
alert_chain_smoke_test.http_post_json = original_post
|
||||
|
||||
self.assertTrue(result.passed)
|
||||
self.assertEqual(
|
||||
calls[0]["url"],
|
||||
"https://awoooi.example/api/v1/webhooks/sentry/error",
|
||||
)
|
||||
issue = calls[0]["payload"]["data"]["issue"]
|
||||
tags = calls[0]["payload"]["data"]["event"]["tags"]
|
||||
self.assertEqual(issue["id"], "awoooi-source-link-canary-run-123")
|
||||
self.assertEqual(issue["title"], "AwoooPSourceLinkCanary")
|
||||
self.assertIn(["source_link_canary", "true"], tags)
|
||||
self.assertIn(["target_incident_id", "INC-20260505-25E744"], tags)
|
||||
self.assertEqual(calls[0]["headers"]["X-AwoooP-Operator-Key"], "secret")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -569,6 +569,39 @@ def _build_sentry_upstream_canary_payload(safe_ref: str) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _build_sentry_source_link_canary_payload(
|
||||
safe_ref: str,
|
||||
*,
|
||||
target_incident_id: str,
|
||||
) -> dict[str, Any]:
|
||||
issue_id = f"awoooi-source-link-canary-{safe_ref}"
|
||||
return {
|
||||
"action": "triggered",
|
||||
"data": {
|
||||
"issue": {
|
||||
"id": issue_id,
|
||||
"shortId": "AWOOOI-CANARY-SOURCE-LINK",
|
||||
"title": "AwoooPSourceLinkCanary",
|
||||
"culprit": "source-correlation-refresh",
|
||||
"level": "info",
|
||||
"project": {"slug": "awoooi"},
|
||||
"permalink": "https://awoooi.wooo.work/zh-TW/awooop/work-items",
|
||||
},
|
||||
"event": {
|
||||
"message": "AwoooP source correlation refresh canary",
|
||||
"platform": "python",
|
||||
"tags": [
|
||||
["awoooi_canary", "true"],
|
||||
["source_link_canary", "true"],
|
||||
["run_ref", safe_ref],
|
||||
["target_incident_id", target_incident_id],
|
||||
],
|
||||
},
|
||||
},
|
||||
"actor": {"type": "application", "name": "AwoooP E2E"},
|
||||
}
|
||||
|
||||
|
||||
def _build_signoz_upstream_canary_payload(safe_ref: str) -> dict[str, Any]:
|
||||
fingerprint = f"source-provider-canary:signoz:{safe_ref}"
|
||||
return {
|
||||
@@ -693,6 +726,68 @@ def send_source_provider_upstream_canary(
|
||||
)
|
||||
|
||||
|
||||
def send_source_link_canary(
|
||||
api_url: str,
|
||||
*,
|
||||
target_incident_id: str,
|
||||
operator_key: str | None,
|
||||
operator_id: str,
|
||||
run_ref: str | None = None,
|
||||
) -> CheckResult:
|
||||
"""Send a Sentry canary meant specifically for source-link refresh proof."""
|
||||
target_id = str(target_incident_id or "").strip()
|
||||
if not target_id:
|
||||
return CheckResult(
|
||||
"Source Link Canary",
|
||||
False,
|
||||
"target_incident_id 未設定;無法建立 source-link refresh canary",
|
||||
)
|
||||
if not operator_key:
|
||||
return CheckResult(
|
||||
"Source Link Canary",
|
||||
False,
|
||||
"AWOOOP_OPERATOR_API_KEY 未設定;無法打入受保護 source-link canary",
|
||||
)
|
||||
|
||||
safe_ref = _safe_run_ref(run_ref)
|
||||
url = f"{api_url}/api/v1/webhooks/sentry/error"
|
||||
payload = _build_sentry_source_link_canary_payload(
|
||||
safe_ref,
|
||||
target_incident_id=target_id,
|
||||
)
|
||||
try:
|
||||
resp = http_post_json(
|
||||
url,
|
||||
payload,
|
||||
headers={
|
||||
"X-AwoooP-Operator-Id": operator_id,
|
||||
"X-AwoooP-Operator-Key": operator_key,
|
||||
},
|
||||
timeout=TIMEOUT,
|
||||
)
|
||||
data = resp.json()
|
||||
except (URLError, TimeoutError, OSError, json.JSONDecodeError) as e:
|
||||
return CheckResult(
|
||||
"Source Link Canary",
|
||||
False,
|
||||
f"sentry source-link canary failed: {_http_error_message(e)}",
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
return CheckResult(
|
||||
"Source Link Canary",
|
||||
False,
|
||||
f"sentry HTTP {resp.status_code}: {data.get('detail', resp.text) if isinstance(data, dict) else resp.text}",
|
||||
)
|
||||
validation_error = _validate_upstream_canary_response("sentry", data)
|
||||
if validation_error:
|
||||
return CheckResult("Source Link Canary", False, validation_error)
|
||||
return CheckResult(
|
||||
"Source Link Canary",
|
||||
True,
|
||||
f"recorded sentry source-link canary event for {target_id}",
|
||||
)
|
||||
|
||||
|
||||
def check_signoz_reachable(signoz_url: str) -> CheckResult:
|
||||
"""Check 4: SigNoz UI 可達"""
|
||||
try:
|
||||
@@ -793,6 +888,7 @@ def run_smoke_test(
|
||||
operator_key: str | None = None,
|
||||
operator_id: str = "gitea-e2e-health",
|
||||
run_ref: str | None = None,
|
||||
source_link_canary_target_incident_id: str | None = None,
|
||||
) -> SmokeTestReport:
|
||||
report = SmokeTestReport()
|
||||
metrics_url = metrics_api_url or api_url
|
||||
@@ -864,6 +960,18 @@ def run_smoke_test(
|
||||
)
|
||||
)
|
||||
|
||||
if source_link_canary_target_incident_id:
|
||||
source_link_result = send_source_link_canary(
|
||||
api_url,
|
||||
target_incident_id=source_link_canary_target_incident_id,
|
||||
operator_key=operator_key,
|
||||
operator_id=operator_id,
|
||||
run_ref=run_ref,
|
||||
)
|
||||
report.add(source_link_result)
|
||||
if fail_fast and not source_link_result.passed and source_link_result.critical:
|
||||
return report
|
||||
|
||||
# Check 4: SigNoz
|
||||
report.add(check_signoz_reachable(SIGNOZ_URL))
|
||||
|
||||
@@ -926,6 +1034,13 @@ def main() -> int:
|
||||
default=os.environ.get("GITHUB_RUN_ID") or os.environ.get("GITEA_RUN_ID"),
|
||||
help="CI run reference stored in heartbeat payload",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source-link-canary-target-incident-id",
|
||||
help=(
|
||||
"Write a Sentry source-link canary for this target Incident so "
|
||||
"source-correlation refresh can use semantically dedicated evidence."
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
report = run_smoke_test(
|
||||
@@ -938,6 +1053,9 @@ def main() -> int:
|
||||
operator_key=os.environ.get(args.operator_key_env),
|
||||
operator_id=args.operator_id,
|
||||
run_ref=args.run_ref,
|
||||
source_link_canary_target_incident_id=(
|
||||
args.source_link_canary_target_incident_id
|
||||
),
|
||||
)
|
||||
|
||||
print("-" * 50)
|
||||
|
||||
Reference in New Issue
Block a user