feat(awooop): persist inbound source envelopes
This commit is contained in:
@@ -35,6 +35,7 @@ from src.models.approval import (
|
||||
)
|
||||
from src.services.anomaly_counter import get_anomaly_counter
|
||||
from src.services.approval_db import get_approval_service
|
||||
from src.services.channel_hub import record_external_alert_event
|
||||
from src.services.openclaw_http_service import get_openclaw_http_service
|
||||
from src.services.sentry_service import get_sentry_service
|
||||
# 2026-04-27 P3.1-T2 by Claude — Tier-2 三服務感知強化:補 SentryWebhookService 簽章驗證
|
||||
@@ -124,16 +125,60 @@ async def handle_sentry_error(
|
||||
|
||||
# 提取錯誤資訊
|
||||
issue_data = payload.get("data", {}).get("issue", {})
|
||||
event_data = payload.get("data", {}).get("event", {})
|
||||
issue_id = issue_data.get("id")
|
||||
source_url = (
|
||||
issue_data.get("permalink")
|
||||
or issue_data.get("web_url")
|
||||
or issue_data.get("url")
|
||||
)
|
||||
|
||||
background_tasks.add_task(
|
||||
record_external_alert_event,
|
||||
project_id="awoooi",
|
||||
provider="sentry",
|
||||
event_id=str(issue_id or issue_data.get("shortId") or "unknown"),
|
||||
stage="received",
|
||||
title=str(issue_data.get("title") or "Sentry issue"),
|
||||
severity=str(issue_data.get("level") or "error"),
|
||||
namespace="sentry",
|
||||
target_resource=str(issue_data.get("culprit") or issue_data.get("project", {}).get("slug") or "unknown"),
|
||||
fingerprint=f"sentry-{issue_id or issue_data.get('shortId') or 'unknown'}",
|
||||
source_url=source_url,
|
||||
labels={
|
||||
"project": issue_data.get("project", {}),
|
||||
"level": issue_data.get("level"),
|
||||
"culprit": issue_data.get("culprit"),
|
||||
},
|
||||
annotations={"message": event_data.get("message")},
|
||||
payload=payload,
|
||||
)
|
||||
|
||||
# Phase 10.2.1: 去重檢查 (10 分鐘內不重複發送)
|
||||
issue_id = issue_data.get("id")
|
||||
sentry_service = get_sentry_service()
|
||||
if not await sentry_service.check_dedup(issue_id, ttl=SENTRY_DEDUP_TTL):
|
||||
background_tasks.add_task(
|
||||
record_external_alert_event,
|
||||
project_id="awoooi",
|
||||
provider="sentry",
|
||||
event_id=str(issue_id or issue_data.get("shortId") or "unknown"),
|
||||
stage="deduplicated",
|
||||
title=str(issue_data.get("title") or "Sentry issue"),
|
||||
severity=str(issue_data.get("level") or "error"),
|
||||
namespace="sentry",
|
||||
target_resource=str(issue_data.get("culprit") or issue_data.get("project", {}).get("slug") or "unknown"),
|
||||
fingerprint=f"sentry-{issue_id or issue_data.get('shortId') or 'unknown'}",
|
||||
source_url=source_url,
|
||||
labels={"project": issue_data.get("project", {}), "level": issue_data.get("level")},
|
||||
annotations={"message": event_data.get("message")},
|
||||
payload={"dedup_ttl": SENTRY_DEDUP_TTL},
|
||||
is_duplicate=True,
|
||||
)
|
||||
return {"status": "deduplicated", "issue_id": issue_id, "ttl": SENTRY_DEDUP_TTL}
|
||||
event_data = payload.get("data", {}).get("event", {})
|
||||
|
||||
error_context = {
|
||||
"issue_id": issue_data.get("id"),
|
||||
"source_url": source_url,
|
||||
"title": issue_data.get("title"),
|
||||
"culprit": issue_data.get("culprit"),
|
||||
"level": issue_data.get("level"),
|
||||
@@ -256,6 +301,29 @@ async def analyze_and_comment(
|
||||
analysis=analysis,
|
||||
anomaly_frequency=frequency_dict,
|
||||
)
|
||||
await record_external_alert_event(
|
||||
project_id="awoooi",
|
||||
provider="sentry",
|
||||
event_id=str(issue_id or error_context.get("issue_id") or "unknown"),
|
||||
stage="approval_linked",
|
||||
title=str(error_context.get("title") or "Sentry issue"),
|
||||
severity=str(error_context.get("level") or "error"),
|
||||
namespace="sentry",
|
||||
target_resource=str(error_context.get("culprit") or error_context.get("project") or "unknown"),
|
||||
fingerprint=f"sentry-{issue_id or error_context.get('issue_id') or 'unknown'}",
|
||||
approval_id=approval_id,
|
||||
source_url=error_context.get("source_url"),
|
||||
labels={
|
||||
"project": error_context.get("project"),
|
||||
"level": error_context.get("level"),
|
||||
},
|
||||
annotations={"message": error_context.get("message")},
|
||||
payload={
|
||||
"anomaly_frequency": frequency_dict,
|
||||
"ai_analyzed": analysis is not None,
|
||||
"ai_provider": analysis.analyzed_by if analysis else None,
|
||||
},
|
||||
)
|
||||
|
||||
# 4. 發送 Telegram 告警 (含頻率資訊)
|
||||
await send_sentry_telegram_alert(
|
||||
|
||||
@@ -18,6 +18,7 @@ AWOOOI API - SignOz Webhook Handler
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import structlog
|
||||
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
|
||||
@@ -37,10 +38,14 @@ from src.models.approval import (
|
||||
)
|
||||
from src.services.anomaly_counter import get_anomaly_counter
|
||||
from src.services.approval_db import get_approval_service
|
||||
from src.services.channel_hub import record_external_alert_event
|
||||
from src.services.incident_service import get_incident_service
|
||||
from src.services.telegram_gateway import get_telegram_gateway
|
||||
from src.utils.timezone import now_taipei_iso
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from src.services.openclaw import LLMAnalysisResult
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/webhooks/signoz", tags=["SignOz Webhook"])
|
||||
@@ -104,6 +109,26 @@ async def handle_signoz_alert(
|
||||
labels = alert.get("labels", {})
|
||||
annotations = alert.get("annotations", {})
|
||||
severity = labels.get("severity", "warning")
|
||||
source_url = alert.get("generatorURL")
|
||||
service_name = labels.get("service_name", labels.get("service", "unknown"))
|
||||
fingerprint = labels.get("fingerprint") or f"signoz-{alert_name}-{service_name}"
|
||||
|
||||
background_tasks.add_task(
|
||||
record_external_alert_event,
|
||||
project_id="awoooi",
|
||||
provider="signoz",
|
||||
event_id=str(fingerprint),
|
||||
stage="received",
|
||||
title=str(alert_name),
|
||||
severity=str(severity),
|
||||
namespace=str(labels.get("namespace", "signoz")),
|
||||
target_resource=str(service_name),
|
||||
fingerprint=str(fingerprint),
|
||||
source_url=source_url,
|
||||
labels=labels,
|
||||
annotations=annotations,
|
||||
payload=alert,
|
||||
)
|
||||
|
||||
# 背景處理
|
||||
background_tasks.add_task(
|
||||
@@ -113,6 +138,8 @@ async def handle_signoz_alert(
|
||||
annotations=annotations,
|
||||
severity=severity,
|
||||
starts_at=alert.get("startsAt"),
|
||||
source_url=source_url,
|
||||
raw_payload=alert,
|
||||
)
|
||||
|
||||
results.append({
|
||||
@@ -133,6 +160,8 @@ async def process_signoz_alert(
|
||||
annotations: dict,
|
||||
severity: str,
|
||||
starts_at: str | None,
|
||||
source_url: str | None = None,
|
||||
raw_payload: dict | None = None,
|
||||
):
|
||||
"""
|
||||
背景處理 SignOz 告警
|
||||
@@ -190,6 +219,7 @@ async def process_signoz_alert(
|
||||
"annotations": annotations,
|
||||
"fingerprint": f"signoz-{alert_name}-{labels.get('service_name', 'unknown')}",
|
||||
}
|
||||
fingerprint = signal_data["fingerprint"]
|
||||
# ADR-037: 傳遞頻率統計到 Incident
|
||||
incident = await incident_service.create_incident_from_signal(
|
||||
signal_data, frequency_stats=anomaly_frequency
|
||||
@@ -229,6 +259,30 @@ async def process_signoz_alert(
|
||||
anomaly_frequency=anomaly_frequency,
|
||||
analysis_result=analysis_result, # 帶入 AI 結果
|
||||
)
|
||||
await record_external_alert_event(
|
||||
project_id="awoooi",
|
||||
provider="signoz",
|
||||
event_id=str(fingerprint),
|
||||
stage="incident_linked",
|
||||
title=str(alert_name),
|
||||
severity=str(severity),
|
||||
namespace=str(labels.get("namespace", "signoz")),
|
||||
target_resource=str(labels.get("service_name", labels.get("service", "unknown"))),
|
||||
fingerprint=str(fingerprint),
|
||||
incident_id=str(incident.incident_id),
|
||||
approval_id=str(approval_id),
|
||||
source_url=source_url or trace_url,
|
||||
labels=labels,
|
||||
annotations=annotations,
|
||||
payload={
|
||||
"raw_alert": raw_payload or {},
|
||||
"trace_url": trace_url,
|
||||
"has_signoz_metrics": bool(signoz_metrics),
|
||||
"ai_provider": ai_provider,
|
||||
"tokens": tokens,
|
||||
"cost": cost,
|
||||
},
|
||||
)
|
||||
|
||||
# =================================================================
|
||||
# Step 5: 發送 Telegram 告警
|
||||
|
||||
@@ -1741,6 +1741,8 @@ async def _process_new_alert_background(
|
||||
incident_id=incident_id,
|
||||
approval_id=str(approval.id),
|
||||
repeat_count=1,
|
||||
labels=traced_alert_labels,
|
||||
annotations=alert_context.get("annotations", {}),
|
||||
)
|
||||
|
||||
if _cs2_auto_approval is not None and _cs2_exec_success is not None:
|
||||
@@ -2017,6 +2019,8 @@ async def _process_new_alert_background(
|
||||
incident_id=incident_id,
|
||||
approval_id=str(approval.id),
|
||||
repeat_count=1,
|
||||
labels=traced_alert_labels,
|
||||
annotations=alert_context.get("annotations", {}),
|
||||
)
|
||||
|
||||
if _cs3_auto_approval is not None and _cs3_exec_success is not None:
|
||||
@@ -2197,6 +2201,8 @@ async def _process_new_alert_background(
|
||||
incident_id=fallback_incident_id,
|
||||
approval_id=str(approval.id),
|
||||
repeat_count=1,
|
||||
labels=traced_alert_labels,
|
||||
annotations=alert_context.get("annotations", {}),
|
||||
)
|
||||
|
||||
await _push_to_telegram_background(
|
||||
@@ -2457,6 +2463,9 @@ async def alertmanager_webhook(
|
||||
stage="received",
|
||||
notification_type=notification_type,
|
||||
alert_category=alert_category,
|
||||
source_url=alert.generatorURL,
|
||||
labels=dict(alert.labels) if alert.labels else {},
|
||||
annotations=dict(alert.annotations) if alert.annotations else {},
|
||||
)
|
||||
|
||||
# ==========================================================================
|
||||
@@ -2556,6 +2565,9 @@ async def alertmanager_webhook(
|
||||
approval_id=str(updated_approval.id),
|
||||
repeat_count=updated_approval.hit_count,
|
||||
is_duplicate=True,
|
||||
source_url=alert.generatorURL,
|
||||
labels=dict(alert.labels) if alert.labels else {},
|
||||
annotations=dict(alert.annotations) if alert.annotations else {},
|
||||
)
|
||||
|
||||
return AlertResponse(
|
||||
@@ -2601,6 +2613,9 @@ async def alertmanager_webhook(
|
||||
notification_type="TYPE-1",
|
||||
alert_category=alert_category,
|
||||
incident_id=_info_incident_id,
|
||||
source_url=alert.generatorURL,
|
||||
labels={**alert.labels, "fingerprint": fingerprint, "alert_id": alert_id},
|
||||
annotations=dict(alert.annotations) if alert.annotations else {},
|
||||
)
|
||||
# 2026-04-15 ogt: TYPE-1 純資訊告警建立後立即關閉
|
||||
# 設計原則: backup/heartbeat/info 告警無需追蹤狀態,通知即完成
|
||||
@@ -2646,6 +2661,9 @@ async def alertmanager_webhook(
|
||||
notification_type=notification_type,
|
||||
alert_category=alert_category,
|
||||
is_duplicate=True,
|
||||
source_url=alert.generatorURL,
|
||||
labels=dict(alert.labels) if alert.labels else {},
|
||||
annotations=dict(alert.annotations) if alert.annotations else {},
|
||||
)
|
||||
return AlertResponse(
|
||||
success=True,
|
||||
|
||||
Reference in New Issue
Block a user