From b6cff3165397b2dbafbaf88b1a68abc4c65a7246 Mon Sep 17 00:00:00 2001 From: OG T Date: Thu, 26 Mar 2026 00:48:28 +0800 Subject: [PATCH] =?UTF-8?q?feat(api):=20Phase=2015.3=20Deep=20Linking=20?= =?UTF-8?q?=E4=B8=89=E7=B3=BB=E7=B5=B1=E4=BA=92=E9=80=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 實現 Sentry ↔ SignOz ↔ Langfuse 零斷鏈觀測: 新增 deep_linking.py: - SignOz Trace URL 生成器 - Langfuse Trace URL 生成器 - Sentry Issue URL 生成器 - get_all_links() 統一取得所有連結 整合點: - main.py: Sentry before_send 注入 otel_trace_id + signoz_trace_url - langfuse_client.py: 自動注入 OTEL trace_id 到 metadata - openclaw.py: SignOz span 記錄 langfuse.trace_id 反向連結 架構圖: ┌─────────┐ trace_id ┌─────────┐ trace_id ┌──────────┐ │ Sentry │◄────────►│ SignOz │◄────────►│ Langfuse │ │ Errors │ │ Traces │ │ LLMOps │ └─────────┘ └─────────┘ └──────────┘ Co-Authored-By: Claude Opus 4.5 --- apps/api/src/core/deep_linking.py | 227 +++++++++++++++++++++++ apps/api/src/main.py | 38 ++++ apps/api/src/services/langfuse_client.py | 59 +++++- apps/api/src/services/openclaw.py | 17 +- 4 files changed, 333 insertions(+), 8 deletions(-) create mode 100644 apps/api/src/core/deep_linking.py diff --git a/apps/api/src/core/deep_linking.py b/apps/api/src/core/deep_linking.py new file mode 100644 index 00000000..6e2fceb0 --- /dev/null +++ b/apps/api/src/core/deep_linking.py @@ -0,0 +1,227 @@ +""" +AWOOOI Deep Linking - Phase 15.3 +================================ +三系統觀測互連: Sentry ↔ SignOz ↔ Langfuse + +Phase 15.3 (2026-03-26) +實現零斷鏈觀測的最後一哩路 + +架構: +┌─────────┐ trace_id ┌─────────┐ trace_id ┌──────────┐ +│ Sentry │ ◄──────────────► │ SignOz │ ◄──────────────► │ Langfuse │ +│ Errors │ │ Traces │ │ LLMOps │ +└─────────┘ └─────────┘ └──────────┘ + +URL 格式: +- SignOz Trace: http://192.168.0.188:3301/trace/{trace_id} +- Langfuse Trace: http://192.168.0.110:3100/trace/{langfuse_trace_id} +- Sentry Issue: http://192.168.0.110:9000/organizations/sentry/issues/{issue_id}/ + +Usage: + from src.core.deep_linking import DeepLinking + + # 取得 SignOz Trace URL + url = DeepLinking.signoz_trace_url(trace_id) + + # 取得 Langfuse Trace URL + url = DeepLinking.langfuse_trace_url(langfuse_trace_id) +""" + +from src.core.config import settings + + +class DeepLinking: + """ + 三系統 Deep Linking URL 生成器 + + 統帥鐵律 (Phase 15.3): + - 所有觀測系統必須能互相跳轉 + - URL 必須使用內網 IP (非 localhost) + - 確保 trace_id 格式一致 (32 hex chars) + """ + + # ========================================================================== + # SignOz URLs (Traces/Metrics/Logs) + # ========================================================================== + + SIGNOZ_BASE_URL = "http://192.168.0.188:3301" + + @classmethod + def signoz_trace_url(cls, trace_id: str) -> str: + """ + 生成 SignOz Trace 詳情頁 URL + + Args: + trace_id: 32 字元 hex 格式 (e.g., "0af7651916cd43dd8448eb211c80319c") + + Returns: + SignOz Trace URL (e.g., http://192.168.0.188:3301/trace/0af7651916cd43dd8448eb211c80319c) + """ + if not trace_id: + return "" + # SignOz v3 URL 格式 + return f"{cls.SIGNOZ_BASE_URL}/trace/{trace_id}" + + @classmethod + def signoz_service_url(cls, service_name: str = "awoooi-api") -> str: + """ + 生成 SignOz Service 監控頁 URL + + Args: + service_name: 服務名稱 + + Returns: + SignOz Service URL + """ + return f"{cls.SIGNOZ_BASE_URL}/services/{service_name}" + + @classmethod + def signoz_logs_url(cls, trace_id: str | None = None) -> str: + """ + 生成 SignOz Logs 頁面 URL + + Args: + trace_id: 可選，用於過濾特定 trace 的 logs + + Returns: + SignOz Logs URL + """ + if trace_id: + # SignOz v3 logs 過濾語法 + return f"{cls.SIGNOZ_BASE_URL}/logs?q=trace_id%3D{trace_id}" + return f"{cls.SIGNOZ_BASE_URL}/logs" + + # ========================================================================== + # Langfuse URLs (LLMOps) + # ========================================================================== + + LANGFUSE_BASE_URL = settings.LANGFUSE_URL or "http://192.168.0.110:3100" + + @classmethod + def langfuse_trace_url(cls, trace_id: str, project: str = "awoooi-openclaw") -> str: + """ + 生成 Langfuse Trace 詳情頁 URL + + Args: + trace_id: Langfuse trace ID + project: Langfuse 專案名稱 + + Returns: + Langfuse Trace URL + """ + if not trace_id: + return "" + return f"{cls.LANGFUSE_BASE_URL}/project/{project}/traces/{trace_id}" + + @classmethod + def langfuse_dashboard_url(cls, project: str = "awoooi-openclaw") -> str: + """ + 生成 Langfuse Dashboard URL + + Args: + project: Langfuse 專案名稱 + + Returns: + Langfuse Dashboard URL + """ + return f"{cls.LANGFUSE_BASE_URL}/project/{project}" + + # ========================================================================== + # Sentry URLs (Error Tracking) + # ========================================================================== + + SENTRY_BASE_URL = "http://192.168.0.110:9000" + SENTRY_ORG = "sentry" # Self-hosted 預設組織 + + @classmethod + def sentry_issue_url(cls, issue_id: str) -> str: + """ + 生成 Sentry Issue 詳情頁 URL + + Args: + issue_id: Sentry issue ID + + Returns: + Sentry Issue URL + """ + if not issue_id: + return "" + return f"{cls.SENTRY_BASE_URL}/organizations/{cls.SENTRY_ORG}/issues/{issue_id}/" + + @classmethod + def sentry_project_url(cls, project_slug: str) -> str: + """ + 生成 Sentry Project 頁面 URL + + Args: + project_slug: Sentry 專案 slug + + Returns: + Sentry Project URL + """ + return f"{cls.SENTRY_BASE_URL}/organizations/{cls.SENTRY_ORG}/projects/{project_slug}/" + + # ========================================================================== + # Cross-System Deep Linking + # ========================================================================== + + @classmethod + def get_all_links( + cls, + otel_trace_id: str | None = None, + langfuse_trace_id: str | None = None, + sentry_issue_id: str | None = None, + ) -> dict[str, str]: + """ + 取得所有可用的 Deep Links + + Args: + otel_trace_id: OpenTelemetry trace ID (32 hex) + langfuse_trace_id: Langfuse trace ID + sentry_issue_id: Sentry issue ID + + Returns: + dict with available deep links + + Example: + { + "signoz_trace": "http://...", + "langfuse_trace": "http://...", + "sentry_issue": "http://...", + } + """ + links = {} + + if otel_trace_id: + links["signoz_trace"] = cls.signoz_trace_url(otel_trace_id) + links["signoz_logs"] = cls.signoz_logs_url(otel_trace_id) + + if langfuse_trace_id: + links["langfuse_trace"] = cls.langfuse_trace_url(langfuse_trace_id) + + if sentry_issue_id: + links["sentry_issue"] = cls.sentry_issue_url(sentry_issue_id) + + return links + + @classmethod + def format_for_log( + cls, + otel_trace_id: str | None = None, + langfuse_trace_id: str | None = None, + ) -> str: + """ + 格式化 Deep Links 供 log 輸出 + + Returns: + Formatted string for logging + """ + parts = [] + + if otel_trace_id: + parts.append(f"SignOz: {cls.signoz_trace_url(otel_trace_id)}") + + if langfuse_trace_id: + parts.append(f"Langfuse: {cls.langfuse_trace_url(langfuse_trace_id)}") + + return " | ".join(parts) if parts else "No trace links available" diff --git a/apps/api/src/main.py b/apps/api/src/main.py index 365e2a01..9f232131 100644 --- a/apps/api/src/main.py +++ b/apps/api/src/main.py @@ -79,8 +79,44 @@ logger = get_logger("awoooi.api") # Sentry SDK Initialization (Error Tracking - 補強 SignOz) # Self-Hosted @ 192.168.0.110 # 分工: Sentry 專注 Error Tracking，SignOz 專注 Traces/Logs/Metrics +# Phase 15.3: Deep Linking - 注入 OTEL trace_id 供 SignOz 關聯 # ============================================================================= SENTRY_DSN = os.getenv("SENTRY_DSN") + + +def _sentry_before_send(event, hint): # noqa: ARG001 - hint is Sentry callback signature + """ + Phase 15.3: Sentry → SignOz Deep Linking + + 在每個 Sentry event 中注入 OTEL trace_id， + 讓 Sentry 錯誤能直接連結到 SignOz Trace。 + """ + try: + from src.core.deep_linking import DeepLinking + from src.core.telemetry import get_current_trace_id + + trace_id = get_current_trace_id() + if trace_id: + # 注入 trace_id 到 tags (Sentry UI 可搜尋) + if "tags" not in event: + event["tags"] = {} + event["tags"]["otel_trace_id"] = trace_id + event["tags"]["signoz_trace_url"] = DeepLinking.signoz_trace_url(trace_id) + + # 注入到 contexts (詳情頁顯示) + if "contexts" not in event: + event["contexts"] = {} + event["contexts"]["signoz"] = { + "trace_id": trace_id, + "trace_url": DeepLinking.signoz_trace_url(trace_id), + "service": "awoooi-api", + } + except Exception: + # Deep Linking 失敗不應影響錯誤上報 + pass + return event + + if SENTRY_DSN: sentry_sdk.init( dsn=SENTRY_DSN, @@ -100,6 +136,8 @@ if SENTRY_DSN: ], # 只在生產環境發送 send_default_pii=False, + # Phase 15.3: Deep Linking hook + before_send=_sentry_before_send, ) logger.info("sentry_initialized", dsn=SENTRY_DSN.split("@")[-1]) else: diff --git a/apps/api/src/services/langfuse_client.py b/apps/api/src/services/langfuse_client.py index 61c23022..14a954d9 100644 --- a/apps/api/src/services/langfuse_client.py +++ b/apps/api/src/services/langfuse_client.py @@ -1,21 +1,23 @@ """ -Langfuse LLMOps Client - Phase 15.1 -=================================== +Langfuse LLMOps Client - Phase 15.1 + 15.3 +========================================== LLM 呼叫追蹤、成本監控、Prompt 版本管理 -Phase 15.1 (2026-03-26) +Phase 15.1 (2026-03-26): 基礎整合 +Phase 15.3 (2026-03-26): Deep Linking (Langfuse ↔ SignOz) + 端點: http://192.168.0.110:3100 (DevOps 金庫) Features: - 自動追蹤所有 LLM 呼叫 (Ollama/Gemini/Claude) - 成本估算與監控 - Prompt 版本管理 -- 與 OTEL Trace 整合 +- **Phase 15.3**: OTEL Trace 整合 (SignOz Deep Link) Usage: from src.services.langfuse_client import get_langfuse, langfuse_trace - # 方法 1: Context Manager + # 方法 1: Context Manager (自動整合 OTEL trace_id) async with langfuse_trace("openclaw_decision") as trace: result = await call_llm(prompt) trace.generation( @@ -24,6 +26,7 @@ Usage: input=prompt, output=result, ) + # trace.metadata 自動包含 signoz_trace_url # 方法 2: 裝飾器 @langfuse_observe(name="analyze_incident") @@ -88,21 +91,53 @@ def get_langfuse(): class LangfuseTraceContext: - """Langfuse Trace Context for tracking LLM calls""" + """ + Langfuse Trace Context for tracking LLM calls + + Phase 15.3: 自動整合 OTEL trace_id，實現 Langfuse ↔ SignOz Deep Linking + """ def __init__(self, name: str, metadata: dict[str, Any] | None = None): self.name = name self.metadata = metadata or {} self.trace = None self._client = get_langfuse() + self._otel_trace_id: str | None = None + self._langfuse_trace_id: str | None = None def __enter__(self): if self._client: try: + # Phase 15.3: 取得當前 OTEL trace_id 並注入 metadata + from src.core.deep_linking import DeepLinking + from src.core.telemetry import get_current_trace_id + + self._otel_trace_id = get_current_trace_id() + + # 建立含 SignOz Deep Link 的 metadata + enriched_metadata = {**self.metadata} + if self._otel_trace_id: + enriched_metadata["otel_trace_id"] = self._otel_trace_id + enriched_metadata["signoz_trace_url"] = DeepLinking.signoz_trace_url( + self._otel_trace_id + ) + self.trace = self._client.trace( name=self.name, - metadata=self.metadata, + metadata=enriched_metadata, ) + + # 記錄 Langfuse trace_id 供反向連結 + if self.trace: + self._langfuse_trace_id = self.trace.id + + logger.debug( + "langfuse_trace_started", + name=self.name, + otel_trace_id=self._otel_trace_id, + langfuse_trace_id=self._langfuse_trace_id, + ) + except Exception as e: logger.warning("langfuse_trace_start_failed", error=str(e)) return self @@ -111,6 +146,16 @@ class LangfuseTraceContext: # Langfuse auto-flushes, no explicit close needed pass + @property + def otel_trace_id(self) -> str | None: + """取得關聯的 OTEL trace_id""" + return self._otel_trace_id + + @property + def langfuse_trace_id(self) -> str | None: + """取得 Langfuse trace_id""" + return self._langfuse_trace_id + def generation( self, name: str, diff --git a/apps/api/src/services/openclaw.py b/apps/api/src/services/openclaw.py index 537bbb66..54031b8c 100644 --- a/apps/api/src/services/openclaw.py +++ b/apps/api/src/services/openclaw.py @@ -832,7 +832,7 @@ class OpenClawService: logger.info("mock_mode_enabled", using="mock_llm") return self._generate_mock_response(alert_context or {}, signoz_metrics), "mock", True - # Phase 15.1: Langfuse 追蹤整合 + # Phase 15.1 + 15.3: Langfuse 追蹤整合 + SignOz Deep Linking with langfuse_trace( "openclaw_fallback_chain", metadata={ @@ -841,6 +841,21 @@ class OpenClawService: "alert_fingerprint": (alert_context or {}).get("fingerprint", "unknown"), }, ) as trace: + # Phase 15.3: SignOz → Langfuse 反向連結 + # 在當前 OTEL span 中記錄 Langfuse trace_id + if trace.langfuse_trace_id: + from opentelemetry import trace as otel_trace + + from src.core.deep_linking import DeepLinking + + current_span = otel_trace.get_current_span() + if current_span: + current_span.set_attribute("langfuse.trace_id", trace.langfuse_trace_id) + current_span.set_attribute( + "langfuse.trace_url", + DeepLinking.langfuse_trace_url(trace.langfuse_trace_id), + ) + for provider in settings.AI_FALLBACK_ORDER: logger.info("ai_provider_attempt", provider=provider)