wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 5 Packages Projects Releases Wiki Activity

feat(api): Phase 15.3 Deep Linking 三系統互連

Browse Source 
實現 Sentry ↔ SignOz ↔ Langfuse 零斷鏈觀測:

新增 deep_linking.py:
- SignOz Trace URL 生成器
- Langfuse Trace URL 生成器
- Sentry Issue URL 生成器
- get_all_links() 統一取得所有連結

整合點:
- main.py: Sentry before_send 注入 otel_trace_id + signoz_trace_url
- langfuse_client.py: 自動注入 OTEL trace_id 到 metadata
- openclaw.py: SignOz span 記錄 langfuse.trace_id 反向連結

架構圖:
┌─────────┐ trace_id ┌─────────┐ trace_id ┌──────────┐
│ Sentry  │◄────────►│ SignOz  │◄────────►│ Langfuse │
│ Errors  │          │ Traces  │          │ LLMOps   │
└─────────┘          └─────────┘          └──────────┘

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
OG T
2026-03-26 00:48:28 +08:00
parent 0d31ccb911
commit b6cff31653
4 changed files with 333 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
apps/api/src/core/deep_linking.py Normal file
View File

@@ -0,0 +1,227 @@
"""
AWOOOI Deep Linking - Phase 15.3
================================
三系統觀測互連: Sentry ↔ SignOz ↔ Langfuse
Phase 15.3 (2026-03-26)
實現零斷鏈觀測的最後一哩路
架構:
┌─────────┐ trace_id ┌─────────┐ trace_id ┌──────────┐
│ Sentry │ ◄──────────────► │ SignOz │ ◄──────────────► │ Langfuse │
│ Errors │ │ Traces │ │ LLMOps │
└─────────┘ └─────────┘ └──────────┘
URL 格式:
- SignOz Trace: http://192.168.0.188:3301/trace/{trace_id}
- Langfuse Trace: http://192.168.0.110:3100/trace/{langfuse_trace_id}
- Sentry Issue: http://192.168.0.110:9000/organizations/sentry/issues/{issue_id}/
Usage:
from src.core.deep_linking import DeepLinking
# 取得 SignOz Trace URL
url = DeepLinking.signoz_trace_url(trace_id)
# 取得 Langfuse Trace URL
url = DeepLinking.langfuse_trace_url(langfuse_trace_id)
"""
from src.core.config import settings
class DeepLinking:
"""
三系統 Deep Linking URL 生成器
統帥鐵律 (Phase 15.3):
- 所有觀測系統必須能互相跳轉
- URL 必須使用內網 IP (非 localhost)
- 確保 trace_id 格式一致 (32 hex chars)
"""
# ==========================================================================
# SignOz URLs (Traces/Metrics/Logs)
# ==========================================================================
SIGNOZ_BASE_URL = "http://192.168.0.188:3301"
@classmethod
def signoz_trace_url(cls, trace_id: str) -> str:
"""
生成 SignOz Trace 詳情頁 URL
Args:
trace_id: 32 字元 hex 格式 (e.g., "0af7651916cd43dd8448eb211c80319c")
Returns:
SignOz Trace URL (e.g., http://192.168.0.188:3301/trace/0af7651916cd43dd8448eb211c80319c)
"""
if not trace_id:
return ""
# SignOz v3 URL 格式
return f"{cls.SIGNOZ_BASE_URL}/trace/{trace_id}"
@classmethod
def signoz_service_url(cls, service_name: str = "awoooi-api") -> str:
"""
生成 SignOz Service 監控頁 URL
Args:
service_name: 服務名稱
Returns:
SignOz Service URL
"""
return f"{cls.SIGNOZ_BASE_URL}/services/{service_name}"
@classmethod
def signoz_logs_url(cls, trace_id: str | None = None) -> str:
"""
生成 SignOz Logs 頁面 URL
Args:
trace_id: 可選，用於過濾特定 trace 的 logs
Returns:
SignOz Logs URL
"""
if trace_id:
# SignOz v3 logs 過濾語法
return f"{cls.SIGNOZ_BASE_URL}/logs?q=trace_id%3D{trace_id}"
return f"{cls.SIGNOZ_BASE_URL}/logs"
# ==========================================================================
# Langfuse URLs (LLMOps)
# ==========================================================================
LANGFUSE_BASE_URL = settings.LANGFUSE_URL or "http://192.168.0.110:3100"
@classmethod
def langfuse_trace_url(cls, trace_id: str, project: str = "awoooi-openclaw") -> str:
"""
生成 Langfuse Trace 詳情頁 URL
Args:
trace_id: Langfuse trace ID
project: Langfuse 專案名稱
Returns:
Langfuse Trace URL
"""
if not trace_id:
return ""
return f"{cls.LANGFUSE_BASE_URL}/project/{project}/traces/{trace_id}"
@classmethod
def langfuse_dashboard_url(cls, project: str = "awoooi-openclaw") -> str:
"""
生成 Langfuse Dashboard URL
Args:
project: Langfuse 專案名稱
Returns:
Langfuse Dashboard URL
"""
return f"{cls.LANGFUSE_BASE_URL}/project/{project}"
# ==========================================================================
# Sentry URLs (Error Tracking)
# ==========================================================================
SENTRY_BASE_URL = "http://192.168.0.110:9000"
SENTRY_ORG = "sentry" # Self-hosted 預設組織
@classmethod
def sentry_issue_url(cls, issue_id: str) -> str:
"""
生成 Sentry Issue 詳情頁 URL
Args:
issue_id: Sentry issue ID
Returns:
Sentry Issue URL
"""
if not issue_id:
return ""
return f"{cls.SENTRY_BASE_URL}/organizations/{cls.SENTRY_ORG}/issues/{issue_id}/"
@classmethod
def sentry_project_url(cls, project_slug: str) -> str:
"""
生成 Sentry Project 頁面 URL
Args:
project_slug: Sentry 專案 slug
Returns:
Sentry Project URL
"""
return f"{cls.SENTRY_BASE_URL}/organizations/{cls.SENTRY_ORG}/projects/{project_slug}/"
# ==========================================================================
# Cross-System Deep Linking
# ==========================================================================
@classmethod
def get_all_links(
cls,
otel_trace_id: str | None = None,
langfuse_trace_id: str | None = None,
sentry_issue_id: str | None = None,
) -> dict[str, str]:
"""
取得所有可用的 Deep Links
Args:
otel_trace_id: OpenTelemetry trace ID (32 hex)
langfuse_trace_id: Langfuse trace ID
sentry_issue_id: Sentry issue ID
Returns:
dict with available deep links
Example:
{
"signoz_trace": "http://...",
"langfuse_trace": "http://...",
"sentry_issue": "http://...",
}
"""
links = {}
if otel_trace_id:
links["signoz_trace"] = cls.signoz_trace_url(otel_trace_id)
links["signoz_logs"] = cls.signoz_logs_url(otel_trace_id)
if langfuse_trace_id:
links["langfuse_trace"] = cls.langfuse_trace_url(langfuse_trace_id)
if sentry_issue_id:
links["sentry_issue"] = cls.sentry_issue_url(sentry_issue_id)
return links
@classmethod
def format_for_log(
cls,
otel_trace_id: str | None = None,
langfuse_trace_id: str | None = None,
) -> str:
"""
格式化 Deep Links 供 log 輸出
Returns:
Formatted string for logging
"""
parts = []
if otel_trace_id:
parts.append(f"SignOz: {cls.signoz_trace_url(otel_trace_id)}")
if langfuse_trace_id:
parts.append(f"Langfuse: {cls.langfuse_trace_url(langfuse_trace_id)}")
return " | ".join(parts) if parts else "No trace links available"

38
apps/api/src/main.py
View File

@@ -79,8 +79,44 @@ logger = get_logger("awoooi.api")
# Sentry SDK Initialization (Error Tracking - 補強 SignOz)
# Self-Hosted @ 192.168.0.110
# 分工: Sentry 專注 Error Tracking，SignOz 專注 Traces/Logs/Metrics
# Phase 15.3: Deep Linking - 注入 OTEL trace_id 供 SignOz 關聯
# =============================================================================
SENTRY_DSN = os.getenv("SENTRY_DSN")
def _sentry_before_send(event, hint): # noqa: ARG001 - hint is Sentry callback signature
"""
Phase 15.3: Sentry → SignOz Deep Linking
在每個 Sentry event 中注入 OTEL trace_id，
讓 Sentry 錯誤能直接連結到 SignOz Trace。
"""
try:
from src.core.deep_linking import DeepLinking
from src.core.telemetry import get_current_trace_id
trace_id = get_current_trace_id()
if trace_id:
# 注入 trace_id 到 tags (Sentry UI 可搜尋)
if "tags" not in event:
event["tags"] = {}
event["tags"]["otel_trace_id"] = trace_id
event["tags"]["signoz_trace_url"] = DeepLinking.signoz_trace_url(trace_id)
# 注入到 contexts (詳情頁顯示)
if "contexts" not in event:
event["contexts"] = {}
event["contexts"]["signoz"] = {
"trace_id": trace_id,
"trace_url": DeepLinking.signoz_trace_url(trace_id),
"service": "awoooi-api",
}
except Exception:
# Deep Linking 失敗不應影響錯誤上報
pass
return event
if SENTRY_DSN:
sentry_sdk.init(
dsn=SENTRY_DSN,
@@ -100,6 +136,8 @@ if SENTRY_DSN:
],
# 只在生產環境發送
send_default_pii=False,
# Phase 15.3: Deep Linking hook
before_send=_sentry_before_send,
)
logger.info("sentry_initialized", dsn=SENTRY_DSN.split("@")[-1])
else:

59
apps/api/src/services/langfuse_client.py
View File

@@ -1,21 +1,23 @@
"""
Langfuse LLMOps Client - Phase 15.1
===================================
Langfuse LLMOps Client - Phase 15.1 + 15.3
==========================================
LLM 呼叫追蹤、成本監控、Prompt 版本管理
Phase 15.1 (2026-03-26)
Phase 15.1 (2026-03-26): 基礎整合
Phase 15.3 (2026-03-26): Deep Linking (Langfuse ↔ SignOz)
端點: http://192.168.0.110:3100 (DevOps 金庫)
Features:
- 自動追蹤所有 LLM 呼叫 (Ollama/Gemini/Claude)
- 成本估算與監控
- Prompt 版本管理
- 與 OTEL Trace 整合
- **Phase 15.3**: OTEL Trace 整合 (SignOz Deep Link)
Usage:
from src.services.langfuse_client import get_langfuse, langfuse_trace
# 方法 1: Context Manager
# 方法 1: Context Manager (自動整合 OTEL trace_id)
async with langfuse_trace("openclaw_decision") as trace:
result = await call_llm(prompt)
trace.generation(
@@ -24,6 +26,7 @@ Usage:
input=prompt,
output=result,
)
# trace.metadata 自動包含 signoz_trace_url
# 方法 2: 裝飾器
@langfuse_observe(name="analyze_incident")
@@ -88,21 +91,53 @@ def get_langfuse():
class LangfuseTraceContext:
"""Langfuse Trace Context for tracking LLM calls"""
"""
Langfuse Trace Context for tracking LLM calls
Phase 15.3: 自動整合 OTEL trace_id，實現 Langfuse ↔ SignOz Deep Linking
"""
def __init__(self, name: str, metadata: dict[str, Any] | None = None):
self.name = name
self.metadata = metadata or {}
self.trace = None
self._client = get_langfuse()
self._otel_trace_id: str | None = None
self._langfuse_trace_id: str | None = None
def __enter__(self):
if self._client:
try:
# Phase 15.3: 取得當前 OTEL trace_id 並注入 metadata
from src.core.deep_linking import DeepLinking
from src.core.telemetry import get_current_trace_id
self._otel_trace_id = get_current_trace_id()
# 建立含 SignOz Deep Link 的 metadata
enriched_metadata = {**self.metadata}
if self._otel_trace_id:
enriched_metadata["otel_trace_id"] = self._otel_trace_id
enriched_metadata["signoz_trace_url"] = DeepLinking.signoz_trace_url(
self._otel_trace_id
)
self.trace = self._client.trace(
name=self.name,
metadata=self.metadata,
metadata=enriched_metadata,
)
# 記錄 Langfuse trace_id 供反向連結
if self.trace:
self._langfuse_trace_id = self.trace.id
logger.debug(
"langfuse_trace_started",
name=self.name,
otel_trace_id=self._otel_trace_id,
langfuse_trace_id=self._langfuse_trace_id,
)
except Exception as e:
logger.warning("langfuse_trace_start_failed", error=str(e))
return self
@@ -111,6 +146,16 @@ class LangfuseTraceContext:
# Langfuse auto-flushes, no explicit close needed
pass
@property
def otel_trace_id(self) -> str | None:
"""取得關聯的 OTEL trace_id"""
return self._otel_trace_id
@property
def langfuse_trace_id(self) -> str | None:
"""取得 Langfuse trace_id"""
return self._langfuse_trace_id
def generation(
self,
name: str,

17
apps/api/src/services/openclaw.py
View File

@@ -832,7 +832,7 @@ class OpenClawService:
logger.info("mock_mode_enabled", using="mock_llm")
return self._generate_mock_response(alert_context or {}, signoz_metrics), "mock", True
# Phase 15.1: Langfuse 追蹤整合
# Phase 15.1 + 15.3: Langfuse 追蹤整合 + SignOz Deep Linking
with langfuse_trace(
"openclaw_fallback_chain",
metadata={
@@ -841,6 +841,21 @@ class OpenClawService:
"alert_fingerprint": (alert_context or {}).get("fingerprint", "unknown"),
},
) as trace:
# Phase 15.3: SignOz → Langfuse 反向連結
# 在當前 OTEL span 中記錄 Langfuse trace_id
if trace.langfuse_trace_id:
from opentelemetry import trace as otel_trace
from src.core.deep_linking import DeepLinking
current_span = otel_trace.get_current_span()
if current_span:
current_span.set_attribute("langfuse.trace_id", trace.langfuse_trace_id)
current_span.set_attribute(
"langfuse.trace_url",
DeepLinking.langfuse_trace_url(trace.langfuse_trace_id),
)
for provider in settings.AI_FALLBACK_ORDER:
logger.info("ai_provider_attempt", provider=provider)