2743 lines
113 KiB
Python
2743 lines
113 KiB
Python
"""
|
||
OpenClaw AI Decision Engine - True LLM + SignOz Integration
|
||
============================================================
|
||
Phase 5: OpenClaw 實體化升級 (2026-03-21)
|
||
統帥校正: SignOz 為唯一全能視力中心
|
||
|
||
Features:
|
||
- 真實 LLM SDK 整合 (告警預設 Ollama GCP-A → GCP-B → 111 → Gemini)
|
||
- SignOz Gold Metrics 即時擷取 (P99/Error/RPS)
|
||
- AIOps Agent 專業人格 (K8s 維運 + SRE RCA 專精)
|
||
- 強制結構化 JSON 輸出 (符合 API 契約)
|
||
- 動態告警上下文注入 + SignOz 數據
|
||
- Shadow Mode 調優指令生成 (日誌輸出，不執行)
|
||
|
||
防禦性工程鐵律:
|
||
- Zero Trust: 預設不信任 LLM 輸出，必須通過 Pydantic 驗證
|
||
- Edge Case: 網路失敗、解析失敗、超時處理
|
||
- SignOz 失敗時優雅降級 (不阻塞主流程)
|
||
"""
|
||
|
||
import hashlib
|
||
import json
|
||
import random
|
||
import re
|
||
import time
|
||
from datetime import datetime
|
||
|
||
import httpx
|
||
import structlog
|
||
|
||
from src.core.config import settings
|
||
from src.core.prompts import NEMOTRON_SYSTEM_PROMPT, OPENCLAW_SYSTEM_PROMPT
|
||
from src.core.redis_client import get_redis
|
||
from src.models.ai import (
|
||
AIRiskLevel,
|
||
OpenClawDecision,
|
||
SuggestedAction,
|
||
)
|
||
from src.services.langfuse_client import langfuse_trace
|
||
from src.services.model_registry import get_model_registry
|
||
from src.services.ollama_failover_manager import get_ollama_failover_manager
|
||
from src.services.signoz_client import GoldMetrics, get_signoz_client
|
||
from src.utils.k8s_naming import normalize_resource_name
|
||
from src.utils.timezone import now_taipei_iso
|
||
|
||
logger = structlog.get_logger(__name__)
|
||
|
||
|
||
# =============================================================================
|
||
# AIOps Agent System Prompt (專業人格 + 仲裁邏輯 + SignOz 數據)
|
||
# =============================================================================
|
||
|
||
# 責任矩陣定義
|
||
RESPONSIBILITY_MATRIX = {
|
||
"FE": "前端團隊 (Frontend)",
|
||
"BE": "後端團隊 (Backend)",
|
||
"INFRA": "基礎設施團隊 (Infrastructure/SRE)",
|
||
"DB": "資料庫團隊 (Database/DBA)",
|
||
"COLLAB": "協同處理 (需多團隊會診)",
|
||
}
|
||
|
||
# 信心度閾值
|
||
CONFIDENCE_THRESHOLD_COLLAB = 0.70 # 低於此閾值自動標記為 COLLAB
|
||
|
||
# OPENCLAW_SYSTEM_PROMPT 已移至 src/core/prompts.py (Phase 17 P2 改進)
|
||
|
||
|
||
# =============================================================================
|
||
# LLM Analysis Result - Using Pydantic for Schema Enforcement
|
||
# =============================================================================
|
||
|
||
# We use OpenClawDecision from models/ai.py for Pydantic validation
|
||
# This alias is for backwards compatibility
|
||
LLMAnalysisResult = OpenClawDecision
|
||
|
||
|
||
# =============================================================================
|
||
# kubectl_command 回填 helper
|
||
# 2026-04-09 Claude Sonnet 4.6: I2 架構Review修復 — 補齊所有 tool 類型，消除兩處重複邏輯 (M3)
|
||
# =============================================================================
|
||
|
||
def _backfill_kubectl_command(proposal: dict, tools: list) -> None:
|
||
"""將 AI tool call 結果回填為可執行的 kubectl_command。
|
||
proposal["kubectl_command"] 若已有值則不覆蓋（LLM 直接填的優先）。
|
||
"""
|
||
if not tools or proposal.get("kubectl_command"):
|
||
return
|
||
_t = tools[0]
|
||
_tool_name = _t.get("tool", "")
|
||
_args = _t.get("args", {})
|
||
_ns = _args.get("namespace", proposal.get("namespace", "awoooi-prod"))
|
||
|
||
if _tool_name == "restart_deployment":
|
||
_deploy = _args.get("deployment_name", proposal.get("target_resource", ""))
|
||
if _deploy:
|
||
proposal["kubectl_command"] = f"kubectl rollout restart deployment/{_deploy} -n {_ns}"
|
||
elif _tool_name == "delete_pod":
|
||
_pod = _args.get("pod_name", "")
|
||
if _pod:
|
||
proposal["kubectl_command"] = f"kubectl delete pod {_pod} -n {_ns}"
|
||
elif _tool_name == "scale_deployment":
|
||
_deploy = _args.get("deployment_name", "")
|
||
_replicas = _args.get("replicas", 2)
|
||
if _deploy:
|
||
proposal["kubectl_command"] = f"kubectl scale deployment/{_deploy} --replicas={_replicas} -n {_ns}"
|
||
elif _tool_name == "delete_deployment":
|
||
_deploy = _args.get("deployment_name", "")
|
||
if _deploy:
|
||
proposal["kubectl_command"] = f"kubectl delete deployment/{_deploy} -n {_ns}"
|
||
elif _tool_name == "drain_node":
|
||
_node = _args.get("node_name", "")
|
||
if _node:
|
||
proposal["kubectl_command"] = f"kubectl drain {_node} --ignore-daemonsets --delete-emptydir-data"
|
||
elif _tool_name == "cordon_node":
|
||
_node = _args.get("node_name", "")
|
||
if _node:
|
||
proposal["kubectl_command"] = f"kubectl cordon {_node}"
|
||
elif _tool_name == "delete_service":
|
||
_svc = _args.get("service_name", "")
|
||
if _svc:
|
||
proposal["kubectl_command"] = f"kubectl delete service/{_svc} -n {_ns}"
|
||
|
||
|
||
# =============================================================================
|
||
# OpenClaw Service
|
||
# =============================================================================
|
||
|
||
def _build_alert_cache_context_hash(alert_context: dict | None) -> str:
|
||
"""Build a stable LLM cache scope for repeat alerts without dynamic annotations."""
|
||
|
||
if not alert_context:
|
||
return ""
|
||
|
||
alertname = alert_context.get("alertname") or alert_context.get("alert_type", "")
|
||
category = alert_context.get("alert_category", "")
|
||
namespace = alert_context.get("namespace", "")
|
||
target = alert_context.get("target_resource", "")
|
||
severity = alert_context.get("severity", "")
|
||
fingerprint = alert_context.get("fingerprint", "")
|
||
return f"{alertname}:{category}:{namespace}:{target}:{severity}:{fingerprint}"
|
||
|
||
|
||
class OpenClawService:
|
||
"""
|
||
OpenClaw AI 決策服務 - True LLM + SignOz Integration
|
||
|
||
實作 AI_FALLBACK_ORDER 備援機制。
|
||
告警/incident 上下文預設套用成本防線，只允許 Ollama GCP-A → GCP-B → 111。
|
||
|
||
新增 SignOz 整合:
|
||
- 自動擷取 Gold Metrics
|
||
- 數據驅動的 RCA 分析
|
||
- 動態 Trace URL 生成
|
||
"""
|
||
|
||
def __init__(self):
|
||
self._http_client: httpx.AsyncClient | None = None
|
||
self._signoz = get_signoz_client()
|
||
|
||
async def _get_client(self) -> httpx.AsyncClient:
|
||
"""取得 HTTP 客戶端
|
||
|
||
2026-04-14 Claude Sonnet 4.6: 從硬編 120s 改用 OPENCLAW_TIMEOUT 設定 (30s)
|
||
對齊 ADR-052 GAP-B4 的 25s + 5s buffer 設計。原 120s 違反 defense-in-depth。
|
||
"""
|
||
if self._http_client is None or self._http_client.is_closed:
|
||
_t = float(settings.OPENCLAW_TIMEOUT)
|
||
self._http_client = httpx.AsyncClient(
|
||
timeout=httpx.Timeout(_t, connect=10.0),
|
||
)
|
||
return self._http_client
|
||
|
||
async def close(self) -> None:
|
||
"""關閉連線"""
|
||
if self._http_client:
|
||
await self._http_client.aclose()
|
||
self._http_client = None
|
||
|
||
def _is_incident_alert_context(self, alert_context: dict | None) -> bool:
|
||
"""Return true when a request came from the alert/incident automation path."""
|
||
if not alert_context:
|
||
return False
|
||
alert_keys = {
|
||
"alert_type",
|
||
"alertname",
|
||
"alert_name",
|
||
"fingerprint",
|
||
"incident_id",
|
||
"severity",
|
||
"signals",
|
||
"target_resource",
|
||
}
|
||
return any(key in alert_context for key in alert_keys)
|
||
|
||
def _cloud_fallback_allowed_for_alert(self, alert_context: dict | None) -> bool:
|
||
"""Cloud fallback is allowed after the ordered Ollama lane for alerts."""
|
||
if alert_context and alert_context.get("allow_cloud_fallback") is False:
|
||
return False
|
||
if not self._is_incident_alert_context(alert_context):
|
||
return True
|
||
return bool(getattr(settings, "ALERT_AI_ALLOW_CLOUD_FALLBACK", True))
|
||
|
||
def _alert_enforces_ollama_first(self, alert_context: dict | None) -> bool:
|
||
"""Alert and AI-governance lanes must try GCP-A/GCP-B/111 before Gemini backup."""
|
||
return (
|
||
bool(alert_context)
|
||
and (
|
||
bool(alert_context.get("enforce_ollama_first"))
|
||
or self._is_incident_alert_context(alert_context)
|
||
)
|
||
and bool(getattr(settings, "ALERT_AI_ENFORCE_OLLAMA_FIRST", True))
|
||
)
|
||
|
||
async def _resolve_alert_provider_order(
|
||
self,
|
||
task_type: str = "diagnose",
|
||
alert_context: dict | None = None,
|
||
cloud_provider_order: list[str] | None = None,
|
||
) -> list[str]:
|
||
"""Resolve GCP-A/GCP-B/111, then OpenClaw/Nemo before Gemini backup."""
|
||
provider_order: list[str] = []
|
||
try:
|
||
route = await get_ollama_failover_manager().select_provider(task_type=task_type)
|
||
provider_order = [
|
||
endpoint.provider_name
|
||
for endpoint in route.all_endpoints_in_order()
|
||
if endpoint.provider_name.startswith("ollama")
|
||
]
|
||
except Exception as route_error:
|
||
logger.warning(
|
||
"alert_ollama_route_lookup_failed",
|
||
error=str(route_error),
|
||
task_type=task_type,
|
||
)
|
||
|
||
if not provider_order:
|
||
provider_order = ["ollama_gcp_a", "ollama_gcp_b", "ollama_local"]
|
||
|
||
deduped: list[str] = []
|
||
for provider_name in provider_order:
|
||
if provider_name and provider_name not in deduped:
|
||
deduped.append(provider_name)
|
||
|
||
if not self._alert_enforces_ollama_first(alert_context):
|
||
return deduped
|
||
|
||
ollama_order = {"ollama_gcp_a": 0, "ollama_gcp_b": 1, "ollama_local": 2}
|
||
ordered_ollama = [
|
||
provider_name
|
||
for provider_name in deduped
|
||
if provider_name in ollama_order
|
||
]
|
||
ordered_ollama.sort(key=lambda provider_name: ollama_order[provider_name])
|
||
if not ordered_ollama:
|
||
ordered_ollama = ["ollama_gcp_a", "ollama_gcp_b", "ollama_local"]
|
||
|
||
if not self._cloud_fallback_allowed_for_alert(alert_context):
|
||
return ordered_ollama
|
||
|
||
cloud_aliases = {"nvidia": "openclaw_nemo"}
|
||
cloud_candidates = {
|
||
cloud_aliases.get(provider_name, provider_name)
|
||
for provider_name in (cloud_provider_order or [])
|
||
}
|
||
try:
|
||
from src.services.ai_control import is_provider_disabled
|
||
if await is_provider_disabled("openclaw_nemo"):
|
||
cloud_candidates.discard("openclaw_nemo")
|
||
else:
|
||
cloud_candidates.add("openclaw_nemo")
|
||
except Exception as control_error:
|
||
logger.warning("alert_openclaw_nemo_control_check_failed", error=str(control_error))
|
||
# Gemini remains the final paid backup, but alert traffic should use
|
||
# OpenClaw/Nemo first whenever the router control plane has not disabled it.
|
||
cloud_candidates.add("gemini")
|
||
cloud_backup = [
|
||
provider_name
|
||
for provider_name in ("openclaw_nemo", "gemini")
|
||
if provider_name in cloud_candidates
|
||
]
|
||
|
||
return ordered_ollama + cloud_backup
|
||
|
||
# =========================================================================
|
||
# SignOz Integration
|
||
# =========================================================================
|
||
|
||
async def get_signoz_context(
|
||
self,
|
||
service_name: str,
|
||
namespace: str = "default",
|
||
alert_timestamp: datetime | None = None,
|
||
) -> tuple[GoldMetrics | None, str]:
|
||
"""
|
||
擷取 SignOz 上下文數據
|
||
|
||
Returns:
|
||
(GoldMetrics, trace_url) or (None, fallback_url)
|
||
"""
|
||
try:
|
||
metrics = await self._signoz.get_gold_metrics(
|
||
service_name=service_name,
|
||
namespace=namespace,
|
||
time_window_minutes=10,
|
||
)
|
||
|
||
trace_url = self._signoz.generate_trace_url(
|
||
service_name=service_name,
|
||
alert_timestamp=alert_timestamp,
|
||
window_minutes=5,
|
||
)
|
||
|
||
logger.info(
|
||
"signoz_context_fetched",
|
||
service=service_name,
|
||
rps=metrics.rps,
|
||
error_rate=metrics.error_rate,
|
||
p99_latency=metrics.p99_latency_ms,
|
||
)
|
||
|
||
return metrics, trace_url
|
||
|
||
except Exception as e:
|
||
logger.warning(
|
||
"signoz_context_fetch_failed",
|
||
service=service_name,
|
||
error=str(e),
|
||
)
|
||
# 降級: 返回 None 和靜態 URL
|
||
fallback_url = f"{settings.SIGNOZ_URL}/traces?service={service_name}"
|
||
return None, fallback_url
|
||
|
||
def generate_auto_tuning_command(
|
||
self,
|
||
alert_type: str,
|
||
target_resource: str,
|
||
namespace: str,
|
||
metrics: GoldMetrics | None = None,
|
||
) -> dict:
|
||
"""
|
||
根據告警類型和 SignOz 數據生成調優指令
|
||
|
||
Shadow Mode: 僅生成指令，不執行
|
||
|
||
Phase 18.1.6: 整合 K8s 資源名稱驗證 (ADR-016)
|
||
|
||
Returns:
|
||
{command: str, description: str, type: str}
|
||
"""
|
||
# Phase 18.1.6: 先正規化資源名稱
|
||
normalized = normalize_resource_name(target_resource, namespace)
|
||
|
||
if not normalized.is_k8s_resource:
|
||
# 非 K8s 資源，返回提示訊息
|
||
logger.info(
|
||
"non_k8s_resource_detected",
|
||
original=target_resource,
|
||
note=normalized.note,
|
||
)
|
||
return {
|
||
"type": "MANUAL",
|
||
"command": f"# 非 K8s 資源: {target_resource}",
|
||
"description": f"此資源不在 K8s 中，需人工處理。{normalized.note or ''}",
|
||
}
|
||
|
||
# 使用正規化後的名稱
|
||
resolved_name = normalized.normalized or target_resource
|
||
resolved_ns = normalized.namespace or namespace
|
||
|
||
if normalized.confidence < 0.8:
|
||
logger.warning(
|
||
"low_confidence_resource_name",
|
||
original=target_resource,
|
||
resolved=resolved_name,
|
||
confidence=normalized.confidence,
|
||
)
|
||
|
||
# 根據告警類型選擇調優策略 (使用正規化後的名稱)
|
||
if "cpu" in alert_type.lower() or "high_cpu" in alert_type.lower():
|
||
# CPU 高 → 擴容或調整 limit
|
||
if metrics and metrics.rps > 100:
|
||
# 高流量場景 → HPA
|
||
return {
|
||
"type": "HPA",
|
||
"command": f"kubectl autoscale deployment {resolved_name} --cpu-percent=70 --min=2 --max=10 -n {resolved_ns}",
|
||
"description": f"SignOz RPS={metrics.rps:.0f}，配置 HPA 應對流量波動",
|
||
}
|
||
else:
|
||
# 低流量但 CPU 高 → 調整資源
|
||
return {
|
||
"type": "RESOURCE_LIMIT",
|
||
"command": f"kubectl set resources deployment/{resolved_name} --limits=cpu=2000m -n {resolved_ns}",
|
||
"description": "增加 CPU limit 緩解資源競爭",
|
||
}
|
||
|
||
elif "memory" in alert_type.lower() or "oom" in alert_type.lower():
|
||
return {
|
||
"type": "RESOURCE_LIMIT",
|
||
"command": f"kubectl set resources deployment/{resolved_name} --limits=memory=1Gi -n {resolved_ns}",
|
||
"description": "增加 Memory limit 防止 OOM",
|
||
}
|
||
|
||
elif "pod_crash" in alert_type.lower() or "crash" in alert_type.lower():
|
||
return {
|
||
"type": "RESTART",
|
||
"command": f"kubectl rollout restart deployment/{resolved_name} -n {resolved_ns}",
|
||
"description": "滾動重啟清除異常狀態",
|
||
}
|
||
|
||
elif "latency" in alert_type.lower() or "slow" in alert_type.lower():
|
||
if metrics and metrics.p99_latency_ms > 500:
|
||
return {
|
||
"type": "SCALE",
|
||
"command": f"kubectl scale deployment {resolved_name} --replicas=+2 -n {resolved_ns}",
|
||
"description": f"SignOz P99={metrics.p99_latency_ms:.0f}ms，擴容分散負載",
|
||
}
|
||
else:
|
||
return {
|
||
"type": "CACHE",
|
||
"command": "# 檢查 Redis 連線池配置",
|
||
"description": "建議增加緩存層減少後端壓力",
|
||
}
|
||
|
||
else:
|
||
# 通用: 滾動重啟
|
||
return {
|
||
"type": "RESTART",
|
||
"command": f"kubectl rollout restart deployment/{resolved_name} -n {resolved_ns}",
|
||
"description": "滾動重啟恢復服務",
|
||
}
|
||
|
||
# =========================================================================
|
||
# =========================================================================
|
||
# OpenClaw (Nemo) 委派仲裁 — 架構鐵律
|
||
# 2026-04-01 ogt: AWOOOI API 不直接打 LLM，委派給 OpenClaw (192.168.0.188:8089)
|
||
# =========================================================================
|
||
|
||
async def _call_openclaw_analyze(
|
||
self,
|
||
incident_id: str,
|
||
severity: str,
|
||
signals: list[dict],
|
||
affected_services: list[str],
|
||
expert_context: dict | None = None,
|
||
) -> dict | None:
|
||
"""
|
||
委派 Incident RCA 給 OpenClaw (Nemo) API
|
||
|
||
Returns:
|
||
proposal_dict if success, None if failed (fallback to direct LLM)
|
||
"""
|
||
try:
|
||
client = await self._get_client()
|
||
import json as _json
|
||
def _to_serializable(obj):
|
||
if isinstance(obj, dict):
|
||
return {k: _to_serializable(v) for k, v in obj.items()}
|
||
if isinstance(obj, list):
|
||
return [_to_serializable(i) for i in obj]
|
||
try:
|
||
_json.dumps(obj)
|
||
return obj
|
||
except (TypeError, ValueError):
|
||
return str(obj)
|
||
|
||
payload = {
|
||
"incident_id": incident_id,
|
||
"severity": severity,
|
||
"signals": _to_serializable(signals[:5]),
|
||
"affected_services": affected_services,
|
||
"expert_context": _to_serializable(expert_context) if expert_context else None,
|
||
}
|
||
# 2026-04-14 Claude Sonnet 4.6: 從硬編 130s 改用 OPENCLAW_TIMEOUT
|
||
# 原 130s 讓 LLM 能卡 2m10s，超過 Ollama 真實返回時間（P95 54s）
|
||
resp = await client.post(
|
||
f"{settings.OPENCLAW_URL}/api/v1/analyze/incident",
|
||
json=payload,
|
||
timeout=httpx.Timeout(float(settings.OPENCLAW_TIMEOUT), connect=5.0),
|
||
)
|
||
resp.raise_for_status()
|
||
data = resp.json()
|
||
|
||
# 驗證必要欄位
|
||
if not data.get("action_title") or not data.get("risk_level"):
|
||
logger.warning("openclaw_analyze_invalid_response", incident_id=incident_id)
|
||
return None
|
||
|
||
try:
|
||
confidence_value = float(data.get("confidence", 0.0))
|
||
except (TypeError, ValueError):
|
||
confidence_value = 0.0
|
||
if data.get("degraded") or data.get("provider") == "openclaw_degraded" or confidence_value < 0.3:
|
||
logger.warning(
|
||
"openclaw_analyze_degraded_response",
|
||
incident_id=incident_id,
|
||
provider=data.get("provider"),
|
||
confidence=data.get("confidence"),
|
||
)
|
||
return None
|
||
|
||
logger.info(
|
||
"openclaw_analyze_success",
|
||
incident_id=incident_id,
|
||
confidence=data.get("confidence", 0),
|
||
risk_level=data.get("risk_level"),
|
||
)
|
||
|
||
# 轉換為 AWOOOI API proposal dict 格式
|
||
return {
|
||
"action": data.get("action_title", "AI 分析"),
|
||
"description": data.get("description", ""),
|
||
"kubectl_command": data.get("kubectl_command") or "",
|
||
"target_resource": data.get("target_resource", "unknown"),
|
||
"namespace": data.get("namespace", "awoooi-prod"),
|
||
"risk_level": data.get("risk_level", "medium"),
|
||
"reasoning": data.get("reasoning", ""),
|
||
"confidence": data.get("confidence", 0.8),
|
||
"primary_responsibility": data.get("primary_responsibility", "INFRA"),
|
||
"optimization_suggestions": data.get("optimization_suggestions", []),
|
||
"signoz_correlation": data.get("signoz_correlation", ""),
|
||
"from_cache": False,
|
||
"provider": "openclaw_nemo",
|
||
"ai_tokens": 0,
|
||
"ai_cost": 0.0,
|
||
}
|
||
|
||
except Exception as e:
|
||
logger.warning(
|
||
"openclaw_analyze_failed",
|
||
incident_id=incident_id,
|
||
error=str(e),
|
||
reason="fallback to direct LLM",
|
||
)
|
||
return None
|
||
|
||
# =========================================================================
|
||
# [ARCHIVED Phase 24 B4 — 2026-04-03 ogt]
|
||
# 以下三個方法 (_call_ollama/_call_gemini/_call_claude) 為舊版 fallback chain
|
||
# 新路徑: USE_AI_ROUTER=true → _call_with_fallback → AIRouterExecutor (ai_router.py)
|
||
# 新 Provider 實作: apps/api/src/services/ai_providers/ (OllamaProvider/GeminiProvider/ClaudeProvider)
|
||
# 回滾保留: USE_AI_ROUTER=false 時仍由 _call_with_fallback (line ~993) 呼叫此區塊
|
||
# 完整移除時機: Phase 24 完整驗收後 (ADR-052 D11)
|
||
# =========================================================================
|
||
|
||
async def _call_ollama(self, prompt: str, *, ollama_only: bool = False) -> tuple[str, bool]:
|
||
"""
|
||
呼叫 Ollama (支援 JSON Mode)。
|
||
|
||
USE_AI_ROUTER=true 正常會走 AIRouterExecutor；這裡是 legacy safety-net。
|
||
2026-05-05 Codex: safety-net 也必須遵守 ADR-110 三層 Ollama
|
||
路由，告警路徑預設只允許 GCP-A/GCP-B/111，不能只打 OLLAMA_URL 後直接掉 Gemini。
|
||
"""
|
||
try:
|
||
client = await self._get_client()
|
||
|
||
# 從 ModelRegistry 取得模型配置
|
||
registry = get_model_registry()
|
||
model_name = registry.get_model("ollama", "rca")
|
||
if ollama_only:
|
||
model_name = getattr(settings, "ALERT_OLLAMA_MODEL", "qwen3:14b")
|
||
options = registry.get_provider_options("ollama")
|
||
timeout_seconds = max(
|
||
float(settings.OPENCLAW_TIMEOUT),
|
||
float(getattr(settings, "OLLAMA_DIAGNOSE_TIMEOUT_SECONDS", settings.OPENCLAW_TIMEOUT)),
|
||
)
|
||
|
||
endpoints: list[tuple[str, str]] = []
|
||
try:
|
||
route = await get_ollama_failover_manager().select_provider()
|
||
endpoints = [
|
||
(endpoint.provider_name, endpoint.url)
|
||
for endpoint in route.all_endpoints_in_order()
|
||
if endpoint.provider_name.startswith("ollama") and endpoint.url
|
||
]
|
||
except Exception as route_error:
|
||
logger.warning(
|
||
"legacy_ollama_route_lookup_failed",
|
||
error=str(route_error),
|
||
)
|
||
|
||
if not endpoints:
|
||
configured_endpoints = [
|
||
("ollama_gcp_a", settings.OLLAMA_URL),
|
||
("ollama_gcp_b", getattr(settings, "OLLAMA_SECONDARY_URL", "")),
|
||
("ollama_local", getattr(settings, "OLLAMA_FALLBACK_URL", "")),
|
||
]
|
||
seen_urls: set[str] = set()
|
||
endpoints = []
|
||
for provider_name, endpoint_url in configured_endpoints:
|
||
if endpoint_url and endpoint_url not in seen_urls:
|
||
endpoints.append((provider_name, endpoint_url))
|
||
seen_urls.add(endpoint_url)
|
||
|
||
if ollama_only:
|
||
allowed_provider_order = {"ollama_gcp_a": 0, "ollama_gcp_b": 1, "ollama_local": 2}
|
||
endpoints = [
|
||
(provider_name, endpoint_url)
|
||
for provider_name, endpoint_url in endpoints
|
||
if provider_name in allowed_provider_order
|
||
]
|
||
endpoints.sort(key=lambda item: allowed_provider_order[item[0]])
|
||
if not endpoints:
|
||
endpoints = [
|
||
("ollama_gcp_a", settings.OLLAMA_URL),
|
||
("ollama_gcp_b", getattr(settings, "OLLAMA_SECONDARY_URL", "")),
|
||
("ollama_local", getattr(settings, "OLLAMA_FALLBACK_URL", "")),
|
||
]
|
||
endpoints = [
|
||
(provider_name, endpoint_url)
|
||
for provider_name, endpoint_url in endpoints
|
||
if endpoint_url
|
||
]
|
||
|
||
last_error = ""
|
||
for provider_name, endpoint_url in endpoints:
|
||
try:
|
||
logger.info(
|
||
"ollama_request_start",
|
||
provider=provider_name,
|
||
url=f"{endpoint_url}/api/generate",
|
||
prompt_length=len(prompt),
|
||
)
|
||
|
||
response = await client.post(
|
||
f"{endpoint_url}/api/generate",
|
||
json={
|
||
"model": model_name,
|
||
"prompt": prompt,
|
||
"stream": False,
|
||
"format": "json", # 強制 JSON 輸出
|
||
"options": {
|
||
"num_predict": options.get("num_predict", 1024),
|
||
"temperature": options.get("temperature", 0.1),
|
||
"top_p": options.get("top_p", 0.9),
|
||
},
|
||
},
|
||
timeout=httpx.Timeout(timeout_seconds, connect=10.0),
|
||
)
|
||
|
||
logger.info(
|
||
"ollama_response_received",
|
||
provider=provider_name,
|
||
status_code=response.status_code,
|
||
)
|
||
|
||
response.raise_for_status()
|
||
data = response.json()
|
||
result = data.get("response", "")
|
||
|
||
logger.info(
|
||
"ollama_response_parsed",
|
||
provider=provider_name,
|
||
response_length=len(result),
|
||
)
|
||
|
||
return result, True
|
||
except httpx.TimeoutException as e:
|
||
last_error = f"{provider_name} timeout: {e}"
|
||
logger.warning("ollama_timeout", provider=provider_name, error=str(e))
|
||
except Exception as e:
|
||
last_error = f"{provider_name} failed: {e}"
|
||
logger.warning(
|
||
"ollama_call_failed",
|
||
provider=provider_name,
|
||
error=str(e),
|
||
error_type=type(e).__name__,
|
||
)
|
||
|
||
return last_error or "all Ollama endpoints failed", False
|
||
|
||
except httpx.TimeoutException as e:
|
||
logger.warning("ollama_timeout", error=str(e))
|
||
return f"Timeout: {e}", False
|
||
|
||
except Exception as e:
|
||
logger.warning(
|
||
"ollama_call_failed",
|
||
error=str(e),
|
||
error_type=type(e).__name__,
|
||
)
|
||
return str(e), False
|
||
|
||
async def _call_gemini(self, prompt: str) -> tuple[str, bool, int, float]:
|
||
"""
|
||
呼叫 Google Gemini (支援 JSON Mode)
|
||
|
||
Returns:
|
||
tuple: (response_text, success, total_tokens, cost_usd)
|
||
- response_text: LLM 回應文本
|
||
- success: 是否成功
|
||
- total_tokens: 使用的 Token 總數
|
||
- cost_usd: 預估成本 (USD)
|
||
|
||
2026-03-29 ogt: 加入 Token/Cost 追蹤
|
||
"""
|
||
if not settings.GEMINI_API_KEY:
|
||
return "GEMINI_API_KEY not configured", False, 0, 0.0
|
||
|
||
try:
|
||
client = await self._get_client()
|
||
|
||
# 從 ModelRegistry 取得模型配置
|
||
registry = get_model_registry()
|
||
model_name = registry.get_model("gemini", "rca")
|
||
|
||
response = await client.post(
|
||
f"https://generativelanguage.googleapis.com/v1beta/models/{model_name}:generateContent",
|
||
headers={"x-goog-api-key": settings.GEMINI_API_KEY},
|
||
json={
|
||
"contents": [{"parts": [{"text": prompt}]}],
|
||
"generationConfig": {
|
||
"temperature": 0.1,
|
||
"maxOutputTokens": 2048,
|
||
"responseMimeType": "application/json", # 強制 JSON 輸出
|
||
},
|
||
},
|
||
timeout=30.0,
|
||
)
|
||
response.raise_for_status()
|
||
data = response.json()
|
||
text = data["candidates"][0]["content"]["parts"][0]["text"]
|
||
|
||
# 2026-03-29 ogt: 擷取 Token 使用量
|
||
usage_metadata = data.get("usageMetadata", {})
|
||
prompt_tokens = usage_metadata.get("promptTokenCount", 0)
|
||
completion_tokens = usage_metadata.get("candidatesTokenCount", 0)
|
||
total_tokens = usage_metadata.get("totalTokenCount", prompt_tokens + completion_tokens)
|
||
|
||
# Gemini 1.5 Flash 定價 (per 1M tokens)
|
||
# Input: $0.075 / 1M, Output: $0.30 / 1M
|
||
cost_usd = (prompt_tokens * 0.000000075) + (completion_tokens * 0.0000003)
|
||
|
||
logger.info(
|
||
"gemini_response_received",
|
||
response_length=len(text),
|
||
prompt_tokens=prompt_tokens,
|
||
completion_tokens=completion_tokens,
|
||
total_tokens=total_tokens,
|
||
cost_usd=f"${cost_usd:.6f}",
|
||
)
|
||
return text, True, total_tokens, cost_usd
|
||
|
||
except Exception as e:
|
||
logger.warning("gemini_call_failed", error=str(e))
|
||
return str(e), False, 0, 0.0
|
||
|
||
async def _call_claude(self, prompt: str) -> tuple[str, bool]:
|
||
"""
|
||
呼叫 Anthropic Claude (使用 Tool Use 強制 JSON)
|
||
"""
|
||
if not settings.CLAUDE_API_KEY:
|
||
return "CLAUDE_API_KEY not configured", False
|
||
|
||
try:
|
||
client = await self._get_client()
|
||
|
||
# Claude 使用 Tool Use 強制結構化輸出
|
||
response = await client.post(
|
||
"https://api.anthropic.com/v1/messages",
|
||
headers={
|
||
"x-api-key": settings.CLAUDE_API_KEY,
|
||
"anthropic-version": "2023-06-01",
|
||
"content-type": "application/json",
|
||
},
|
||
json={
|
||
"model": get_model_registry().get_model("claude", "rca"),
|
||
"max_tokens": 2048,
|
||
"messages": [{"role": "user", "content": prompt}],
|
||
"tools": [{
|
||
"name": "submit_analysis",
|
||
"description": "Submit the RCA analysis result in structured format",
|
||
"input_schema": {
|
||
"type": "object",
|
||
"properties": {
|
||
"action_title": {"type": "string"},
|
||
"description": {"type": "string"},
|
||
"suggested_action": {"type": "string", "enum": ["RESTART_DEPLOYMENT", "DELETE_POD", "SCALE_DEPLOYMENT", "APPLY_HPA", "TUNE_RESOURCES", "INVESTIGATE", "OBSERVE", "NO_ACTION"]},
|
||
"kubectl_command": {"type": "string"},
|
||
"target_resource": {"type": "string"},
|
||
"namespace": {"type": "string"},
|
||
"risk_level": {"type": "string", "enum": ["low", "medium", "critical"]},
|
||
"blast_radius": {
|
||
"type": "object",
|
||
"properties": {
|
||
"affected_pods": {"type": "integer"},
|
||
"estimated_downtime": {"type": "string"},
|
||
"related_services": {"type": "array", "items": {"type": "string"}},
|
||
"data_impact": {"type": "string", "enum": ["NONE", "READ_ONLY", "WRITE", "DESTRUCTIVE"]}
|
||
},
|
||
"required": ["affected_pods", "estimated_downtime", "related_services", "data_impact"]
|
||
},
|
||
"reasoning": {"type": "string"},
|
||
"deviation_analysis": {"type": "string"},
|
||
"confidence": {"type": "number"},
|
||
"affected_services": {"type": "array", "items": {"type": "string"}}
|
||
},
|
||
"required": ["action_title", "description", "suggested_action", "kubectl_command", "target_resource", "namespace", "risk_level", "blast_radius", "reasoning", "confidence"]
|
||
}
|
||
}],
|
||
"tool_choice": {"type": "tool", "name": "submit_analysis"},
|
||
},
|
||
timeout=30.0,
|
||
)
|
||
response.raise_for_status()
|
||
data = response.json()
|
||
|
||
# 從 Tool Use 回應中提取 JSON
|
||
for block in data.get("content", []):
|
||
if block.get("type") == "tool_use" and block.get("name") == "submit_analysis":
|
||
tool_input = block.get("input", {})
|
||
logger.info("claude_tool_use_response", input_keys=list(tool_input.keys()))
|
||
return json.dumps(tool_input), True
|
||
|
||
# Fallback: 嘗試從 text 內容提取
|
||
for block in data.get("content", []):
|
||
if block.get("type") == "text":
|
||
return block.get("text", ""), True
|
||
|
||
return "No valid response from Claude", False
|
||
|
||
except Exception as e:
|
||
logger.warning("claude_call_failed", error=str(e))
|
||
return str(e), False
|
||
|
||
# 2026-03-29 ogt: _call_nvidia 已移至 nvidia_provider.py (ARCHIVED)
|
||
# 符合模組化規範 - 所有 NVIDIA API 呼叫統一由 NvidiaProvider / OpenClawNemoProvider 處理
|
||
|
||
# =========================================================================
|
||
# [END ARCHIVED Phase 24 B4]
|
||
# =========================================================================
|
||
|
||
# =========================================================================
|
||
# Mock LLM - Intelligent Fallback with SignOz Data
|
||
# =========================================================================
|
||
|
||
def _generate_mock_response(
|
||
self,
|
||
alert_context: dict,
|
||
signoz_metrics: GoldMetrics | None = None,
|
||
) -> str:
|
||
"""
|
||
規則引擎降級回應 (v8.0) — 生產用途，不是假數據
|
||
|
||
從 alert_rules.yaml 載入規則進行匹配，AI 分析失敗時的正式降級路徑。
|
||
命中 generic_fallback 時會回傳 rule_id="generic_fallback"，
|
||
由上層 async 方法（_call_with_fallback）觸發 auto_generate_rule() 學習新規則。
|
||
|
||
Returns:
|
||
(json_str, rule_id) tuple
|
||
|
||
2026-04-09 ogt: 重構為規則引擎，移除 if/elif 硬編碼
|
||
2026-04-09 ogt: S2-4 架構師審查 — 修正 Mock 語意混淆，澄清為規則引擎生產路徑
|
||
"""
|
||
from src.services.alert_rule_engine import match_rule
|
||
|
||
time.sleep(random.uniform(0.3, 0.8)) # 模擬思考延遲
|
||
|
||
# SignOz 數據整合
|
||
signoz_correlation = "SignOz 數據擷取中..."
|
||
if signoz_metrics:
|
||
signoz_correlation = (
|
||
f"RPS={signoz_metrics.rps:.1f} ({signoz_metrics.rps_trend}), "
|
||
f"Error={signoz_metrics.error_rate:.2f}%, "
|
||
f"P99={signoz_metrics.p99_latency_ms:.0f}ms"
|
||
)
|
||
|
||
mock_response = match_rule(alert_context)
|
||
if mock_response is None:
|
||
# match_rule 不應該回傳 None（有通用兜底），但防禦性處理
|
||
alert_type = alert_context.get("alert_type", "custom")
|
||
target = alert_context.get("target_resource", "unknown")
|
||
namespace = alert_context.get("namespace", "awoooi-prod")
|
||
mock_response = {
|
||
"action_title": f"重新啟動 {target} 服務",
|
||
"description": f"⚙️ 規則匹配: {target} 發生異常，需進一步診斷確認根因。",
|
||
"suggested_action": "RESTART_DEPLOYMENT",
|
||
"kubectl_command": f"kubectl rollout restart deployment/{target} -n {namespace}",
|
||
"target_resource": target,
|
||
"namespace": namespace,
|
||
"risk_level": "medium",
|
||
"blast_radius": {
|
||
"affected_pods": 1,
|
||
"estimated_downtime": "5-15 min",
|
||
"related_services": [target],
|
||
"data_impact": "NONE",
|
||
},
|
||
"primary_responsibility": "COLLAB",
|
||
"responsibility_reasoning": "告警資訊不足，建議多團隊協同排查",
|
||
"secondary_teams": ["BE", "INFRA"],
|
||
"optimization_suggestions": [],
|
||
"reasoning": f"[規則匹配] 根據告警 {alert_type} 先重啟恢復服務。",
|
||
"deviation_analysis": "監控指標顯示異常偏離基準線",
|
||
"confidence": 0.0,
|
||
"affected_services": [target],
|
||
"signoz_correlation": signoz_correlation,
|
||
}
|
||
|
||
# 補充 SignOz 關聯資訊（規則引擎不持有 signoz_metrics）
|
||
mock_response["signoz_correlation"] = signoz_correlation
|
||
if signoz_metrics:
|
||
mock_response["description"] += f" {signoz_metrics.to_summary()}"
|
||
|
||
rule_id = mock_response.get("rule_id", "unknown")
|
||
logger.info(
|
||
"mock_llm_response_generated",
|
||
rule_id=rule_id,
|
||
action_title=mock_response["action_title"],
|
||
risk_level=mock_response["risk_level"],
|
||
primary_responsibility=mock_response["primary_responsibility"],
|
||
confidence=mock_response["confidence"],
|
||
signoz_integrated=signoz_metrics is not None,
|
||
is_mock=True,
|
||
)
|
||
|
||
# 2026-04-09 ogt: rule_id 回傳給上層 async 方法觸發自動規則生成
|
||
# 不在此 sync 方法中呼叫 asyncio，避免 event loop 混用問題 (S1-1 架構師審查)
|
||
return json.dumps(mock_response), rule_id
|
||
|
||
# =========================================================================
|
||
# LLM Cache Layer (憲法要求: 嚴禁無快取裸奔)
|
||
# =========================================================================
|
||
|
||
def _generate_cache_key(self, prompt: str, context_hash: str = "") -> str:
|
||
"""
|
||
生成 LLM 快取鍵
|
||
|
||
有告警上下文時，使用 prompt family + 穩定告警維度，避免 annotations /
|
||
SignOz 即時數值讓同一告警每 20 秒打穿快取；沒有上下文時仍用完整 prompt。
|
||
"""
|
||
if context_hash:
|
||
prompt_family_source = (
|
||
"openclaw_alert_analysis"
|
||
if "## Alert Data:" in prompt
|
||
else prompt[:512]
|
||
)
|
||
prompt_family = hashlib.sha256(prompt_family_source.encode()).hexdigest()[:8]
|
||
content = f"{prompt_family}:{context_hash}"
|
||
else:
|
||
content = prompt
|
||
hash_digest = hashlib.sha256(content.encode()).hexdigest()[:16]
|
||
return f"llm_cache:{hash_digest}"
|
||
|
||
async def _call_with_cache(
|
||
self,
|
||
prompt: str,
|
||
alert_context: dict | None = None,
|
||
signoz_metrics: GoldMetrics | None = None,
|
||
cache_ttl: int = 3600, # 1 hour default
|
||
) -> tuple[str, str, bool, bool, int, float]:
|
||
"""
|
||
帶快取的 LLM 呼叫包裝器
|
||
|
||
憲法條款: 必須使用快取保護算力資源
|
||
|
||
Args:
|
||
prompt: LLM prompt
|
||
alert_context: 告警上下文
|
||
signoz_metrics: SignOz 指標
|
||
cache_ttl: 快取存活時間 (秒)
|
||
|
||
Returns:
|
||
(response, provider, success, from_cache, total_tokens, cost_usd)
|
||
|
||
2026-03-29 ogt: 加入 Token/Cost 追蹤
|
||
"""
|
||
# 生成快取鍵 (基於 prompt + alert_context hash)
|
||
# 2026-04-16 ogt + Claude Sonnet 4.6: 修復 — alertname 才是主要識別符
|
||
# 舊版用 alert_type:target_resource → 不同告警 (e.g. PostgreSQLDiskGrowth vs PodCrashLoop)
|
||
# 在 alert_type="custom" 時共用同一快取鍵 → 全部回傳相同 LLM 結果
|
||
context_hash = _build_alert_cache_context_hash(alert_context)
|
||
|
||
cache_key = self._generate_cache_key(prompt, context_hash)
|
||
|
||
# 1. 嘗試從快取讀取
|
||
try:
|
||
redis_client = get_redis()
|
||
cached = await redis_client.get(cache_key)
|
||
if cached:
|
||
cached_data = json.loads(cached)
|
||
logger.info(
|
||
"llm_cache_hit",
|
||
cache_key=cache_key[:20],
|
||
provider=cached_data.get("provider", "cached"),
|
||
)
|
||
return (
|
||
cached_data["response"],
|
||
f"{cached_data['provider']}_cached",
|
||
True,
|
||
True, # from_cache
|
||
0, # tokens (cache hit, no new tokens)
|
||
0.0, # cost (cache hit, no cost)
|
||
)
|
||
except Exception as e:
|
||
logger.warning("llm_cache_read_failed", error=str(e))
|
||
|
||
# 2. Cache Miss - 呼叫 LLM
|
||
logger.info("llm_cache_miss", cache_key=cache_key[:20])
|
||
response, provider, success, total_tokens, cost_usd = await self._call_with_fallback(
|
||
prompt, alert_context, signoz_metrics
|
||
)
|
||
|
||
# 3. 成功則寫入快取
|
||
if success:
|
||
try:
|
||
redis_client = get_redis()
|
||
cache_data = {
|
||
"response": response,
|
||
"provider": provider,
|
||
"cached_at": now_taipei_iso(),
|
||
}
|
||
await redis_client.set(
|
||
cache_key,
|
||
json.dumps(cache_data, ensure_ascii=False),
|
||
ex=cache_ttl,
|
||
)
|
||
logger.info(
|
||
"llm_cache_write",
|
||
cache_key=cache_key[:20],
|
||
provider=provider,
|
||
ttl=cache_ttl,
|
||
)
|
||
except Exception as e:
|
||
logger.warning("llm_cache_write_failed", error=str(e))
|
||
|
||
return response, provider, success, False, total_tokens, cost_usd # from_cache=False
|
||
|
||
# =========================================================================
|
||
# Public LLM Interface (ILLMProvider Protocol)
|
||
# =========================================================================
|
||
|
||
async def call(
|
||
self,
|
||
prompt: str,
|
||
alert_context: dict | None = None,
|
||
) -> tuple[str, str, bool]:
|
||
"""
|
||
呼叫 LLM (ILLMProvider Protocol 實作)
|
||
|
||
#39 Error Analyzer Agent 使用此方法
|
||
|
||
Args:
|
||
prompt: 完整的 prompt
|
||
alert_context: 可選的告警上下文（含 incident_id/signals，
|
||
供 OPENCLAW_NEMO provider 使用結構化 API）
|
||
|
||
Returns:
|
||
(response, provider, success)
|
||
|
||
2026-04-16 ogt + Claude Sonnet 4.6: 修復 — _call_with_fallback 回傳 5 值，
|
||
call() 回傳 3 值，避免呼叫端 (diagnostician_agent:107) ValueError
|
||
2026-04-16 ogt + Claude Sonnet 4.6: 加入 alert_context — 讓 diagnostician
|
||
能傳 snapshot 結構化資料給 OPENCLAW_NEMO，避免 prompt 被截斷為 garbage
|
||
"""
|
||
response, provider, success, _tokens, _cost = await self._call_with_fallback(
|
||
prompt, alert_context=alert_context
|
||
)
|
||
return response, provider, success
|
||
|
||
# =========================================================================
|
||
# Fallback Chain
|
||
# =========================================================================
|
||
|
||
async def _call_with_fallback(
|
||
self,
|
||
prompt: str,
|
||
alert_context: dict | None = None,
|
||
signoz_metrics: GoldMetrics | None = None,
|
||
) -> tuple[str, str, bool, int, float]:
|
||
"""
|
||
依 AI_FALLBACK_ORDER 順序呼叫 AI
|
||
|
||
若 MOCK_MODE=True，直接回傳模擬結果。
|
||
若所有 Provider 失敗，fallback 到 Mock。
|
||
|
||
Returns:
|
||
tuple: (response, provider, success, total_tokens, cost_usd)
|
||
|
||
Phase 15.1: 整合 Langfuse LLMOps 追蹤
|
||
2026-03-29 ogt: 加入 Token/Cost 追蹤
|
||
2026-04-02 ogt: Phase 24 ADR-052 絞殺者包裝 — USE_AI_ROUTER 新舊並存
|
||
"""
|
||
# =================================================================
|
||
# Phase 24 ADR-052: 絞殺者分支 (Strangler Fig)
|
||
# USE_AI_ROUTER=true → 新 AIRouterExecutor 路由
|
||
# USE_AI_ROUTER=false → 舊 if/else fallback chain (現狀)
|
||
# 回滾: kubectl set env deployment/awoooi-api USE_AI_ROUTER=false
|
||
# Phase 24 C: Redis 狀態覆蓋 env var (/ai router on/off)
|
||
# =================================================================
|
||
# Redis 狀態優先 (Phase 24 C — 2026-04-03 ogt)
|
||
_use_ai_router = settings.USE_AI_ROUTER
|
||
try:
|
||
from src.services.ai_control import get_ai_router_enabled
|
||
_redis_override = await get_ai_router_enabled()
|
||
if _redis_override is not None:
|
||
_use_ai_router = _redis_override
|
||
except Exception:
|
||
pass
|
||
|
||
if _use_ai_router:
|
||
try:
|
||
# 2026-04-02 ogt: C2 修復 — 呼叫 AIRouter.route() 智慧路由 (非靜態 order)
|
||
# D1 意圖分類路由、D7 隱私保護 (DIAGNOSE/CODE_REVIEW 強制 local) 生效
|
||
from src.services.ai_router import (
|
||
IntentType,
|
||
get_ai_executor,
|
||
get_ai_router,
|
||
)
|
||
router = get_ai_router()
|
||
executor = get_ai_executor()
|
||
|
||
# Step 1: 取得路由決策 (含意圖分類 + 複雜度評分)
|
||
decision = await router.route(prompt, alert_context)
|
||
|
||
# Step 2: 從 RoutingDecision 建立 provider_order (主 + fallback)
|
||
# Phase 24 C: Redis primary_provider 覆蓋路由決策
|
||
provider_order = [decision.selected_provider.value] + [
|
||
p.value for p, _ in decision.fallback_chain
|
||
if p.value != decision.selected_provider.value
|
||
]
|
||
try:
|
||
from src.services.ai_control import (
|
||
get_primary_provider,
|
||
is_provider_disabled,
|
||
)
|
||
_primary = await get_primary_provider()
|
||
if _primary and _primary != decision.selected_provider.value:
|
||
# 把 primary 移到首位 (保留原始 fallback)
|
||
provider_order = [_primary] + [p for p in provider_order if p != _primary]
|
||
# 過濾被停用的 Provider
|
||
# C2 修復 (2026-04-03 首席架構師審查): Python 3.11 不支援 list comprehension 中 await
|
||
_filtered = []
|
||
for _p in provider_order:
|
||
if not await is_provider_disabled(_p):
|
||
_filtered.append(_p)
|
||
if _filtered:
|
||
provider_order = _filtered
|
||
except Exception as _e:
|
||
logger.warning("ai_control_override_failed", error=str(_e))
|
||
|
||
if self._alert_enforces_ollama_first(alert_context):
|
||
original_provider_order = list(provider_order)
|
||
provider_order = await self._resolve_alert_provider_order(
|
||
task_type=decision.intent.value if decision.intent else "diagnose",
|
||
alert_context=alert_context,
|
||
cloud_provider_order=original_provider_order,
|
||
)
|
||
logger.info(
|
||
"alert_ollama_first_provider_order",
|
||
original_provider_order=original_provider_order,
|
||
provider_order=provider_order,
|
||
cloud_fallback_allowed=self._cloud_fallback_allowed_for_alert(alert_context),
|
||
)
|
||
|
||
# Step 3: D7 隱私 — CODE_REVIEW 強制 local
|
||
# 2026-04-15 ogt: DIAGNOSE 移除 require_local（v4.3 決策：NIM 為主力，無隱私問題）
|
||
# ai_router.py v4.3 已明確：「NIM 從 Phase 22 起就是主力，無隱私問題」
|
||
# require_local=True 對 DIAGNOSE 只會讓所有 provider 被 privacy_skip → 永遠失敗
|
||
require_local = decision.intent in (IntentType.CODE_REVIEW,)
|
||
|
||
# 2026-04-29 ogt + Claude Code: 注入 task_type 讓 Ollama 用正確 timeout
|
||
# 根因: ai_providers/ollama.py:77 讀 context["task_type"] 決定 timeout
|
||
# - "diagnose"/"force_local" → OLLAMA_DIAGNOSE_TIMEOUT_SECONDS
|
||
# - 其他/未注入 → OPENCLAW_TIMEOUT=30s（不夠 qwen2.5:7b 推理）
|
||
# webhooks alert_context 從未注入 task_type → Ollama fallback 永遠 30s timeout
|
||
# 對齊 decision.intent 後 Ollama fallback 真正能跑完
|
||
exec_context = dict(alert_context) if alert_context else {}
|
||
if decision.intent == IntentType.DIAGNOSE:
|
||
exec_context["task_type"] = "diagnose"
|
||
if self._alert_enforces_ollama_first(alert_context):
|
||
exec_context.setdefault("task_type", "diagnose")
|
||
exec_context.setdefault("ollama_model", getattr(settings, "ALERT_OLLAMA_MODEL", "qwen3:14b"))
|
||
exec_context["allow_gcp_heavy_model"] = bool(
|
||
exec_context.get("allow_gcp_heavy_model", True)
|
||
)
|
||
exec_context["alert_requires_ollama_before_cloud"] = True
|
||
|
||
result = await executor.execute(
|
||
prompt=prompt,
|
||
provider_order=provider_order,
|
||
context=exec_context,
|
||
cache_ttl=3600,
|
||
require_local=require_local,
|
||
)
|
||
logger.info(
|
||
"phase24_ai_router_used",
|
||
provider=result.provider,
|
||
success=result.success,
|
||
latency_ms=round(result.latency_ms, 1),
|
||
intent=decision.intent.value,
|
||
routing_reason=decision.routing_reason,
|
||
)
|
||
return result.raw_response, result.provider, result.success, result.tokens, result.cost_usd
|
||
except Exception as e:
|
||
# AIRouter 失敗時 fallback 到舊路徑 (安全網)
|
||
logger.warning("phase24_ai_router_fallback_to_legacy", error=str(e))
|
||
|
||
# Mock Mode: 開發測試用
|
||
if settings.MOCK_MODE:
|
||
logger.info("mock_mode_enabled", using="mock_llm")
|
||
_mock_json, _rule_id = self._generate_mock_response(alert_context or {}, signoz_metrics)
|
||
if _rule_id == "generic_fallback":
|
||
import asyncio
|
||
|
||
from src.services.alert_rule_engine import auto_generate_rule
|
||
try:
|
||
asyncio.create_task(auto_generate_rule(
|
||
alert_context or {},
|
||
ollama_url=settings.OLLAMA_URL,
|
||
model=settings.OPENCLAW_DEFAULT_MODEL,
|
||
gemini_api_key=(
|
||
getattr(settings, "GEMINI_API_KEY", "")
|
||
if self._cloud_fallback_allowed_for_alert(alert_context)
|
||
else ""
|
||
),
|
||
))
|
||
except Exception as _e:
|
||
logger.warning("auto_rule_trigger_failed", error=str(_e))
|
||
return _mock_json, "mock", True, 0, 0.0
|
||
|
||
# Phase 15.1 + 15.3: Langfuse 追蹤整合 + SignOz Deep Linking
|
||
with langfuse_trace(
|
||
"openclaw_fallback_chain",
|
||
metadata={
|
||
"prompt_length": len(prompt),
|
||
"fallback_order": settings.AI_FALLBACK_ORDER,
|
||
"alert_fingerprint": (alert_context or {}).get("fingerprint", "unknown"),
|
||
},
|
||
) as trace:
|
||
# Phase 15.3: SignOz → Langfuse 反向連結
|
||
# 在當前 OTEL span 中記錄 Langfuse trace_id
|
||
if trace.langfuse_trace_id:
|
||
from opentelemetry import trace as otel_trace
|
||
|
||
from src.core.deep_linking import DeepLinking
|
||
|
||
current_span = otel_trace.get_current_span()
|
||
if current_span:
|
||
current_span.set_attribute("langfuse.trace_id", trace.langfuse_trace_id)
|
||
current_span.set_attribute(
|
||
"langfuse.trace_url",
|
||
DeepLinking.langfuse_trace_url(trace.langfuse_trace_id),
|
||
)
|
||
|
||
# Phase 13.2: Rate Limiter 整合 (2026-03-26)
|
||
# 防止雲端 API 用量暴衝，超限自動降級
|
||
from src.services.ai_rate_limiter import get_ai_rate_limiter
|
||
rate_limiter = get_ai_rate_limiter()
|
||
|
||
legacy_provider_order = list(settings.AI_FALLBACK_ORDER)
|
||
if self._alert_enforces_ollama_first(alert_context):
|
||
legacy_provider_order = ["ollama"]
|
||
if self._cloud_fallback_allowed_for_alert(alert_context):
|
||
legacy_provider_order.append("gemini")
|
||
logger.info(
|
||
"legacy_alert_ollama_first_provider_order",
|
||
provider_order=legacy_provider_order,
|
||
cloud_fallback_allowed=self._cloud_fallback_allowed_for_alert(alert_context),
|
||
)
|
||
|
||
for provider in legacy_provider_order:
|
||
# Rate Limit 檢查 (nvidia/gemini/claude 需檢查，ollama 不限)
|
||
# 2026-03-30 ogt: 加入 nvidia (RPM=5 限制)
|
||
if provider in ("nvidia", "gemini", "claude"):
|
||
allowed, reason = await rate_limiter.check_and_increment(provider)
|
||
if not allowed:
|
||
logger.warning(
|
||
"ai_rate_limit_skip",
|
||
provider=provider,
|
||
reason=reason,
|
||
)
|
||
continue # 跳過此 provider，嘗試下一個
|
||
|
||
logger.info("ai_provider_attempt", provider=provider)
|
||
|
||
start_time = time.time()
|
||
model_name = self._get_model_name(provider)
|
||
|
||
# 2026-03-29 ogt: Gemini 回傳 4 值 (含 token/cost)，其他 Provider 補 0
|
||
total_tokens = 0
|
||
cost_usd = 0.0
|
||
|
||
if provider == "ollama":
|
||
response, success = await self._call_ollama(
|
||
prompt,
|
||
ollama_only=self._alert_enforces_ollama_first(alert_context),
|
||
)
|
||
elif provider == "gemini":
|
||
response, success, total_tokens, cost_usd = await self._call_gemini(prompt)
|
||
elif provider == "nvidia":
|
||
# 2026-03-29 ogt: 使用 NvidiaProvider.chat() (模組化規範)
|
||
from src.services.nvidia_provider import get_nvidia_provider
|
||
nvidia_provider = get_nvidia_provider()
|
||
response, success, total_tokens, cost_usd = await nvidia_provider.chat(prompt, use_json_mode=True)
|
||
elif provider == "claude":
|
||
response, success = await self._call_claude(prompt)
|
||
else:
|
||
logger.warning("unknown_ai_provider", provider=provider)
|
||
continue
|
||
|
||
latency_ms = (time.time() - start_time) * 1000
|
||
|
||
# Langfuse: 記錄每次 LLM 呼叫
|
||
trace.generation(
|
||
name=f"{provider}_call",
|
||
model=model_name,
|
||
input=prompt[:500], # 截斷避免過長
|
||
output=response[:500] if success else f"ERROR: {response[:200]}",
|
||
metadata={
|
||
"success": success,
|
||
"latency_ms": round(latency_ms, 2),
|
||
"provider": provider,
|
||
"total_tokens": total_tokens,
|
||
"cost_usd": cost_usd,
|
||
},
|
||
)
|
||
|
||
if success:
|
||
logger.info(
|
||
"ai_provider_success",
|
||
provider=provider,
|
||
latency_ms=latency_ms,
|
||
total_tokens=total_tokens,
|
||
cost_usd=f"${cost_usd:.6f}",
|
||
)
|
||
# Langfuse: 記錄成功評分
|
||
trace.score(name="provider_success", value=1.0, comment=f"Success via {provider}")
|
||
|
||
# 2026-03-29 ogt: 記錄累積成本 (Gemini/Claude)
|
||
if cost_usd > 0:
|
||
await rate_limiter.record_cost(provider, cost_usd)
|
||
|
||
return response, provider, True, total_tokens, cost_usd
|
||
|
||
logger.warning("ai_provider_failed_fallback", provider=provider, latency_ms=latency_ms)
|
||
|
||
# 所有 Provider 失敗時，fallback 到 Mock (優雅降級)
|
||
logger.warning("all_providers_failed_using_mock", fallback="mock_llm")
|
||
trace.score(name="provider_success", value=0.0, comment="All providers failed, using mock")
|
||
_mock_json, _rule_id = self._generate_mock_response(alert_context or {}, signoz_metrics)
|
||
if _rule_id == "generic_fallback":
|
||
import asyncio
|
||
|
||
from src.services.alert_rule_engine import auto_generate_rule
|
||
try:
|
||
asyncio.create_task(auto_generate_rule(
|
||
alert_context or {},
|
||
ollama_url=settings.OLLAMA_URL,
|
||
model=settings.OPENCLAW_DEFAULT_MODEL,
|
||
gemini_api_key=(
|
||
getattr(settings, "GEMINI_API_KEY", "")
|
||
if self._cloud_fallback_allowed_for_alert(alert_context)
|
||
else ""
|
||
),
|
||
))
|
||
except Exception as _e:
|
||
logger.warning("auto_rule_trigger_failed", error=str(_e))
|
||
return _mock_json, "mock_fallback", True, 0, 0.0
|
||
|
||
def _get_model_name(self, provider: str) -> str:
|
||
"""取得 provider 對應的模型名稱 (從 ModelRegistry)"""
|
||
registry = get_model_registry()
|
||
return registry.get_model(provider, "rca")
|
||
|
||
# =========================================================================
|
||
# Response Parsing (防禦性解析)
|
||
# =========================================================================
|
||
|
||
def _extract_json_from_response(self, text: str) -> str | None:
|
||
"""從 LLM 回應中提取 JSON (含啟發式修補)"""
|
||
# 0. 清理開頭結尾空白
|
||
text = text.strip()
|
||
if not text:
|
||
return None
|
||
|
||
# 1. 嘗試直接解析
|
||
try:
|
||
json.loads(text)
|
||
return text
|
||
except json.JSONDecodeError:
|
||
pass
|
||
|
||
# 2. 嘗試從 markdown code block 提取
|
||
patterns = [
|
||
r"```json\s*([\s\S]*?)\s*```",
|
||
r"```\s*([\s\S]*?)\s*```",
|
||
r"(\{[\s\S]*\})", # 貪婪匹配最大括號對
|
||
]
|
||
|
||
for pattern in patterns:
|
||
match = re.search(pattern, text)
|
||
if match:
|
||
candidate = match.group(1) if "(" in pattern else match.group(0)
|
||
candidate = candidate.strip()
|
||
try:
|
||
json.loads(candidate)
|
||
return candidate
|
||
except json.JSONDecodeError:
|
||
# 3. 啟發式修補: 如果結尾缺少括號，嘗試補齊
|
||
if candidate.startswith("{") and not candidate.endswith("}"):
|
||
for i in range(1, 5): # 嘗試補 1-5 個括號/引號
|
||
try:
|
||
repaired = candidate + '"' * (i - 1) + "}" * i
|
||
json.loads(repaired)
|
||
logger.info("json_repaired_heuristically", level=i)
|
||
return repaired
|
||
except json.JSONDecodeError:
|
||
continue
|
||
continue
|
||
|
||
# 4. 極端情況: 找出最後一個有效 key
|
||
if "{" in text:
|
||
start_idx = text.find("{")
|
||
candidate = text[start_idx:]
|
||
# 暴力去除非法尾綴 (如 \t\t...)
|
||
candidate = re.sub(r"[ \t\r\n]+$", "", candidate)
|
||
if not candidate.endswith("}"):
|
||
candidate += '"}' # 嘗試最簡單的閉合
|
||
try:
|
||
json.loads(candidate)
|
||
return candidate
|
||
except json.JSONDecodeError:
|
||
pass
|
||
|
||
return None
|
||
|
||
def _validate_deployment_inventory(
|
||
self,
|
||
result: "OpenClawDecision | None",
|
||
k8s_inventory: str,
|
||
k8s_ns: str,
|
||
) -> None:
|
||
"""
|
||
2026-04-19 ogt + Claude Opus 4.7 (抽取自 analyze_alert):
|
||
幻覺 deployment 名偵測與降級。雙路徑共用(analyze_alert + generate_incident_proposal)。
|
||
|
||
根因: NEMOTRON 即使 prompt 含 inventory 仍會拿 namespace 當 deployment 名
|
||
→ 執行 kubectl rollout restart deployment/awoooi-prod → "not found"
|
||
修復: 正則抽出 kubectl 指令的 deployment 名,對照 inventory 白名單;
|
||
不在白名單 → 降級為 NO_ACTION + 轉純調查 get deploy + 信心 0。
|
||
"""
|
||
if not result or not k8s_inventory:
|
||
return
|
||
_inventory_names = {n.strip() for n in k8s_inventory.split(",") if n.strip()}
|
||
if not _inventory_names:
|
||
return
|
||
_kcmd = (result.kubectl_command or "").lower()
|
||
import re as _re
|
||
_m = _re.search(r"deployment[/\s]+([a-z0-9][a-z0-9-]*)", _kcmd)
|
||
if not _m:
|
||
return
|
||
_deploy_guess = _m.group(1)
|
||
if _deploy_guess in _inventory_names:
|
||
return
|
||
|
||
logger.warning(
|
||
"openclaw_deployment_hallucination_detected",
|
||
hallucinated=_deploy_guess,
|
||
inventory=sorted(_inventory_names),
|
||
original_kubectl_cmd=result.kubectl_command,
|
||
original_action=(
|
||
result.suggested_action.value
|
||
if hasattr(result.suggested_action, "value")
|
||
else str(result.suggested_action)
|
||
),
|
||
namespace=k8s_ns,
|
||
)
|
||
# 降級為安全調查動作,不執行破壞性操作
|
||
try:
|
||
result.kubectl_command = f"kubectl get deploy -n {k8s_ns}"
|
||
except Exception:
|
||
pass
|
||
try:
|
||
result.target_resource = "unknown(hallucinated)"
|
||
except Exception:
|
||
pass
|
||
try:
|
||
result.suggested_action = SuggestedAction.NO_ACTION
|
||
except Exception:
|
||
pass
|
||
try:
|
||
result.action_title = f"[安全降級] 調查 {k8s_ns} 真實資源狀態"
|
||
except Exception:
|
||
pass
|
||
try:
|
||
result.description = (
|
||
f"[安全降級] 原 LLM 建議的 deployment '{_deploy_guess}' 不在叢集 inventory "
|
||
f"({', '.join(sorted(_inventory_names))})。"
|
||
f"已降級為純調查動作(kubectl get deploy),請手動確認實際問題資源。"
|
||
)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
result.confidence = 0.0
|
||
except Exception:
|
||
pass
|
||
# 2026-04-21 ogt: 降級後風險強制 LOW，避免 NO_ACTION 因原 HIGH/CRITICAL risk
|
||
# 留在 PENDING 等 Telegram 批准（已無執行內容，等待毫無意義）
|
||
try:
|
||
result.risk_level = AIRiskLevel.LOW
|
||
except Exception:
|
||
pass
|
||
|
||
def _parse_analysis_result(self, raw_response: str) -> OpenClawDecision | None:
|
||
"""
|
||
解析 LLM 分析結果 - 使用 Pydantic Schema Enforcement
|
||
|
||
關鍵：blast_radius 為 REQUIRED，使用 AIBlastRadius Pydantic 模型驗證
|
||
"""
|
||
json_str = self._extract_json_from_response(raw_response)
|
||
if not json_str:
|
||
logger.error("json_extraction_failed", raw_response=raw_response[:200])
|
||
return None
|
||
|
||
try:
|
||
data = json.loads(json_str)
|
||
|
||
# Step 1: 確保 blast_radius 存在且為正確格式
|
||
if "blast_radius" not in data or not isinstance(data["blast_radius"], dict):
|
||
data["blast_radius"] = {
|
||
"affected_pods": 1,
|
||
"estimated_downtime": "~30s",
|
||
"related_services": data.get("affected_services", []),
|
||
"data_impact": "NONE"
|
||
}
|
||
else:
|
||
# 確保 blast_radius 內的必填欄位存在
|
||
br = data["blast_radius"]
|
||
if "affected_pods" not in br:
|
||
br["affected_pods"] = 1
|
||
if "estimated_downtime" not in br:
|
||
br["estimated_downtime"] = "~30s"
|
||
if "related_services" not in br:
|
||
br["related_services"] = data.get("affected_services", [])
|
||
if "data_impact" not in br:
|
||
br["data_impact"] = "NONE"
|
||
|
||
# Step 2: 填補其他可選欄位
|
||
if "action_title" not in data:
|
||
data["action_title"] = data.get("action", "未知操作")
|
||
if "target_resource" not in data:
|
||
data["target_resource"] = "unknown"
|
||
if "suggested_action" not in data:
|
||
data["suggested_action"] = "NO_ACTION"
|
||
|
||
# Step 2.5: 2026-04-01 Claude Code - 斷片補全 (信心度必須誠實)
|
||
# 🔴 禁止填入假信心度！截斷 = 0.0，讓 auto-approve 正確判斷
|
||
# 2026-04-12 ogt: NIM/Ollama 有時回傳字串 "0.85"，先嘗試 parse 再判斷
|
||
if "confidence" in data and isinstance(data["confidence"], str):
|
||
try:
|
||
data["confidence"] = float(data["confidence"])
|
||
except (ValueError, TypeError):
|
||
data.pop("confidence", None)
|
||
if "confidence" not in data or not isinstance(data["confidence"], int | float):
|
||
data["confidence"] = 0.0 # 截斷/缺失 → 0.0，不可偽造
|
||
if "risk_level" not in data:
|
||
data["risk_level"] = "low"
|
||
# 2026-04-19 ogt + Claude Opus 4.7 修 AP-3:
|
||
# primary_responsibility 有時 LLM 填空字串/None → resp_display 顯示「❓ 未知」
|
||
# 強制正規化: 空/None/不在白名單 → 用 kubectl 有無推 INFRA 或 BE (非「未知」)
|
||
_valid_resp = {"FE", "BE", "INFRA", "DB", "COLLAB"}
|
||
_cur_resp = str(data.get("primary_responsibility") or "").strip().upper()
|
||
if _cur_resp not in _valid_resp:
|
||
data["primary_responsibility"] = "INFRA" if "kubectl" in str(data) else "BE"
|
||
if "suggested_action" not in data:
|
||
data["suggested_action"] = "RESTART_DEPLOYMENT" if "restart" in str(data).lower() else "NO_ACTION"
|
||
if "reasoning" not in data:
|
||
data["reasoning"] = "AI 產出欄位缺失，系統自動補全以維持運作。"
|
||
|
||
# Step 3: 使用 Pydantic 驗證 (會自動正規化 risk_level, data_impact 等)
|
||
decision = OpenClawDecision(**data)
|
||
|
||
logger.info(
|
||
"pydantic_validation_success",
|
||
action_title=decision.action_title,
|
||
risk_level=decision.risk_level.value,
|
||
blast_radius_pods=decision.blast_radius.affected_pods,
|
||
)
|
||
|
||
return decision
|
||
|
||
except Exception as e:
|
||
logger.error(
|
||
"pydantic_validation_failed",
|
||
error=str(e),
|
||
json_str=json_str[:300],
|
||
)
|
||
return None
|
||
|
||
# =========================================================================
|
||
# Main Analysis Methods
|
||
# =========================================================================
|
||
|
||
async def analyze_alert(
|
||
self,
|
||
alert_context: dict,
|
||
) -> tuple[LLMAnalysisResult | None, str, str, GoldMetrics | None, str, int, float]:
|
||
"""
|
||
分析告警並產生 RCA 結果 (含 SignOz 整合)
|
||
|
||
Args:
|
||
alert_context: 告警上下文 (alert_type, severity, target_resource, etc.)
|
||
|
||
Returns:
|
||
(analysis_result, ai_provider, raw_response, signoz_metrics, signoz_trace_url, total_tokens, cost_usd)
|
||
|
||
2026-03-29 ogt: 加入 Token/Cost 追蹤
|
||
"""
|
||
# Step 0: 擷取 SignOz 上下文
|
||
service_name = alert_context.get("target_resource", "unknown")
|
||
namespace = alert_context.get("namespace", "default")
|
||
|
||
signoz_metrics, signoz_trace_url = await self.get_signoz_context(
|
||
service_name=service_name,
|
||
namespace=namespace,
|
||
)
|
||
|
||
# 將 SignOz 數據加入 prompt
|
||
signoz_context = ""
|
||
if signoz_metrics:
|
||
signoz_context = f"""
|
||
## 📊 SignOz Real-time Metrics (Last 10 min)
|
||
{signoz_metrics.to_summary()}
|
||
|
||
Trace URL: {signoz_trace_url}
|
||
"""
|
||
|
||
# Step 0.5: 擷取 K8s 叢集真實資源清單（Checkpoint-2 webhook path）
|
||
# 2026-04-17 ogt + Claude Sonnet 4.6: 防止 NemoTron 幻覺 deployment/awoooi-service
|
||
# 根因：webhook path 沒有叢集上下文 → LLM 盲猜資源名稱 → kubectl not found → trust 0 永遠
|
||
# 修復：每次分析前先拉真實 Deployment 清單，注入 prompt 強制 LLM 對齊
|
||
_k8s_ns = alert_context.get("namespace", "awoooi-prod")
|
||
_k8s_inventory = await _fetch_k8s_inventory_for_openclaw(namespace=_k8s_ns)
|
||
k8s_section = (
|
||
f"\n\n## 🔒 叢集實際資源清單（{_k8s_ns}）\n"
|
||
f"kubectl_command 與 target_resource **必須**從以下名稱選擇，不可自行編造：\n"
|
||
f"{_k8s_inventory}\n"
|
||
if _k8s_inventory
|
||
else "\n\n## ⚠️ 無法取得叢集清單，target_resource 請依 alertname 推斷，勿編造。\n"
|
||
)
|
||
|
||
# P2.1 fix 2026-04-24 ogt + Claude Sonnet 4.6: 提取 MCP evidence_summary 注入 prompt
|
||
# diagnosis_context 由 decision_manager 在呼叫前填入（pre_decision_investigator 產出）
|
||
_raw_evidence = alert_context.get("diagnosis_context", "") or ""
|
||
if _raw_evidence and not _raw_evidence.startswith("<raw_evidence>"):
|
||
_raw_evidence = f"<raw_evidence>\n{_raw_evidence}\n</raw_evidence>"
|
||
evidence_section = f"\n\n## 🔬 MCP 實時環境證據\n{_raw_evidence}\n" if _raw_evidence else ""
|
||
|
||
# 格式化告警為 Prompt (2026-03-31 ogt: 強力截斷以符合 NVIDIA 4K 限制)
|
||
# 優先保留 System Prompt，截斷 Alert Data
|
||
available_len = 3500 - len(OPENCLAW_SYSTEM_PROMPT) - len(signoz_context) - len(k8s_section) - len(evidence_section)
|
||
if available_len < 500:
|
||
# 如果 SignOz 太長，也截斷它
|
||
signoz_context = signoz_context[:500] + "... (truncated)"
|
||
available_len = 3500 - len(OPENCLAW_SYSTEM_PROMPT) - len(signoz_context) - len(k8s_section) - len(evidence_section)
|
||
|
||
alert_json = json.dumps(alert_context, ensure_ascii=False, indent=2)
|
||
if len(alert_json) > available_len:
|
||
alert_json = alert_json[:available_len] + "... (truncated)"
|
||
|
||
full_prompt = OPENCLAW_SYSTEM_PROMPT + signoz_context + k8s_section + evidence_section + "\n\n## Alert Data:\n" + alert_json
|
||
|
||
logger.info(
|
||
"openclaw_alert_analysis_start",
|
||
alert_type=alert_context.get("alert_type"),
|
||
target=alert_context.get("target_resource"),
|
||
signoz_available=signoz_metrics is not None,
|
||
)
|
||
|
||
# 呼叫 LLM (使用快取層保護算力)
|
||
raw_response, provider, success, from_cache, total_tokens, cost_usd = await self._call_with_cache(
|
||
full_prompt,
|
||
alert_context,
|
||
signoz_metrics,
|
||
cache_ttl=1800, # 30 min for alert analysis
|
||
)
|
||
|
||
if not success:
|
||
logger.error("openclaw_all_providers_failed")
|
||
return None, provider, raw_response, signoz_metrics, signoz_trace_url, 0, 0.0
|
||
|
||
if from_cache:
|
||
logger.info("openclaw_using_cached_response", provider=provider)
|
||
|
||
logger.info(
|
||
"openclaw_llm_response_received",
|
||
provider=provider,
|
||
response_length=len(raw_response),
|
||
)
|
||
|
||
# 解析結果
|
||
result = self._parse_analysis_result(raw_response)
|
||
|
||
# 2026-04-18 → 2026-04-19: 幻覺 deployment 名偵測與降級 (共用 helper)
|
||
self._validate_deployment_inventory(result, _k8s_inventory, _k8s_ns)
|
||
|
||
if result:
|
||
logger.info(
|
||
"openclaw_analysis_complete",
|
||
action_title=result.action_title,
|
||
risk_level=result.risk_level,
|
||
confidence=result.confidence,
|
||
provider=provider,
|
||
signoz_integrated=signoz_metrics is not None,
|
||
)
|
||
else:
|
||
logger.warning(
|
||
"openclaw_analysis_parse_failed",
|
||
raw_response=raw_response[:300],
|
||
)
|
||
|
||
return result, provider, raw_response, signoz_metrics, signoz_trace_url, total_tokens, cost_usd
|
||
|
||
# =========================================================================
|
||
# Phase 6.4: LLM Proposal Generation
|
||
# =========================================================================
|
||
|
||
async def generate_incident_proposal(
|
||
self,
|
||
incident_id: str,
|
||
severity: str,
|
||
signals: list[dict],
|
||
affected_services: list[str],
|
||
expert_context: dict | None = None,
|
||
) -> tuple[dict | None, str, bool]:
|
||
"""
|
||
為 Incident 生成 LLM-based 修復提案
|
||
|
||
Phase 6.4: 賦予大腦「生成解決方案」的思考能力
|
||
2026-03-27: 整合 Expert System 診斷上下文
|
||
|
||
Args:
|
||
incident_id: Incident ID
|
||
severity: 嚴重度 (P0/P1/P2/P3)
|
||
signals: 關聯的告警訊號
|
||
affected_services: 受影響服務
|
||
expert_context: Expert System 初步診斷 (可選)
|
||
- initial_diagnosis: 規則匹配結果
|
||
- diagnosis_description: 診斷描述
|
||
- suggested_diagnosis_commands: 建議診斷指令
|
||
- expert_confidence: 信心分數
|
||
- requires_human_review: 是否需人工介入
|
||
|
||
Returns:
|
||
(proposal_dict, provider, success)
|
||
proposal_dict 包含:
|
||
- action: 建議動作
|
||
- description: 動作描述
|
||
- kubectl_command: kubectl 指令
|
||
- risk_level: 風險等級
|
||
- reasoning: LLM 推理過程
|
||
"""
|
||
# 建構 prompt (2026-03-31 ogt: Nemotron-mini context 較小，限制數量與長度)
|
||
signal_summary = "\n".join([
|
||
f"- {s.get('alert_name', 'unknown')}: {str(s.get('description', 'N/A'))[:100]}..."
|
||
for s in signals[:3] # 最多 3 筆，每筆最多 100 字元
|
||
])
|
||
|
||
target = affected_services[0] if affected_services else "unknown-service"
|
||
|
||
# 擷取 SignOz 指標
|
||
signoz_metrics, signoz_trace_url = await self.get_signoz_context(
|
||
service_name=target,
|
||
namespace="awoooi-prod",
|
||
)
|
||
|
||
signoz_context = ""
|
||
if signoz_metrics:
|
||
signoz_context = f"""
|
||
## 📊 SignOz Real-time Metrics
|
||
{signoz_metrics.to_summary()}
|
||
"""
|
||
|
||
# 2026-03-27: 整合 Expert System 診斷上下文
|
||
# 2026-03-26: ADR-030 Phase 2 - 加入 K8s/SignOz 診斷上下文
|
||
expert_diagnosis_context = ""
|
||
if expert_context:
|
||
diagnosis_cmds = expert_context.get("suggested_diagnosis_commands", [])
|
||
diagnosis_cmds_str = "\n".join([f" - `{cmd}`" for cmd in diagnosis_cmds]) if diagnosis_cmds else " - (無)"
|
||
|
||
# ADR-030: 加入完整診斷上下文 (如果有)，並限制長度以符合 4K Context
|
||
full_diagnosis = str(expert_context.get("diagnosis_context", ""))[:800]
|
||
if len(str(expert_context.get("diagnosis_context", ""))) > 800:
|
||
full_diagnosis += "... (truncated)"
|
||
diagnosis_signals = expert_context.get("diagnosis_signals", [])
|
||
signals_summary = ""
|
||
if diagnosis_signals:
|
||
signals_summary = "\n".join([
|
||
f" - [{s.get('severity', 'info').upper()}] {s.get('source', 'unknown')}: {s.get('message', 'N/A')[:100]}"
|
||
for s in diagnosis_signals[:5]
|
||
])
|
||
|
||
expert_diagnosis_context = f"""
|
||
## 🔍 Expert System Initial Diagnosis
|
||
- **Matched Rule**: {expert_context.get('initial_diagnosis', 'unknown')}
|
||
- **Diagnosis**: {expert_context.get('diagnosis_description', 'N/A')}
|
||
- **Confidence**: {expert_context.get('expert_confidence', 0.0):.0%}
|
||
- **Requires Human Review**: {'Yes' if expert_context.get('requires_human_review') else 'No'}
|
||
- **Suggested Diagnosis Commands**:
|
||
{diagnosis_cmds_str}
|
||
|
||
{f'''## 🩺 K8s/SignOz Deep Diagnosis (ADR-030)
|
||
{full_diagnosis}
|
||
|
||
### Diagnosis Signals
|
||
{signals_summary if signals_summary else " - (No signals detected)"}
|
||
''' if full_diagnosis else ''}
|
||
|
||
**IMPORTANT**: The Expert System and Diagnostic Aggregator have provided context.
|
||
Consider this data but apply your own analysis. If Expert says "human review required",
|
||
provide diagnostic guidance rather than automated fixes.
|
||
"""
|
||
|
||
# 2026-03-31 ogt: 針對 NVIDIA Nemo-4B 使用超精簡 Prompt
|
||
registry = get_model_registry()
|
||
is_nemo = "nvidia" in (registry.get_model("nvidia", "rca") or "").lower()
|
||
base_prompt = NEMOTRON_SYSTEM_PROMPT if is_nemo else OPENCLAW_SYSTEM_PROMPT
|
||
|
||
proposal_prompt = f"""{base_prompt}
|
||
|
||
{signoz_context}
|
||
{expert_diagnosis_context}
|
||
|
||
## 🚨 Incident Context
|
||
- **Incident ID**: {incident_id}
|
||
- **Severity**: {severity}
|
||
- **Affected Services**: {', '.join(affected_services)}
|
||
- **Signal Count**: {len(signals)}
|
||
|
||
## 📋 Alert Signals
|
||
{signal_summary}
|
||
|
||
## 🎯 Your Task
|
||
Based on the above incident, signals, and Expert System diagnosis, generate a remediation proposal.
|
||
You MUST respond with ONLY valid JSON following the schema above.
|
||
Focus on:
|
||
1. Root cause analysis based on signals, SignOz data, and Expert diagnosis
|
||
2. Specific kubectl command to remediate (or diagnostic command if root cause unclear)
|
||
3. Risk assessment for the proposed action
|
||
4. Preventive recommendations
|
||
5. If Expert System flagged "human review required", prioritize diagnostic commands over fixes
|
||
"""
|
||
|
||
logger.info(
|
||
"proposal_generation_start",
|
||
incident_id=incident_id,
|
||
severity=severity,
|
||
signal_count=len(signals),
|
||
signoz_available=signoz_metrics is not None,
|
||
)
|
||
|
||
# 2026-04-01 ogt: 架構鐵律 — OpenClaw (Nemo) 是 AI 大腦，優先委派仲裁
|
||
# AWOOOI K8s Pod 不直接打 Ollama/NVIDIA，避免並發逾時
|
||
openclaw_result = await self._call_openclaw_analyze(
|
||
incident_id, severity, signals, affected_services, expert_context
|
||
)
|
||
if openclaw_result is not None:
|
||
return openclaw_result, "openclaw_nemo", True
|
||
|
||
# 使用快取呼叫 LLM
|
||
alert_context = {
|
||
"incident_id": incident_id,
|
||
"alert_type": signals[0].get("alert_name", "incident") if signals else "incident",
|
||
"target_resource": target,
|
||
"severity": severity,
|
||
}
|
||
|
||
# 2026-03-29 ogt: 修復 tuple unpacking (Token/Cost 追蹤)
|
||
raw_response, provider, success, from_cache, ai_tokens, ai_cost = await self._call_with_cache(
|
||
proposal_prompt,
|
||
alert_context,
|
||
signoz_metrics,
|
||
cache_ttl=3600, # 1 hour for proposals
|
||
)
|
||
|
||
if not success:
|
||
logger.error(
|
||
"proposal_generation_failed",
|
||
incident_id=incident_id,
|
||
provider=provider,
|
||
)
|
||
return None, provider, False
|
||
|
||
# 解析 LLM 結果
|
||
result = self._parse_analysis_result(raw_response)
|
||
|
||
# 2026-04-19 ogt + Claude Opus 4.7: 同 analyze_alert 也需幻覺驗證
|
||
# 此路徑沒有 inventory 預抓,動態抓
|
||
_k8s_ns_for_validate = alert_context.get("namespace", "awoooi-prod") if "alert_context" in dir() else "awoooi-prod"
|
||
try:
|
||
_k8s_inv = await _fetch_k8s_inventory_for_openclaw(namespace=_k8s_ns_for_validate)
|
||
except Exception:
|
||
_k8s_inv = ""
|
||
self._validate_deployment_inventory(result, _k8s_inv, _k8s_ns_for_validate)
|
||
|
||
if result:
|
||
logger.info(
|
||
"proposal_generation_complete",
|
||
incident_id=incident_id,
|
||
action_title=result.action_title,
|
||
risk_level=result.risk_level,
|
||
provider=provider,
|
||
from_cache=from_cache,
|
||
)
|
||
|
||
# 轉換為 proposal dict (optimization_suggestions 是 list[dict])
|
||
# 2026-03-29 ogt: 加入 ai_tokens/ai_cost 追蹤
|
||
proposal_dict = {
|
||
"action": result.action_title,
|
||
"description": result.description,
|
||
"kubectl_command": result.kubectl_command,
|
||
"target_resource": result.target_resource,
|
||
"namespace": result.namespace,
|
||
"risk_level": result.risk_level,
|
||
"reasoning": result.reasoning,
|
||
"confidence": result.confidence,
|
||
"primary_responsibility": result.primary_responsibility,
|
||
"optimization_suggestions": [
|
||
{
|
||
"type": s.get("type", "UNKNOWN"),
|
||
"description": s.get("description", ""),
|
||
"kubectl_or_config": s.get("kubectl_or_config", ""),
|
||
}
|
||
for s in result.optimization_suggestions
|
||
],
|
||
"signoz_correlation": result.signoz_correlation,
|
||
"from_cache": from_cache,
|
||
"provider": provider,
|
||
"model": self._get_model_name(provider), # 2026-04-04 ogt: 底層模型名稱
|
||
"ai_tokens": ai_tokens,
|
||
"ai_cost": ai_cost,
|
||
}
|
||
return proposal_dict, provider, True
|
||
|
||
logger.warning(
|
||
"proposal_parse_failed",
|
||
incident_id=incident_id,
|
||
raw_response=raw_response[:300],
|
||
)
|
||
return None, provider, False
|
||
|
||
# =========================================================================
|
||
# Phase 22: OpenClaw + Nemotron 協作 (ADR-044)
|
||
# 2026-03-31 Claude Code: 統帥批准實作
|
||
# =========================================================================
|
||
|
||
async def _maybe_run_openclaw_agent_loop_shadow(
|
||
self,
|
||
*,
|
||
proposal: dict,
|
||
incident_id: str,
|
||
severity: str,
|
||
signals: list[dict],
|
||
affected_services: list[str],
|
||
expert_context: dict | None = None,
|
||
) -> None:
|
||
"""
|
||
ADR-105 P1: read-only Agent Loop shadow investigation.
|
||
|
||
This is intentionally non-decisive: it proves MCP tool_use/audit wiring
|
||
with local models and read-only tools, then falls back silently to the
|
||
regular proposal when disabled or unavailable.
|
||
"""
|
||
if not settings.ENABLE_OPENCLAW_AGENT_LOOP_SHADOW:
|
||
return
|
||
|
||
try:
|
||
from src.plugins.mcp.registry import get_provider_registry
|
||
from src.services.ai_providers.agent_loop import AgentToolExecutor
|
||
from src.services.ai_providers.permissions import is_read_only_tool
|
||
from src.services.ai_router import get_ai_registry
|
||
|
||
ai_registry = get_ai_registry()
|
||
provider = ai_registry.get("ollama") or ai_registry.get("ollama_local")
|
||
if provider is None or not hasattr(provider, "analyze_with_tools"):
|
||
logger.warning(
|
||
"openclaw_agent_loop_shadow_skipped",
|
||
incident_id=incident_id,
|
||
reason="no_local_tool_provider",
|
||
)
|
||
return
|
||
|
||
mcp_registry = get_provider_registry()
|
||
providers = {p.name: p for p in mcp_registry.all()}
|
||
allowed_servers = {"kubernetes", "prometheus", "signoz", "database", "rag", "grafana"}
|
||
available_tools = []
|
||
for mcp_provider in providers.values():
|
||
if mcp_provider.name not in allowed_servers:
|
||
continue
|
||
try:
|
||
provider_tools = await mcp_provider.list_tools()
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"openclaw_agent_loop_tool_list_failed",
|
||
incident_id=incident_id,
|
||
provider=mcp_provider.name,
|
||
error=str(exc),
|
||
)
|
||
continue
|
||
available_tools.extend(
|
||
tool for tool in provider_tools
|
||
if tool.server_name in allowed_servers and is_read_only_tool(tool)
|
||
)
|
||
|
||
if not available_tools:
|
||
logger.warning(
|
||
"openclaw_agent_loop_shadow_skipped",
|
||
incident_id=incident_id,
|
||
reason="no_readonly_tools",
|
||
)
|
||
return
|
||
|
||
executor = AgentToolExecutor(
|
||
available_tools=available_tools,
|
||
providers=providers,
|
||
agent_role="openclaw",
|
||
incident_id=incident_id,
|
||
flywheel_node="reason",
|
||
)
|
||
shadow_prompt = self._build_agent_loop_shadow_prompt(
|
||
proposal=proposal,
|
||
incident_id=incident_id,
|
||
severity=severity,
|
||
signals=signals,
|
||
affected_services=affected_services,
|
||
expert_context=expert_context,
|
||
)
|
||
result = await provider.analyze_with_tools(
|
||
prompt=shadow_prompt,
|
||
available_tools=available_tools,
|
||
tool_executor=executor.execute,
|
||
max_iterations=settings.OPENCLAW_AGENT_LOOP_MAX_ITERATIONS,
|
||
agent_role="openclaw",
|
||
context={
|
||
"incident_id": incident_id,
|
||
"severity": severity,
|
||
"task_type": "diagnose",
|
||
},
|
||
)
|
||
structured_shadow = self._parse_agent_loop_shadow_response(result.raw_response or "")
|
||
proposal["agent_loop_shadow"] = {
|
||
"enabled": True,
|
||
"success": result.success,
|
||
"provider": result.provider,
|
||
"tokens": result.tokens,
|
||
"latency_ms": round(result.latency_ms, 1),
|
||
"error": result.error,
|
||
"decision_impact": "none",
|
||
"structured": structured_shadow,
|
||
"confidence_delta": structured_shadow.get("confidence_delta", 0.0),
|
||
"preview": (result.raw_response or "")[:700],
|
||
}
|
||
logger.info(
|
||
"openclaw_agent_loop_shadow_complete",
|
||
incident_id=incident_id,
|
||
provider=result.provider,
|
||
success=result.success,
|
||
tools_available=len(available_tools),
|
||
latency_ms=round(result.latency_ms, 1),
|
||
confidence_delta=structured_shadow.get("confidence_delta", 0.0),
|
||
parse_status=structured_shadow.get("parse_status"),
|
||
)
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"openclaw_agent_loop_shadow_failed",
|
||
incident_id=incident_id,
|
||
error=str(exc),
|
||
)
|
||
|
||
@classmethod
|
||
def _parse_agent_loop_shadow_response(cls, raw_response: str) -> dict:
|
||
"""
|
||
Normalize read-only Agent Loop output into durable metadata.
|
||
|
||
The shadow result is intentionally non-decisive. Downstream code can
|
||
inspect this structure for quality review, but it must not override the
|
||
main proposal until ADR-105 canary graduation.
|
||
"""
|
||
text = (raw_response or "").strip()
|
||
if not text:
|
||
return {
|
||
"parse_status": "empty",
|
||
"root_cause_check": "",
|
||
"evidence_used": [],
|
||
"confidence_delta": 0.0,
|
||
"missing_evidence": [],
|
||
"human_or_ai_next_step": "",
|
||
}
|
||
|
||
payload = cls._extract_json_object(text)
|
||
if not isinstance(payload, dict):
|
||
return {
|
||
"parse_status": "unparsed",
|
||
"root_cause_check": "",
|
||
"evidence_used": [],
|
||
"confidence_delta": 0.0,
|
||
"missing_evidence": [],
|
||
"human_or_ai_next_step": "",
|
||
"raw_preview": text[:700],
|
||
}
|
||
|
||
return {
|
||
"parse_status": "ok",
|
||
"root_cause_check": cls._clip_shadow_text(payload.get("root_cause_check"), max_chars=500),
|
||
"evidence_used": cls._coerce_shadow_list(payload.get("evidence_used"), max_items=5),
|
||
"confidence_delta": cls._coerce_agent_loop_confidence_delta(
|
||
payload.get("confidence_delta", 0.0)
|
||
),
|
||
"missing_evidence": cls._coerce_shadow_list(payload.get("missing_evidence"), max_items=5),
|
||
"human_or_ai_next_step": cls._clip_shadow_text(
|
||
payload.get("human_or_ai_next_step"), max_chars=500
|
||
),
|
||
}
|
||
|
||
@staticmethod
|
||
def _extract_json_object(text: str) -> dict | None:
|
||
"""Extract the first JSON object from plain or fenced LLM output."""
|
||
candidates = [text]
|
||
fenced = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, flags=re.DOTALL | re.IGNORECASE)
|
||
if fenced:
|
||
candidates.insert(0, fenced.group(1))
|
||
object_match = re.search(r"\{.*\}", text, flags=re.DOTALL)
|
||
if object_match:
|
||
candidates.append(object_match.group(0))
|
||
|
||
for candidate in candidates:
|
||
try:
|
||
parsed = json.loads(candidate)
|
||
except (TypeError, json.JSONDecodeError):
|
||
continue
|
||
if isinstance(parsed, dict):
|
||
return parsed
|
||
return None
|
||
|
||
@staticmethod
|
||
def _clip_shadow_text(value: object, *, max_chars: int) -> str:
|
||
if value is None:
|
||
return ""
|
||
return str(value).strip()[:max_chars]
|
||
|
||
@classmethod
|
||
def _coerce_shadow_list(cls, value: object, *, max_items: int) -> list[str]:
|
||
if value is None:
|
||
return []
|
||
if isinstance(value, list):
|
||
items = value
|
||
else:
|
||
items = [value]
|
||
normalized = []
|
||
for item in items:
|
||
clipped = cls._clip_shadow_text(item, max_chars=240)
|
||
if clipped:
|
||
normalized.append(clipped)
|
||
if len(normalized) >= max_items:
|
||
break
|
||
return normalized
|
||
|
||
@staticmethod
|
||
def _coerce_agent_loop_confidence_delta(value: object) -> float:
|
||
"""
|
||
Keep canary deltas conservative: metadata may lower confidence later,
|
||
but positive boosts are recorded as 0 until the shadow path graduates.
|
||
"""
|
||
try:
|
||
delta = float(value)
|
||
except (TypeError, ValueError):
|
||
return 0.0
|
||
return round(max(min(delta, 0.0), -0.15), 3)
|
||
|
||
def _build_agent_loop_shadow_prompt(
|
||
self,
|
||
*,
|
||
proposal: dict,
|
||
incident_id: str,
|
||
severity: str,
|
||
signals: list[dict],
|
||
affected_services: list[str],
|
||
expert_context: dict | None = None,
|
||
) -> str:
|
||
"""Build a compact read-only investigation prompt for Agent Loop shadow mode."""
|
||
return f"""你是 OpenClaw 的唯讀 shadow investigator。你可以使用 MCP 工具查證，但不得要求任何寫入、重啟、刪除或通知動作。
|
||
|
||
請只回傳 JSON：
|
||
{{
|
||
"root_cause_check": "你對目前根因的查證結論",
|
||
"evidence_used": ["最多 5 條具體證據"],
|
||
"confidence_delta": -0.1,
|
||
"missing_evidence": ["還缺什麼證據"],
|
||
"human_or_ai_next_step": "下一個安全步驟"
|
||
}}
|
||
|
||
Incident: {incident_id}
|
||
Severity: {severity}
|
||
Affected services: {json.dumps(affected_services, ensure_ascii=False)}
|
||
Signals: {json.dumps(signals[:5], ensure_ascii=False, default=str)}
|
||
Current proposal: {json.dumps(proposal, ensure_ascii=False, default=str)}
|
||
Expert context: {json.dumps(expert_context or {}, ensure_ascii=False, default=str)}
|
||
"""
|
||
|
||
async def generate_incident_proposal_with_tools(
|
||
self,
|
||
incident_id: str,
|
||
severity: str,
|
||
signals: list[dict],
|
||
affected_services: list[str],
|
||
expert_context: dict | None = None,
|
||
) -> tuple[dict | None, str, bool]:
|
||
"""
|
||
Phase 22: OpenClaw + Nemotron 協作生成修復提案
|
||
|
||
架構:
|
||
- OpenClaw = 仲裁者 (Arbitrator) - 決定「為什麼」和「風險等級」
|
||
- Nemotron = 執行者 (Executor) - 決定「怎麼做」和「具體指令」
|
||
|
||
觸發條件:
|
||
- LOW 風險 → 僅 OpenClaw，跳過 Nemotron
|
||
- MEDIUM/HIGH/CRITICAL → OpenClaw + Nemotron 雙軌
|
||
|
||
Args:
|
||
incident_id: Incident ID
|
||
severity: 嚴重度 (P0/P1/P2/P3)
|
||
signals: 關聯的告警訊號
|
||
affected_services: 受影響服務
|
||
expert_context: Expert System 初步診斷 (可選)
|
||
|
||
Returns:
|
||
(proposal_dict, provider, success)
|
||
proposal_dict 新增:
|
||
- nemotron_enabled: bool
|
||
- nemotron_tools: list[dict] (如果啟用)
|
||
- nemotron_validation: str
|
||
- nemotron_latency_ms: float
|
||
"""
|
||
# Feature Flag 檢查
|
||
if not settings.ENABLE_NEMOTRON_COLLABORATION:
|
||
logger.info(
|
||
"nemotron_collaboration_disabled",
|
||
incident_id=incident_id,
|
||
reason="Feature flag disabled",
|
||
)
|
||
return await self.generate_incident_proposal(
|
||
incident_id, severity, signals, affected_services, expert_context
|
||
)
|
||
|
||
# Step 1: OpenClaw 仲裁
|
||
proposal, provider, success = await self.generate_incident_proposal(
|
||
incident_id, severity, signals, affected_services, expert_context
|
||
)
|
||
|
||
if not success or proposal is None:
|
||
return proposal, provider, success
|
||
|
||
await self._maybe_run_openclaw_agent_loop_shadow(
|
||
proposal=proposal,
|
||
incident_id=incident_id,
|
||
severity=severity,
|
||
signals=signals,
|
||
affected_services=affected_services,
|
||
expert_context=expert_context,
|
||
)
|
||
|
||
# Step 2: 判斷是否需要 Nemotron
|
||
risk_level = proposal.get("risk_level", "low").lower()
|
||
if risk_level == "low":
|
||
proposal["nemotron_enabled"] = False
|
||
logger.info(
|
||
"nemotron_skipped_low_risk",
|
||
incident_id=incident_id,
|
||
risk_level=risk_level,
|
||
)
|
||
return proposal, provider, True
|
||
|
||
# Step 3: 呼叫 Nemotron Tool Calling — 🔴 必須等到有結果，不可跳過
|
||
# 2026-04-07 ogt: 統帥指示 Nemotron 不能跳過，必須等到處理完成
|
||
logger.info(
|
||
"nemotron_collaboration_start",
|
||
incident_id=incident_id,
|
||
risk_level=risk_level,
|
||
)
|
||
|
||
max_retries = 2
|
||
last_error = None
|
||
for attempt in range(1, max_retries + 1):
|
||
try:
|
||
nemotron_result = await self._call_nemotron_tools(
|
||
incident_id=incident_id,
|
||
reasoning=proposal.get("reasoning", ""),
|
||
target_resource=proposal.get("target_resource", ""),
|
||
suggested_action=proposal.get("action", ""),
|
||
namespace=proposal.get("namespace", "awoooi-prod"),
|
||
)
|
||
|
||
proposal["nemotron_enabled"] = True
|
||
proposal["nemotron_tools"] = nemotron_result.get("tools", [])
|
||
proposal["nemotron_validation"] = nemotron_result.get("validation", "⏳ 驗證中")
|
||
proposal["nemotron_latency_ms"] = nemotron_result.get("latency_ms", 0.0)
|
||
proposal["nemotron_tool_model"] = nemotron_result.get("tool_model", "")
|
||
proposal["nemotron_tool_backend"] = nemotron_result.get("tool_backend", "")
|
||
|
||
# 2026-04-09 Claude Sonnet 4.6: 將 Nemotron tool call 回填為 kubectl_command
|
||
# 根本問題修復：approval_records.action 需要可執行指令才能被 parse_operation_from_action 解析
|
||
_backfill_kubectl_command(proposal, proposal["nemotron_tools"])
|
||
|
||
logger.info(
|
||
"nemotron_collaboration_complete",
|
||
incident_id=incident_id,
|
||
tools_count=len(proposal["nemotron_tools"]),
|
||
validation=proposal["nemotron_validation"],
|
||
latency_ms=proposal["nemotron_latency_ms"],
|
||
attempt=attempt,
|
||
)
|
||
last_error = None
|
||
break # 成功，跳出重試迴圈
|
||
|
||
except Exception as e:
|
||
last_error = e
|
||
logger.warning(
|
||
"nemotron_collaboration_retry",
|
||
incident_id=incident_id,
|
||
error=str(e),
|
||
attempt=attempt,
|
||
max_retries=max_retries,
|
||
)
|
||
if attempt < max_retries:
|
||
import asyncio
|
||
await asyncio.sleep(2) # 重試前等 2 秒
|
||
|
||
# 重試全部失敗 — fallback 到 Gemini 模擬 tool calling
|
||
# 2026-04-08 ogt: NIM 完全不可用時，改用 Gemini 產生執行方案（不可跳過）
|
||
if last_error is not None:
|
||
logger.error(
|
||
"nemotron_collaboration_exhausted",
|
||
incident_id=incident_id,
|
||
error=str(last_error),
|
||
retries=max_retries,
|
||
)
|
||
logger.info("nemotron_fallback_gemini_start", incident_id=incident_id)
|
||
gemini_fallback_result = await self._call_nemotron_tools_via_gemini(
|
||
incident_id=incident_id,
|
||
reasoning=proposal.get("reasoning", ""),
|
||
target_resource=proposal.get("target_resource", ""),
|
||
suggested_action=proposal.get("action", ""),
|
||
namespace=proposal.get("namespace", "awoooi-prod"),
|
||
)
|
||
proposal["nemotron_enabled"] = True
|
||
proposal["nemotron_tools"] = gemini_fallback_result.get("tools", [])
|
||
proposal["nemotron_validation"] = gemini_fallback_result.get("validation", "⚠️ Gemini 代理")
|
||
proposal["nemotron_latency_ms"] = gemini_fallback_result.get("latency_ms", 0.0)
|
||
proposal["nemotron_tool_model"] = "gemini-fallback"
|
||
proposal["nemotron_tool_backend"] = "Gemini 雲端"
|
||
|
||
# 2026-04-09 Claude Sonnet 4.6: Gemini fallback 同樣回填 kubectl_command
|
||
_backfill_kubectl_command(proposal, proposal["nemotron_tools"])
|
||
|
||
return proposal, provider, True
|
||
|
||
async def _call_nemotron_tools(
|
||
self,
|
||
incident_id: str,
|
||
reasoning: str,
|
||
target_resource: str,
|
||
suggested_action: str,
|
||
namespace: str = "awoooi-prod",
|
||
) -> dict:
|
||
"""
|
||
呼叫 Nemotron 執行 Tool Calling
|
||
|
||
Args:
|
||
incident_id: Incident ID
|
||
reasoning: OpenClaw 推理結果
|
||
target_resource: 目標資源名稱
|
||
suggested_action: OpenClaw 建議的操作
|
||
namespace: K8s namespace
|
||
|
||
Returns:
|
||
{
|
||
"tools": [{"tool": str, "args": dict, "valid": bool}],
|
||
"validation": str,
|
||
"latency_ms": float
|
||
}
|
||
"""
|
||
import asyncio
|
||
|
||
from src.services.nvidia_provider import get_nvidia_provider
|
||
|
||
nvidia = get_nvidia_provider()
|
||
start_time = time.time()
|
||
|
||
# 建構 Tool Calling prompt
|
||
# 2026-04-09 Claude Code: 明確指示 deployment_name，防止 Nemotron 填 placeholder
|
||
tool_prompt = f"""根據以下 AI 分析結果，生成對應的 kubectl 操作指令：
|
||
|
||
## Incident 上下文
|
||
- Incident ID: {incident_id}
|
||
- 目標資源 (deployment_name): {target_resource}
|
||
- Namespace: {namespace}
|
||
|
||
## OpenClaw 分析
|
||
- 建議操作: {suggested_action}
|
||
- 推理過程: {reasoning[:500]}
|
||
|
||
## 你的任務
|
||
使用提供的工具生成 kubectl 操作。
|
||
**重要**: deployment_name 必須填入 "{target_resource}"，不可使用 placeholder。
|
||
"""
|
||
|
||
# 定義可用 Tools (K8s 操作)
|
||
k8s_tools = [
|
||
{
|
||
"type": "function",
|
||
"function": {
|
||
"name": "restart_deployment",
|
||
"description": "重啟 Deployment (rollout restart)",
|
||
"parameters": {
|
||
"type": "object",
|
||
"properties": {
|
||
"deployment_name": {"type": "string"},
|
||
"namespace": {"type": "string", "default": "awoooi-prod"},
|
||
},
|
||
"required": ["deployment_name"],
|
||
},
|
||
},
|
||
},
|
||
{
|
||
"type": "function",
|
||
"function": {
|
||
"name": "scale_deployment",
|
||
"description": "調整 Deployment 副本數",
|
||
"parameters": {
|
||
"type": "object",
|
||
"properties": {
|
||
"deployment_name": {"type": "string"},
|
||
"replicas": {"type": "integer"},
|
||
"namespace": {"type": "string", "default": "awoooi-prod"},
|
||
},
|
||
"required": ["deployment_name", "replicas"],
|
||
},
|
||
},
|
||
},
|
||
{
|
||
"type": "function",
|
||
"function": {
|
||
"name": "delete_pod",
|
||
"description": "刪除 Pod (強制重建)",
|
||
"parameters": {
|
||
"type": "object",
|
||
"properties": {
|
||
"pod_name": {"type": "string"},
|
||
"namespace": {"type": "string", "default": "awoooi-prod"},
|
||
},
|
||
"required": ["pod_name"],
|
||
},
|
||
},
|
||
},
|
||
]
|
||
|
||
try:
|
||
# 2026-04-07 ogt: 統帥指示不可跳過 Nemotron，用 120 秒寬裕超時
|
||
timeout = 120
|
||
|
||
result = await asyncio.wait_for(
|
||
nvidia.tool_call(
|
||
messages=[{"role": "user", "content": tool_prompt}],
|
||
tools=k8s_tools,
|
||
),
|
||
timeout=timeout,
|
||
)
|
||
|
||
latency_ms = (time.time() - start_time) * 1000
|
||
|
||
# 解析 Tool Calling 結果
|
||
tools = []
|
||
validation_passed = True
|
||
|
||
if result and hasattr(result, "tool_calls") and result.tool_calls:
|
||
for tc in result.tool_calls:
|
||
tool_entry = {
|
||
"tool": tc.tool_name if hasattr(tc, "tool_name") else str(tc.get("name", "unknown")),
|
||
"args": tc.arguments if hasattr(tc, "arguments") else tc.get("arguments", {}),
|
||
"valid": tc.valid if hasattr(tc, "valid") else True,
|
||
}
|
||
tools.append(tool_entry)
|
||
if not tool_entry["valid"]:
|
||
validation_passed = False
|
||
elif result and isinstance(result, dict) and result.get("tool_calls"):
|
||
for tc in result["tool_calls"]:
|
||
tool_entry = {
|
||
"tool": tc.get("name", "unknown"),
|
||
"args": tc.get("arguments", {}),
|
||
"valid": True,
|
||
}
|
||
tools.append(tool_entry)
|
||
|
||
# 2026-04-09 Claude Code: 修正 Nemotron 回傳 placeholder 問題
|
||
# 若 deployment_name 是 placeholder 或空值，用 target_resource 覆蓋
|
||
_PLACEHOLDERS = {"<deployment_name>", "<target>", "", "unknown", "null", "None"}
|
||
for t in tools:
|
||
args = t.get("args", {})
|
||
if isinstance(args, dict):
|
||
dn = args.get("deployment_name", "")
|
||
if not dn or str(dn).strip("<>") in _PLACEHOLDERS or dn.startswith("<"):
|
||
args["deployment_name"] = target_resource
|
||
logger.info(
|
||
"nemotron_placeholder_corrected",
|
||
incident_id=incident_id,
|
||
original=dn,
|
||
corrected=target_resource,
|
||
)
|
||
|
||
validation_status = "✅ 驗證通過" if validation_passed and tools else "❌ 驗證失敗"
|
||
|
||
return {
|
||
"tools": tools,
|
||
"validation": validation_status,
|
||
"latency_ms": latency_ms,
|
||
}
|
||
|
||
except TimeoutError:
|
||
latency_ms = (time.time() - start_time) * 1000
|
||
logger.error(
|
||
"nemotron_tool_call_timeout",
|
||
incident_id=incident_id,
|
||
timeout_seconds=timeout,
|
||
)
|
||
# 超時也拋出，讓外層重試
|
||
raise
|
||
|
||
except Exception as e:
|
||
latency_ms = (time.time() - start_time) * 1000
|
||
logger.error(
|
||
"nemotron_tool_call_error",
|
||
incident_id=incident_id,
|
||
error=str(e),
|
||
)
|
||
raise
|
||
|
||
async def _call_nemotron_tools_via_gemini(
|
||
self,
|
||
incident_id: str,
|
||
reasoning: str,
|
||
target_resource: str,
|
||
suggested_action: str,
|
||
namespace: str = "awoooi-prod",
|
||
) -> dict:
|
||
"""
|
||
NIM 完全不可用時，由 Gemini 代理產生 tool calling 執行方案。
|
||
2026-04-08 ogt: NIM timeout 後的唯一 fallback，不可跳過。
|
||
|
||
Returns: {"tools": [...], "validation": str, "latency_ms": float}
|
||
"""
|
||
import time as _time
|
||
start_time = _time.time()
|
||
|
||
prompt = f"""你是 K8s SRE 專家。根據以下分析，輸出對應的 kubectl 操作指令（JSON 格式）。
|
||
|
||
Incident ID: {incident_id}
|
||
目標資源: {target_resource}
|
||
Namespace: {namespace}
|
||
建議操作: {suggested_action}
|
||
分析摘要: {reasoning[:300]}
|
||
|
||
請輸出以下 JSON 格式（只輸出 JSON，不要其他文字）：
|
||
{{
|
||
"tool_name": "restart_deployment 或 scale_deployment 或 no_action",
|
||
"deployment_name": "部署名稱",
|
||
"namespace": "{namespace}",
|
||
"reason": "一句話說明原因"
|
||
}}"""
|
||
|
||
try:
|
||
text, success, _, _ = await self._call_gemini(prompt)
|
||
latency_ms = (_time.time() - start_time) * 1000
|
||
|
||
if not success:
|
||
logger.warning("nemotron_gemini_fallback_failed", incident_id=incident_id, error=text)
|
||
return {"tools": [], "validation": "❌ NIM + Gemini 均不可用", "latency_ms": latency_ms}
|
||
|
||
import json as _json
|
||
data = _json.loads(text)
|
||
tool_name = data.get("tool_name", "no_action")
|
||
tools = []
|
||
if tool_name != "no_action":
|
||
tools = [{
|
||
"tool": tool_name,
|
||
"args": {
|
||
"deployment_name": data.get("deployment_name", target_resource),
|
||
"namespace": data.get("namespace", namespace),
|
||
},
|
||
"valid": True,
|
||
}]
|
||
|
||
logger.info(
|
||
"nemotron_gemini_fallback_success",
|
||
incident_id=incident_id,
|
||
tool=tool_name,
|
||
latency_ms=latency_ms,
|
||
)
|
||
return {
|
||
"tools": tools,
|
||
"validation": "✅ Gemini 代理驗證通過",
|
||
"latency_ms": latency_ms,
|
||
}
|
||
|
||
except Exception as e:
|
||
latency_ms = (_time.time() - start_time) * 1000
|
||
logger.error("nemotron_gemini_fallback_error", incident_id=incident_id, error=str(e))
|
||
return {"tools": [], "validation": f"❌ Gemini 代理失敗: {str(e)[:50]}", "latency_ms": latency_ms}
|
||
|
||
# =========================================================================
|
||
# Shadow Mode Auto-Tuning
|
||
# =========================================================================
|
||
|
||
async def execute_auto_tuning(
|
||
self,
|
||
approval_id: str,
|
||
kubectl_command: str,
|
||
description: str,
|
||
) -> dict:
|
||
"""
|
||
執行自動調優 (Shadow Mode: 僅日誌輸出)
|
||
|
||
統帥鐵律: Shadow Mode 下嚴禁實際執行 K8s 命令
|
||
|
||
Args:
|
||
approval_id: 簽核單 ID
|
||
kubectl_command: kubectl 指令
|
||
description: 操作描述
|
||
|
||
Returns:
|
||
{executed: bool, shadow_mode: bool, command: str, log: str}
|
||
"""
|
||
if settings.SHADOW_MODE_ENABLED:
|
||
# Shadow Mode: 僅記錄，不執行
|
||
log_message = f"[SHADOW MODE] AI 生成的調優指令：{kubectl_command}"
|
||
logger.info(
|
||
"shadow_mode_auto_tuning",
|
||
approval_id=approval_id,
|
||
command=kubectl_command,
|
||
description=description,
|
||
executed=False,
|
||
)
|
||
print(f"\n{'='*60}")
|
||
print(log_message)
|
||
print(f"描述: {description}")
|
||
print(f"簽核單: {approval_id}")
|
||
print(f"{'='*60}\n")
|
||
|
||
return {
|
||
"executed": False,
|
||
"shadow_mode": True,
|
||
"command": kubectl_command,
|
||
"description": description,
|
||
"log": log_message,
|
||
}
|
||
else:
|
||
# 生產模式: 實際執行 (需要額外安全檢查)
|
||
logger.warning(
|
||
"auto_tuning_execution_attempted",
|
||
approval_id=approval_id,
|
||
command=kubectl_command,
|
||
message="Production execution not yet implemented - requires multi-sig approval",
|
||
)
|
||
return {
|
||
"executed": False,
|
||
"shadow_mode": False,
|
||
"command": kubectl_command,
|
||
"description": description,
|
||
"log": "Production execution requires multi-sig approval",
|
||
}
|
||
|
||
|
||
# =============================================================================
|
||
# Singleton
|
||
# =============================================================================
|
||
|
||
_openclaw: OpenClawService | None = None
|
||
|
||
|
||
def get_openclaw() -> OpenClawService:
|
||
"""取得全域 OpenClaw 實例"""
|
||
global _openclaw
|
||
if _openclaw is None:
|
||
_openclaw = OpenClawService()
|
||
return _openclaw
|
||
|
||
|
||
async def close_openclaw() -> None:
|
||
"""關閉 OpenClaw 連線"""
|
||
global _openclaw
|
||
if _openclaw:
|
||
await _openclaw.close()
|
||
_openclaw = None
|
||
|
||
|
||
async def _fetch_k8s_inventory_for_openclaw(
|
||
namespace: str = "awoooi-prod",
|
||
timeout_sec: float = 3.0,
|
||
) -> str:
|
||
"""
|
||
取得 K8s 叢集實際 Deployment/StatefulSet 清單，注入 analyze_alert prompt。
|
||
|
||
2026-04-17 ogt + Claude Sonnet 4.6 (Checkpoint-2 webhook path):
|
||
- 根因：NemoTron 在 webhook path 收不到叢集清單 → 幻覺 deployment/awoooi-service
|
||
- 修復：analyze_alert 前拉取真實資源名，注入 prompt，強制 LLM 從清單選擇
|
||
- 超時/失敗 → 返回 ""（prompt 仍正常但無鎖定效果，不中斷主流程）
|
||
- 只執行唯讀 get 指令，不修改叢集
|
||
|
||
Returns:
|
||
"awoooi-api, awoooi-web, ..." 格式字串，失敗時返回 ""
|
||
"""
|
||
import asyncio as _asyncio
|
||
|
||
import structlog as _structlog
|
||
_logger = _structlog.get_logger(__name__)
|
||
try:
|
||
cmd = (
|
||
f"kubectl get deployments,statefulsets -n {namespace} "
|
||
"-o jsonpath='{.items[*].metadata.name}' 2>/dev/null"
|
||
)
|
||
proc = await _asyncio.create_subprocess_shell(
|
||
cmd,
|
||
stdout=_asyncio.subprocess.PIPE,
|
||
stderr=_asyncio.subprocess.PIPE,
|
||
)
|
||
try:
|
||
stdout, _ = await _asyncio.wait_for(proc.communicate(), timeout=timeout_sec)
|
||
except TimeoutError:
|
||
proc.kill()
|
||
_logger.warning("k8s_inventory_timeout_openclaw", namespace=namespace)
|
||
return ""
|
||
raw = (stdout or b"").decode("utf-8", errors="replace").strip()
|
||
if not raw:
|
||
return ""
|
||
names = [n.strip() for n in raw.split() if n.strip()]
|
||
return ", ".join(names)
|
||
except Exception as _e:
|
||
_logger.warning("k8s_inventory_failed_openclaw", namespace=namespace, error=str(_e))
|
||
return ""
|
||
|
||
|
||
# =============================================================================
|
||
# Phase 5 + SignOz Integration Complete
|
||
# =============================================================================
|