From 3eb3051a73cc2c22bd9c112c01c0b3b3b9e84a58 Mon Sep 17 00:00:00 2001 From: OG T Date: Sun, 29 Mar 2026 22:17:18 +0800 Subject: [PATCH] =?UTF-8?q?fix(ci):=20=E4=BF=AE=E5=BE=A9=20docker=20socket?= =?UTF-8?q?=20=E9=87=8D=E8=A4=87=E6=8E=9B=E8=BC=89=20(1774793847)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/src/services/intent_classifier.py | 136 +++++++++- apps/api/tests/test_intent_classifier.py | 288 +++++++++++++++++++++ docs/LOGBOOK.md | 32 ++- 3 files changed, 445 insertions(+), 11 deletions(-) create mode 100644 apps/api/tests/test_intent_classifier.py diff --git a/apps/api/src/services/intent_classifier.py b/apps/api/src/services/intent_classifier.py index a62ef4bf..23cdf75d 100644 --- a/apps/api/src/services/intent_classifier.py +++ b/apps/api/src/services/intent_classifier.py @@ -21,18 +21,40 @@ K8s 操作意圖分類器，用於智能路由模型選擇 from __future__ import annotations +import json import re +import time from dataclasses import dataclass, field from enum import Enum from typing import Protocol, runtime_checkable +import httpx import structlog +from src.core.config import settings from src.services.model_registry import get_model_registry logger = structlog.get_logger(__name__) +# LLM 分類 Prompt 模板 (Phase 13.4) +_LLM_CLASSIFY_PROMPT = """你是 K8s 操作意圖分類專家。根據以下輸入，判斷用戶的操作意圖。 + +可選意圖類型: +- restart: 重啟 Pod/Deployment/StatefulSet +- scale: 擴縮容、HPA 調整 +- config: ConfigMap/Secret/ENV 變更 +- diagnose: 日誌查詢、健康檢查、RCA +- delete: 刪除資源（高風險） +- rollback: 回滾版本 +- unknown: 無法判斷 + +輸入: {text} + +請以 JSON 格式回答，只輸出 JSON: +{{"intent": "<類型>", "confidence": <0.0-1.0>, "reasoning": "<判斷依據>"}}""" + + # ============================================================================= # 意圖類型定義 (Phase 13.3 #85) # ============================================================================= @@ -503,9 +525,9 @@ class IntentClassifier: async def _llm_classify(self, text: str) -> IntentResult: """ - LLM 分類 (方案 B) + LLM 分類 (方案 B) - Phase 13.4 - 目標延遲: < 100ms (使用 qwen2.5:1b) + 目標延遲: < 100ms (使用 qwen2.5:1b 或配置的 intent 模型) Args: text: 已轉小寫的輸入文字 @@ -513,20 +535,114 @@ class IntentClassifier: Returns: IntentResult: 分類結果 - Note: - 目前返回 UNKNOWN，待 Ollama qwen2.5:1b 部署後啟用 + 2026-03-30 Claude Code: 實作 Ollama 整合 """ - # TODO: 整合 Ollama qwen2.5:1b (Phase 13.4) - # 預計使用 text 呼叫 Ollama API 進行分類 - # 目前先返回 UNKNOWN，規則引擎已能處理大部分情況 - del text # 預留給 LLM 分類使用，避免 unused-parameter 警告 + start_time = time.time() + + try: + # 建構 Prompt + prompt = _LLM_CLASSIFY_PROMPT.format(text=text) + + # 取得模型配置 + model_name = self.llm_model # qwen2.5:1b 或配置值 + + # 呼叫 Ollama + async with httpx.AsyncClient() as client: + response = await client.post( + f"{settings.OLLAMA_URL}/api/generate", + json={ + "model": model_name, + "prompt": prompt, + "stream": False, + "format": "json", + "options": { + "num_predict": 128, # 意圖分類只需短回應 + "temperature": 0.0, # 確定性輸出 + "top_p": 0.9, + }, + }, + timeout=httpx.Timeout(5.0, connect=2.0), # 嚴格超時 + ) + + response.raise_for_status() + data = response.json() + result_text = data.get("response", "") + + # 解析 JSON 回應 + elapsed_ms = (time.time() - start_time) * 1000 + + try: + parsed = json.loads(result_text) + intent_str = parsed.get("intent", "unknown").lower() + confidence = float(parsed.get("confidence", 0.5)) + reasoning = parsed.get("reasoning", "") + + # 映射到 IntentType + intent = self._parse_intent_type(intent_str) + + logger.info( + "intent_llm_classified", + intent=intent.value, + confidence=confidence, + elapsed_ms=round(elapsed_ms, 1), + model=model_name, + ) + + return IntentResult( + intent=intent, + confidence=confidence, + method="llm", + matched_keywords=[], + detected_resources=[], + reasoning=reasoning, + ) + + except (json.JSONDecodeError, KeyError, ValueError) as e: + logger.warning( + "intent_llm_parse_failed", + error=str(e), + response_preview=result_text[:100], + ) + return self._llm_fallback_result("JSON 解析失敗") + + except httpx.TimeoutException: + elapsed_ms = (time.time() - start_time) * 1000 + logger.warning( + "intent_llm_timeout", + elapsed_ms=round(elapsed_ms, 1), + ) + return self._llm_fallback_result("LLM 超時") + + except Exception as e: + logger.warning( + "intent_llm_error", + error=str(e), + error_type=type(e).__name__, + ) + return self._llm_fallback_result(f"LLM 錯誤: {type(e).__name__}") + + def _parse_intent_type(self, intent_str: str) -> IntentType: + """解析意圖字串為 IntentType""" + intent_map = { + "restart": IntentType.RESTART, + "scale": IntentType.SCALE, + "config": IntentType.CONFIG, + "diagnose": IntentType.DIAGNOSE, + "delete": IntentType.DELETE, + "rollback": IntentType.ROLLBACK, + "unknown": IntentType.UNKNOWN, + } + return intent_map.get(intent_str.lower(), IntentType.UNKNOWN) + + def _llm_fallback_result(self, reason: str) -> IntentResult: + """LLM 失敗時的 fallback 結果""" return IntentResult( intent=IntentType.UNKNOWN, - confidence=0.0, # 🔴 LLM 未啟用，非 AI 分析 + confidence=0.0, method="llm", matched_keywords=[], detected_resources=[], - reasoning="LLM 分類尚未啟用", + reasoning=reason, ) def get_supported_intents(self) -> list[dict]: diff --git a/apps/api/tests/test_intent_classifier.py b/apps/api/tests/test_intent_classifier.py new file mode 100644 index 00000000..5f186036 --- /dev/null +++ b/apps/api/tests/test_intent_classifier.py @@ -0,0 +1,288 @@ +""" +Intent Classifier Tests - Phase 13.4 Ollama 整合 +================================================ +2026-03-30 Claude Code: Intent Classifier LLM 整合測試 + +測試範圍: +- _llm_classify: LLM 分類邏輯 +- _parse_intent_type: 意圖解析 +- _llm_fallback_result: 失敗回退 +- classify: 完整分類流程 +""" + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch +import json + +from src.services.intent_classifier import ( + IntentClassifier, + IntentType, + IntentResult, + RiskLevel, + get_intent_classifier, +) + + +# ============================================================================= +# Fixtures +# ============================================================================= + + +@pytest.fixture +def classifier(): + """測試用 IntentClassifier""" + return IntentClassifier() + + +@pytest.fixture +def mock_ollama_response(): + """模擬 Ollama 回應""" + return { + "response": json.dumps({ + "intent": "restart", + "confidence": 0.85, + "reasoning": "用戶要求重啟 Pod" + }) + } + + +# ============================================================================= +# Test Cases +# ============================================================================= + + +class TestParseIntentType: + """_parse_intent_type 測試""" + + def test_parse_restart(self, classifier): + """測試解析 restart""" + assert classifier._parse_intent_type("restart") == IntentType.RESTART + assert classifier._parse_intent_type("RESTART") == IntentType.RESTART + + def test_parse_scale(self, classifier): + """測試解析 scale""" + assert classifier._parse_intent_type("scale") == IntentType.SCALE + + def test_parse_config(self, classifier): + """測試解析 config""" + assert classifier._parse_intent_type("config") == IntentType.CONFIG + + def test_parse_diagnose(self, classifier): + """測試解析 diagnose""" + assert classifier._parse_intent_type("diagnose") == IntentType.DIAGNOSE + + def test_parse_delete(self, classifier): + """測試解析 delete""" + assert classifier._parse_intent_type("delete") == IntentType.DELETE + + def test_parse_rollback(self, classifier): + """測試解析 rollback""" + assert classifier._parse_intent_type("rollback") == IntentType.ROLLBACK + + def test_parse_unknown(self, classifier): + """測試解析 unknown""" + assert classifier._parse_intent_type("unknown") == IntentType.UNKNOWN + assert classifier._parse_intent_type("invalid") == IntentType.UNKNOWN + assert classifier._parse_intent_type("") == IntentType.UNKNOWN + + +class TestLlmFallbackResult: + """_llm_fallback_result 測試""" + + def test_fallback_result(self, classifier): + """測試 fallback 結果""" + result = classifier._llm_fallback_result("test error") + + assert result.intent == IntentType.UNKNOWN + assert result.confidence == 0.0 + assert result.method == "llm" + assert result.reasoning == "test error" + assert result.matched_keywords == [] + assert result.detected_resources == [] + + +class TestKeywordClassify: + """關鍵字分類測試""" + + def test_restart_keywords(self, classifier): + """測試重啟關鍵字""" + result = classifier.classify_sync("重啟 api-server pod") + assert result.intent == IntentType.RESTART + assert result.method == "keyword" + # 關鍵字匹配信心度為 0 (非 AI 分析) + assert result.confidence == 0.0 + assert "重啟" in result.matched_keywords + + def test_scale_keywords(self, classifier): + """測試擴縮容關鍵字""" + result = classifier.classify_sync("擴展 deployment 副本數到 5") + assert result.intent == IntentType.SCALE + assert result.method == "keyword" + + def test_diagnose_keywords(self, classifier): + """測試診斷關鍵字""" + result = classifier.classify_sync("查看 pod 日誌") + assert result.intent == IntentType.DIAGNOSE + assert result.method == "keyword" + + def test_delete_keywords(self, classifier): + """測試刪除關鍵字""" + result = classifier.classify_sync("刪除這個 pod") + assert result.intent == IntentType.DELETE + assert result.method == "keyword" + + def test_unknown_text(self, classifier): + """測試無法識別的文字""" + result = classifier.classify_sync("今天天氣真好") + assert result.intent == IntentType.UNKNOWN + + +class TestLlmClassify: + """_llm_classify 測試""" + + @pytest.mark.asyncio + async def test_llm_success(self, classifier, mock_ollama_response): + """測試 LLM 成功分類""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = mock_ollama_response + mock_response.raise_for_status = MagicMock() + + with patch("httpx.AsyncClient") as mock_client: + mock_instance = AsyncMock() + mock_instance.post.return_value = mock_response + mock_instance.__aenter__.return_value = mock_instance + mock_instance.__aexit__.return_value = None + mock_client.return_value = mock_instance + + result = await classifier._llm_classify("重啟 pod") + + assert result.intent == IntentType.RESTART + assert result.confidence == 0.85 + assert result.method == "llm" + + @pytest.mark.asyncio + async def test_llm_timeout(self, classifier): + """測試 LLM 超時""" + import httpx + + with patch("httpx.AsyncClient") as mock_client: + mock_instance = AsyncMock() + mock_instance.post.side_effect = httpx.TimeoutException("timeout") + mock_instance.__aenter__.return_value = mock_instance + mock_instance.__aexit__.return_value = None + mock_client.return_value = mock_instance + + result = await classifier._llm_classify("重啟 pod") + + assert result.intent == IntentType.UNKNOWN + assert result.confidence == 0.0 + assert "超時" in result.reasoning + + @pytest.mark.asyncio + async def test_llm_invalid_json(self, classifier): + """測試 LLM 返回無效 JSON""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"response": "not valid json"} + mock_response.raise_for_status = MagicMock() + + with patch("httpx.AsyncClient") as mock_client: + mock_instance = AsyncMock() + mock_instance.post.return_value = mock_response + mock_instance.__aenter__.return_value = mock_instance + mock_instance.__aexit__.return_value = None + mock_client.return_value = mock_instance + + result = await classifier._llm_classify("重啟 pod") + + assert result.intent == IntentType.UNKNOWN + assert "解析失敗" in result.reasoning + + +class TestClassify: + """完整分類流程測試""" + + @pytest.mark.asyncio + async def test_keyword_with_llm_fallback(self, classifier): + """測試關鍵字匹配 + LLM fallback""" + # 由於關鍵字信心度為 0，會嘗試 LLM + # LLM 可能超時或失敗，最終返回關鍵字結果 + result = await classifier.classify("重啟 api pod") + + # 意圖應該是 RESTART (來自關鍵字或 LLM) + assert result.intent == IntentType.RESTART + # method 可能是 keyword (LLM 超時) 或 llm (LLM 成功) + assert result.method in ["keyword", "llm"] + + @pytest.mark.asyncio + async def test_llm_used_when_available(self, classifier, mock_ollama_response): + """測試 LLM 可用時使用 LLM 結果""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = mock_ollama_response + mock_response.raise_for_status = MagicMock() + + with patch("httpx.AsyncClient") as mock_client: + mock_instance = AsyncMock() + mock_instance.post.return_value = mock_response + mock_instance.__aenter__.return_value = mock_instance + mock_instance.__aexit__.return_value = None + mock_client.return_value = mock_instance + + result = await classifier.classify("重啟 pod") + + # LLM 成功時，應該使用 LLM 結果 (信心度 0.85 > 0) + assert result.intent == IntentType.RESTART + assert result.method == "llm" + assert result.confidence == 0.85 + + +class TestGetIntentClassifier: + """Singleton 測試""" + + def test_singleton(self): + """測試 singleton 模式""" + c1 = get_intent_classifier() + c2 = get_intent_classifier() + assert c1 is c2 + + +class TestIntentResult: + """IntentResult 測試""" + + def test_dataclass_fields(self): + """測試 dataclass 欄位""" + result = IntentResult( + intent=IntentType.RESTART, + confidence=0.9, + method="llm", + matched_keywords=["重啟", "pod"], + detected_resources=["api-server"], + reasoning="匹配重啟關鍵字", + ) + + assert result.intent == IntentType.RESTART + assert result.confidence == 0.9 + assert result.method == "llm" + assert result.risk_level == RiskLevel.MEDIUM # auto-set by __post_init__ + assert "重啟" in result.matched_keywords + assert "api-server" in result.detected_resources + + def test_risk_level_auto_set(self): + """測試風險等級自動設定""" + # DELETE 應該是 CRITICAL + delete_result = IntentResult( + intent=IntentType.DELETE, + confidence=0.8, + method="llm", + ) + assert delete_result.risk_level == RiskLevel.CRITICAL + + # DIAGNOSE 應該是 LOW + diagnose_result = IntentResult( + intent=IntentType.DIAGNOSE, + confidence=0.8, + method="llm", + ) + assert diagnose_result.risk_level == RiskLevel.LOW diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index bbfa3eb9..0eb7e61c 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -5,10 +5,11 @@ --- -## 📍 當前狀態 (2026-03-30 00:30 台北) +## 📍 當前狀態 (2026-03-30 01:00 台北) | 項目 | 狀態 | |------|------| +| **Intent Classifier** | ✅ **Ollama 整合完成** (21 測試通過) | | **Learning Service** | ✅ **Playbook 信心度調整完成** (13 測試通過) | | **🔴 ADR-039 Gitea 遷移** | 🔄 **執行中** (方案 B - GitHub → Gitea CI/CD) | | **Gitea CI/CD** | ✅ **已設置** (cd.yaml + e2e-health.yaml) | @@ -45,6 +46,35 @@ | **Wave 2 Worker HPA** | ✅ **已部署** (min:1 max:3, CPU 70%) | | **Wave C-D 監控** | ✅ **全部完成** (generate + discover + coverage_report) | +## ✅ Intent Classifier Ollama 整合 (2026-03-30 01:00 台北) + +### 實作內容 (Phase 13.4) + +| 功能 | 說明 | +|------|------| +| `_llm_classify` | 呼叫 Ollama 進行意圖分類 | +| `_parse_intent_type` | 解析 LLM 返回的意圖字串 | +| `_llm_fallback_result` | LLM 失敗時的 fallback | +| `_LLM_CLASSIFY_PROMPT` | 結構化 Prompt 模板 | + +### 設計特點 + +| 特性 | 說明 | +|------|------| +| **超時控制** | 5 秒超時 (目標 < 100ms) | +| **JSON Mode** | 強制 Ollama 返回 JSON | +| **Fallback** | LLM 失敗時使用關鍵字結果 | +| **確定性** | temperature: 0.0 | + +### 測試覆蓋 + +- **21 測試案例全部通過** +- LLM 成功/超時/解析失敗測試 +- 關鍵字分類測試 +- 完整流程測試 + +--- + ## ✅ Learning Service 信心度調整 (2026-03-30 00:30 台北) ### 實作內容