# apps/api/tests/test_ai_router_failover_integration.py | 2026-04-25 @ Asia/Taipei # 2026-04-25 P1.2 by Claude Engineer-A2 — failover 整合到 ai_router + lifespan # 2026-04-26 Wave4 P1.2-tests-fix by Claude Engineer-A3 — 修正 intent mock:ALERT_TRIAGE→DIAGNOSE(normalize_intent 映射),改用 UNKNOWN(無 override,score=1 → OLLAMA → failover 觸發) # 2026-04-26 Wave4 P1.2-tests-fix-v2 by Claude Opus 4.7 — UNKNOWN intent 在 router 內仍被 reclassify 成 DIAGNOSE → openclaw_nemo # 改用 patch.object(router, "_select_provider_and_model") 直接強制初始路由為 OLLAMA,繞過 normalize / alert detection 邏輯 # 2026-05-03 ogt: ADR-110 GCP 三層容災,_make_health host 更新為 GCP-A Primary """ AIRouter × OllamaFailoverManager 整合測試 ========================================== 測試覆蓋: 1. 初步路由選 OLLAMA → failover_manager 重評 → decision 使用 failover 結果 2. failover 回傳 GEMINI primary → decision.selected_provider == GEMINI 3. failover 的 fallback_chain 正確轉換到 decision.fallback_chain 4. 初步路由選 NEMOTRON → failover_manager 不被呼叫 5. 初步路由選 OPENCLAW_NEMO → failover_manager 不被呼叫 6. failover_manager 發生例外 → fail-open,保留原始 provider 測試分類:unit(mock OllamaFailoverManager,無 Redis / DB 依賴) """ from __future__ import annotations from unittest.mock import AsyncMock, MagicMock, patch import pytest from src.services.ai_router import AIProviderEnum, AIRouter, reset_ai_router from src.services.ollama_failover_manager import OllamaEndpoint, OllamaRoutingResult from src.services.ollama_health_monitor import HealthReport, HealthStatus # ============================================================================= # Fixtures / Helpers # ============================================================================= @pytest.fixture(autouse=True) def reset_router_singleton(): """每個測試前後重置 AIRouter singleton,避免 failover_manager mock 殘留""" yield reset_ai_router() def _make_health(status: HealthStatus) -> HealthReport: return HealthReport(status=status, host="http://34.143.170.20:11434", latency_ms=500.0) # GCP-A Primary(ADR-110) def _make_failover_result( primary_provider: str, primary_model: str, fallback: list[tuple[str, str]] | None = None, ) -> OllamaRoutingResult: """建立 OllamaRoutingResult 測試物件""" fb_endpoints = [ OllamaEndpoint(url="", provider_name=p, model=m) for p, m in (fallback or []) ] return OllamaRoutingResult( primary=OllamaEndpoint(url="", provider_name=primary_provider, model=primary_model), fallback_chain=fb_endpoints, routing_reason=f"test: {primary_provider}", health_gcp_a=_make_health(HealthStatus.OFFLINE), # ADR-110:欄位改為 health_gcp_a ) def _make_router_with_mock_failover(mock_failover_manager) -> AIRouter: """建立 AIRouter,並替換其 _failover_manager""" router = AIRouter() router._failover_manager = mock_failover_manager return router # ============================================================================= # Test 1: OLLAMA 路由 → failover_manager 重評 → 使用 GEMINI # ============================================================================= @pytest.mark.asyncio async def test_router_uses_failover_when_ollama_initial_provider(): """初步路由選 OLLAMA → 應走 failover_manager 重評,decision.selected_provider == GEMINI""" mock_fm = MagicMock() mock_fm.select_provider = AsyncMock( return_value=_make_failover_result( primary_provider="gemini", primary_model="gemini-1.5-flash", fallback=[("ollama_local", "qwen2.5:7b-instruct"), ("nemotron", "nvidia/nemotron-mini-4b-instruct")], ) ) router = _make_router_with_mock_failover(mock_fm) # 2026-04-26 Wave4 v2 by Claude Opus 4.7 — 直接 mock _select_provider_and_model 強制初始 OLLAMA with patch.object( router, "_select_provider_and_model", return_value=(AIProviderEnum.OLLAMA, "qwen3:8b", "test forced ollama"), ): decision = await router.route("test alert message") assert decision.selected_provider == AIProviderEnum.GEMINI assert decision.selected_model == "gemini-1.5-flash" mock_fm.select_provider.assert_awaited_once() # ============================================================================= # Test 2: fallback_chain 正確轉換 # ============================================================================= @pytest.mark.asyncio async def test_router_failover_fallback_chain_converted(): """failover_manager 回傳 fallback_chain → decision.fallback_chain 包含 OLLAMA_LOCAL""" mock_fm = MagicMock() mock_fm.select_provider = AsyncMock( return_value=_make_failover_result( primary_provider="gemini", primary_model="gemini-1.5-flash", fallback=[ ("ollama_local", "qwen2.5:7b-instruct"), ("nemotron", "nvidia/nemotron-mini-4b-instruct"), ("claude", "claude-haiku-4-5-20251001"), ], ) ) router = _make_router_with_mock_failover(mock_fm) # 2026-04-26 Wave4 v2 by Claude Opus 4.7 — 直接 mock _select_provider_and_model 強制初始 OLLAMA with patch.object( router, "_select_provider_and_model", return_value=(AIProviderEnum.OLLAMA, "qwen3:8b", "test forced ollama"), ): decision = await router.route("test alert message") fb_providers = [p for p, _ in decision.fallback_chain] assert AIProviderEnum.OLLAMA_LOCAL in fb_providers, ( f"OLLAMA_LOCAL not in fallback_chain: {fb_providers}" ) assert AIProviderEnum.NEMOTRON in fb_providers assert AIProviderEnum.CLAUDE in fb_providers # ============================================================================= # Test 3: NEMOTRON 路由 → failover_manager 不被呼叫 # ============================================================================= @pytest.mark.asyncio async def test_router_does_not_use_failover_for_nemotron(): """初步路由選 NEMOTRON(tool_calling)→ failover_manager.select_provider 不應被呼叫""" mock_fm = MagicMock() mock_fm.select_provider = AsyncMock() router = _make_router_with_mock_failover(mock_fm) # 強制 intent = DIAGNOSE(→ OPENCLAW_NEMO),再用 context_hint 跳過 LLM # 但 NEMOTRON 只由 route_tool_calling() 觸發,route() 最多到 OPENCLAW_NEMO # 改用 QUERY → OLLAMA 的 override,然後驗 failover 被觸發(這不是 NEMOTRON 測試) # 正確測試:強制 CRITICAL → CLAUDE,驗 failover 不被呼叫 with patch.object(router._intent_classifier, "classify") as mock_classify: from src.services.intent_classifier import IntentResult, IntentType, RiskLevel from src.services.complexity_scorer import ComplexityScore mock_classify.return_value = IntentResult( intent=IntentType.DELETE, confidence=1.0, method="keyword", matched_keywords=["delete"], detected_resources=[], reasoning="test", risk_level=RiskLevel.CRITICAL, ) with patch.object(router._complexity_scorer, "score") as mock_score: mock_score.return_value = ComplexityScore(score=5, features={}) decision = await router.route("delete this service") # CRITICAL risk → CLAUDE,failover_manager 不應被呼叫 assert decision.selected_provider == AIProviderEnum.CLAUDE mock_fm.select_provider.assert_not_awaited() # ============================================================================= # Test 4: OPENCLAW_NEMO 路由 → failover_manager 不被呼叫 # ============================================================================= @pytest.mark.asyncio async def test_router_uses_failover_for_diagnose_ollama_primary(): """2026-04-29: DIAGNOSE intent → OLLAMA → failover_manager 應被呼叫 推翻 A2 後 DIAGNOSE primary 為 OLLAMA(本地優先鐵律) failover_manager 對 OLLAMA primary 會檢查健康度(111 vs 188 CPU 備援切換) """ mock_fm = MagicMock() mock_fm.select_provider = AsyncMock() # mock 回傳保持原 provider(無切換) mock_decision = MagicMock() mock_decision.primary.provider_name = "ollama" mock_decision.primary.model = "qwen2.5:7b-instruct" mock_decision.fallback_chain = [] mock_fm.select_provider.return_value = mock_decision router = _make_router_with_mock_failover(mock_fm) decision = await router.route( "diagnose service crash", context={"intent_hint": "diagnose"}, ) # 推翻 A2:DIAGNOSE primary 是 OLLAMA assert decision.selected_provider == AIProviderEnum.OLLAMA # OLLAMA primary 觸發 failover_manager 健康檢查(111 vs 188) mock_fm.select_provider.assert_awaited() # ============================================================================= # Test 5: failover_manager 發生例外 → fail-open,保留原始 OLLAMA # ============================================================================= @pytest.mark.asyncio async def test_router_failopen_when_failover_manager_raises(): """failover_manager.select_provider 拋出例外 → fail-open,decision 仍然成功(使用原始 OLLAMA)""" mock_fm = MagicMock() mock_fm.select_provider = AsyncMock(side_effect=RuntimeError("redis timeout")) router = _make_router_with_mock_failover(mock_fm) # 2026-04-26 Wave4 v2 by Claude Opus 4.7 — 直接 mock _select_provider_and_model 強制初始 OLLAMA # → failover 觸發 → raises RuntimeError → fail-open → 保留 OLLAMA with patch.object( router, "_select_provider_and_model", return_value=(AIProviderEnum.OLLAMA, "qwen3:8b", "test forced ollama"), ): # 不應 raise,應 fail-open decision = await router.route("test alert message") # fail-open → 保留 OLLAMA(原始 initial decision) assert decision.selected_provider == AIProviderEnum.OLLAMA # fallback_chain 仍然存在(來自 _build_fallback_chain) assert len(decision.fallback_chain) > 0