# apps/api/tests/test_ollama_failover_manager.py | 2026-04-27 @ Asia/Taipei # Created 2026-04-25 P1.1c by Claude Engineer-C # 2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復（路由矩陣更新） # 2026-04-27 波次對齊 by Claude Sonnet 4.6 — 統帥鐵律：唯一 Ollama=111，188 完全移出 # 2026-05-03 ogt: ADR-110 GCP 三層容災架構，URL 常數更新為 GCP-A/B/Local，新增三層容災場景 # 2026-05-19 Codex: GCP-A healthy fast path 不等待 Local 111 health timeout """ OllamaFailoverManager 單元測試 - P1.1c v4.0 ============================================= 測試覆蓋（新路由矩陣：ADR-110 GCP 三層容災，2026-05-03）： - GCP-A HEALTHY → primary=ollama_gcp_a - GCP-A OFFLINE + GCP-B HEALTHY → primary=ollama_gcp_b - GCP-A OFFLINE + GCP-B OFFLINE + Local HEALTHY → primary=ollama_local - 全部 OFFLINE → primary=Gemini - Gemini quota exceeded → primary=Nemotron，fallback=[Claude] - select_provider 只 check GCP-A（primary URL） - clear_cache() / notify_recovery() 方法 - OllamaRoutingResult.health_111 backward-compat property（實際欄位 health_gcp_a） 測試分類：unit（mock OllamaHealthMonitor，無 DB 依賴） """ from __future__ import annotations from unittest.mock import AsyncMock, MagicMock, patch import pytest from src.services.ollama_failover_manager import ( OllamaFailoverManager, OllamaRoutingResult, get_ollama_failover_manager, reset_ollama_failover_manager, ) from src.services.ollama_health_monitor import HealthReport, HealthStatus # ============================================================================= # Fixtures # ============================================================================= URL_GCP_A = "http://34.143.170.20:11434" # GCP-A Primary (SSD) URL_GCP_B = "http://34.21.145.224:11434" # GCP-B Secondary (SSD) URL_LOCAL = "http://192.168.0.111:11434" # Local HDD Fallback（後備） # 向下相容別名（舊測試引用 URL_111 時仍可用） URL_111 = URL_GCP_A @pytest.fixture(autouse=True) def reset_singleton(): yield reset_ollama_failover_manager() def _make_health(status: HealthStatus, url: str = URL_111) -> HealthReport: return HealthReport(status=status, host=url, latency_ms=500.0) def _make_manager( url_primary: str = URL_GCP_A, url_secondary: str = URL_GCP_B, url_fallback: str = URL_LOCAL, ) -> OllamaFailoverManager: """建立 manager，settings mock 為 GCP 三層容災 URL（ADR-110）""" mock_settings = MagicMock() mock_settings.OLLAMA_URL = url_primary mock_settings.OLLAMA_SECONDARY_URL = url_secondary mock_settings.OLLAMA_FALLBACK_URL = url_fallback mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct" mock_monitor = MagicMock() manager = OllamaFailoverManager(health_monitor=mock_monitor) manager._settings = mock_settings return manager # ============================================================================= # _decide_route 決策矩陣 # ============================================================================= def _offline_health(url: str = URL_GCP_A) -> HealthReport: """建立 OFFLINE 的 HealthReport""" return HealthReport(status=HealthStatus.OFFLINE, host=url, latency_ms=0.0) class TestDecideRoute: """_decide_route 路由邏輯純函數測試（ADR-110 三層容災：GCP-A → GCP-B → Local → Gemini）""" def _setup(self) -> OllamaFailoverManager: return _make_manager() # ------------------------------------------------------------------ # GCP-A HEALTHY → primary=GCP-A # ------------------------------------------------------------------ def test_gcp_a_healthy_primary_is_ollama_gcp_a(self): """ADR-110：GCP-A HEALTHY → primary=ollama_gcp_a（SSD 主力）""" manager = self._setup() h_gcp_a = _make_health(HealthStatus.HEALTHY, URL_GCP_A) h_gcp_b = _offline_health(URL_GCP_B) h_local = _offline_health(URL_LOCAL) result = manager._decide_route( health_gcp_a=h_gcp_a, health_gcp_b=h_gcp_b, health_local=h_local, url_gcp_a=URL_GCP_A, url_gcp_b=URL_GCP_B, url_local=URL_LOCAL, ) assert result.primary.provider_name in ("ollama_gcp_a", "ollama") assert result.primary.url == URL_GCP_A def test_gcp_a_healthy_fallback_includes_gemini(self): """GCP-A HEALTHY 時 fallback 包含 Gemini""" manager = self._setup() h_gcp_a = _make_health(HealthStatus.HEALTHY, URL_GCP_A) h_gcp_b = _offline_health(URL_GCP_B) h_local = _offline_health(URL_LOCAL) result = manager._decide_route( health_gcp_a=h_gcp_a, health_gcp_b=h_gcp_b, health_local=h_local, url_gcp_a=URL_GCP_A, url_gcp_b=URL_GCP_B, url_local=URL_LOCAL, ) provider_names = [e.provider_name for e in result.fallback_chain] assert "gemini" in provider_names # ------------------------------------------------------------------ # GCP-A SLOW → primary=GCP-A（高負載時仍優先保住本地/代理 Ollama） # ------------------------------------------------------------------ def test_gcp_a_slow_primary_is_ollama_gcp_a(self): """GCP-A SLOW → primary=GCP-A，避免過早切 Gemini""" manager = self._setup() h_gcp_a = _make_health(HealthStatus.SLOW, URL_GCP_A) h_gcp_b = _offline_health(URL_GCP_B) h_local = _offline_health(URL_LOCAL) result = manager._decide_route( health_gcp_a=h_gcp_a, health_gcp_b=h_gcp_b, health_local=h_local, url_gcp_a=URL_GCP_A, url_gcp_b=URL_GCP_B, url_local=URL_LOCAL, ) assert result.primary.provider_name == "ollama_gcp_a" # ------------------------------------------------------------------ # GCP-A DEGRADED → primary=Gemini # ------------------------------------------------------------------ def test_gcp_a_degraded_primary_is_gemini(self): """GCP-A DEGRADED → primary=Gemini""" manager = self._setup() h_gcp_a = _make_health(HealthStatus.DEGRADED, URL_GCP_A) h_gcp_b = _offline_health(URL_GCP_B) h_local = _offline_health(URL_LOCAL) result = manager._decide_route( health_gcp_a=h_gcp_a, health_gcp_b=h_gcp_b, health_local=h_local, url_gcp_a=URL_GCP_A, url_gcp_b=URL_GCP_B, url_local=URL_LOCAL, ) assert result.primary.provider_name == "gemini" def test_gcp_a_degraded_fallback_includes_nemotron_claude(self): """GCP-A DEGRADED fallback 應包含 Nemotron 和 Claude""" manager = self._setup() h_gcp_a = _make_health(HealthStatus.DEGRADED, URL_GCP_A) h_gcp_b = _offline_health(URL_GCP_B) h_local = _offline_health(URL_LOCAL) result = manager._decide_route( health_gcp_a=h_gcp_a, health_gcp_b=h_gcp_b, health_local=h_local, url_gcp_a=URL_GCP_A, url_gcp_b=URL_GCP_B, url_local=URL_LOCAL, ) provider_names = [e.provider_name for e in result.fallback_chain] assert "nemotron" in provider_names assert "claude" in provider_names # ------------------------------------------------------------------ # GCP-A OFFLINE → primary=Gemini # ------------------------------------------------------------------ def test_gcp_a_offline_primary_is_gemini(self): """GCP-A OFFLINE → primary=Gemini""" manager = self._setup() h_gcp_a = _make_health(HealthStatus.OFFLINE, URL_GCP_A) h_gcp_b = _offline_health(URL_GCP_B) h_local = _offline_health(URL_LOCAL) result = manager._decide_route( health_gcp_a=h_gcp_a, health_gcp_b=h_gcp_b, health_local=h_local, url_gcp_a=URL_GCP_A, url_gcp_b=URL_GCP_B, url_local=URL_LOCAL, ) assert result.primary.provider_name == "gemini" def test_gcp_a_offline_fallback_includes_nemotron_claude(self): """GCP-A OFFLINE 時，fallback 包含 Nemotron, Claude""" manager = self._setup() h_gcp_a = _make_health(HealthStatus.OFFLINE, URL_GCP_A) h_gcp_b = _offline_health(URL_GCP_B) h_local = _offline_health(URL_LOCAL) result = manager._decide_route( health_gcp_a=h_gcp_a, health_gcp_b=h_gcp_b, health_local=h_local, url_gcp_a=URL_GCP_A, url_gcp_b=URL_GCP_B, url_local=URL_LOCAL, ) provider_names = [e.provider_name for e in result.fallback_chain] assert "nemotron" in provider_names assert "claude" in provider_names # ------------------------------------------------------------------ # routing_reason 記錄 # ------------------------------------------------------------------ def test_routing_reason_contains_status(self): """routing_reason 應包含 GCP-A 的狀態資訊""" manager = self._setup() h_gcp_a = _make_health(HealthStatus.OFFLINE, URL_GCP_A) h_gcp_b = _offline_health(URL_GCP_B) h_local = _offline_health(URL_LOCAL) result = manager._decide_route( health_gcp_a=h_gcp_a, health_gcp_b=h_gcp_b, health_local=h_local, url_gcp_a=URL_GCP_A, url_gcp_b=URL_GCP_B, url_local=URL_LOCAL, ) reason_lower = result.routing_reason.lower() assert ( "offline" in reason_lower or "gcp" in reason_lower or "gemini" in reason_lower ) # ============================================================================= # select_provider()：GCP-A healthy fast path # ============================================================================= class TestSelectProvider: """select_provider() 三層容災健康檢查。""" def _make_three_layer_mock( self, gcp_a_status: HealthStatus = HealthStatus.HEALTHY, gcp_b_status: HealthStatus = HealthStatus.OFFLINE, local_status: HealthStatus = HealthStatus.OFFLINE, ): """建立三層健康 mock：按呼叫順序返回 GCP-A / GCP-B / Local 健康報告""" side_effect_map = { URL_GCP_A: _make_health(gcp_a_status, URL_GCP_A), URL_GCP_B: _make_health(gcp_b_status, URL_GCP_B), URL_LOCAL: _make_health(local_status, URL_LOCAL), } async def _check_side_effect(url): return side_effect_map.get(url, HealthReport(status=HealthStatus.OFFLINE, host=url, latency_ms=0.0)) mock_monitor = AsyncMock() mock_monitor.check = AsyncMock(side_effect=_check_side_effect) mock_settings = MagicMock() mock_settings.OLLAMA_URL = URL_GCP_A mock_settings.OLLAMA_SECONDARY_URL = URL_GCP_B mock_settings.OLLAMA_FALLBACK_URL = URL_LOCAL mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct" manager = OllamaFailoverManager(health_monitor=mock_monitor) manager._settings = mock_settings return manager, mock_monitor @pytest.mark.asyncio async def test_select_provider_gcp_a_healthy_checks_primary_only(self): """GCP-A healthy 時不等待 GCP-B / Local 111，避免 routing 被 111 timeout 拖慢。""" manager, mock_monitor = self._make_three_layer_mock( gcp_a_status=HealthStatus.HEALTHY, ) with patch.object(manager, "_write_failover_audit", return_value=None): result = await manager.select_provider() assert result.primary.provider_name == "ollama_gcp_a" assert result.health_gcp_b is None assert result.health_local is None assert mock_monitor.check.call_count == 1 called_urls = {call.args[0] for call in mock_monitor.check.call_args_list} assert URL_GCP_A in called_urls assert URL_GCP_B not in called_urls assert URL_LOCAL not in called_urls @pytest.mark.asyncio async def test_select_provider_checks_fallback_hosts_when_gcp_a_not_healthy(self): """GCP-A 不健康時仍檢查 GCP-B / Local，保留三層容災。""" manager, mock_monitor = self._make_three_layer_mock( gcp_a_status=HealthStatus.OFFLINE, gcp_b_status=HealthStatus.HEALTHY, local_status=HealthStatus.OFFLINE, ) with patch.object(manager, "_write_failover_audit", return_value=None): result = await manager.select_provider() assert result.primary.provider_name == "ollama_gcp_b" assert mock_monitor.check.call_count == 3 called_urls = {call.args[0] for call in mock_monitor.check.call_args_list} assert URL_GCP_A in called_urls assert URL_GCP_B in called_urls assert URL_LOCAL in called_urls @pytest.mark.asyncio async def test_select_provider_gcp_a_healthy_primary_ollama(self): """GCP-A HEALTHY → primary=ollama_gcp_a（或向下相容 ollama）""" manager, _ = self._make_three_layer_mock(gcp_a_status=HealthStatus.HEALTHY) with patch.object(manager, "_write_failover_audit", return_value=None): result = await manager.select_provider() assert result.primary.provider_name in ("ollama_gcp_a", "ollama") @pytest.mark.asyncio async def test_select_provider_returns_routing_result(self): """select_provider 返回 OllamaRoutingResult 類型（三層全 OFFLINE → Gemini）""" manager, _ = self._make_three_layer_mock( gcp_a_status=HealthStatus.OFFLINE, gcp_b_status=HealthStatus.OFFLINE, local_status=HealthStatus.OFFLINE, ) # 必須 mock Redis pool（_check_gemini_quota 走 fail-closed 路徑會切到 Nemotron 而非 Gemini） with patch.object(manager, "_write_failover_audit", return_value=None), \ patch.object(manager, "_check_gemini_quota", AsyncMock(return_value=True)), \ patch( "src.services.failover_alerter.get_failover_alerter", return_value=MagicMock( alert_failover=AsyncMock(), alert_gemini_quota_exceeded=AsyncMock(), ), ): result = await manager.select_provider() assert isinstance(result, OllamaRoutingResult) # 三層全 OFFLINE + Gemini quota OK → primary=Gemini assert result.primary.provider_name == "gemini" @pytest.mark.asyncio async def test_audit_not_written_when_gcp_a_healthy(self): """GCP-A 正常時不觸發 failover audit""" manager, _ = self._make_three_layer_mock(gcp_a_status=HealthStatus.HEALTHY) audit_called = [False] async def _spy_audit(result): audit_called[0] = result.primary.provider_name not in ("ollama_gcp_a", "ollama") with patch.object(manager, "_write_failover_audit", side_effect=_spy_audit): await manager.select_provider() # GCP-A HEALTHY，不應有 failover 事件 assert audit_called[0] is False # ============================================================================= # clear_cache() / notify_recovery() # ============================================================================= class TestRecoveryAPI: """clear_cache() / notify_recovery() 方法""" @pytest.mark.asyncio async def test_clear_cache_calls_redis_delete(self): """clear_cache() 呼叫 redis.delete 清除 health monitor 快取""" manager = _make_manager() mock_redis = AsyncMock() mock_redis.delete = AsyncMock() with patch("src.services.ollama_failover_manager.OllamaFailoverManager.clear_cache") as mock_clear: mock_clear.return_value = None await manager.clear_cache() mock_clear.assert_called_once() @pytest.mark.asyncio async def test_clear_cache_fails_gracefully(self): """Redis import 失敗時，clear_cache() 內部 try/except 攔截，靜默不 crash""" manager = _make_manager() # 模擬 get_redis 拋 ImportError（Redis 不可用） # clear_cache 有 try/except Exception，應靜默吸收 with patch( "src.services.ollama_failover_manager.get_redis", side_effect=ImportError("no redis"), create=True, ): # 不應 raise await manager.clear_cache() def test_notify_recovery_does_not_raise(self): """notify_recovery() 只寫 structlog，不應 raise""" manager = _make_manager() # 不應 raise（舊呼叫方式仍支援） manager.notify_recovery("ollama_111") manager.notify_recovery("ollama_gcp_a") # ============================================================================= # OllamaRoutingResult # ============================================================================= class TestOllamaRoutingResult: """OllamaRoutingResult 輔助方法""" def test_all_endpoints_in_order(self): from src.services.ollama_failover_manager import OllamaEndpoint primary = OllamaEndpoint(url=URL_GCP_A, provider_name="ollama_gcp_a", model="m1") fb1 = OllamaEndpoint(url="", provider_name="gemini", model="gemini-1.5-flash") fb2 = OllamaEndpoint(url="", provider_name="nemotron", model="m3") result = OllamaRoutingResult( primary=primary, fallback_chain=[fb1, fb2], routing_reason="test", health_gcp_a=_make_health(HealthStatus.HEALTHY), ) ordered = result.all_endpoints_in_order() assert ordered[0].provider_name == "ollama_gcp_a" assert ordered[1].provider_name == "gemini" assert ordered[2].provider_name == "nemotron" def test_to_dict_structure(self): from src.services.ollama_failover_manager import OllamaEndpoint primary = OllamaEndpoint(url=URL_GCP_A, provider_name="ollama_gcp_a", model="qwen") result = OllamaRoutingResult( primary=primary, fallback_chain=[], routing_reason="GCP-A HEALTHY", health_gcp_a=_make_health(HealthStatus.HEALTHY), ) d = result.to_dict() assert d["primary"]["provider"] == "ollama_gcp_a" assert d["routing_reason"] == "GCP-A HEALTHY" assert isinstance(d["fallback_chain"], list) def test_health_111_backward_compat_property(self): """health_111 是 backward-compat property，指向 health_gcp_a""" from src.services.ollama_failover_manager import OllamaEndpoint primary = OllamaEndpoint(url=URL_GCP_A, provider_name="ollama_gcp_a", model="qwen") h = _make_health(HealthStatus.HEALTHY) result = OllamaRoutingResult( primary=primary, fallback_chain=[], routing_reason="test", health_gcp_a=h, ) # health_111 property 應指向 health_gcp_a assert result.health_111 is result.health_gcp_a def test_health_gcp_b_and_local_optional(self): """health_gcp_b 和 health_local 為 optional None（未傳時）""" from src.services.ollama_failover_manager import OllamaEndpoint primary = OllamaEndpoint(url=URL_GCP_A, provider_name="ollama_gcp_a", model="qwen") result = OllamaRoutingResult( primary=primary, fallback_chain=[], routing_reason="test", health_gcp_a=_make_health(HealthStatus.HEALTHY), # health_gcp_b / health_local 不傳，應為 None ) assert result.health_gcp_b is None assert result.health_local is None # ============================================================================= # ADR-110 三層容災場景（2026-05-03 ogt 新增） # GCP-A → GCP-B → Local → Gemini 四段容災路由 # ============================================================================= class TestThreeLayerFailover: """ADR-110 三層容災場景：GCP-A → GCP-B → Local → Gemini""" def _make_manager_with_health( self, gcp_a: HealthStatus, gcp_b: HealthStatus, local: HealthStatus, ) -> OllamaFailoverManager: """建立三層健康 mock manager（按 URL 路由 health status）""" health_map = { URL_GCP_A: HealthReport(status=gcp_a, host=URL_GCP_A, latency_ms=500.0), URL_GCP_B: HealthReport(status=gcp_b, host=URL_GCP_B, latency_ms=500.0), URL_LOCAL: HealthReport(status=local, host=URL_LOCAL, latency_ms=500.0), } async def _check(url): return health_map.get(url, HealthReport(status=HealthStatus.OFFLINE, host=url)) mock_monitor = AsyncMock() mock_monitor.check = AsyncMock(side_effect=_check) mock_settings = MagicMock() mock_settings.OLLAMA_URL = URL_GCP_A mock_settings.OLLAMA_SECONDARY_URL = URL_GCP_B mock_settings.OLLAMA_FALLBACK_URL = URL_LOCAL mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct" mock_settings.GEMINI_DAILY_QUOTA = 1000 manager = OllamaFailoverManager(health_monitor=mock_monitor) manager._settings = mock_settings return manager @pytest.mark.asyncio async def test_gcp_a_healthy_uses_gcp_a(self): """場景1：GCP-A HEALTHY → primary=GCP-A（SSD 主力）""" manager = self._make_manager_with_health( gcp_a=HealthStatus.HEALTHY, gcp_b=HealthStatus.OFFLINE, local=HealthStatus.OFFLINE, ) with patch.object(manager, "_write_failover_audit", return_value=None), \ patch.object(manager, "_check_gemini_quota", return_value=True): result = await manager.select_provider() assert result.primary.url == URL_GCP_A or result.primary.provider_name in ("ollama_gcp_a", "ollama") @pytest.mark.asyncio async def test_gcp_a_offline_gcp_b_healthy_uses_gcp_b(self): """場景2：GCP-A OFFLINE + GCP-B HEALTHY → primary=GCP-B""" manager = self._make_manager_with_health( gcp_a=HealthStatus.OFFLINE, gcp_b=HealthStatus.HEALTHY, local=HealthStatus.OFFLINE, ) with patch.object(manager, "_write_failover_audit", return_value=None), \ patch.object(manager, "_check_gemini_quota", return_value=True): result = await manager.select_provider() # GCP-A 掛了，應切到 GCP-B assert result.primary.url == URL_GCP_B or result.primary.provider_name in ("ollama_gcp_b", "ollama_gcp_a") @pytest.mark.asyncio async def test_gcp_a_gcp_b_offline_local_healthy_uses_local(self): """場景3：GCP-A OFFLINE + GCP-B OFFLINE + Local HEALTHY → primary=Local(111)""" manager = self._make_manager_with_health( gcp_a=HealthStatus.OFFLINE, gcp_b=HealthStatus.OFFLINE, local=HealthStatus.HEALTHY, ) with patch.object(manager, "_write_failover_audit", return_value=None), \ patch.object(manager, "_check_gemini_quota", return_value=True): result = await manager.select_provider() # GCP-A/B 皆掛，切到 Local assert result.primary.url == URL_LOCAL or result.primary.provider_name in ("ollama_local", "ollama") @pytest.mark.asyncio async def test_all_offline_uses_gemini(self): """場景4：三層全 OFFLINE → primary=Gemini（最終雲端備援）""" manager = self._make_manager_with_health( gcp_a=HealthStatus.OFFLINE, gcp_b=HealthStatus.OFFLINE, local=HealthStatus.OFFLINE, ) with patch.object(manager, "_write_failover_audit", return_value=None), \ patch.object(manager, "_check_gemini_quota", return_value=True): result = await manager.select_provider() assert result.primary.provider_name == "gemini" @pytest.mark.asyncio async def test_all_offline_gemini_quota_exceeded_uses_nemotron(self): """場景5：三層全 OFFLINE + Gemini quota 耗盡 → primary=Nemotron""" manager = self._make_manager_with_health( gcp_a=HealthStatus.OFFLINE, gcp_b=HealthStatus.OFFLINE, local=HealthStatus.OFFLINE, ) with patch.object(manager, "_write_failover_audit", return_value=None), \ patch.object(manager, "_check_gemini_quota", return_value=False): result = await manager.select_provider() assert result.primary.provider_name == "nemotron" # ============================================================================= # Singleton # ============================================================================= def test_singleton_returns_same_instance(): m1 = get_ollama_failover_manager() m2 = get_ollama_failover_manager() assert m1 is m2 def test_reset_singleton_gives_new_instance(): m1 = get_ollama_failover_manager() reset_ollama_failover_manager() m2 = get_ollama_failover_manager() assert m1 is not m2 # ============================================================================= # B1: _write_failover_audit 改用 structlog（不再寫 DB） # 2026-04-25 critic-fix Part2 by Claude Engineer-C2 # ============================================================================= class TestWriteFailoverAudit: """B1 修復驗證：_write_failover_audit 使用 structlog，不依賴 AuditLog model""" @pytest.mark.asyncio async def test_audit_uses_structlog_not_db(self): """_write_failover_audit 應呼叫 structlog，不呼叫 DB""" manager = _make_manager() from src.services.ollama_failover_manager import ( OllamaEndpoint, OllamaRoutingResult, ) result = OllamaRoutingResult( primary=OllamaEndpoint(url="", provider_name="gemini", model="gemini-1.5-flash"), fallback_chain=[], routing_reason="GCP-A OFFLINE → 切 Gemini", health_gcp_a=_make_health(HealthStatus.OFFLINE), ) # 只要不 raise 就是成功（DB path 已移除，structlog path 無 DB 依賴） await manager._write_failover_audit(result) @pytest.mark.asyncio async def test_audit_skipped_when_gcp_a_healthy(self): """GCP-A HEALTHY 時 early return，不記錄 failover""" manager = _make_manager() from src.services.ollama_failover_manager import ( OllamaEndpoint, OllamaRoutingResult, ) result = OllamaRoutingResult( primary=OllamaEndpoint(url=URL_GCP_A, provider_name="ollama_gcp_a", model="qwen"), fallback_chain=[], routing_reason="GCP-A HEALTHY → 主 GCP-A", health_gcp_a=_make_health(HealthStatus.HEALTHY), ) # primary=ollama_gcp_a → early return，不執行任何 DB/log await manager._write_failover_audit(result) # 不應 raise # ============================================================================= # B2: AIProviderEnum.OLLAMA_LOCAL 存在 # 2026-05-06 Codex — 188 不再作為 Ollama Provider # ============================================================================= class TestAIProviderEnumOllamaLocal: """B2 修復驗證：AIProviderEnum.OLLAMA_LOCAL 存在且 PROVIDER_LATENCY_BUDGET 有對應值""" def test_ollama_local_enum_exists(self): from src.services.ai_router import AIProviderEnum assert AIProviderEnum.OLLAMA_LOCAL.value == "ollama_local" def test_ollama_local_in_latency_budget(self): from src.services.ai_router import PROVIDER_LATENCY_BUDGET, AIProviderEnum assert AIProviderEnum.OLLAMA_LOCAL in PROVIDER_LATENCY_BUDGET assert PROVIDER_LATENCY_BUDGET[AIProviderEnum.OLLAMA_LOCAL] == 90000 # ============================================================================= # H4: asyncio.gather return_exceptions=True # 2026-04-25 critic-fix Part2 by Claude Engineer-C2 # ============================================================================= class TestGatherReturnExceptions: """H4 修復驗證：三層主機 check 拋例外時不炸整個 select_provider""" @pytest.mark.asyncio async def test_gather_exception_in_all_hosts_treated_as_offline(self): """三台主機 check 全部拋例外 → 視為 OFFLINE，select_provider 正常返回 Gemini""" mock_monitor = AsyncMock() mock_monitor.check = AsyncMock( side_effect=RuntimeError("network error") ) mock_settings = MagicMock() mock_settings.OLLAMA_URL = URL_GCP_A mock_settings.OLLAMA_SECONDARY_URL = URL_GCP_B mock_settings.OLLAMA_FALLBACK_URL = URL_LOCAL mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct" mock_settings.GEMINI_DAILY_QUOTA = 1000 manager = OllamaFailoverManager(health_monitor=mock_monitor) manager._settings = mock_settings with patch.object(manager, "_write_failover_audit", return_value=None), \ patch.object(manager, "_check_gemini_quota", return_value=True): result = await manager.select_provider() # 三層全部 exception → OFFLINE → primary=gemini assert result.primary.provider_name == "gemini" @pytest.mark.asyncio async def test_gcp_a_healthy_select_provider_primary_ollama(self): """GCP-A HEALTHY → primary=ollama_gcp_a，select_provider 正常返回""" async def _check_side_effect(url): if url == URL_GCP_A: return _make_health(HealthStatus.HEALTHY, URL_GCP_A) return HealthReport(status=HealthStatus.OFFLINE, host=url, latency_ms=0.0) mock_monitor = AsyncMock() mock_monitor.check = AsyncMock(side_effect=_check_side_effect) mock_settings = MagicMock() mock_settings.OLLAMA_URL = URL_GCP_A mock_settings.OLLAMA_SECONDARY_URL = URL_GCP_B mock_settings.OLLAMA_FALLBACK_URL = URL_LOCAL mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct" mock_settings.GEMINI_DAILY_QUOTA = 1000 manager = OllamaFailoverManager(health_monitor=mock_monitor) manager._settings = mock_settings with patch.object(manager, "_write_failover_audit", return_value=None), \ patch.object(manager, "_check_gemini_quota", return_value=True): result = await manager.select_provider() # GCP-A HEALTHY → primary=ollama_gcp_a（或 backward-compat ollama） assert result.primary.provider_name in ("ollama_gcp_a", "ollama") # ============================================================================= # H7: Gemini 帳單熔斷 # 2026-04-25 critic-fix Part2 by Claude Engineer-C2 # ============================================================================= class TestGeminiQuota: """H7 修復驗證：Gemini 每日配額熔斷""" @pytest.mark.asyncio async def test_gemini_quota_under_limit(self): """count=500 < quota=1000 → 返回 True（允許走 Gemini） 2026-04-26 Wave5 B3-fix by Claude Engineer-A4 — 改用 pipeline mock（atomic 修復後） 原 GET/INCR/EXPIRE 三步已改為 pipeline.set(NX)+incr，mock 跟著更新。 """ manager = _make_manager() manager._settings.GEMINI_DAILY_QUOTA = 1000 # pipeline mock：SET NX 返回 True（首次），INCR 返回 501（500+1，未達 quota=1000） mock_pipe = MagicMock() mock_pipe.set = MagicMock(return_value=mock_pipe) mock_pipe.incr = MagicMock(return_value=mock_pipe) mock_pipe.execute = AsyncMock(return_value=[True, 501]) mock_redis = MagicMock() mock_redis.pipeline = MagicMock(return_value=mock_pipe) with patch("src.core.redis_client.get_redis", return_value=mock_redis): ok = await manager._check_gemini_quota() assert ok is True mock_pipe.execute.assert_awaited_once() @pytest.mark.asyncio async def test_gemini_quota_exactly_at_limit(self): """count=1001 > quota=1000 → 返回 False（熔斷，不再呼叫 Gemini） 2026-04-26 Wave5 B3-fix by Claude Engineer-A4 — 改用 pipeline mock（atomic 修復後） pipeline.incr 返回 1001（> quota=1000），應返回 False。 """ manager = _make_manager() manager._settings.GEMINI_DAILY_QUOTA = 1000 mock_pipe = MagicMock() mock_pipe.set = MagicMock(return_value=mock_pipe) mock_pipe.incr = MagicMock(return_value=mock_pipe) mock_pipe.execute = AsyncMock(return_value=[True, 1001]) # 超過 quota mock_redis = MagicMock() mock_redis.pipeline = MagicMock(return_value=mock_pipe) with patch("src.core.redis_client.get_redis", return_value=mock_redis): ok = await manager._check_gemini_quota() assert ok is False @pytest.mark.asyncio async def test_gemini_quota_redis_unavailable_fail_closed(self): """Redis 掛掉 → 返回 False（2026-04-27 Wave8-X2 fail-closed，違反費用鐵律的修復）""" manager = _make_manager() with patch( "src.core.redis_client.get_redis", side_effect=RuntimeError("Redis unavailable"), ), patch( "src.services.failover_alerter.get_failover_alerter", return_value=MagicMock(alert_gemini_quota_exceeded=AsyncMock()), ): ok = await manager._check_gemini_quota() # fail-closed：Redis 異常時拒絕 Gemini，避免費用失控（违反 feedback_cost_change_approval.md） assert ok is False @pytest.mark.asyncio async def test_select_provider_quota_exceeded_uses_nemotron(self): """select_provider：Gemini quota 超過 → primary 改為 Nemotron（三層全 OFFLINE 情境）""" mock_monitor = AsyncMock() mock_monitor.check = AsyncMock( return_value=_make_health(HealthStatus.OFFLINE, URL_GCP_A) ) mock_settings = MagicMock() mock_settings.OLLAMA_URL = URL_GCP_A mock_settings.OLLAMA_SECONDARY_URL = URL_GCP_B mock_settings.OLLAMA_FALLBACK_URL = URL_LOCAL mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct" mock_settings.GEMINI_DAILY_QUOTA = 1000 manager = OllamaFailoverManager(health_monitor=mock_monitor) manager._settings = mock_settings with patch.object(manager, "_write_failover_audit", return_value=None), \ patch.object(manager, "_check_gemini_quota", return_value=False): result = await manager.select_provider() # quota 超過 → 不走 Gemini，改走 Nemotron assert result.primary.provider_name == "nemotron" @pytest.mark.asyncio async def test_select_provider_quota_exceeded_all_offline_uses_nemotron(self): """select_provider：Gemini quota 超過 + 三層全 OFFLINE → primary=Nemotron""" async def _all_offline(url): return HealthReport(status=HealthStatus.OFFLINE, host=url, latency_ms=0.0) mock_monitor = AsyncMock() mock_monitor.check = AsyncMock(side_effect=_all_offline) mock_settings = MagicMock() mock_settings.OLLAMA_URL = URL_GCP_A mock_settings.OLLAMA_SECONDARY_URL = URL_GCP_B mock_settings.OLLAMA_FALLBACK_URL = URL_LOCAL mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct" mock_settings.GEMINI_DAILY_QUOTA = 1000 manager = OllamaFailoverManager(health_monitor=mock_monitor) manager._settings = mock_settings with patch.object(manager, "_write_failover_audit", return_value=None), \ patch.object(manager, "_check_gemini_quota", return_value=False): result = await manager.select_provider() assert result.primary.provider_name == "nemotron"