888 lines
35 KiB
Python
888 lines
35 KiB
Python
# apps/api/tests/test_ollama_failover_manager.py | 2026-04-27 @ Asia/Taipei
|
||
# Created 2026-04-25 P1.1c by Claude Engineer-C
|
||
# 2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復（路由矩陣更新）
|
||
# 2026-04-27 波次對齊 by Claude Sonnet 4.6 — 統帥鐵律：唯一 Ollama=111，188 完全移出
|
||
# 2026-05-03 ogt: ADR-110 GCP 三層容災架構，URL 常數更新為 GCP-A/B/Local，新增三層容災場景
|
||
# 2026-05-19 Codex: GCP-A healthy fast path 不等待 Local 111 health timeout
|
||
"""
|
||
OllamaFailoverManager 單元測試 - P1.1c v4.0
|
||
=============================================
|
||
測試覆蓋（新路由矩陣：ADR-110 GCP 三層容災，2026-05-03）：
|
||
- GCP-A HEALTHY → primary=ollama_gcp_a
|
||
- GCP-A OFFLINE + GCP-B HEALTHY → primary=ollama_gcp_b
|
||
- GCP-A OFFLINE + GCP-B OFFLINE + Local HEALTHY → primary=ollama_local
|
||
- 全部 OFFLINE → primary=Gemini
|
||
- Gemini quota exceeded → primary=Nemotron，fallback=[Claude]
|
||
- select_provider 只 check GCP-A（primary URL）
|
||
- clear_cache() / notify_recovery() 方法
|
||
- OllamaRoutingResult.health_111 backward-compat property（實際欄位 health_gcp_a）
|
||
|
||
測試分類：unit（mock OllamaHealthMonitor，無 DB 依賴）
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from src.services.ollama_failover_manager import (
|
||
OllamaFailoverManager,
|
||
OllamaRoutingResult,
|
||
get_ollama_failover_manager,
|
||
reset_ollama_failover_manager,
|
||
)
|
||
from src.services.ollama_health_monitor import HealthReport, HealthStatus
|
||
|
||
# =============================================================================
|
||
# Fixtures
|
||
# =============================================================================
|
||
|
||
URL_GCP_A = "http://34.143.170.20:11434" # GCP-A Primary (SSD)
|
||
URL_GCP_B = "http://34.21.145.224:11434" # GCP-B Secondary (SSD)
|
||
URL_LOCAL = "http://192.168.0.111:11434" # Local HDD Fallback（後備）
|
||
# 向下相容別名（舊測試引用 URL_111 時仍可用）
|
||
URL_111 = URL_GCP_A
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def reset_singleton():
|
||
yield
|
||
reset_ollama_failover_manager()
|
||
|
||
|
||
def _make_health(status: HealthStatus, url: str = URL_111) -> HealthReport:
|
||
return HealthReport(status=status, host=url, latency_ms=500.0)
|
||
|
||
|
||
def _make_manager(
|
||
url_primary: str = URL_GCP_A,
|
||
url_secondary: str = URL_GCP_B,
|
||
url_fallback: str = URL_LOCAL,
|
||
) -> OllamaFailoverManager:
|
||
"""建立 manager，settings mock 為 GCP 三層容災 URL（ADR-110）"""
|
||
mock_settings = MagicMock()
|
||
mock_settings.OLLAMA_URL = url_primary
|
||
mock_settings.OLLAMA_SECONDARY_URL = url_secondary
|
||
mock_settings.OLLAMA_FALLBACK_URL = url_fallback
|
||
mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct"
|
||
|
||
mock_monitor = MagicMock()
|
||
manager = OllamaFailoverManager(health_monitor=mock_monitor)
|
||
manager._settings = mock_settings
|
||
return manager
|
||
|
||
|
||
# =============================================================================
|
||
# _decide_route 決策矩陣
|
||
# =============================================================================
|
||
|
||
|
||
def _offline_health(url: str = URL_GCP_A) -> HealthReport:
|
||
"""建立 OFFLINE 的 HealthReport"""
|
||
return HealthReport(status=HealthStatus.OFFLINE, host=url, latency_ms=0.0)
|
||
|
||
|
||
class TestDecideRoute:
|
||
"""_decide_route 路由邏輯純函數測試（ADR-110 三層容災：GCP-A → GCP-B → Local → Gemini）"""
|
||
|
||
def _setup(self) -> OllamaFailoverManager:
|
||
return _make_manager()
|
||
|
||
# ------------------------------------------------------------------
|
||
# GCP-A HEALTHY → primary=GCP-A
|
||
# ------------------------------------------------------------------
|
||
|
||
def test_gcp_a_healthy_primary_is_ollama_gcp_a(self):
|
||
"""ADR-110：GCP-A HEALTHY → primary=ollama_gcp_a（SSD 主力）"""
|
||
manager = self._setup()
|
||
h_gcp_a = _make_health(HealthStatus.HEALTHY, URL_GCP_A)
|
||
h_gcp_b = _offline_health(URL_GCP_B)
|
||
h_local = _offline_health(URL_LOCAL)
|
||
|
||
result = manager._decide_route(
|
||
health_gcp_a=h_gcp_a,
|
||
health_gcp_b=h_gcp_b,
|
||
health_local=h_local,
|
||
url_gcp_a=URL_GCP_A,
|
||
url_gcp_b=URL_GCP_B,
|
||
url_local=URL_LOCAL,
|
||
)
|
||
|
||
assert result.primary.provider_name in ("ollama_gcp_a", "ollama")
|
||
assert result.primary.url == URL_GCP_A
|
||
|
||
def test_gcp_a_healthy_fallback_includes_gemini(self):
|
||
"""GCP-A HEALTHY 時 fallback 包含 Gemini"""
|
||
manager = self._setup()
|
||
h_gcp_a = _make_health(HealthStatus.HEALTHY, URL_GCP_A)
|
||
h_gcp_b = _offline_health(URL_GCP_B)
|
||
h_local = _offline_health(URL_LOCAL)
|
||
|
||
result = manager._decide_route(
|
||
health_gcp_a=h_gcp_a,
|
||
health_gcp_b=h_gcp_b,
|
||
health_local=h_local,
|
||
url_gcp_a=URL_GCP_A,
|
||
url_gcp_b=URL_GCP_B,
|
||
url_local=URL_LOCAL,
|
||
)
|
||
|
||
provider_names = [e.provider_name for e in result.fallback_chain]
|
||
assert "gemini" in provider_names
|
||
|
||
# ------------------------------------------------------------------
|
||
# GCP-A SLOW → primary=GCP-A（高負載時仍優先保住本地/代理 Ollama）
|
||
# ------------------------------------------------------------------
|
||
|
||
def test_gcp_a_slow_primary_is_ollama_gcp_a(self):
|
||
"""GCP-A SLOW → primary=GCP-A，避免過早切 Gemini"""
|
||
manager = self._setup()
|
||
h_gcp_a = _make_health(HealthStatus.SLOW, URL_GCP_A)
|
||
h_gcp_b = _offline_health(URL_GCP_B)
|
||
h_local = _offline_health(URL_LOCAL)
|
||
|
||
result = manager._decide_route(
|
||
health_gcp_a=h_gcp_a,
|
||
health_gcp_b=h_gcp_b,
|
||
health_local=h_local,
|
||
url_gcp_a=URL_GCP_A,
|
||
url_gcp_b=URL_GCP_B,
|
||
url_local=URL_LOCAL,
|
||
)
|
||
|
||
assert result.primary.provider_name == "ollama_gcp_a"
|
||
|
||
# ------------------------------------------------------------------
|
||
# GCP-A DEGRADED → primary=Gemini
|
||
# ------------------------------------------------------------------
|
||
|
||
def test_gcp_a_degraded_primary_is_gemini(self):
|
||
"""GCP-A DEGRADED → primary=Gemini"""
|
||
manager = self._setup()
|
||
h_gcp_a = _make_health(HealthStatus.DEGRADED, URL_GCP_A)
|
||
h_gcp_b = _offline_health(URL_GCP_B)
|
||
h_local = _offline_health(URL_LOCAL)
|
||
|
||
result = manager._decide_route(
|
||
health_gcp_a=h_gcp_a,
|
||
health_gcp_b=h_gcp_b,
|
||
health_local=h_local,
|
||
url_gcp_a=URL_GCP_A,
|
||
url_gcp_b=URL_GCP_B,
|
||
url_local=URL_LOCAL,
|
||
)
|
||
|
||
assert result.primary.provider_name == "gemini"
|
||
|
||
def test_gcp_a_degraded_fallback_includes_nemotron_claude(self):
|
||
"""GCP-A DEGRADED fallback 應包含 Nemotron 和 Claude"""
|
||
manager = self._setup()
|
||
h_gcp_a = _make_health(HealthStatus.DEGRADED, URL_GCP_A)
|
||
h_gcp_b = _offline_health(URL_GCP_B)
|
||
h_local = _offline_health(URL_LOCAL)
|
||
|
||
result = manager._decide_route(
|
||
health_gcp_a=h_gcp_a,
|
||
health_gcp_b=h_gcp_b,
|
||
health_local=h_local,
|
||
url_gcp_a=URL_GCP_A,
|
||
url_gcp_b=URL_GCP_B,
|
||
url_local=URL_LOCAL,
|
||
)
|
||
|
||
provider_names = [e.provider_name for e in result.fallback_chain]
|
||
assert "nemotron" in provider_names
|
||
assert "claude" in provider_names
|
||
|
||
# ------------------------------------------------------------------
|
||
# GCP-A OFFLINE → primary=Gemini
|
||
# ------------------------------------------------------------------
|
||
|
||
def test_gcp_a_offline_primary_is_gemini(self):
|
||
"""GCP-A OFFLINE → primary=Gemini"""
|
||
manager = self._setup()
|
||
h_gcp_a = _make_health(HealthStatus.OFFLINE, URL_GCP_A)
|
||
h_gcp_b = _offline_health(URL_GCP_B)
|
||
h_local = _offline_health(URL_LOCAL)
|
||
|
||
result = manager._decide_route(
|
||
health_gcp_a=h_gcp_a,
|
||
health_gcp_b=h_gcp_b,
|
||
health_local=h_local,
|
||
url_gcp_a=URL_GCP_A,
|
||
url_gcp_b=URL_GCP_B,
|
||
url_local=URL_LOCAL,
|
||
)
|
||
|
||
assert result.primary.provider_name == "gemini"
|
||
|
||
def test_gcp_a_offline_fallback_includes_nemotron_claude(self):
|
||
"""GCP-A OFFLINE 時，fallback 包含 Nemotron, Claude"""
|
||
manager = self._setup()
|
||
h_gcp_a = _make_health(HealthStatus.OFFLINE, URL_GCP_A)
|
||
h_gcp_b = _offline_health(URL_GCP_B)
|
||
h_local = _offline_health(URL_LOCAL)
|
||
|
||
result = manager._decide_route(
|
||
health_gcp_a=h_gcp_a,
|
||
health_gcp_b=h_gcp_b,
|
||
health_local=h_local,
|
||
url_gcp_a=URL_GCP_A,
|
||
url_gcp_b=URL_GCP_B,
|
||
url_local=URL_LOCAL,
|
||
)
|
||
|
||
provider_names = [e.provider_name for e in result.fallback_chain]
|
||
assert "nemotron" in provider_names
|
||
assert "claude" in provider_names
|
||
|
||
# ------------------------------------------------------------------
|
||
# routing_reason 記錄
|
||
# ------------------------------------------------------------------
|
||
|
||
def test_routing_reason_contains_status(self):
|
||
"""routing_reason 應包含 GCP-A 的狀態資訊"""
|
||
manager = self._setup()
|
||
h_gcp_a = _make_health(HealthStatus.OFFLINE, URL_GCP_A)
|
||
h_gcp_b = _offline_health(URL_GCP_B)
|
||
h_local = _offline_health(URL_LOCAL)
|
||
|
||
result = manager._decide_route(
|
||
health_gcp_a=h_gcp_a,
|
||
health_gcp_b=h_gcp_b,
|
||
health_local=h_local,
|
||
url_gcp_a=URL_GCP_A,
|
||
url_gcp_b=URL_GCP_B,
|
||
url_local=URL_LOCAL,
|
||
)
|
||
|
||
reason_lower = result.routing_reason.lower()
|
||
assert (
|
||
"offline" in reason_lower
|
||
or "gcp" in reason_lower
|
||
or "gemini" in reason_lower
|
||
)
|
||
|
||
|
||
# =============================================================================
|
||
# select_provider()：GCP-A healthy fast path
|
||
# =============================================================================
|
||
|
||
|
||
class TestSelectProvider:
|
||
"""select_provider() 三層容災健康檢查。"""
|
||
|
||
def _make_three_layer_mock(
|
||
self,
|
||
gcp_a_status: HealthStatus = HealthStatus.HEALTHY,
|
||
gcp_b_status: HealthStatus = HealthStatus.OFFLINE,
|
||
local_status: HealthStatus = HealthStatus.OFFLINE,
|
||
):
|
||
"""建立三層健康 mock：按呼叫順序返回 GCP-A / GCP-B / Local 健康報告"""
|
||
side_effect_map = {
|
||
URL_GCP_A: _make_health(gcp_a_status, URL_GCP_A),
|
||
URL_GCP_B: _make_health(gcp_b_status, URL_GCP_B),
|
||
URL_LOCAL: _make_health(local_status, URL_LOCAL),
|
||
}
|
||
|
||
async def _check_side_effect(url):
|
||
return side_effect_map.get(url, HealthReport(status=HealthStatus.OFFLINE, host=url, latency_ms=0.0))
|
||
|
||
mock_monitor = AsyncMock()
|
||
mock_monitor.check = AsyncMock(side_effect=_check_side_effect)
|
||
|
||
mock_settings = MagicMock()
|
||
mock_settings.OLLAMA_URL = URL_GCP_A
|
||
mock_settings.OLLAMA_SECONDARY_URL = URL_GCP_B
|
||
mock_settings.OLLAMA_FALLBACK_URL = URL_LOCAL
|
||
mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct"
|
||
|
||
manager = OllamaFailoverManager(health_monitor=mock_monitor)
|
||
manager._settings = mock_settings
|
||
return manager, mock_monitor
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_select_provider_gcp_a_healthy_checks_primary_only(self):
|
||
"""GCP-A healthy 時不等待 GCP-B / Local 111，避免 routing 被 111 timeout 拖慢。"""
|
||
manager, mock_monitor = self._make_three_layer_mock(
|
||
gcp_a_status=HealthStatus.HEALTHY,
|
||
)
|
||
|
||
with patch.object(manager, "_write_failover_audit", return_value=None):
|
||
result = await manager.select_provider()
|
||
|
||
assert result.primary.provider_name == "ollama_gcp_a"
|
||
assert result.health_gcp_b is None
|
||
assert result.health_local is None
|
||
assert mock_monitor.check.call_count == 1
|
||
called_urls = {call.args[0] for call in mock_monitor.check.call_args_list}
|
||
assert URL_GCP_A in called_urls
|
||
assert URL_GCP_B not in called_urls
|
||
assert URL_LOCAL not in called_urls
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_select_provider_checks_fallback_hosts_when_gcp_a_not_healthy(self):
|
||
"""GCP-A 不健康時仍檢查 GCP-B / Local，保留三層容災。"""
|
||
manager, mock_monitor = self._make_three_layer_mock(
|
||
gcp_a_status=HealthStatus.OFFLINE,
|
||
gcp_b_status=HealthStatus.HEALTHY,
|
||
local_status=HealthStatus.OFFLINE,
|
||
)
|
||
|
||
with patch.object(manager, "_write_failover_audit", return_value=None):
|
||
result = await manager.select_provider()
|
||
|
||
assert result.primary.provider_name == "ollama_gcp_b"
|
||
assert mock_monitor.check.call_count == 3
|
||
called_urls = {call.args[0] for call in mock_monitor.check.call_args_list}
|
||
assert URL_GCP_A in called_urls
|
||
assert URL_GCP_B in called_urls
|
||
assert URL_LOCAL in called_urls
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_select_provider_gcp_a_healthy_primary_ollama(self):
|
||
"""GCP-A HEALTHY → primary=ollama_gcp_a（或向下相容 ollama）"""
|
||
manager, _ = self._make_three_layer_mock(gcp_a_status=HealthStatus.HEALTHY)
|
||
|
||
with patch.object(manager, "_write_failover_audit", return_value=None):
|
||
result = await manager.select_provider()
|
||
|
||
assert result.primary.provider_name in ("ollama_gcp_a", "ollama")
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_select_provider_returns_routing_result(self):
|
||
"""select_provider 返回 OllamaRoutingResult 類型（三層全 OFFLINE → Gemini）"""
|
||
manager, _ = self._make_three_layer_mock(
|
||
gcp_a_status=HealthStatus.OFFLINE,
|
||
gcp_b_status=HealthStatus.OFFLINE,
|
||
local_status=HealthStatus.OFFLINE,
|
||
)
|
||
|
||
# 必須 mock Redis pool（_check_gemini_quota 走 fail-closed 路徑會切到 Nemotron 而非 Gemini）
|
||
with patch.object(manager, "_write_failover_audit", return_value=None), \
|
||
patch.object(manager, "_check_gemini_quota", AsyncMock(return_value=True)), \
|
||
patch(
|
||
"src.services.failover_alerter.get_failover_alerter",
|
||
return_value=MagicMock(
|
||
alert_failover=AsyncMock(),
|
||
alert_gemini_quota_exceeded=AsyncMock(),
|
||
),
|
||
):
|
||
result = await manager.select_provider()
|
||
|
||
assert isinstance(result, OllamaRoutingResult)
|
||
# 三層全 OFFLINE + Gemini quota OK → primary=Gemini
|
||
assert result.primary.provider_name == "gemini"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_audit_not_written_when_gcp_a_healthy(self):
|
||
"""GCP-A 正常時不觸發 failover audit"""
|
||
manager, _ = self._make_three_layer_mock(gcp_a_status=HealthStatus.HEALTHY)
|
||
|
||
audit_called = [False]
|
||
|
||
async def _spy_audit(result):
|
||
audit_called[0] = result.primary.provider_name not in ("ollama_gcp_a", "ollama")
|
||
|
||
with patch.object(manager, "_write_failover_audit", side_effect=_spy_audit):
|
||
await manager.select_provider()
|
||
|
||
# GCP-A HEALTHY，不應有 failover 事件
|
||
assert audit_called[0] is False
|
||
|
||
|
||
# =============================================================================
|
||
# clear_cache() / notify_recovery()
|
||
# =============================================================================
|
||
|
||
|
||
class TestRecoveryAPI:
|
||
"""clear_cache() / notify_recovery() 方法"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_clear_cache_calls_redis_delete(self):
|
||
"""clear_cache() 呼叫 redis.delete 清除 health monitor 快取"""
|
||
manager = _make_manager()
|
||
mock_redis = AsyncMock()
|
||
mock_redis.delete = AsyncMock()
|
||
|
||
with patch("src.services.ollama_failover_manager.OllamaFailoverManager.clear_cache") as mock_clear:
|
||
mock_clear.return_value = None
|
||
await manager.clear_cache()
|
||
mock_clear.assert_called_once()
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_clear_cache_fails_gracefully(self):
|
||
"""Redis import 失敗時，clear_cache() 內部 try/except 攔截，靜默不 crash"""
|
||
manager = _make_manager()
|
||
|
||
# 模擬 get_redis 拋 ImportError（Redis 不可用）
|
||
# clear_cache 有 try/except Exception，應靜默吸收
|
||
with patch(
|
||
"src.services.ollama_failover_manager.get_redis",
|
||
side_effect=ImportError("no redis"),
|
||
create=True,
|
||
):
|
||
# 不應 raise
|
||
await manager.clear_cache()
|
||
|
||
def test_notify_recovery_does_not_raise(self):
|
||
"""notify_recovery() 只寫 structlog，不應 raise"""
|
||
manager = _make_manager()
|
||
# 不應 raise（舊呼叫方式仍支援）
|
||
manager.notify_recovery("ollama_111")
|
||
manager.notify_recovery("ollama_gcp_a")
|
||
|
||
|
||
# =============================================================================
|
||
# OllamaRoutingResult
|
||
# =============================================================================
|
||
|
||
|
||
class TestOllamaRoutingResult:
|
||
"""OllamaRoutingResult 輔助方法"""
|
||
|
||
def test_all_endpoints_in_order(self):
|
||
from src.services.ollama_failover_manager import OllamaEndpoint
|
||
primary = OllamaEndpoint(url=URL_GCP_A, provider_name="ollama_gcp_a", model="m1")
|
||
fb1 = OllamaEndpoint(url="", provider_name="gemini", model="gemini-1.5-flash")
|
||
fb2 = OllamaEndpoint(url="", provider_name="nemotron", model="m3")
|
||
|
||
result = OllamaRoutingResult(
|
||
primary=primary,
|
||
fallback_chain=[fb1, fb2],
|
||
routing_reason="test",
|
||
health_gcp_a=_make_health(HealthStatus.HEALTHY),
|
||
)
|
||
|
||
ordered = result.all_endpoints_in_order()
|
||
assert ordered[0].provider_name == "ollama_gcp_a"
|
||
assert ordered[1].provider_name == "gemini"
|
||
assert ordered[2].provider_name == "nemotron"
|
||
|
||
def test_to_dict_structure(self):
|
||
from src.services.ollama_failover_manager import OllamaEndpoint
|
||
primary = OllamaEndpoint(url=URL_GCP_A, provider_name="ollama_gcp_a", model="qwen")
|
||
result = OllamaRoutingResult(
|
||
primary=primary,
|
||
fallback_chain=[],
|
||
routing_reason="GCP-A HEALTHY",
|
||
health_gcp_a=_make_health(HealthStatus.HEALTHY),
|
||
)
|
||
d = result.to_dict()
|
||
assert d["primary"]["provider"] == "ollama_gcp_a"
|
||
assert d["routing_reason"] == "GCP-A HEALTHY"
|
||
assert isinstance(d["fallback_chain"], list)
|
||
|
||
def test_health_111_backward_compat_property(self):
|
||
"""health_111 是 backward-compat property，指向 health_gcp_a"""
|
||
from src.services.ollama_failover_manager import OllamaEndpoint
|
||
primary = OllamaEndpoint(url=URL_GCP_A, provider_name="ollama_gcp_a", model="qwen")
|
||
h = _make_health(HealthStatus.HEALTHY)
|
||
result = OllamaRoutingResult(
|
||
primary=primary,
|
||
fallback_chain=[],
|
||
routing_reason="test",
|
||
health_gcp_a=h,
|
||
)
|
||
# health_111 property 應指向 health_gcp_a
|
||
assert result.health_111 is result.health_gcp_a
|
||
|
||
def test_health_gcp_b_and_local_optional(self):
|
||
"""health_gcp_b 和 health_local 為 optional None（未傳時）"""
|
||
from src.services.ollama_failover_manager import OllamaEndpoint
|
||
primary = OllamaEndpoint(url=URL_GCP_A, provider_name="ollama_gcp_a", model="qwen")
|
||
result = OllamaRoutingResult(
|
||
primary=primary,
|
||
fallback_chain=[],
|
||
routing_reason="test",
|
||
health_gcp_a=_make_health(HealthStatus.HEALTHY),
|
||
# health_gcp_b / health_local 不傳，應為 None
|
||
)
|
||
assert result.health_gcp_b is None
|
||
assert result.health_local is None
|
||
|
||
|
||
# =============================================================================
|
||
# ADR-110 三層容災場景（2026-05-03 ogt 新增）
|
||
# GCP-A → GCP-B → Local → Gemini 四段容災路由
|
||
# =============================================================================
|
||
|
||
|
||
class TestThreeLayerFailover:
|
||
"""ADR-110 三層容災場景：GCP-A → GCP-B → Local → Gemini"""
|
||
|
||
def _make_manager_with_health(
|
||
self,
|
||
gcp_a: HealthStatus,
|
||
gcp_b: HealthStatus,
|
||
local: HealthStatus,
|
||
) -> OllamaFailoverManager:
|
||
"""建立三層健康 mock manager（按 URL 路由 health status）"""
|
||
health_map = {
|
||
URL_GCP_A: HealthReport(status=gcp_a, host=URL_GCP_A, latency_ms=500.0),
|
||
URL_GCP_B: HealthReport(status=gcp_b, host=URL_GCP_B, latency_ms=500.0),
|
||
URL_LOCAL: HealthReport(status=local, host=URL_LOCAL, latency_ms=500.0),
|
||
}
|
||
|
||
async def _check(url):
|
||
return health_map.get(url, HealthReport(status=HealthStatus.OFFLINE, host=url))
|
||
|
||
mock_monitor = AsyncMock()
|
||
mock_monitor.check = AsyncMock(side_effect=_check)
|
||
|
||
mock_settings = MagicMock()
|
||
mock_settings.OLLAMA_URL = URL_GCP_A
|
||
mock_settings.OLLAMA_SECONDARY_URL = URL_GCP_B
|
||
mock_settings.OLLAMA_FALLBACK_URL = URL_LOCAL
|
||
mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct"
|
||
mock_settings.GEMINI_DAILY_QUOTA = 1000
|
||
|
||
manager = OllamaFailoverManager(health_monitor=mock_monitor)
|
||
manager._settings = mock_settings
|
||
return manager
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_gcp_a_healthy_uses_gcp_a(self):
|
||
"""場景1：GCP-A HEALTHY → primary=GCP-A（SSD 主力）"""
|
||
manager = self._make_manager_with_health(
|
||
gcp_a=HealthStatus.HEALTHY,
|
||
gcp_b=HealthStatus.OFFLINE,
|
||
local=HealthStatus.OFFLINE,
|
||
)
|
||
with patch.object(manager, "_write_failover_audit", return_value=None), \
|
||
patch.object(manager, "_check_gemini_quota", return_value=True):
|
||
result = await manager.select_provider()
|
||
|
||
assert result.primary.url == URL_GCP_A or result.primary.provider_name in ("ollama_gcp_a", "ollama")
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_gcp_a_offline_gcp_b_healthy_uses_gcp_b(self):
|
||
"""場景2：GCP-A OFFLINE + GCP-B HEALTHY → primary=GCP-B"""
|
||
manager = self._make_manager_with_health(
|
||
gcp_a=HealthStatus.OFFLINE,
|
||
gcp_b=HealthStatus.HEALTHY,
|
||
local=HealthStatus.OFFLINE,
|
||
)
|
||
with patch.object(manager, "_write_failover_audit", return_value=None), \
|
||
patch.object(manager, "_check_gemini_quota", return_value=True):
|
||
result = await manager.select_provider()
|
||
|
||
# GCP-A 掛了，應切到 GCP-B
|
||
assert result.primary.url == URL_GCP_B or result.primary.provider_name in ("ollama_gcp_b", "ollama_gcp_a")
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_gcp_a_gcp_b_offline_local_healthy_uses_local(self):
|
||
"""場景3：GCP-A OFFLINE + GCP-B OFFLINE + Local HEALTHY → primary=Local(111)"""
|
||
manager = self._make_manager_with_health(
|
||
gcp_a=HealthStatus.OFFLINE,
|
||
gcp_b=HealthStatus.OFFLINE,
|
||
local=HealthStatus.HEALTHY,
|
||
)
|
||
with patch.object(manager, "_write_failover_audit", return_value=None), \
|
||
patch.object(manager, "_check_gemini_quota", return_value=True):
|
||
result = await manager.select_provider()
|
||
|
||
# GCP-A/B 皆掛，切到 Local
|
||
assert result.primary.url == URL_LOCAL or result.primary.provider_name in ("ollama_local", "ollama")
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_all_offline_uses_gemini(self):
|
||
"""場景4：三層全 OFFLINE → primary=Gemini（最終雲端備援）"""
|
||
manager = self._make_manager_with_health(
|
||
gcp_a=HealthStatus.OFFLINE,
|
||
gcp_b=HealthStatus.OFFLINE,
|
||
local=HealthStatus.OFFLINE,
|
||
)
|
||
with patch.object(manager, "_write_failover_audit", return_value=None), \
|
||
patch.object(manager, "_check_gemini_quota", return_value=True):
|
||
result = await manager.select_provider()
|
||
|
||
assert result.primary.provider_name == "gemini"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_all_offline_gemini_quota_exceeded_uses_nemotron(self):
|
||
"""場景5：三層全 OFFLINE + Gemini quota 耗盡 → primary=Nemotron"""
|
||
manager = self._make_manager_with_health(
|
||
gcp_a=HealthStatus.OFFLINE,
|
||
gcp_b=HealthStatus.OFFLINE,
|
||
local=HealthStatus.OFFLINE,
|
||
)
|
||
with patch.object(manager, "_write_failover_audit", return_value=None), \
|
||
patch.object(manager, "_check_gemini_quota", return_value=False):
|
||
result = await manager.select_provider()
|
||
|
||
assert result.primary.provider_name == "nemotron"
|
||
|
||
|
||
# =============================================================================
|
||
# Singleton
|
||
# =============================================================================
|
||
|
||
|
||
def test_singleton_returns_same_instance():
|
||
m1 = get_ollama_failover_manager()
|
||
m2 = get_ollama_failover_manager()
|
||
assert m1 is m2
|
||
|
||
|
||
def test_reset_singleton_gives_new_instance():
|
||
m1 = get_ollama_failover_manager()
|
||
reset_ollama_failover_manager()
|
||
m2 = get_ollama_failover_manager()
|
||
assert m1 is not m2
|
||
|
||
|
||
# =============================================================================
|
||
# B1: _write_failover_audit 改用 structlog（不再寫 DB）
|
||
# 2026-04-25 critic-fix Part2 by Claude Engineer-C2
|
||
# =============================================================================
|
||
|
||
|
||
class TestWriteFailoverAudit:
|
||
"""B1 修復驗證：_write_failover_audit 使用 structlog，不依賴 AuditLog model"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_audit_uses_structlog_not_db(self):
|
||
"""_write_failover_audit 應呼叫 structlog，不呼叫 DB"""
|
||
manager = _make_manager()
|
||
from src.services.ollama_failover_manager import (
|
||
OllamaEndpoint,
|
||
OllamaRoutingResult,
|
||
)
|
||
|
||
result = OllamaRoutingResult(
|
||
primary=OllamaEndpoint(url="", provider_name="gemini", model="gemini-1.5-flash"),
|
||
fallback_chain=[],
|
||
routing_reason="GCP-A OFFLINE → 切 Gemini",
|
||
health_gcp_a=_make_health(HealthStatus.OFFLINE),
|
||
)
|
||
|
||
# 只要不 raise 就是成功（DB path 已移除，structlog path 無 DB 依賴）
|
||
await manager._write_failover_audit(result)
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_audit_skipped_when_gcp_a_healthy(self):
|
||
"""GCP-A HEALTHY 時 early return，不記錄 failover"""
|
||
manager = _make_manager()
|
||
from src.services.ollama_failover_manager import (
|
||
OllamaEndpoint,
|
||
OllamaRoutingResult,
|
||
)
|
||
|
||
result = OllamaRoutingResult(
|
||
primary=OllamaEndpoint(url=URL_GCP_A, provider_name="ollama_gcp_a", model="qwen"),
|
||
fallback_chain=[],
|
||
routing_reason="GCP-A HEALTHY → 主 GCP-A",
|
||
health_gcp_a=_make_health(HealthStatus.HEALTHY),
|
||
)
|
||
|
||
# primary=ollama_gcp_a → early return，不執行任何 DB/log
|
||
await manager._write_failover_audit(result) # 不應 raise
|
||
|
||
|
||
# =============================================================================
|
||
# B2: AIProviderEnum.OLLAMA_LOCAL 存在
|
||
# 2026-05-06 Codex — 188 不再作為 Ollama Provider
|
||
# =============================================================================
|
||
|
||
|
||
class TestAIProviderEnumOllamaLocal:
|
||
"""B2 修復驗證：AIProviderEnum.OLLAMA_LOCAL 存在且 PROVIDER_LATENCY_BUDGET 有對應值"""
|
||
|
||
def test_ollama_local_enum_exists(self):
|
||
from src.services.ai_router import AIProviderEnum
|
||
assert AIProviderEnum.OLLAMA_LOCAL.value == "ollama_local"
|
||
|
||
def test_ollama_local_in_latency_budget(self):
|
||
from src.services.ai_router import PROVIDER_LATENCY_BUDGET, AIProviderEnum
|
||
assert AIProviderEnum.OLLAMA_LOCAL in PROVIDER_LATENCY_BUDGET
|
||
assert PROVIDER_LATENCY_BUDGET[AIProviderEnum.OLLAMA_LOCAL] == 90000
|
||
|
||
|
||
# =============================================================================
|
||
# H4: asyncio.gather return_exceptions=True
|
||
# 2026-04-25 critic-fix Part2 by Claude Engineer-C2
|
||
# =============================================================================
|
||
|
||
|
||
class TestGatherReturnExceptions:
|
||
"""H4 修復驗證：三層主機 check 拋例外時不炸整個 select_provider"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_gather_exception_in_all_hosts_treated_as_offline(self):
|
||
"""三台主機 check 全部拋例外 → 視為 OFFLINE，select_provider 正常返回 Gemini"""
|
||
mock_monitor = AsyncMock()
|
||
mock_monitor.check = AsyncMock(
|
||
side_effect=RuntimeError("network error")
|
||
)
|
||
|
||
mock_settings = MagicMock()
|
||
mock_settings.OLLAMA_URL = URL_GCP_A
|
||
mock_settings.OLLAMA_SECONDARY_URL = URL_GCP_B
|
||
mock_settings.OLLAMA_FALLBACK_URL = URL_LOCAL
|
||
mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct"
|
||
mock_settings.GEMINI_DAILY_QUOTA = 1000
|
||
|
||
manager = OllamaFailoverManager(health_monitor=mock_monitor)
|
||
manager._settings = mock_settings
|
||
|
||
with patch.object(manager, "_write_failover_audit", return_value=None), \
|
||
patch.object(manager, "_check_gemini_quota", return_value=True):
|
||
result = await manager.select_provider()
|
||
|
||
# 三層全部 exception → OFFLINE → primary=gemini
|
||
assert result.primary.provider_name == "gemini"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_gcp_a_healthy_select_provider_primary_ollama(self):
|
||
"""GCP-A HEALTHY → primary=ollama_gcp_a，select_provider 正常返回"""
|
||
async def _check_side_effect(url):
|
||
if url == URL_GCP_A:
|
||
return _make_health(HealthStatus.HEALTHY, URL_GCP_A)
|
||
return HealthReport(status=HealthStatus.OFFLINE, host=url, latency_ms=0.0)
|
||
|
||
mock_monitor = AsyncMock()
|
||
mock_monitor.check = AsyncMock(side_effect=_check_side_effect)
|
||
|
||
mock_settings = MagicMock()
|
||
mock_settings.OLLAMA_URL = URL_GCP_A
|
||
mock_settings.OLLAMA_SECONDARY_URL = URL_GCP_B
|
||
mock_settings.OLLAMA_FALLBACK_URL = URL_LOCAL
|
||
mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct"
|
||
mock_settings.GEMINI_DAILY_QUOTA = 1000
|
||
|
||
manager = OllamaFailoverManager(health_monitor=mock_monitor)
|
||
manager._settings = mock_settings
|
||
|
||
with patch.object(manager, "_write_failover_audit", return_value=None), \
|
||
patch.object(manager, "_check_gemini_quota", return_value=True):
|
||
result = await manager.select_provider()
|
||
|
||
# GCP-A HEALTHY → primary=ollama_gcp_a（或 backward-compat ollama）
|
||
assert result.primary.provider_name in ("ollama_gcp_a", "ollama")
|
||
|
||
|
||
# =============================================================================
|
||
# H7: Gemini 帳單熔斷
|
||
# 2026-04-25 critic-fix Part2 by Claude Engineer-C2
|
||
# =============================================================================
|
||
|
||
|
||
class TestGeminiQuota:
|
||
"""H7 修復驗證：Gemini 每日配額熔斷"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_gemini_quota_under_limit(self):
|
||
"""count=500 < quota=1000 → 返回 True（允許走 Gemini）
|
||
2026-04-26 Wave5 B3-fix by Claude Engineer-A4 — 改用 pipeline mock（atomic 修復後）
|
||
原 GET/INCR/EXPIRE 三步已改為 pipeline.set(NX)+incr，mock 跟著更新。
|
||
"""
|
||
manager = _make_manager()
|
||
manager._settings.GEMINI_DAILY_QUOTA = 1000
|
||
|
||
# pipeline mock：SET NX 返回 True（首次），INCR 返回 501（500+1，未達 quota=1000）
|
||
mock_pipe = MagicMock()
|
||
mock_pipe.set = MagicMock(return_value=mock_pipe)
|
||
mock_pipe.incr = MagicMock(return_value=mock_pipe)
|
||
mock_pipe.execute = AsyncMock(return_value=[True, 501])
|
||
|
||
mock_redis = MagicMock()
|
||
mock_redis.pipeline = MagicMock(return_value=mock_pipe)
|
||
|
||
with patch("src.core.redis_client.get_redis", return_value=mock_redis):
|
||
ok = await manager._check_gemini_quota()
|
||
|
||
assert ok is True
|
||
mock_pipe.execute.assert_awaited_once()
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_gemini_quota_exactly_at_limit(self):
|
||
"""count=1001 > quota=1000 → 返回 False（熔斷，不再呼叫 Gemini）
|
||
2026-04-26 Wave5 B3-fix by Claude Engineer-A4 — 改用 pipeline mock（atomic 修復後）
|
||
pipeline.incr 返回 1001（> quota=1000），應返回 False。
|
||
"""
|
||
manager = _make_manager()
|
||
manager._settings.GEMINI_DAILY_QUOTA = 1000
|
||
|
||
mock_pipe = MagicMock()
|
||
mock_pipe.set = MagicMock(return_value=mock_pipe)
|
||
mock_pipe.incr = MagicMock(return_value=mock_pipe)
|
||
mock_pipe.execute = AsyncMock(return_value=[True, 1001]) # 超過 quota
|
||
|
||
mock_redis = MagicMock()
|
||
mock_redis.pipeline = MagicMock(return_value=mock_pipe)
|
||
|
||
with patch("src.core.redis_client.get_redis", return_value=mock_redis):
|
||
ok = await manager._check_gemini_quota()
|
||
|
||
assert ok is False
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_gemini_quota_redis_unavailable_fail_closed(self):
|
||
"""Redis 掛掉 → 返回 False（2026-04-27 Wave8-X2 fail-closed，違反費用鐵律的修復）"""
|
||
manager = _make_manager()
|
||
|
||
with patch(
|
||
"src.core.redis_client.get_redis",
|
||
side_effect=RuntimeError("Redis unavailable"),
|
||
), patch(
|
||
"src.services.failover_alerter.get_failover_alerter",
|
||
return_value=MagicMock(alert_gemini_quota_exceeded=AsyncMock()),
|
||
):
|
||
ok = await manager._check_gemini_quota()
|
||
|
||
# fail-closed：Redis 異常時拒絕 Gemini，避免費用失控（违反 feedback_cost_change_approval.md）
|
||
assert ok is False
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_select_provider_quota_exceeded_uses_nemotron(self):
|
||
"""select_provider：Gemini quota 超過 → primary 改為 Nemotron（三層全 OFFLINE 情境）"""
|
||
mock_monitor = AsyncMock()
|
||
mock_monitor.check = AsyncMock(
|
||
return_value=_make_health(HealthStatus.OFFLINE, URL_GCP_A)
|
||
)
|
||
|
||
mock_settings = MagicMock()
|
||
mock_settings.OLLAMA_URL = URL_GCP_A
|
||
mock_settings.OLLAMA_SECONDARY_URL = URL_GCP_B
|
||
mock_settings.OLLAMA_FALLBACK_URL = URL_LOCAL
|
||
mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct"
|
||
mock_settings.GEMINI_DAILY_QUOTA = 1000
|
||
|
||
manager = OllamaFailoverManager(health_monitor=mock_monitor)
|
||
manager._settings = mock_settings
|
||
|
||
with patch.object(manager, "_write_failover_audit", return_value=None), \
|
||
patch.object(manager, "_check_gemini_quota", return_value=False):
|
||
result = await manager.select_provider()
|
||
|
||
# quota 超過 → 不走 Gemini，改走 Nemotron
|
||
assert result.primary.provider_name == "nemotron"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_select_provider_quota_exceeded_all_offline_uses_nemotron(self):
|
||
"""select_provider：Gemini quota 超過 + 三層全 OFFLINE → primary=Nemotron"""
|
||
async def _all_offline(url):
|
||
return HealthReport(status=HealthStatus.OFFLINE, host=url, latency_ms=0.0)
|
||
|
||
mock_monitor = AsyncMock()
|
||
mock_monitor.check = AsyncMock(side_effect=_all_offline)
|
||
|
||
mock_settings = MagicMock()
|
||
mock_settings.OLLAMA_URL = URL_GCP_A
|
||
mock_settings.OLLAMA_SECONDARY_URL = URL_GCP_B
|
||
mock_settings.OLLAMA_FALLBACK_URL = URL_LOCAL
|
||
mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct"
|
||
mock_settings.GEMINI_DAILY_QUOTA = 1000
|
||
|
||
manager = OllamaFailoverManager(health_monitor=mock_monitor)
|
||
manager._settings = mock_settings
|
||
|
||
with patch.object(manager, "_write_failover_audit", return_value=None), \
|
||
patch.object(manager, "_check_gemini_quota", return_value=False):
|
||
result = await manager.select_provider()
|
||
|
||
assert result.primary.provider_name == "nemotron"
|