ADR-092 P1 飛輪閉環的 Ollama 失敗轉移子系統,全部 Engineer-A2/C/C2 補上。 新服務 (1581 行): - ollama_health_monitor.py (356):3 層健康檢測(TCP/HTTP/推理) - ollama_failover_manager.py (571):111→188 自動切換 + Redis 持久化 + recovery callback - ollama_auto_recovery.py (436):30s 背景監控 + 連續 3 次 HEALTHY → 切回 + clear_cache - failover_alerter.py (218):P1.5 Telegram 容災告警 服務整合: - ai_router.py: AIProviderEnum.OLLAMA_188 + 120s budget + failover fallback chain - main.py lifespan: 啟動時 wire callback + start recovery,關閉時優雅 stop - config.py: OLLAMA_FALLBACK_URL / OLLAMA_HEALTH_CHECK_MODEL / GEMINI_DAILY_QUOTA(帳單熔斷) K8s 配置: - 04-configmap.yaml.patch-188-fallback:注入 OLLAMA_FALLBACK_URL=http://192.168.0.188:11434 測試 (2082 行): - test_ollama_health_monitor.py (402) - test_ollama_failover_manager.py (707) - test_ollama_auto_recovery.py (580) - test_ai_router_failover_integration.py (257) - test_lifespan_failover_wiring.py (136) 依賴鏈:service 三件套 + ai_router + main.py 一起 commit,缺一就 ImportError。 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
137 lines
5.0 KiB
Python
137 lines
5.0 KiB
Python
# apps/api/tests/test_lifespan_failover_wiring.py | 2026-04-25 @ Asia/Taipei
|
||
# 2026-04-25 P1.2 by Claude Engineer-A2 — failover 整合到 ai_router + lifespan
|
||
"""
|
||
Ollama Failover Lifespan Wiring 測試
|
||
=====================================
|
||
驗收:
|
||
1. get_ollama_failover_manager singleton 呼叫了 set_recovery_callback
|
||
2. get_ollama_auto_recovery_service singleton 的 start() 被呼叫
|
||
3. shutdown 時 recovery service 的 stop() 被呼叫
|
||
4. failover_manager.set_recovery_callback 收到的是 recovery_svc.set_current_primary
|
||
|
||
測試分類:unit(mock get_ollama_failover_manager / get_ollama_auto_recovery_service)
|
||
不啟動完整 FastAPI app(避免 DB / Redis 依賴),直接測試 wiring 邏輯。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
|
||
# =============================================================================
|
||
# Wiring 邏輯測試
|
||
# =============================================================================
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_lifespan_wires_recovery_callback_and_starts_service():
|
||
"""
|
||
lifespan startup 邏輯:
|
||
- failover_mgr.set_recovery_callback(recovery_svc.set_current_primary) 被呼叫
|
||
- recovery_svc.start() 被呼叫
|
||
"""
|
||
mock_failover_mgr = MagicMock()
|
||
mock_failover_mgr.set_recovery_callback = MagicMock()
|
||
|
||
mock_recovery_svc = MagicMock()
|
||
mock_recovery_svc.set_current_primary = AsyncMock()
|
||
mock_recovery_svc.start = AsyncMock()
|
||
mock_recovery_svc.stop = AsyncMock()
|
||
|
||
with (
|
||
patch(
|
||
"src.services.ollama_failover_manager.get_ollama_failover_manager",
|
||
return_value=mock_failover_mgr,
|
||
),
|
||
patch(
|
||
"src.services.ollama_auto_recovery.get_ollama_auto_recovery_service",
|
||
return_value=mock_recovery_svc,
|
||
),
|
||
):
|
||
# 直接執行 lifespan 邏輯(不啟動完整 FastAPI)
|
||
from src.services.ollama_failover_manager import get_ollama_failover_manager
|
||
from src.services.ollama_auto_recovery import get_ollama_auto_recovery_service
|
||
|
||
_failover_mgr = get_ollama_failover_manager()
|
||
_recovery_svc = get_ollama_auto_recovery_service()
|
||
|
||
_failover_mgr.set_recovery_callback(_recovery_svc.set_current_primary)
|
||
await _recovery_svc.start()
|
||
|
||
# 驗收 1: set_recovery_callback 被呼叫一次
|
||
mock_failover_mgr.set_recovery_callback.assert_called_once_with(
|
||
mock_recovery_svc.set_current_primary
|
||
)
|
||
|
||
# 驗收 2: start() 被呼叫一次
|
||
mock_recovery_svc.start.assert_awaited_once()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_lifespan_stops_recovery_service_on_shutdown():
|
||
"""
|
||
lifespan shutdown 邏輯:
|
||
- recovery_svc.stop() 被呼叫
|
||
"""
|
||
mock_recovery_svc = MagicMock()
|
||
mock_recovery_svc.stop = AsyncMock()
|
||
|
||
with patch(
|
||
"src.services.ollama_auto_recovery.get_ollama_auto_recovery_service",
|
||
return_value=mock_recovery_svc,
|
||
):
|
||
from src.services.ollama_auto_recovery import get_ollama_auto_recovery_service
|
||
svc = get_ollama_auto_recovery_service()
|
||
await svc.stop()
|
||
|
||
mock_recovery_svc.stop.assert_awaited_once()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_set_recovery_callback_wires_correct_method():
|
||
"""
|
||
set_recovery_callback 收到的 callable 是 recovery_svc.set_current_primary
|
||
確保 wiring 的物件同一性正確
|
||
"""
|
||
from src.services.ollama_failover_manager import (
|
||
OllamaFailoverManager,
|
||
reset_ollama_failover_manager,
|
||
)
|
||
from src.services.ollama_auto_recovery import (
|
||
OllamaAutoRecoveryService,
|
||
reset_ollama_auto_recovery_service,
|
||
)
|
||
|
||
try:
|
||
mock_monitor = MagicMock()
|
||
failover_mgr = OllamaFailoverManager(health_monitor=mock_monitor)
|
||
|
||
mock_health_monitor = MagicMock()
|
||
mock_failover_mgr_inner = MagicMock()
|
||
recovery_svc = OllamaAutoRecoveryService(
|
||
health_monitor=mock_health_monitor,
|
||
failover_manager=mock_failover_mgr_inner,
|
||
)
|
||
|
||
# Wire callback
|
||
failover_mgr.set_recovery_callback(recovery_svc.set_current_primary)
|
||
|
||
# 驗收:_recovery_callback 是 recovery_svc.set_current_primary
|
||
# 注意:bound methods 每次屬性存取都建立新物件,必須用 __func__ + __self__ 比對
|
||
cb = failover_mgr._recovery_callback
|
||
assert cb is not None
|
||
assert cb.__func__ is recovery_svc.set_current_primary.__func__
|
||
assert cb.__self__ is recovery_svc
|
||
|
||
# 驗收:呼叫 callback 等同於呼叫 set_current_primary
|
||
# (用 mock 驗證真實 set_current_primary 被呼叫)
|
||
with patch.object(recovery_svc, "set_current_primary", new_callable=AsyncMock) as mock_scp:
|
||
failover_mgr.set_recovery_callback(recovery_svc.set_current_primary)
|
||
await failover_mgr._recovery_callback("gemini")
|
||
mock_scp.assert_awaited_once_with("gemini")
|
||
finally:
|
||
reset_ollama_failover_manager()
|
||
reset_ollama_auto_recovery_service()
|