Files
awoooi/apps/api/tests/test_lifespan_failover_wiring.py
Your Name 55c6b4e2d9 feat(p1): Ollama 多層容災系統 — P1.1 健康檢測 + P1.2 ai_router 整合 + P1.5 容災告警
ADR-092 P1 飛輪閉環的 Ollama 失敗轉移子系統,全部 Engineer-A2/C/C2 補上。

新服務 (1581 行):
- ollama_health_monitor.py (356):3 層健康檢測(TCP/HTTP/推理)
- ollama_failover_manager.py (571):111→188 自動切換 + Redis 持久化 + recovery callback
- ollama_auto_recovery.py (436):30s 背景監控 + 連續 3 次 HEALTHY → 切回 + clear_cache
- failover_alerter.py (218):P1.5 Telegram 容災告警

服務整合:
- ai_router.py: AIProviderEnum.OLLAMA_188 + 120s budget + failover fallback chain
- main.py lifespan: 啟動時 wire callback + start recovery,關閉時優雅 stop
- config.py: OLLAMA_FALLBACK_URL / OLLAMA_HEALTH_CHECK_MODEL / GEMINI_DAILY_QUOTA(帳單熔斷)

K8s 配置:
- 04-configmap.yaml.patch-188-fallback:注入 OLLAMA_FALLBACK_URL=http://192.168.0.188:11434

測試 (2082 行):
- test_ollama_health_monitor.py (402)
- test_ollama_failover_manager.py (707)
- test_ollama_auto_recovery.py (580)
- test_ai_router_failover_integration.py (257)
- test_lifespan_failover_wiring.py (136)

依賴鏈:service 三件套 + ai_router + main.py 一起 commit,缺一就 ImportError。

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-26 20:18:33 +08:00

137 lines
5.0 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# apps/api/tests/test_lifespan_failover_wiring.py | 2026-04-25 @ Asia/Taipei
# 2026-04-25 P1.2 by Claude Engineer-A2 — failover 整合到 ai_router + lifespan
"""
Ollama Failover Lifespan Wiring 測試
=====================================
驗收:
1. get_ollama_failover_manager singleton 呼叫了 set_recovery_callback
2. get_ollama_auto_recovery_service singleton 的 start() 被呼叫
3. shutdown 時 recovery service 的 stop() 被呼叫
4. failover_manager.set_recovery_callback 收到的是 recovery_svc.set_current_primary
測試分類unitmock get_ollama_failover_manager / get_ollama_auto_recovery_service
不啟動完整 FastAPI app避免 DB / Redis 依賴),直接測試 wiring 邏輯。
"""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
# =============================================================================
# Wiring 邏輯測試
# =============================================================================
@pytest.mark.asyncio
async def test_lifespan_wires_recovery_callback_and_starts_service():
"""
lifespan startup 邏輯:
- failover_mgr.set_recovery_callback(recovery_svc.set_current_primary) 被呼叫
- recovery_svc.start() 被呼叫
"""
mock_failover_mgr = MagicMock()
mock_failover_mgr.set_recovery_callback = MagicMock()
mock_recovery_svc = MagicMock()
mock_recovery_svc.set_current_primary = AsyncMock()
mock_recovery_svc.start = AsyncMock()
mock_recovery_svc.stop = AsyncMock()
with (
patch(
"src.services.ollama_failover_manager.get_ollama_failover_manager",
return_value=mock_failover_mgr,
),
patch(
"src.services.ollama_auto_recovery.get_ollama_auto_recovery_service",
return_value=mock_recovery_svc,
),
):
# 直接執行 lifespan 邏輯(不啟動完整 FastAPI
from src.services.ollama_failover_manager import get_ollama_failover_manager
from src.services.ollama_auto_recovery import get_ollama_auto_recovery_service
_failover_mgr = get_ollama_failover_manager()
_recovery_svc = get_ollama_auto_recovery_service()
_failover_mgr.set_recovery_callback(_recovery_svc.set_current_primary)
await _recovery_svc.start()
# 驗收 1: set_recovery_callback 被呼叫一次
mock_failover_mgr.set_recovery_callback.assert_called_once_with(
mock_recovery_svc.set_current_primary
)
# 驗收 2: start() 被呼叫一次
mock_recovery_svc.start.assert_awaited_once()
@pytest.mark.asyncio
async def test_lifespan_stops_recovery_service_on_shutdown():
"""
lifespan shutdown 邏輯:
- recovery_svc.stop() 被呼叫
"""
mock_recovery_svc = MagicMock()
mock_recovery_svc.stop = AsyncMock()
with patch(
"src.services.ollama_auto_recovery.get_ollama_auto_recovery_service",
return_value=mock_recovery_svc,
):
from src.services.ollama_auto_recovery import get_ollama_auto_recovery_service
svc = get_ollama_auto_recovery_service()
await svc.stop()
mock_recovery_svc.stop.assert_awaited_once()
@pytest.mark.asyncio
async def test_set_recovery_callback_wires_correct_method():
"""
set_recovery_callback 收到的 callable 是 recovery_svc.set_current_primary
確保 wiring 的物件同一性正確
"""
from src.services.ollama_failover_manager import (
OllamaFailoverManager,
reset_ollama_failover_manager,
)
from src.services.ollama_auto_recovery import (
OllamaAutoRecoveryService,
reset_ollama_auto_recovery_service,
)
try:
mock_monitor = MagicMock()
failover_mgr = OllamaFailoverManager(health_monitor=mock_monitor)
mock_health_monitor = MagicMock()
mock_failover_mgr_inner = MagicMock()
recovery_svc = OllamaAutoRecoveryService(
health_monitor=mock_health_monitor,
failover_manager=mock_failover_mgr_inner,
)
# Wire callback
failover_mgr.set_recovery_callback(recovery_svc.set_current_primary)
# 驗收_recovery_callback 是 recovery_svc.set_current_primary
# 注意bound methods 每次屬性存取都建立新物件,必須用 __func__ + __self__ 比對
cb = failover_mgr._recovery_callback
assert cb is not None
assert cb.__func__ is recovery_svc.set_current_primary.__func__
assert cb.__self__ is recovery_svc
# 驗收:呼叫 callback 等同於呼叫 set_current_primary
# (用 mock 驗證真實 set_current_primary 被呼叫)
with patch.object(recovery_svc, "set_current_primary", new_callable=AsyncMock) as mock_scp:
failover_mgr.set_recovery_callback(recovery_svc.set_current_primary)
await failover_mgr._recovery_callback("gemini")
mock_scp.assert_awaited_once_with("gemini")
finally:
reset_ollama_failover_manager()
reset_ollama_auto_recovery_service()