387 lines
15 KiB
Python
387 lines
15 KiB
Python
# apps/api/tests/test_model_version_probe.py
|
||
# 2026-04-27 P3.2.1 by Claude
|
||
"""
|
||
model_version_probe 單元測試
|
||
==============================
|
||
測試覆蓋：
|
||
- probe_ollama_version: 成功 / model not found / HTTP 錯誤 / timeout
|
||
- probe_gemini_version: 成功 / API key 未設定 / HTTP 錯誤
|
||
- probe_claude_version: 成功 / API key 未設定
|
||
- probe_openclaw_nemo_version: 成功（找到 model） / 成功（model not in tags，graceful fallback）
|
||
- probe_all_providers: 並行 + return_exceptions（部分失敗不 crash）
|
||
|
||
測試分類：unit（mock httpx + settings，無 DB / Redis 依賴）
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from datetime import datetime, timedelta, timezone
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
from src.services.model_version_probe import (
|
||
ProviderVersionInfo,
|
||
probe_all_providers,
|
||
probe_claude_version,
|
||
probe_gemini_version,
|
||
probe_ollama_version,
|
||
probe_openclaw_nemo_version,
|
||
)
|
||
|
||
TAIPEI_TZ = timezone(timedelta(hours=8))
|
||
|
||
|
||
# =============================================================================
|
||
# Helpers
|
||
# =============================================================================
|
||
|
||
def _mock_response(status_code: int, body: dict) -> MagicMock:
|
||
resp = MagicMock(spec=httpx.Response)
|
||
resp.status_code = status_code
|
||
resp.json.return_value = body
|
||
resp.raise_for_status = MagicMock()
|
||
if status_code >= 400:
|
||
resp.raise_for_status.side_effect = httpx.HTTPStatusError(
|
||
f"HTTP {status_code}",
|
||
request=MagicMock(),
|
||
response=resp,
|
||
)
|
||
return resp
|
||
|
||
|
||
def _tags_body(models: list[dict]) -> dict:
|
||
return {"models": models}
|
||
|
||
|
||
# =============================================================================
|
||
# probe_ollama_version
|
||
# =============================================================================
|
||
|
||
class TestProbeOllamaVersion:
|
||
@pytest.mark.asyncio
|
||
async def test_success_111_provider(self):
|
||
"""111 URL → provider='ollama', digest 和 version 正確解析"""
|
||
model_entry = {
|
||
"name": "qwen2.5:7b-instruct",
|
||
"modified_at": "2026-04-01T00:00:00Z",
|
||
"digest": "sha256:abc123",
|
||
}
|
||
resp = _mock_response(200, _tags_body([model_entry]))
|
||
|
||
async def _fake_get(url, **kwargs):
|
||
return resp
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||
mock_client.get = AsyncMock(side_effect=_fake_get)
|
||
|
||
with patch("httpx.AsyncClient", return_value=mock_client):
|
||
info = await probe_ollama_version(
|
||
"http://192.168.0.111:11434", "qwen2.5:7b-instruct"
|
||
)
|
||
|
||
assert info.provider == "ollama"
|
||
assert info.model == "qwen2.5:7b-instruct"
|
||
assert info.version == "2026-04-01T00:00:00Z"
|
||
assert info.digest == "sha256:abc123"
|
||
assert isinstance(info.captured_at, datetime)
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_success_188_provider(self):
|
||
"""188 URL → provider='ollama_188'"""
|
||
model_entry = {
|
||
"name": "deepseek-r1:14b",
|
||
"modified_at": "2026-04-02T00:00:00Z",
|
||
"digest": "sha256:def456",
|
||
}
|
||
resp = _mock_response(200, _tags_body([model_entry]))
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||
mock_client.get = AsyncMock(return_value=resp)
|
||
|
||
with patch("httpx.AsyncClient", return_value=mock_client):
|
||
info = await probe_ollama_version(
|
||
"http://192.168.0.188:11434", "deepseek-r1:14b"
|
||
)
|
||
|
||
assert info.provider == "ollama_188"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_model_not_found_raises(self):
|
||
"""model 不在清單 → ValueError"""
|
||
resp = _mock_response(200, _tags_body([{"name": "other-model:7b", "modified_at": "", "digest": ""}]))
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||
mock_client.get = AsyncMock(return_value=resp)
|
||
|
||
with patch("httpx.AsyncClient", return_value=mock_client):
|
||
with pytest.raises(ValueError, match="not found"):
|
||
await probe_ollama_version(
|
||
"http://192.168.0.111:11434", "qwen2.5:7b-instruct"
|
||
)
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_http_error_propagates(self):
|
||
"""HTTP 500 → HTTPStatusError 上拋"""
|
||
resp = _mock_response(500, {})
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||
mock_client.get = AsyncMock(return_value=resp)
|
||
|
||
with patch("httpx.AsyncClient", return_value=mock_client):
|
||
with pytest.raises(httpx.HTTPStatusError):
|
||
await probe_ollama_version(
|
||
"http://192.168.0.111:11434", "qwen2.5:7b-instruct"
|
||
)
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_timeout_propagates(self):
|
||
"""連線 timeout → TimeoutException 上拋"""
|
||
mock_client = AsyncMock()
|
||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||
mock_client.get = AsyncMock(side_effect=httpx.TimeoutException("timeout"))
|
||
|
||
with patch("httpx.AsyncClient", return_value=mock_client):
|
||
with pytest.raises(httpx.TimeoutException):
|
||
await probe_ollama_version(
|
||
"http://192.168.0.111:11434", "qwen2.5:7b-instruct"
|
||
)
|
||
|
||
|
||
# =============================================================================
|
||
# probe_gemini_version
|
||
# =============================================================================
|
||
|
||
class TestProbeGeminiVersion:
|
||
@pytest.mark.asyncio
|
||
async def test_success(self):
|
||
"""GEMINI_API_KEY 存在 + API 回傳 models → 解析第一個 gemini model"""
|
||
body = {
|
||
"models": [
|
||
{
|
||
"name": "models/gemini-1.5-flash",
|
||
"supportedGenerationMethods": ["generateContent"],
|
||
},
|
||
]
|
||
}
|
||
resp = _mock_response(200, body)
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||
mock_client.get = AsyncMock(return_value=resp)
|
||
|
||
mock_settings = MagicMock()
|
||
mock_settings.GEMINI_API_KEY = "fake-key"
|
||
|
||
with patch("src.services.model_version_probe.settings", mock_settings), \
|
||
patch("httpx.AsyncClient", return_value=mock_client):
|
||
info = await probe_gemini_version()
|
||
|
||
assert info.provider == "gemini"
|
||
assert "gemini" in info.model
|
||
assert info.digest is None
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_missing_api_key_raises(self):
|
||
"""GEMINI_API_KEY 未設定 → RuntimeError"""
|
||
mock_settings = MagicMock()
|
||
mock_settings.GEMINI_API_KEY = ""
|
||
|
||
with patch("src.services.model_version_probe.settings", mock_settings):
|
||
with pytest.raises(RuntimeError, match="GEMINI_API_KEY"):
|
||
await probe_gemini_version()
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_http_error_propagates(self):
|
||
"""Gemini API 回 403 → HTTPStatusError"""
|
||
resp = _mock_response(403, {})
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||
mock_client.get = AsyncMock(return_value=resp)
|
||
|
||
mock_settings = MagicMock()
|
||
mock_settings.GEMINI_API_KEY = "fake-key"
|
||
|
||
with patch("src.services.model_version_probe.settings", mock_settings), \
|
||
patch("httpx.AsyncClient", return_value=mock_client):
|
||
with pytest.raises(httpx.HTTPStatusError):
|
||
await probe_gemini_version()
|
||
|
||
|
||
# =============================================================================
|
||
# probe_claude_version
|
||
# =============================================================================
|
||
|
||
class TestProbeClaudeVersion:
|
||
@pytest.mark.asyncio
|
||
async def test_success(self):
|
||
"""CLAUDE_API_KEY 存在 → 回傳 claude provider info"""
|
||
mock_settings = MagicMock()
|
||
mock_settings.CLAUDE_API_KEY = "sk-fake"
|
||
|
||
with patch("src.services.model_version_probe.settings", mock_settings):
|
||
info = await probe_claude_version()
|
||
|
||
assert info.provider == "claude"
|
||
assert "claude" in info.model
|
||
assert info.version == info.model
|
||
assert info.digest is None
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_missing_api_key_raises(self):
|
||
"""CLAUDE_API_KEY 未設定 → RuntimeError"""
|
||
mock_settings = MagicMock()
|
||
mock_settings.CLAUDE_API_KEY = ""
|
||
|
||
with patch("src.services.model_version_probe.settings", mock_settings):
|
||
with pytest.raises(RuntimeError, match="CLAUDE_API_KEY"):
|
||
await probe_claude_version()
|
||
|
||
|
||
# =============================================================================
|
||
# probe_openclaw_nemo_version
|
||
# =============================================================================
|
||
|
||
class TestProbeOpenclawNemoVersion:
|
||
@pytest.mark.asyncio
|
||
async def test_success_model_found(self):
|
||
"""model 在 /api/tags 清單 → 正確解析"""
|
||
model_entry = {
|
||
"name": "deepseek-r1:14b",
|
||
"modified_at": "2026-04-03T00:00:00Z",
|
||
"digest": "sha256:nemo999",
|
||
}
|
||
resp = _mock_response(200, _tags_body([model_entry]))
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||
mock_client.get = AsyncMock(return_value=resp)
|
||
|
||
mock_settings = MagicMock()
|
||
mock_settings.OPENCLAW_DEFAULT_MODEL = "deepseek-r1:14b"
|
||
mock_settings.OLLAMA_FALLBACK_URL = "http://192.168.0.188:11434"
|
||
|
||
with patch("src.services.model_version_probe.settings", mock_settings), \
|
||
patch("httpx.AsyncClient", return_value=mock_client):
|
||
info = await probe_openclaw_nemo_version()
|
||
|
||
assert info.provider == "openclaw_nemo"
|
||
assert info.model == "deepseek-r1:14b"
|
||
assert info.digest == "sha256:nemo999"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_model_not_in_tags_graceful(self):
|
||
"""model 不在清單 → graceful fallback（不 raise，version=model name）"""
|
||
resp = _mock_response(200, _tags_body([{"name": "other:7b", "modified_at": "", "digest": ""}]))
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||
mock_client.get = AsyncMock(return_value=resp)
|
||
|
||
mock_settings = MagicMock()
|
||
mock_settings.OPENCLAW_DEFAULT_MODEL = "deepseek-r1:14b"
|
||
mock_settings.OLLAMA_FALLBACK_URL = "http://192.168.0.188:11434"
|
||
|
||
with patch("src.services.model_version_probe.settings", mock_settings), \
|
||
patch("httpx.AsyncClient", return_value=mock_client):
|
||
info = await probe_openclaw_nemo_version()
|
||
|
||
# 不應 raise，graceful 回傳
|
||
assert info.provider == "openclaw_nemo"
|
||
assert info.version == "deepseek-r1:14b"
|
||
assert info.digest is None
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_missing_model_config_raises(self):
|
||
"""OPENCLAW_DEFAULT_MODEL 未設定 → RuntimeError"""
|
||
mock_settings = MagicMock()
|
||
mock_settings.OPENCLAW_DEFAULT_MODEL = ""
|
||
|
||
with patch("src.services.model_version_probe.settings", mock_settings):
|
||
with pytest.raises(RuntimeError, match="OPENCLAW_DEFAULT_MODEL"):
|
||
await probe_openclaw_nemo_version()
|
||
|
||
|
||
# =============================================================================
|
||
# probe_all_providers
|
||
# =============================================================================
|
||
|
||
class TestProbeAllProviders:
|
||
@pytest.mark.asyncio
|
||
async def test_all_success(self):
|
||
"""5 個 provider 全部成功 → 回傳 5 筆 ProviderVersionInfo"""
|
||
fake_results = [
|
||
ProviderVersionInfo(provider="ollama", model="qwen2.5:7b-instruct", version="v1"),
|
||
ProviderVersionInfo(provider="ollama_188", model="qwen2.5:7b-instruct", version="v1"),
|
||
ProviderVersionInfo(provider="gemini", model="gemini-1.5-flash", version="gemini-1.5-flash"),
|
||
ProviderVersionInfo(provider="claude", model="claude-haiku-4-5-20251001", version="claude-haiku-4-5-20251001"),
|
||
ProviderVersionInfo(provider="openclaw_nemo", model="deepseek-r1:14b", version="v1"),
|
||
]
|
||
|
||
with patch("src.services.model_version_probe.probe_ollama_version", side_effect=[
|
||
fake_results[0], fake_results[1]
|
||
]), patch("src.services.model_version_probe.probe_gemini_version", return_value=fake_results[2]), \
|
||
patch("src.services.model_version_probe.probe_claude_version", return_value=fake_results[3]), \
|
||
patch("src.services.model_version_probe.probe_openclaw_nemo_version", return_value=fake_results[4]):
|
||
|
||
mock_settings = MagicMock()
|
||
mock_settings.OLLAMA_URL = "http://192.168.0.111:11434"
|
||
mock_settings.OLLAMA_FALLBACK_URL = "http://192.168.0.188:11434"
|
||
mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct"
|
||
|
||
with patch("src.services.model_version_probe.settings", mock_settings):
|
||
results = await probe_all_providers()
|
||
|
||
assert len(results) == 5
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_partial_failure_no_crash(self):
|
||
"""2 個 provider 失敗 → 只回傳成功的 3 筆，不 crash"""
|
||
good = ProviderVersionInfo(provider="ollama", model="qwen2.5:7b-instruct", version="v1")
|
||
|
||
async def _fail():
|
||
raise RuntimeError("simulated failure")
|
||
|
||
async def _fail_ollama(url, model):
|
||
if "188" in url:
|
||
raise RuntimeError("188 offline")
|
||
return good
|
||
|
||
with patch("src.services.model_version_probe.probe_ollama_version", side_effect=_fail_ollama), \
|
||
patch("src.services.model_version_probe.probe_gemini_version", side_effect=_fail), \
|
||
patch("src.services.model_version_probe.probe_claude_version", return_value=ProviderVersionInfo(
|
||
provider="claude", model="claude-haiku-4-5-20251001", version="claude-haiku-4-5-20251001"
|
||
)), \
|
||
patch("src.services.model_version_probe.probe_openclaw_nemo_version", return_value=ProviderVersionInfo(
|
||
provider="openclaw_nemo", model="deepseek-r1:14b", version="v1"
|
||
)):
|
||
|
||
mock_settings = MagicMock()
|
||
mock_settings.OLLAMA_URL = "http://192.168.0.111:11434"
|
||
mock_settings.OLLAMA_FALLBACK_URL = "http://192.168.0.188:11434"
|
||
mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct"
|
||
|
||
with patch("src.services.model_version_probe.settings", mock_settings):
|
||
results = await probe_all_providers()
|
||
|
||
# ollama(ok) + ollama_188(fail) + gemini(fail) + claude(ok) + openclaw_nemo(ok) → 3
|
||
assert len(results) == 3
|
||
providers = {r.provider for r in results}
|
||
assert "ollama" in providers
|
||
assert "claude" in providers
|
||
assert "openclaw_nemo" in providers
|