awoooi/apps/api/tests/test_model_version_probe.py

# apps/api/tests/test_model_version_probe.py
# 2026-04-27 P3.2.1 by Claude
# 2026-05-03 ogt: ADR-110 GCP 三層容災，更新 probe URL 為 GCP-A Primary
"""
model_version_probe 單元測試
==============================
測試覆蓋：
- probe_ollama_version: 成功 / model not found / HTTP 錯誤 / timeout
- probe_gemini_version: 成功 / API key 未設定 / HTTP 錯誤
- probe_claude_version: 成功 / API key 未設定
- probe_openclaw_nemo_version: 成功（找到 model） / 成功（model not in tags，graceful fallback）
- probe_all_providers: 並行 + return_exceptions（部分失敗不 crash）

測試分類：unit（mock httpx + settings，無 DB / Redis 依賴）
"""
from __future__ import annotations

from datetime import datetime, timedelta, timezone
from unittest.mock import AsyncMock, MagicMock, patch

import httpx
import pytest

from src.services.model_version_probe import (
    ProviderVersionInfo,
    probe_all_providers,
    probe_claude_version,
    probe_gemini_version,
    probe_ollama_version,
    probe_openclaw_nemo_version,
)

TAIPEI_TZ = timezone(timedelta(hours=8))


# =============================================================================
# Helpers
# =============================================================================

def _mock_response(status_code: int, body: dict) -> MagicMock:
    resp = MagicMock(spec=httpx.Response)
    resp.status_code = status_code
    resp.json.return_value = body
    resp.raise_for_status = MagicMock()
    if status_code >= 400:
        resp.raise_for_status.side_effect = httpx.HTTPStatusError(
            f"HTTP {status_code}",
            request=MagicMock(),
            response=resp,
        )
    return resp


def _tags_body(models: list[dict]) -> dict:
    return {"models": models}


# =============================================================================
# probe_ollama_version
# =============================================================================

class TestProbeOllamaVersion:
    @pytest.mark.asyncio
    async def test_success_gcp_a_provider(self):
        """GCP-A URL → provider='ollama', digest 和 version 正確解析（ADR-110）"""
        model_entry = {
            "name": "qwen2.5:7b-instruct",
            "modified_at": "2026-04-01T00:00:00Z",
            "digest": "sha256:abc123",
        }
        resp = _mock_response(200, _tags_body([model_entry]))

        async def _fake_get(url, **kwargs):
            return resp

        mock_client = AsyncMock()
        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
        mock_client.__aexit__ = AsyncMock(return_value=False)
        mock_client.get = AsyncMock(side_effect=_fake_get)

        with patch("httpx.AsyncClient", return_value=mock_client):
            info = await probe_ollama_version(
                "http://34.143.170.20:11434", "qwen2.5:7b-instruct"
            )

        assert info.provider == "ollama"
        assert info.model == "qwen2.5:7b-instruct"
        assert info.version == "2026-04-01T00:00:00Z"
        assert info.digest == "sha256:abc123"
        assert isinstance(info.captured_at, datetime)

    @pytest.mark.asyncio
    async def test_success_188_provider(self):
        """188 URL → provider='ollama_188'"""
        model_entry = {
            "name": "deepseek-r1:14b",
            "modified_at": "2026-04-02T00:00:00Z",
            "digest": "sha256:def456",
        }
        resp = _mock_response(200, _tags_body([model_entry]))

        mock_client = AsyncMock()
        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
        mock_client.__aexit__ = AsyncMock(return_value=False)
        mock_client.get = AsyncMock(return_value=resp)

        with patch("httpx.AsyncClient", return_value=mock_client):
            info = await probe_ollama_version(
                "http://192.168.0.188:11434", "deepseek-r1:14b"
            )

        assert info.provider == "ollama_188"

    @pytest.mark.asyncio
    async def test_model_not_found_raises(self):
        """model 不在清單 → ValueError"""
        resp = _mock_response(200, _tags_body([{"name": "other-model:7b", "modified_at": "", "digest": ""}]))

        mock_client = AsyncMock()
        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
        mock_client.__aexit__ = AsyncMock(return_value=False)
        mock_client.get = AsyncMock(return_value=resp)

        with patch("httpx.AsyncClient", return_value=mock_client):
            with pytest.raises(ValueError, match="not found"):
                await probe_ollama_version(
                    "http://34.143.170.20:11434", "qwen2.5:7b-instruct"
                )

    @pytest.mark.asyncio
    async def test_http_error_propagates(self):
        """HTTP 500 → HTTPStatusError 上拋"""
        resp = _mock_response(500, {})

        mock_client = AsyncMock()
        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
        mock_client.__aexit__ = AsyncMock(return_value=False)
        mock_client.get = AsyncMock(return_value=resp)

        with patch("httpx.AsyncClient", return_value=mock_client):
            with pytest.raises(httpx.HTTPStatusError):
                await probe_ollama_version(
                    "http://34.143.170.20:11434", "qwen2.5:7b-instruct"
                )

    @pytest.mark.asyncio
    async def test_timeout_propagates(self):
        """連線 timeout → TimeoutException 上拋"""
        mock_client = AsyncMock()
        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
        mock_client.__aexit__ = AsyncMock(return_value=False)
        mock_client.get = AsyncMock(side_effect=httpx.TimeoutException("timeout"))

        with patch("httpx.AsyncClient", return_value=mock_client):
            with pytest.raises(httpx.TimeoutException):
                await probe_ollama_version(
                    "http://34.143.170.20:11434", "qwen2.5:7b-instruct"
                )


# =============================================================================
# probe_gemini_version
# =============================================================================

class TestProbeGeminiVersion:
    @pytest.mark.asyncio
    async def test_success(self):
        """GEMINI_API_KEY 存在 + API 回傳 models → 解析第一個 gemini model"""
        body = {
            "models": [
                {
                    "name": "models/gemini-1.5-flash",
                    "supportedGenerationMethods": ["generateContent"],
                },
            ]
        }
        resp = _mock_response(200, body)

        mock_client = AsyncMock()
        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
        mock_client.__aexit__ = AsyncMock(return_value=False)
        mock_client.get = AsyncMock(return_value=resp)

        mock_settings = MagicMock()
        mock_settings.GEMINI_API_KEY = "fake-key"

        with patch("src.services.model_version_probe.settings", mock_settings), \
             patch("httpx.AsyncClient", return_value=mock_client):
            info = await probe_gemini_version()

        assert info.provider == "gemini"
        assert "gemini" in info.model
        assert info.digest is None
        mock_client.get.assert_awaited_once()
        _, kwargs = mock_client.get.await_args
        assert kwargs["headers"] == {"x-goog-api-key": "fake-key"}
        assert kwargs["params"] == {"pageSize": 50}
        assert "fake-key" not in mock_client.get.await_args.args[0]
        assert "fake-key" not in str(kwargs["params"])

    @pytest.mark.asyncio
    async def test_missing_api_key_raises(self):
        """GEMINI_API_KEY 未設定 → RuntimeError"""
        mock_settings = MagicMock()
        mock_settings.GEMINI_API_KEY = ""

        with patch("src.services.model_version_probe.settings", mock_settings):
            with pytest.raises(RuntimeError, match="GEMINI_API_KEY"):
                await probe_gemini_version()

    @pytest.mark.asyncio
    async def test_http_error_propagates(self):
        """Gemini API 回 403 → HTTPStatusError"""
        resp = _mock_response(403, {})

        mock_client = AsyncMock()
        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
        mock_client.__aexit__ = AsyncMock(return_value=False)
        mock_client.get = AsyncMock(return_value=resp)

        mock_settings = MagicMock()
        mock_settings.GEMINI_API_KEY = "fake-key"

        with patch("src.services.model_version_probe.settings", mock_settings), \
             patch("httpx.AsyncClient", return_value=mock_client):
            with pytest.raises(httpx.HTTPStatusError):
                await probe_gemini_version()


# =============================================================================
# probe_claude_version
# =============================================================================

class TestProbeClaudeVersion:
    @pytest.mark.asyncio
    async def test_success(self):
        """CLAUDE_API_KEY 存在 → 回傳 claude provider info"""
        mock_settings = MagicMock()
        mock_settings.CLAUDE_API_KEY = "sk-fake"

        with patch("src.services.model_version_probe.settings", mock_settings):
            info = await probe_claude_version()

        assert info.provider == "claude"
        assert "claude" in info.model
        assert info.version == info.model
        assert info.digest is None

    @pytest.mark.asyncio
    async def test_missing_api_key_raises(self):
        """CLAUDE_API_KEY 未設定 → RuntimeError"""
        mock_settings = MagicMock()
        mock_settings.CLAUDE_API_KEY = ""

        with patch("src.services.model_version_probe.settings", mock_settings):
            with pytest.raises(RuntimeError, match="CLAUDE_API_KEY"):
                await probe_claude_version()


# =============================================================================
# probe_openclaw_nemo_version
# =============================================================================

class TestProbeOpenclawNemoVersion:
    @pytest.mark.asyncio
    async def test_success_model_found(self):
        """model 在 /api/tags 清單 → 正確解析"""
        model_entry = {
            "name": "deepseek-r1:14b",
            "modified_at": "2026-04-03T00:00:00Z",
            "digest": "sha256:nemo999",
        }
        resp = _mock_response(200, _tags_body([model_entry]))

        mock_client = AsyncMock()
        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
        mock_client.__aexit__ = AsyncMock(return_value=False)
        mock_client.get = AsyncMock(return_value=resp)

        mock_settings = MagicMock()
        mock_settings.OPENCLAW_DEFAULT_MODEL = "deepseek-r1:14b"
        mock_settings.OLLAMA_FALLBACK_URL = "http://192.168.0.188:11434"

        with patch("src.services.model_version_probe.settings", mock_settings), \
             patch("httpx.AsyncClient", return_value=mock_client):
            info = await probe_openclaw_nemo_version()

        assert info.provider == "openclaw_nemo"
        assert info.model == "deepseek-r1:14b"
        assert info.digest == "sha256:nemo999"

    @pytest.mark.asyncio
    async def test_model_not_in_tags_graceful(self):
        """model 不在清單 → graceful fallback（不 raise，version=model name）"""
        resp = _mock_response(200, _tags_body([{"name": "other:7b", "modified_at": "", "digest": ""}]))

        mock_client = AsyncMock()
        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
        mock_client.__aexit__ = AsyncMock(return_value=False)
        mock_client.get = AsyncMock(return_value=resp)

        mock_settings = MagicMock()
        mock_settings.OPENCLAW_DEFAULT_MODEL = "deepseek-r1:14b"
        mock_settings.OLLAMA_FALLBACK_URL = "http://192.168.0.188:11434"

        with patch("src.services.model_version_probe.settings", mock_settings), \
             patch("httpx.AsyncClient", return_value=mock_client):
            info = await probe_openclaw_nemo_version()

        # 不應 raise，graceful 回傳
        assert info.provider == "openclaw_nemo"
        assert info.version == "deepseek-r1:14b"
        assert info.digest is None

    @pytest.mark.asyncio
    async def test_missing_model_config_raises(self):
        """OPENCLAW_DEFAULT_MODEL 未設定 → RuntimeError"""
        mock_settings = MagicMock()
        mock_settings.OPENCLAW_DEFAULT_MODEL = ""

        with patch("src.services.model_version_probe.settings", mock_settings):
            with pytest.raises(RuntimeError, match="OPENCLAW_DEFAULT_MODEL"):
                await probe_openclaw_nemo_version()


# =============================================================================
# probe_all_providers
# =============================================================================

class TestProbeAllProviders:
    @pytest.mark.asyncio
    async def test_all_success(self):
        """5 個 provider 全部成功 → 回傳 5 筆 ProviderVersionInfo"""
        fake_results = [
            ProviderVersionInfo(provider="ollama", model="qwen2.5:7b-instruct", version="v1"),
            ProviderVersionInfo(provider="ollama_188", model="qwen2.5:7b-instruct", version="v1"),
            ProviderVersionInfo(provider="gemini", model="gemini-1.5-flash", version="gemini-1.5-flash"),
            ProviderVersionInfo(provider="claude", model="claude-haiku-4-5-20251001", version="claude-haiku-4-5-20251001"),
            ProviderVersionInfo(provider="openclaw_nemo", model="deepseek-r1:14b", version="v1"),
        ]

        with patch("src.services.model_version_probe.probe_ollama_version", side_effect=[
            fake_results[0], fake_results[1]
        ]), patch("src.services.model_version_probe.probe_gemini_version", return_value=fake_results[2]), \
           patch("src.services.model_version_probe.probe_claude_version", return_value=fake_results[3]), \
           patch("src.services.model_version_probe.probe_openclaw_nemo_version", return_value=fake_results[4]):

            mock_settings = MagicMock()
            mock_settings.OLLAMA_URL = "http://34.143.170.20:11434"       # GCP-A（ADR-110）
            mock_settings.OLLAMA_FALLBACK_URL = "http://192.168.0.188:11434"
            mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct"

            with patch("src.services.model_version_probe.settings", mock_settings):
                results = await probe_all_providers()

        assert len(results) == 5

    @pytest.mark.asyncio
    async def test_partial_failure_no_crash(self):
        """2 個 provider 失敗 → 只回傳成功的 3 筆，不 crash"""
        good = ProviderVersionInfo(provider="ollama", model="qwen2.5:7b-instruct", version="v1")

        async def _fail():
            raise RuntimeError("simulated failure")

        async def _fail_ollama(url, model):
            if "188" in url:
                raise RuntimeError("188 offline")
            return good

        with patch("src.services.model_version_probe.probe_ollama_version", side_effect=_fail_ollama), \
             patch("src.services.model_version_probe.probe_gemini_version", side_effect=_fail), \
             patch("src.services.model_version_probe.probe_claude_version", return_value=ProviderVersionInfo(
                 provider="claude", model="claude-haiku-4-5-20251001", version="claude-haiku-4-5-20251001"
             )), \
             patch("src.services.model_version_probe.probe_openclaw_nemo_version", return_value=ProviderVersionInfo(
                 provider="openclaw_nemo", model="deepseek-r1:14b", version="v1"
             )):

            mock_settings = MagicMock()
            mock_settings.OLLAMA_URL = "http://34.143.170.20:11434"       # GCP-A（ADR-110）
            mock_settings.OLLAMA_FALLBACK_URL = "http://192.168.0.188:11434"
            mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct"

            with patch("src.services.model_version_probe.settings", mock_settings):
                results = await probe_all_providers()

        # ollama(ok) + ollama_188(fail) + gemini(fail) + claude(ok) + openclaw_nemo(ok) → 3
        assert len(results) == 3
        providers = {r.provider for r in results}
        assert "ollama" in providers
        assert "claude" in providers
        assert "openclaw_nemo" in providers