294 lines
12 KiB
Python
294 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
tests/test_code_review_claude_routing.py
|
||
CodeReviewPipeline._openclaw_assess Ollama-first / cloud fallback 路由測試
|
||
|
||
驗收項目：
|
||
- Ollama 成功 → 不呼叫 Claude / Gemini / Elephant
|
||
- Ollama 失敗 + flag=true + Claude 可用 → 走 Claude
|
||
- Ollama 失敗 + Claude 不可用/失敗 → Gemini 只作備援
|
||
- Ollama + Gemini 都失敗 → 最終 Elephant 接手
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import importlib
|
||
import os
|
||
import sys
|
||
import types
|
||
from unittest.mock import MagicMock
|
||
|
||
import pytest
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# 共用工具
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
def _reload_pipeline():
|
||
"""重新載入 pipeline 模組（讓 module-level CODE_REVIEW_USE_CLAUDE flag 即時生效）"""
|
||
import services.code_review_pipeline_service as svc_mod
|
||
importlib.reload(svc_mod)
|
||
return svc_mod
|
||
|
||
|
||
def _make_pipeline(svc_mod, commit="abc12345"):
|
||
return svc_mod.CodeReviewPipeline(
|
||
commit_sha=commit,
|
||
changed_files=["services/foo.py"],
|
||
branch="main",
|
||
deploy_type="sync",
|
||
)
|
||
|
||
|
||
def _stub_anthropic(monkeypatch, svc_mod, *, available: bool, success: bool = True,
|
||
content: str = "CLAUDE-RESULT", error: str = None):
|
||
"""注入假的 anthropic_service 全域單例到 services.anthropic_service。
|
||
目標：避免 monkeypatch sys.modules 失敗（pipeline 內是 from services.anthropic_service import anthropic_service）
|
||
"""
|
||
fake_resp = MagicMock()
|
||
fake_resp.success = success
|
||
fake_resp.content = content if success else ""
|
||
fake_resp.input_tokens = 200
|
||
fake_resp.output_tokens = 100
|
||
fake_resp.cache_creation_tokens = 50
|
||
fake_resp.cache_read_tokens = 150
|
||
fake_resp.cache_hit = True
|
||
fake_resp.error = error
|
||
|
||
fake_svc = MagicMock()
|
||
fake_svc.is_available.return_value = available
|
||
fake_svc.generate.return_value = fake_resp
|
||
|
||
# 動態造一個假 module 並寫入 sys.modules（覆蓋既有 import 結果）
|
||
fake_module = types.ModuleType('services.anthropic_service')
|
||
fake_module.anthropic_service = fake_svc
|
||
monkeypatch.setitem(sys.modules, 'services.anthropic_service', fake_module)
|
||
return fake_svc
|
||
|
||
|
||
def _stub_logger(monkeypatch):
|
||
"""避免 log_ai_call 真寫 DB"""
|
||
import services.ai_call_logger as logger_mod
|
||
monkeypatch.setattr(logger_mod, '_write_to_db', lambda state: None)
|
||
monkeypatch.setenv('AI_CALL_LOGGING_ENABLED', 'false')
|
||
|
||
|
||
def _stub_ollama(monkeypatch, *, success: bool = True,
|
||
content: str = "OLLAMA-RESULT",
|
||
error: str = "all hosts failed"):
|
||
import services.ollama_service as ollama_mod
|
||
|
||
fake_resp = MagicMock()
|
||
fake_resp.success = success
|
||
fake_resp.content = content if success else ""
|
||
fake_resp.model = "qwen3:14b"
|
||
fake_resp.error = None if success else error
|
||
fake_resp.host = "http://34.143.170.20:11434"
|
||
fake_resp.input_tokens = 30 if success else 0
|
||
fake_resp.output_tokens = 12 if success else 0
|
||
|
||
fake_service = MagicMock()
|
||
fake_service.generate.return_value = fake_resp
|
||
fake_cls = MagicMock(return_value=fake_service)
|
||
monkeypatch.setattr(ollama_mod, "OllamaService", fake_cls)
|
||
return fake_cls, fake_service
|
||
|
||
|
||
def _stub_gemini_and_elephant(monkeypatch, *,
|
||
gemini_text: str = "GEMINI-RESULT",
|
||
elephant_text: str = "ELEPHANT-RESULT",
|
||
gemini_works: bool = True):
|
||
"""攔截 _openclaw_assess 內的 import google.generativeai / elephant_service。
|
||
|
||
pipeline 內是 lazy import，所以注入到 sys.modules 即可生效。
|
||
"""
|
||
# 1) Fake google.generativeai
|
||
fake_genai = types.ModuleType('google.generativeai')
|
||
fake_types = types.SimpleNamespace(GenerationConfig=lambda **kw: MagicMock())
|
||
fake_genai.types = fake_types
|
||
fake_genai.configure = lambda **kw: None
|
||
|
||
if gemini_works:
|
||
fake_resp = MagicMock()
|
||
fake_resp.text = gemini_text
|
||
fake_resp.usage_metadata = MagicMock(prompt_token_count=10, candidates_token_count=5)
|
||
fake_model = MagicMock()
|
||
fake_model.generate_content.return_value = fake_resp
|
||
fake_genai.GenerativeModel = MagicMock(return_value=fake_model)
|
||
else:
|
||
fake_genai.GenerativeModel = MagicMock(side_effect=RuntimeError("gemini broken"))
|
||
|
||
# google.generativeai 是子模組；注入它和父模組
|
||
fake_google = types.ModuleType('google')
|
||
fake_google.generativeai = fake_genai
|
||
monkeypatch.setitem(sys.modules, 'google', fake_google)
|
||
monkeypatch.setitem(sys.modules, 'google.generativeai', fake_genai)
|
||
|
||
# 2) Fake elephant_service
|
||
fake_eresp = MagicMock(success=True, content=elephant_text,
|
||
input_tokens=20, output_tokens=10, error=None)
|
||
fake_elephant = MagicMock()
|
||
fake_elephant.generate.return_value = fake_eresp
|
||
fake_eservice_mod = types.ModuleType('services.elephant_service')
|
||
fake_eservice_mod.elephant_service = fake_elephant
|
||
monkeypatch.setitem(sys.modules, 'services.elephant_service', fake_eservice_mod)
|
||
|
||
return fake_genai, fake_elephant
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Tests
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
def test_default_uses_ollama_first(monkeypatch):
|
||
"""CODE_REVIEW_USE_CLAUDE=false（預設）→ Ollama 成功時 Gemini 不被呼叫"""
|
||
monkeypatch.setenv('CODE_REVIEW_USE_CLAUDE', 'false')
|
||
monkeypatch.setenv('GEMINI_API_KEY', 'test-key')
|
||
_stub_logger(monkeypatch)
|
||
|
||
svc_mod = _reload_pipeline()
|
||
_ollama_cls, fake_ollama = _stub_ollama(monkeypatch)
|
||
fake_claude = _stub_anthropic(monkeypatch, svc_mod, available=True)
|
||
fake_genai, fake_elephant = _stub_gemini_and_elephant(monkeypatch)
|
||
|
||
pipeline = _make_pipeline(svc_mod)
|
||
result = pipeline._openclaw_assess(
|
||
files={"services/foo.py": "def x(): pass"},
|
||
findings=[],
|
||
)
|
||
|
||
assert result == "OLLAMA-RESULT"
|
||
fake_ollama.generate.assert_called_once()
|
||
fake_claude.generate.assert_not_called()
|
||
fake_genai.GenerativeModel.assert_not_called()
|
||
fake_elephant.generate.assert_not_called()
|
||
|
||
|
||
def test_flag_true_still_uses_ollama_before_claude(monkeypatch):
|
||
"""flag=true 也不得跳過 Ollama；Ollama 成功時 Claude/Gemini 都不呼叫"""
|
||
monkeypatch.setenv('CODE_REVIEW_USE_CLAUDE', 'true')
|
||
monkeypatch.setenv('GEMINI_API_KEY', 'test-key')
|
||
_stub_logger(monkeypatch)
|
||
|
||
svc_mod = _reload_pipeline()
|
||
_stub_ollama(monkeypatch)
|
||
fake_claude = _stub_anthropic(monkeypatch, svc_mod, available=True,
|
||
success=True, content="CLAUDE-RESULT")
|
||
fake_genai, fake_elephant = _stub_gemini_and_elephant(monkeypatch)
|
||
|
||
pipeline = _make_pipeline(svc_mod)
|
||
result = pipeline._openclaw_assess(
|
||
files={"services/foo.py": "def x(): pass"},
|
||
findings=[{"severity": "HIGH", "file": "services/foo.py",
|
||
"description": "x", "type": "bug"}],
|
||
)
|
||
|
||
assert result == "OLLAMA-RESULT"
|
||
fake_claude.generate.assert_not_called()
|
||
fake_genai.GenerativeModel.assert_not_called()
|
||
fake_elephant.generate.assert_not_called()
|
||
|
||
|
||
def test_ollama_failure_flag_true_uses_claude_backup(monkeypatch):
|
||
"""Ollama 失敗 + flag=true + Claude 可用 → Claude 才接手"""
|
||
monkeypatch.setenv('CODE_REVIEW_USE_CLAUDE', 'true')
|
||
monkeypatch.setenv('GEMINI_API_KEY', 'test-key')
|
||
_stub_logger(monkeypatch)
|
||
|
||
svc_mod = _reload_pipeline()
|
||
_stub_ollama(monkeypatch, success=False)
|
||
fake_claude = _stub_anthropic(monkeypatch, svc_mod, available=True,
|
||
success=True, content="CLAUDE-RESULT")
|
||
fake_genai, fake_elephant = _stub_gemini_and_elephant(monkeypatch)
|
||
|
||
pipeline = _make_pipeline(svc_mod)
|
||
result = pipeline._openclaw_assess(
|
||
files={"services/foo.py": "def x(): pass"},
|
||
findings=[],
|
||
)
|
||
|
||
assert result == "CLAUDE-RESULT"
|
||
fake_claude.generate.assert_called_once()
|
||
fake_genai.GenerativeModel.assert_not_called()
|
||
fake_elephant.generate.assert_not_called()
|
||
|
||
call_kwargs = fake_claude.generate.call_args.kwargs
|
||
assert call_kwargs['cache_system'] is True
|
||
assert call_kwargs['temperature'] == 0.2
|
||
assert call_kwargs['model'] == 'claude-opus-4-7'
|
||
|
||
|
||
def test_ollama_failure_flag_false_uses_gemini_backup(monkeypatch):
|
||
"""Ollama 失敗且 Claude flag=false → Gemini 才作備援"""
|
||
monkeypatch.setenv('CODE_REVIEW_USE_CLAUDE', 'false')
|
||
monkeypatch.setenv('GEMINI_API_KEY', 'test-key')
|
||
_stub_logger(monkeypatch)
|
||
|
||
svc_mod = _reload_pipeline()
|
||
_stub_ollama(monkeypatch, success=False)
|
||
fake_claude = _stub_anthropic(monkeypatch, svc_mod, available=True)
|
||
fake_genai, fake_elephant = _stub_gemini_and_elephant(monkeypatch)
|
||
|
||
pipeline = _make_pipeline(svc_mod)
|
||
result = pipeline._openclaw_assess(
|
||
files={"services/foo.py": "def x(): pass"},
|
||
findings=[],
|
||
)
|
||
|
||
assert result == "GEMINI-RESULT"
|
||
fake_claude.generate.assert_not_called()
|
||
fake_genai.GenerativeModel.assert_called_once()
|
||
fake_elephant.generate.assert_not_called()
|
||
|
||
|
||
def test_ollama_failure_claude_unavailable_uses_gemini_backup(monkeypatch):
|
||
"""Ollama 失敗 + flag=true 但 Claude unavailable → Gemini 才作備援"""
|
||
monkeypatch.setenv('CODE_REVIEW_USE_CLAUDE', 'true')
|
||
monkeypatch.setenv('GEMINI_API_KEY', 'test-key')
|
||
_stub_logger(monkeypatch)
|
||
|
||
svc_mod = _reload_pipeline()
|
||
_stub_ollama(monkeypatch, success=False)
|
||
fake_claude = _stub_anthropic(monkeypatch, svc_mod, available=False)
|
||
fake_genai, fake_elephant = _stub_gemini_and_elephant(monkeypatch)
|
||
|
||
pipeline = _make_pipeline(svc_mod)
|
||
result = pipeline._openclaw_assess(
|
||
files={"services/foo.py": "def x(): pass"},
|
||
findings=[],
|
||
)
|
||
|
||
assert result == "GEMINI-RESULT"
|
||
fake_claude.generate.assert_not_called()
|
||
fake_genai.GenerativeModel.assert_called_once()
|
||
fake_elephant.generate.assert_not_called()
|
||
|
||
|
||
def test_full_fallback_chain_after_ollama_failure(monkeypatch):
|
||
"""Ollama 失敗 + Claude 失敗 + Gemini 失敗 → 最終 Elephant 接手"""
|
||
monkeypatch.setenv('CODE_REVIEW_USE_CLAUDE', 'true')
|
||
monkeypatch.setenv('GEMINI_API_KEY', 'test-key')
|
||
_stub_logger(monkeypatch)
|
||
|
||
svc_mod = _reload_pipeline()
|
||
_stub_ollama(monkeypatch, success=False)
|
||
fake_claude = _stub_anthropic(monkeypatch, svc_mod, available=True,
|
||
success=False, error="claude down")
|
||
fake_genai, fake_elephant = _stub_gemini_and_elephant(
|
||
monkeypatch, gemini_works=False,
|
||
)
|
||
|
||
pipeline = _make_pipeline(svc_mod)
|
||
result = pipeline._openclaw_assess(
|
||
files={"services/foo.py": "def x(): pass"},
|
||
findings=[],
|
||
)
|
||
|
||
assert result == "ELEPHANT-RESULT"
|
||
fake_claude.generate.assert_called_once()
|
||
fake_elephant.generate.assert_called_once()
|