f5b19cf108
- 新增 _promote_playbook: 高評分提升信心度 +0.1 - 新增 _demote_playbook: 低評分降低信心度 -0.15 - 新增 find_by_source_incident: 按 incident_id 查詢 Playbook - 新增 adjust_confidence: 信心度調整 + 狀態自動轉換 - 新增 Playbook.failure_rate 屬性 自動狀態轉換: - ai_confidence >= 0.9 + DRAFT → 自動 APPROVED - ai_confidence < 0.3 + failure_rate > 50% → 自動 DEPRECATED 測試: 13 案例全部通過 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
263 lines
8.7 KiB
Python
263 lines
8.7 KiB
Python
"""
|
|
Learning Service Tests - Playbook 信心度調整
|
|
=============================================
|
|
2026-03-30 Claude Code: Learning Service 信心度調整功能測試
|
|
|
|
測試範圍:
|
|
- _promote_playbook: 高評分提升信心度
|
|
- _demote_playbook: 低評分降低信心度
|
|
- adjust_confidence: 信心度調整邊界條件
|
|
- find_by_source_incident: 按 incident_id 查詢
|
|
"""
|
|
|
|
import pytest
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
from src.models.playbook import Playbook, PlaybookStatus
|
|
from src.services.learning_service import LearningService
|
|
|
|
|
|
# =============================================================================
|
|
# Fixtures
|
|
# =============================================================================
|
|
|
|
|
|
@pytest.fixture
|
|
def sample_playbook():
|
|
"""測試用 Playbook"""
|
|
return Playbook(
|
|
playbook_id="PB-20260330-TEST01",
|
|
name="Test Playbook",
|
|
description="Test playbook for learning service",
|
|
status=PlaybookStatus.DRAFT,
|
|
ai_confidence=0.5,
|
|
source_incident_ids=["INC-20260330-001"],
|
|
success_count=3,
|
|
failure_count=1,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def high_confidence_playbook():
|
|
"""高信心度 Playbook (接近自動升級)"""
|
|
return Playbook(
|
|
playbook_id="PB-20260330-HIGH01",
|
|
name="High Confidence Playbook",
|
|
description="Near auto-approve threshold",
|
|
status=PlaybookStatus.DRAFT,
|
|
ai_confidence=0.85,
|
|
source_incident_ids=["INC-20260330-002"],
|
|
success_count=8,
|
|
failure_count=1,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def low_confidence_playbook():
|
|
"""低信心度 Playbook (接近棄用)"""
|
|
return Playbook(
|
|
playbook_id="PB-20260330-LOW01",
|
|
name="Low Confidence Playbook",
|
|
description="Near deprecation threshold",
|
|
status=PlaybookStatus.APPROVED,
|
|
ai_confidence=0.35,
|
|
source_incident_ids=["INC-20260330-003"],
|
|
success_count=2,
|
|
failure_count=5,
|
|
)
|
|
|
|
|
|
# =============================================================================
|
|
# Test Cases
|
|
# =============================================================================
|
|
|
|
|
|
class TestPlaybookConfidenceModel:
|
|
"""Playbook 模型信心度相關測試"""
|
|
|
|
def test_failure_rate_empty(self):
|
|
"""測試無執行記錄時的失敗率"""
|
|
pb = Playbook(name="test", description="test")
|
|
assert pb.failure_rate == 0.0
|
|
assert pb.success_rate == 0.0
|
|
|
|
def test_failure_rate_calculation(self):
|
|
"""測試失敗率計算"""
|
|
pb = Playbook(
|
|
name="test",
|
|
description="test",
|
|
success_count=3,
|
|
failure_count=2,
|
|
)
|
|
assert pb.failure_rate == 0.4 # 2/5
|
|
assert pb.success_rate == 0.6 # 3/5
|
|
|
|
def test_total_executions(self):
|
|
"""測試總執行次數"""
|
|
pb = Playbook(
|
|
name="test",
|
|
description="test",
|
|
success_count=10,
|
|
failure_count=5,
|
|
)
|
|
assert pb.total_executions == 15
|
|
|
|
|
|
class TestFindBySourceIncident:
|
|
"""find_by_source_incident 測試"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_existing_incident(self, sample_playbook):
|
|
"""測試找到匹配的 Playbook"""
|
|
mock_repo = MagicMock()
|
|
mock_repo.find_by_source_incident = AsyncMock(return_value=[sample_playbook])
|
|
|
|
# 直接測試 mock 返回
|
|
result = await mock_repo.find_by_source_incident("INC-20260330-001")
|
|
|
|
assert len(result) == 1
|
|
assert result[0].playbook_id == "PB-20260330-TEST01"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_no_match(self):
|
|
"""測試找不到匹配的 Playbook"""
|
|
mock_repo = MagicMock()
|
|
mock_repo.find_by_source_incident = AsyncMock(return_value=[])
|
|
|
|
result = await mock_repo.find_by_source_incident("INC-NONEXISTENT")
|
|
assert len(result) == 0
|
|
|
|
|
|
class TestPromotePlaybook:
|
|
"""_promote_playbook 測試"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_promote_existing_playbook(self, sample_playbook):
|
|
"""測試提升現有 Playbook 信心度"""
|
|
# 模擬 adjust_confidence 返回更新後的 playbook
|
|
updated_playbook = Playbook(**sample_playbook.model_dump())
|
|
updated_playbook.ai_confidence = 0.6 # +0.1
|
|
|
|
mock_repo = MagicMock()
|
|
mock_repo.find_by_source_incident = AsyncMock(return_value=[sample_playbook])
|
|
mock_repo.adjust_confidence = AsyncMock(return_value=updated_playbook)
|
|
|
|
# 直接測試 mock 的行為邏輯
|
|
playbooks = await mock_repo.find_by_source_incident("INC-20260330-001")
|
|
assert len(playbooks) == 1
|
|
|
|
result = await mock_repo.adjust_confidence(
|
|
playbook_id=playbooks[0].playbook_id,
|
|
delta=0.1,
|
|
reason="test",
|
|
)
|
|
assert result.ai_confidence == 0.6
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_promote_auto_approve(self, high_confidence_playbook):
|
|
"""測試高信心度自動升級為 APPROVED"""
|
|
# ai_confidence: 0.85 + 0.1 = 0.95 >= 0.9 → 應該自動升級
|
|
updated_playbook = Playbook(**high_confidence_playbook.model_dump())
|
|
updated_playbook.ai_confidence = 0.95
|
|
updated_playbook.status = PlaybookStatus.APPROVED
|
|
updated_playbook.approved_by = "auto_learning"
|
|
|
|
mock_repo = MagicMock()
|
|
mock_repo.find_by_source_incident = AsyncMock(
|
|
return_value=[high_confidence_playbook]
|
|
)
|
|
mock_repo.adjust_confidence = AsyncMock(return_value=updated_playbook)
|
|
|
|
# 驗證狀態轉換邏輯
|
|
assert updated_playbook.ai_confidence >= 0.9
|
|
assert updated_playbook.status == PlaybookStatus.APPROVED
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_promote_no_playbook(self):
|
|
"""測試找不到 Playbook 時返回 False"""
|
|
mock_repo = MagicMock()
|
|
mock_repo.find_by_source_incident = AsyncMock(return_value=[])
|
|
|
|
with patch(
|
|
"src.repositories.playbook_repository.get_playbook_repository",
|
|
return_value=mock_repo,
|
|
):
|
|
service = LearningService()
|
|
result = await service._promote_playbook("INC-NONEXISTENT")
|
|
|
|
assert result is False
|
|
|
|
|
|
class TestDemotePlaybook:
|
|
"""_demote_playbook 測試"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_demote_existing_playbook(self, sample_playbook):
|
|
"""測試降低現有 Playbook 信心度"""
|
|
# 模擬 adjust_confidence 返回更新後的 playbook
|
|
updated_playbook = Playbook(**sample_playbook.model_dump())
|
|
updated_playbook.ai_confidence = 0.35 # -0.15
|
|
|
|
mock_repo = MagicMock()
|
|
mock_repo.find_by_source_incident = AsyncMock(return_value=[sample_playbook])
|
|
mock_repo.adjust_confidence = AsyncMock(return_value=updated_playbook)
|
|
|
|
# 驗證更新後的信心度
|
|
assert updated_playbook.ai_confidence == 0.35
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_demote_auto_deprecate(self, low_confidence_playbook):
|
|
"""測試低信心度 + 高失敗率自動棄用"""
|
|
# ai_confidence: 0.35 - 0.15 = 0.2 < 0.3
|
|
# failure_rate: 5/7 = 71% > 50%
|
|
# → 應該自動棄用
|
|
updated_playbook = Playbook(**low_confidence_playbook.model_dump())
|
|
updated_playbook.ai_confidence = 0.2
|
|
updated_playbook.status = PlaybookStatus.DEPRECATED
|
|
|
|
# 驗證棄用條件
|
|
assert updated_playbook.ai_confidence < 0.3
|
|
assert low_confidence_playbook.failure_rate > 0.5
|
|
assert updated_playbook.status == PlaybookStatus.DEPRECATED
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_demote_no_playbook(self):
|
|
"""測試找不到 Playbook 時返回 False"""
|
|
mock_repo = MagicMock()
|
|
mock_repo.find_by_source_incident = AsyncMock(return_value=[])
|
|
|
|
with patch(
|
|
"src.repositories.playbook_repository.get_playbook_repository",
|
|
return_value=mock_repo,
|
|
):
|
|
service = LearningService()
|
|
result = await service._demote_playbook("INC-NONEXISTENT")
|
|
|
|
assert result is False
|
|
|
|
|
|
class TestConfidenceBoundaries:
|
|
"""信心度邊界條件測試"""
|
|
|
|
def test_confidence_upper_bound(self):
|
|
"""測試信心度上限 (不超過 1.0)"""
|
|
pb = Playbook(
|
|
name="test",
|
|
description="test",
|
|
ai_confidence=0.98,
|
|
)
|
|
# 模擬 +0.1 後應該是 1.0 而不是 1.08
|
|
new_confidence = max(0.0, min(1.0, pb.ai_confidence + 0.1))
|
|
assert new_confidence == 1.0
|
|
|
|
def test_confidence_lower_bound(self):
|
|
"""測試信心度下限 (不低於 0.0)"""
|
|
pb = Playbook(
|
|
name="test",
|
|
description="test",
|
|
ai_confidence=0.1,
|
|
)
|
|
# 模擬 -0.15 後應該是 0.0 而不是 -0.05
|
|
new_confidence = max(0.0, min(1.0, pb.ai_confidence - 0.15))
|
|
assert new_confidence == 0.0
|