59c9eff83a
- F401: 移除未使用的 imports (TerminalSessionStatus, AutoApproveDecision, TerminalSession)
- I001: 修正 import blocks 排序
- C401: set(generator) → {set comprehension}
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
363 lines
13 KiB
Python
363 lines
13 KiB
Python
"""
|
|
ADR-030 Auto Approve Policy Tests
|
|
=================================
|
|
測試自動執行策略服務
|
|
|
|
版本: v1.0
|
|
建立: 2026-03-26 (台北時區)
|
|
建立者: Claude Code (ADR-030 Phase 4)
|
|
"""
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
from src.models.playbook import (
|
|
ActionType,
|
|
Playbook,
|
|
PlaybookStatus,
|
|
RepairStep,
|
|
RiskLevel,
|
|
SymptomPattern,
|
|
)
|
|
from src.services.auto_approve import (
|
|
AutoApproveConfig,
|
|
AutoApprovePolicy,
|
|
AutoApproveReason,
|
|
)
|
|
from src.services.playbook_rag import PlaybookMatch
|
|
|
|
|
|
class TestAutoApprovePolicy:
|
|
"""AutoApprovePolicy 單元測試"""
|
|
|
|
@pytest.fixture
|
|
def mock_trust_manager(self):
|
|
"""建立 mock TrustScoreManager"""
|
|
manager = MagicMock()
|
|
# 預設信任分數為 5
|
|
manager.get_trust_record.return_value = MagicMock(score=5)
|
|
return manager
|
|
|
|
@pytest.fixture
|
|
def policy(self, mock_trust_manager):
|
|
return AutoApprovePolicy(trust_manager=mock_trust_manager)
|
|
|
|
def create_proposal_data(
|
|
self,
|
|
risk_level: str = "low",
|
|
confidence: float = 0.95,
|
|
action: str = "kubectl rollout restart deployment/test-app -n prod",
|
|
) -> dict:
|
|
"""建立測試用 proposal_data"""
|
|
return {
|
|
"risk_level": risk_level,
|
|
"confidence": confidence,
|
|
"action": action,
|
|
}
|
|
|
|
def create_playbook(
|
|
self,
|
|
success_count: int = 10,
|
|
failure_count: int = 0,
|
|
risk_level: RiskLevel = RiskLevel.LOW,
|
|
) -> Playbook:
|
|
"""建立測試用 Playbook"""
|
|
return Playbook(
|
|
playbook_id="PB-TEST-001",
|
|
name="Test Playbook",
|
|
description="For testing",
|
|
status=PlaybookStatus.APPROVED,
|
|
symptom_pattern=SymptomPattern(
|
|
alert_names=["HighCPU"],
|
|
affected_services=["test-app"],
|
|
),
|
|
repair_steps=[
|
|
RepairStep(
|
|
step_number=1,
|
|
action_type=ActionType.KUBECTL,
|
|
command="kubectl rollout restart deployment/{target}",
|
|
risk_level=risk_level,
|
|
),
|
|
],
|
|
success_count=success_count,
|
|
failure_count=failure_count,
|
|
)
|
|
|
|
def create_match(
|
|
self,
|
|
playbook_id: str = "PB-TEST-001",
|
|
similarity: float = 0.95,
|
|
) -> PlaybookMatch:
|
|
"""建立測試用 PlaybookMatch"""
|
|
return PlaybookMatch(
|
|
playbook_id=playbook_id,
|
|
similarity_score=similarity,
|
|
match_type="hybrid",
|
|
)
|
|
|
|
# =========================================================================
|
|
# 通過條件測試
|
|
# =========================================================================
|
|
|
|
def test_approve_all_conditions_met(self, policy):
|
|
"""所有條件都滿足時應該批准"""
|
|
proposal = self.create_proposal_data(
|
|
risk_level="low",
|
|
confidence=0.95,
|
|
)
|
|
playbook = self.create_playbook(success_count=10, failure_count=0)
|
|
match = self.create_match(similarity=0.95)
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is True
|
|
assert decision.reason == AutoApproveReason.PLAYBOOK_MATCH
|
|
|
|
def test_approve_with_high_confidence(self, policy):
|
|
"""高信心度應該通過"""
|
|
proposal = self.create_proposal_data(confidence=0.99)
|
|
playbook = self.create_playbook(success_count=10)
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is True
|
|
assert decision.confidence == 0.99
|
|
|
|
# =========================================================================
|
|
# 拒絕條件測試
|
|
# =========================================================================
|
|
|
|
def test_reject_critical_risk(self, policy):
|
|
"""CRITICAL 風險永遠拒絕"""
|
|
proposal = self.create_proposal_data(risk_level="critical")
|
|
playbook = self.create_playbook()
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is False
|
|
assert decision.reason == AutoApproveReason.CRITICAL_OPERATION
|
|
|
|
def test_reject_high_risk(self, policy):
|
|
"""HIGH 風險應該拒絕"""
|
|
proposal = self.create_proposal_data(risk_level="high")
|
|
playbook = self.create_playbook()
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is False
|
|
assert decision.reason == AutoApproveReason.HIGH_RISK
|
|
|
|
def test_reject_medium_risk(self, policy):
|
|
"""MEDIUM 風險應該拒絕 (預設只允許 low)"""
|
|
proposal = self.create_proposal_data(risk_level="medium")
|
|
playbook = self.create_playbook()
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is False
|
|
assert decision.reason == AutoApproveReason.HIGH_RISK
|
|
|
|
def test_reject_low_trust_score(self, policy, mock_trust_manager):
|
|
"""低信任分數應該拒絕"""
|
|
mock_trust_manager.get_trust_record.return_value = MagicMock(score=2)
|
|
|
|
proposal = self.create_proposal_data()
|
|
playbook = self.create_playbook()
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is False
|
|
assert decision.reason == AutoApproveReason.LOW_TRUST
|
|
|
|
def test_reject_low_confidence(self, policy):
|
|
"""低信心度應該拒絕"""
|
|
proposal = self.create_proposal_data(confidence=0.7)
|
|
playbook = self.create_playbook()
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is False
|
|
assert decision.reason == AutoApproveReason.LOW_TRUST
|
|
assert "Confidence" in decision.reason_detail
|
|
|
|
def test_reject_no_playbook(self, policy):
|
|
"""無 Playbook 時應該拒絕"""
|
|
proposal = self.create_proposal_data()
|
|
|
|
decision = policy.evaluate(proposal, None, None)
|
|
|
|
assert decision.should_auto_approve is False
|
|
assert decision.reason == AutoApproveReason.NO_PLAYBOOK
|
|
|
|
def test_reject_low_playbook_success_rate(self, policy):
|
|
"""Playbook 成功率低應該拒絕"""
|
|
proposal = self.create_proposal_data()
|
|
playbook = self.create_playbook(
|
|
success_count=10,
|
|
failure_count=5, # 66.7% success rate
|
|
)
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is False
|
|
assert decision.reason == AutoApproveReason.LOW_SUCCESS_RATE
|
|
|
|
def test_reject_insufficient_history(self, policy):
|
|
"""Playbook 執行次數不足應該拒絕"""
|
|
proposal = self.create_proposal_data()
|
|
playbook = self.create_playbook(
|
|
success_count=2, # < 3
|
|
failure_count=0,
|
|
)
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is False
|
|
assert decision.reason == AutoApproveReason.INSUFFICIENT_HISTORY
|
|
|
|
# =========================================================================
|
|
# 邊界條件測試
|
|
# =========================================================================
|
|
|
|
def test_boundary_trust_score_exactly_5(self, policy, mock_trust_manager):
|
|
"""信任分數剛好等於 5 (邊界值)"""
|
|
mock_trust_manager.get_trust_record.return_value = MagicMock(score=5)
|
|
|
|
proposal = self.create_proposal_data()
|
|
playbook = self.create_playbook()
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is True
|
|
assert decision.trust_score == 5
|
|
|
|
def test_boundary_confidence_exactly_90(self, policy):
|
|
"""信心度剛好等於 90% (邊界值)"""
|
|
proposal = self.create_proposal_data(confidence=0.90)
|
|
playbook = self.create_playbook()
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is True
|
|
assert decision.confidence == 0.90
|
|
|
|
def test_boundary_success_count_exactly_3(self, policy):
|
|
"""成功次數剛好等於 3 (邊界值)"""
|
|
proposal = self.create_proposal_data()
|
|
playbook = self.create_playbook(success_count=3, failure_count=0)
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is True
|
|
assert decision.playbook_success_count == 3
|
|
|
|
def test_boundary_success_rate_exactly_95(self, policy):
|
|
"""成功率剛好等於 95% (邊界值)"""
|
|
proposal = self.create_proposal_data()
|
|
playbook = self.create_playbook(
|
|
success_count=19,
|
|
failure_count=1, # 95% exactly
|
|
)
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is True
|
|
|
|
# =========================================================================
|
|
# 配置測試
|
|
# =========================================================================
|
|
|
|
def test_disabled_policy_rejects_all(self, mock_trust_manager):
|
|
"""停用的策略應該拒絕所有請求"""
|
|
config = AutoApproveConfig(enabled=False)
|
|
policy = AutoApprovePolicy(config=config, trust_manager=mock_trust_manager)
|
|
|
|
proposal = self.create_proposal_data()
|
|
playbook = self.create_playbook()
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
assert decision.should_auto_approve is False
|
|
assert "disabled" in decision.reason_detail.lower()
|
|
|
|
def test_custom_allowed_risk_levels(self, mock_trust_manager):
|
|
"""自訂允許的風險等級"""
|
|
config = AutoApproveConfig(allowed_risk_levels=["low", "medium"])
|
|
policy = AutoApprovePolicy(config=config, trust_manager=mock_trust_manager)
|
|
|
|
proposal = self.create_proposal_data(risk_level="medium")
|
|
playbook = self.create_playbook()
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
|
|
# medium 風險現在應該被允許
|
|
assert decision.should_auto_approve is True
|
|
|
|
# =========================================================================
|
|
# 資料結構測試
|
|
# =========================================================================
|
|
|
|
def test_decision_to_dict(self, policy):
|
|
"""Decision.to_dict() 應該正常工作"""
|
|
proposal = self.create_proposal_data()
|
|
playbook = self.create_playbook()
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
data = decision.to_dict()
|
|
|
|
assert "should_auto_approve" in data
|
|
assert "reason" in data
|
|
assert "reason_detail" in data
|
|
assert "risk_level" in data
|
|
assert "trust_score" in data
|
|
assert "confidence" in data
|
|
assert "decided_at" in data
|
|
|
|
def test_decision_to_audit_log(self, policy):
|
|
"""Decision.to_audit_log() 應該正常工作"""
|
|
proposal = self.create_proposal_data()
|
|
playbook = self.create_playbook()
|
|
match = self.create_match()
|
|
|
|
decision = policy.evaluate(proposal, playbook, match)
|
|
log = decision.to_audit_log()
|
|
|
|
assert "AUTO_APPROVED" in log or "REQUIRES_HUMAN" in log
|
|
assert "risk=" in log
|
|
assert "trust=" in log
|
|
|
|
|
|
class TestAutoApproveReason:
|
|
"""AutoApproveReason Enum 測試"""
|
|
|
|
def test_all_reasons_exist(self):
|
|
"""確認所有原因類型都存在"""
|
|
# 批准原因
|
|
assert AutoApproveReason.PLAYBOOK_MATCH.value == "playbook_match"
|
|
assert AutoApproveReason.TRUST_SCORE.value == "trust_score"
|
|
assert AutoApproveReason.LOW_RISK.value == "low_risk"
|
|
|
|
# 拒絕原因
|
|
assert AutoApproveReason.HIGH_RISK.value == "high_risk"
|
|
assert AutoApproveReason.CRITICAL_OPERATION.value == "critical_operation"
|
|
assert AutoApproveReason.LOW_TRUST.value == "low_trust"
|
|
assert AutoApproveReason.NO_PLAYBOOK.value == "no_playbook"
|
|
assert AutoApproveReason.LOW_SUCCESS_RATE.value == "low_success_rate"
|
|
assert AutoApproveReason.INSUFFICIENT_HISTORY.value == "insufficient_history"
|