229 lines
8.9 KiB
Python
229 lines
8.9 KiB
Python
# apps/api/tests/test_cs1_auto_execute.py
|
||
# 2026-04-27 ogt + Claude Sonnet 4.6 — CS1 LLM 高信心度自動執行邏輯單元測試
|
||
"""
|
||
測試覆蓋：
|
||
1. confidence=0.90 + low risk + kubectl 有值 → execute_approved_action 被呼叫
|
||
2. confidence=0.70 → 不執行（低信心度）
|
||
3. confidence=0.85 + CRITICAL → 不執行
|
||
4. confidence=0.90 + DESTRUCTIVE_PATTERN → 不執行
|
||
5. confidence=0.90 + NO_ACTION → 不執行
|
||
6. confidence=0.90 執行失敗（exception）→ 降級 PENDING 不 crash
|
||
|
||
測試分類：unit（mock ApprovalExecutionService / service，無 DB / Redis 依賴）
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from src.models.ai import AIBlastRadius, AIDataImpact, AIRiskLevel, OpenClawDecision, SuggestedAction
|
||
from src.models.approval import RiskLevel
|
||
|
||
|
||
# =============================================================================
|
||
# Helpers
|
||
# =============================================================================
|
||
|
||
|
||
def _make_analysis(
|
||
confidence: float = 0.90,
|
||
kubectl_command: str = "kubectl rollout restart deployment/api -n prod",
|
||
risk_level_str: str = "low",
|
||
suggested_action: SuggestedAction = SuggestedAction.RESTART_DEPLOYMENT,
|
||
) -> OpenClawDecision:
|
||
return OpenClawDecision(
|
||
action_title="Restart deployment",
|
||
kubectl_command=kubectl_command,
|
||
description="Auto restart",
|
||
risk_level=risk_level_str,
|
||
suggested_action=suggested_action,
|
||
confidence=confidence,
|
||
blast_radius=AIBlastRadius(
|
||
affected_pods=1,
|
||
estimated_downtime="~30s",
|
||
related_services=[],
|
||
data_impact=AIDataImpact.NONE,
|
||
),
|
||
target_resource="deployment/api",
|
||
affected_services=[],
|
||
deviation_analysis="none",
|
||
primary_responsibility="COLLAB",
|
||
)
|
||
|
||
|
||
def _run_cs1_block(
|
||
analysis_result: OpenClawDecision | None,
|
||
risk_level: RiskLevel,
|
||
exec_side_effect=None,
|
||
) -> tuple[MagicMock, MagicMock]:
|
||
"""
|
||
從 webhooks.py CS1 auto-execute 邏輯提取的同等邏輯，
|
||
直接呼叫，驗證 execute_approved_action 的呼叫情況。
|
||
|
||
回傳 (mock_executor_class, mock_execute_method)
|
||
"""
|
||
from src.services.action_parser import is_safe_kubectl_action
|
||
|
||
mock_exec_instance = MagicMock()
|
||
if exec_side_effect is not None:
|
||
mock_exec_instance.execute_approved_action = AsyncMock(side_effect=exec_side_effect)
|
||
else:
|
||
mock_exec_instance.execute_approved_action = AsyncMock(return_value=True)
|
||
|
||
mock_executor_cls = MagicMock(return_value=mock_exec_instance)
|
||
|
||
with (
|
||
patch("src.services.approval_execution.ApprovalExecutionService", mock_executor_cls),
|
||
patch("src.models.approval.ApprovalRequest", MagicMock()),
|
||
patch("src.models.approval.ApprovalStatus", MagicMock()),
|
||
):
|
||
# Replicate the exact condition logic from webhooks.py CS1 block
|
||
_non_destructive_actions = {"NO_ACTION", "INVESTIGATE", "OBSERVE"}
|
||
_sa_val = (
|
||
analysis_result.suggested_action.value
|
||
if analysis_result and hasattr(analysis_result.suggested_action, "value")
|
||
else str(getattr(analysis_result, "suggested_action", ""))
|
||
)
|
||
|
||
if analysis_result:
|
||
_cs1_kubectl = analysis_result.kubectl_command.strip() if analysis_result.kubectl_command else ""
|
||
_cs1_can_auto = (
|
||
bool(_cs1_kubectl)
|
||
and analysis_result.confidence >= 0.85
|
||
and risk_level != RiskLevel.CRITICAL
|
||
and _sa_val not in _non_destructive_actions
|
||
and is_safe_kubectl_action(_cs1_kubectl)
|
||
)
|
||
if _cs1_can_auto:
|
||
import asyncio
|
||
|
||
from src.models.approval import ApprovalRequest, ApprovalStatus
|
||
from src.services.approval_execution import ApprovalExecutionService
|
||
|
||
_cs1_auto_approval = MagicMock()
|
||
_cs1_executor = ApprovalExecutionService()
|
||
asyncio.run(_cs1_executor.execute_approved_action(_cs1_auto_approval))
|
||
|
||
return mock_executor_cls, mock_exec_instance
|
||
|
||
|
||
# =============================================================================
|
||
# Tests
|
||
# =============================================================================
|
||
|
||
|
||
class TestCS1AutoExecuteConditions:
|
||
"""測試 CS1 自動執行的觸發條件"""
|
||
|
||
def test_high_confidence_low_risk_executes(self):
|
||
"""confidence=0.90 + LOW risk + kubectl 有值 → execute 被呼叫"""
|
||
analysis = _make_analysis(confidence=0.90)
|
||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||
mock_exec.execute_approved_action.assert_called_once()
|
||
|
||
def test_low_confidence_does_not_execute(self):
|
||
"""confidence=0.70 → 不執行"""
|
||
analysis = _make_analysis(confidence=0.70)
|
||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||
mock_exec.execute_approved_action.assert_not_called()
|
||
|
||
def test_boundary_confidence_085_executes(self):
|
||
"""confidence=0.85 剛好等於門檻 → 執行"""
|
||
analysis = _make_analysis(confidence=0.85)
|
||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||
mock_exec.execute_approved_action.assert_called_once()
|
||
|
||
def test_critical_risk_does_not_execute(self):
|
||
"""confidence=0.90 + CRITICAL → 不執行"""
|
||
analysis = _make_analysis(confidence=0.90, risk_level_str="critical")
|
||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.CRITICAL)
|
||
mock_exec.execute_approved_action.assert_not_called()
|
||
|
||
def test_destructive_pattern_does_not_execute(self):
|
||
"""kubectl 含 destructive pattern → 不執行"""
|
||
from src.services.auto_approve import _DESTRUCTIVE_PATTERNS
|
||
# 取第一個 pattern 構造一個含危險詞的指令
|
||
bad_pattern = _DESTRUCTIVE_PATTERNS[0]
|
||
analysis = _make_analysis(
|
||
confidence=0.90,
|
||
kubectl_command=f"kubectl {bad_pattern} deployment/api -n prod",
|
||
)
|
||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||
mock_exec.execute_approved_action.assert_not_called()
|
||
|
||
def test_single_delete_pod_executes(self):
|
||
"""單一 Pod delete 是可恢復操作，parser 不應誤殺"""
|
||
analysis = _make_analysis(
|
||
confidence=0.90,
|
||
kubectl_command="kubectl delete pod api-xxx-yyy -n prod",
|
||
)
|
||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||
mock_exec.execute_approved_action.assert_called_once()
|
||
|
||
def test_no_action_does_not_execute(self):
|
||
"""suggested_action=NO_ACTION → 不執行"""
|
||
analysis = _make_analysis(
|
||
confidence=0.90,
|
||
suggested_action=SuggestedAction.NO_ACTION,
|
||
)
|
||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||
mock_exec.execute_approved_action.assert_not_called()
|
||
|
||
def test_investigate_does_not_execute(self):
|
||
"""suggested_action=INVESTIGATE → 不執行"""
|
||
analysis = _make_analysis(
|
||
confidence=0.90,
|
||
suggested_action=SuggestedAction.INVESTIGATE,
|
||
)
|
||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||
mock_exec.execute_approved_action.assert_not_called()
|
||
|
||
|
||
class TestCS1AutoExecuteFailureDegradation:
|
||
"""測試執行失敗時的降級行為"""
|
||
|
||
def test_execution_exception_does_not_crash(self):
|
||
"""execute_approved_action 拋 exception → 捕捉後繼續，不 crash"""
|
||
analysis = _make_analysis(confidence=0.90)
|
||
|
||
# 直接測試條件邏輯，確保例外被吞掉
|
||
from src.services.action_parser import is_safe_kubectl_action
|
||
|
||
_non_destructive_actions = {"NO_ACTION", "INVESTIGATE", "OBSERVE"}
|
||
_sa_val = analysis.suggested_action.value
|
||
_cs1_kubectl = analysis.kubectl_command.strip()
|
||
_cs1_can_auto = (
|
||
bool(_cs1_kubectl)
|
||
and analysis.confidence >= 0.85
|
||
and RiskLevel.LOW != RiskLevel.CRITICAL
|
||
and _sa_val not in _non_destructive_actions
|
||
and is_safe_kubectl_action(_cs1_kubectl)
|
||
)
|
||
assert _cs1_can_auto, "前置條件必須為 True 才能測試降級"
|
||
|
||
raised = False
|
||
try:
|
||
import asyncio
|
||
|
||
async def _simulate():
|
||
# 模擬整個 if _cs1_can_auto: try/except 區塊
|
||
if _cs1_can_auto:
|
||
try:
|
||
raise RuntimeError("executor exploded")
|
||
except Exception:
|
||
pass # 降級：維持 PENDING
|
||
|
||
asyncio.run(_simulate())
|
||
except Exception:
|
||
raised = True
|
||
|
||
assert not raised, "例外應被 CS1 try/except 吞掉，不應傳播"
|
||
|
||
def test_empty_kubectl_does_not_execute(self):
|
||
"""kubectl_command 為空字串 → 不執行"""
|
||
analysis = _make_analysis(confidence=0.90, kubectl_command="")
|
||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||
mock_exec.execute_approved_action.assert_not_called()
|