""" Tier 1: Schema Validation Tests - ADR-018 ========================================== CI 必跑測試,使用 Golden Responses 驗證: - LLM 輸出符合 Schema - kubectl 語法有效 - 風險等級為有效 Enum 版本: v1.0 建立: 2026-03-26 (台北時區) """ import pytest from tests.llm_testing.golden_responses import ( DEFAULT_GOLDEN_RESPONSES, GoldenResponseManager, ) from tests.llm_testing.property_validators import ( extract_kubectl_from_text, extract_risk_level_from_text, validate_kubectl_syntax, validate_response_length, validate_risk_level, ) from tests.llm_testing.schema_validators import ( validate_proposal_schema, ) # ============================================================================= # Fixtures # ============================================================================= @pytest.fixture def golden_manager(): """Golden Response 管理器""" manager = GoldenResponseManager() # 確保有預設資料 if not manager.get_all(): for name, data in DEFAULT_GOLDEN_RESPONSES.items(): manager._responses[name] = data return manager # ============================================================================= # Tier 1: Schema Validation Tests # ============================================================================= class TestSchemaValidation: """Schema 驗證測試 - 使用 Golden Responses""" @pytest.mark.parametrize("test_name", list(DEFAULT_GOLDEN_RESPONSES.keys())) def test_golden_response_schema(self, test_name: str): """ 驗證 Golden Response 符合 Schema 這是 Tier 1 核心測試: - 不呼叫 LLM - 驗證預錄的 Golden Response - 確保 Schema 定義正確 """ golden = DEFAULT_GOLDEN_RESPONSES[test_name] response = golden["response"] is_valid, error_msg, output = validate_proposal_schema(response) assert is_valid, f"[{test_name}] Schema 驗證失敗: {error_msg}" assert output is not None assert output.risk_level in {"LOW", "MEDIUM", "HIGH", "CRITICAL"} @pytest.mark.parametrize("test_name", list(DEFAULT_GOLDEN_RESPONSES.keys())) def test_response_length(self, test_name: str): """驗證回應長度在合理範圍""" golden = DEFAULT_GOLDEN_RESPONSES[test_name] response = golden["response"] result = validate_response_length(response, min_length=10, max_length=1000) assert result.is_valid, f"[{test_name}] {result.message}" class TestPropertyValidation: """屬性驗證測試""" @pytest.mark.parametrize( "command,expected_valid", [ ("kubectl get pods", True), ("kubectl rollout restart deployment/api", True), ("kubectl delete pod test-pod", True), ("kubectl scale deployment/api --replicas=3", True), ("kubectl", False), # 缺少動詞 ("docker run nginx", False), # 不是 kubectl ("", False), # 空字串 ], ) def test_kubectl_syntax_validation(self, command: str, expected_valid: bool): """驗證 kubectl 語法檢查器""" result = validate_kubectl_syntax(command) assert result.is_valid == expected_valid, f"Command: {command}, {result.message}" @pytest.mark.parametrize( "risk_level,expected_valid", [ ("LOW", True), ("MEDIUM", True), ("HIGH", True), ("CRITICAL", True), ("low", True), # 大小寫不敏感 ("High", True), ("高風險", True), # 中文 ("INVALID", False), ("", False), ], ) def test_risk_level_validation(self, risk_level: str, expected_valid: bool): """驗證風險等級檢查器""" result = validate_risk_level(risk_level) assert result.is_valid == expected_valid, f"Risk: {risk_level}, {result.message}" class TestExtractors: """提取器測試""" @pytest.mark.parametrize( "text,expected", [ ("kubectl get pods", "kubectl get pods"), ("執行 kubectl rollout restart deployment/api", "kubectl rollout restart deployment/api"), ("```bash\nkubectl delete pod test\n```", "kubectl delete pod test"), ("沒有命令", None), ], ) def test_extract_kubectl(self, text: str, expected: str | None): """測試 kubectl 提取""" result = extract_kubectl_from_text(text) if expected: assert result is not None assert "kubectl" in result else: assert result is None @pytest.mark.parametrize( "text,expected", [ ("風險等級: HIGH", "HIGH"), ("這是 CRITICAL 操作", "CRITICAL"), ("低風險操作", "LOW"), ("無風險資訊", None), ], ) def test_extract_risk_level(self, text: str, expected: str | None): """測試風險等級提取""" result = extract_risk_level_from_text(text) assert result == expected # ============================================================================= # 測試報告 # ============================================================================= def test_tier1_summary(): """ Tier 1 測試摘要 此測試確保: 1. 所有 Golden Responses 符合 Schema 2. kubectl 語法驗證正確 3. 風險等級驗證正確 """ total_golden = len(DEFAULT_GOLDEN_RESPONSES) valid_count = 0 for _name, golden in DEFAULT_GOLDEN_RESPONSES.items(): is_valid, _, _ = validate_proposal_schema(golden["response"]) if is_valid: valid_count += 1 assert valid_count == total_golden, f"Golden Response 驗證: {valid_count}/{total_golden}"