awoooi/apps/api/tests/test_llm_tier1_schema.py

"""
Tier 1: Schema Validation Tests - ADR-018
==========================================
CI 必跑測試，使用 Golden Responses

驗證:
- LLM 輸出符合 Schema
- kubectl 語法有效
- 風險等級為有效 Enum

版本: v1.0
建立: 2026-03-26 (台北時區)
"""

import pytest

from tests.llm_testing.golden_responses import (
    DEFAULT_GOLDEN_RESPONSES,
    GoldenResponseManager,
)
from tests.llm_testing.property_validators import (
    extract_kubectl_from_text,
    extract_risk_level_from_text,
    validate_kubectl_syntax,
    validate_response_length,
    validate_risk_level,
)
from tests.llm_testing.schema_validators import (
    validate_proposal_schema,
)

# =============================================================================
# Fixtures
# =============================================================================


@pytest.fixture
def golden_manager():
    """Golden Response 管理器"""
    manager = GoldenResponseManager()
    # 確保有預設資料
    if not manager.get_all():
        for name, data in DEFAULT_GOLDEN_RESPONSES.items():
            manager._responses[name] = data
    return manager


# =============================================================================
# Tier 1: Schema Validation Tests
# =============================================================================


class TestSchemaValidation:
    """Schema 驗證測試 - 使用 Golden Responses"""

    @pytest.mark.parametrize("test_name", list(DEFAULT_GOLDEN_RESPONSES.keys()))
    def test_golden_response_schema(self, test_name: str):
        """
        驗證 Golden Response 符合 Schema

        這是 Tier 1 核心測試:
        - 不呼叫 LLM
        - 驗證預錄的 Golden Response
        - 確保 Schema 定義正確
        """
        golden = DEFAULT_GOLDEN_RESPONSES[test_name]
        response = golden["response"]

        is_valid, error_msg, output = validate_proposal_schema(response)

        assert is_valid, f"[{test_name}] Schema 驗證失敗: {error_msg}"
        assert output is not None
        assert output.risk_level in {"LOW", "MEDIUM", "HIGH", "CRITICAL"}

    @pytest.mark.parametrize("test_name", list(DEFAULT_GOLDEN_RESPONSES.keys()))
    def test_response_length(self, test_name: str):
        """驗證回應長度在合理範圍"""
        golden = DEFAULT_GOLDEN_RESPONSES[test_name]
        response = golden["response"]

        result = validate_response_length(response, min_length=10, max_length=1000)
        assert result.is_valid, f"[{test_name}] {result.message}"


class TestPropertyValidation:
    """屬性驗證測試"""

    @pytest.mark.parametrize(
        "command,expected_valid",
        [
            ("kubectl get pods", True),
            ("kubectl rollout restart deployment/api", True),
            ("kubectl delete pod test-pod", True),
            ("kubectl scale deployment/api --replicas=3", True),
            ("kubectl", False),  # 缺少動詞
            ("docker run nginx", False),  # 不是 kubectl
            ("", False),  # 空字串
        ],
    )
    def test_kubectl_syntax_validation(self, command: str, expected_valid: bool):
        """驗證 kubectl 語法檢查器"""
        result = validate_kubectl_syntax(command)
        assert result.is_valid == expected_valid, f"Command: {command}, {result.message}"

    @pytest.mark.parametrize(
        "risk_level,expected_valid",
        [
            ("LOW", True),
            ("MEDIUM", True),
            ("HIGH", True),
            ("CRITICAL", True),
            ("low", True),  # 大小寫不敏感
            ("High", True),
            ("高風險", True),  # 中文
            ("INVALID", False),
            ("", False),
        ],
    )
    def test_risk_level_validation(self, risk_level: str, expected_valid: bool):
        """驗證風險等級檢查器"""
        result = validate_risk_level(risk_level)
        assert result.is_valid == expected_valid, f"Risk: {risk_level}, {result.message}"


class TestExtractors:
    """提取器測試"""

    @pytest.mark.parametrize(
        "text,expected",
        [
            ("kubectl get pods", "kubectl get pods"),
            ("執行 kubectl rollout restart deployment/api", "kubectl rollout restart deployment/api"),
            ("```bash\nkubectl delete pod test\n```", "kubectl delete pod test"),
            ("沒有命令", None),
        ],
    )
    def test_extract_kubectl(self, text: str, expected: str | None):
        """測試 kubectl 提取"""
        result = extract_kubectl_from_text(text)
        if expected:
            assert result is not None
            assert "kubectl" in result
        else:
            assert result is None

    @pytest.mark.parametrize(
        "text,expected",
        [
            ("風險等級: HIGH", "HIGH"),
            ("這是 CRITICAL 操作", "CRITICAL"),
            ("低風險操作", "LOW"),
            ("無風險資訊", None),
        ],
    )
    def test_extract_risk_level(self, text: str, expected: str | None):
        """測試風險等級提取"""
        result = extract_risk_level_from_text(text)
        assert result == expected


# =============================================================================
# 測試報告
# =============================================================================


def test_tier1_summary():
    """
    Tier 1 測試摘要

    此測試確保:
    1. 所有 Golden Responses 符合 Schema
    2. kubectl 語法驗證正確
    3. 風險等級驗證正確
    """
    total_golden = len(DEFAULT_GOLDEN_RESPONSES)
    valid_count = 0

    for _name, golden in DEFAULT_GOLDEN_RESPONSES.items():
        is_valid, _, _ = validate_proposal_schema(golden["response"])
        if is_valid:
            valid_count += 1

    assert valid_count == total_golden, f"Golden Response 驗證: {valid_count}/{total_golden}"