awoooi/apps/api/src/services/audit_sink.py

"""
Audit Sink with PII/Secret Redaction
======================================
AwoooP Phase 4.4: Audit log 寫入前的 sanitization pipeline（ADR-116）
2026-05-04 ogt + Claude Sonnet 4.6

設計原則：
- audit log 不記錄 raw LLM input/output，只記 hash + schema validation result
- PII / secret pattern 硬攔（不可被 caller 繞過）
- 攔截清單：GCP IP、PostgreSQL password、Telegram token、SSH key、Bearer token 等
- redaction 後原值不可還原（替換為 [REDACTED:<type>]）
- 所有 audit 寫入透過此 sink（禁止其他 service 直接 INSERT audit_logs）

使用：
    from src.services.audit_sink import write_audit

    await write_audit(
        project_id="awoooi",
        action="run.completed",
        resource_type="run",
        resource_id=str(run_id),
        details={"trace_id": trace_id, "cost_usd": 0.012},
    )
"""

from __future__ import annotations

import hashlib
import json
import re
from typing import Any

import structlog

logger = structlog.get_logger(__name__)


# ─────────────────────────────────────────────────────────────────────────────
# Redaction patterns（ADR-116 P1-08）
# ─────────────────────────────────────────────────────────────────────────────

# 每個 pattern: (compiled_re, replacement_tag)
_REDACTION_PATTERNS: list[tuple[re.Pattern[str], str]] = [
    # Telegram bot token（數字:英數字母混合 32~64 字元）
    (re.compile(r"\d{8,12}:[A-Za-z0-9_-]{32,64}"), "TELEGRAM_TOKEN"),

    # PostgreSQL connection string
    (re.compile(r"postgresql(?:\+asyncpg)?://[^:]+:[^@]+@[^/\s]+"), "PG_DSN"),

    # Generic password in URL / config
    (re.compile(r"(?i)(?:password|passwd|pwd)\s*[:=]\s*\S+"), "PASSWORD"),

    # Bearer / Authorization header value
    (re.compile(r"(?i)(?:bearer|token)\s+[A-Za-z0-9\-._~+/]+=*"), "BEARER_TOKEN"),

    # AWS / GCP / NVIDIA API key patterns
    (re.compile(r"(?i)(?:api[_-]?key|apikey)\s*[:=]\s*[A-Za-z0-9\-._]{20,}"), "API_KEY"),

    # Private GCP internal IPs（ADR-116 禁止 GCP 內網 IP 進 log）
    (re.compile(r"\b10\.\d{1,3}\.\d{1,3}\.\d{1,3}\b"), "INTERNAL_IP"),
    (re.compile(r"\b172\.(?:1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3}\b"), "INTERNAL_IP"),
    (re.compile(r"\b192\.168\.\d{1,3}\.\d{1,3}\b"), "INTERNAL_IP"),

    # SSH private key
    (re.compile(r"-----BEGIN (?:RSA|EC|OPENSSH) PRIVATE KEY-----[\s\S]*?-----END [A-Z ]+ PRIVATE KEY-----"), "SSH_PRIVATE_KEY"),

    # JWT（三段 base64 以 . 分隔）
    (re.compile(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+"), "JWT_TOKEN"),

    # Hex secret >= 32 位（可能是 HMAC key / session token）
    (re.compile(r"\b[0-9a-f]{64}\b"), "HEX_SECRET_64"),
]

# 欄位名稱黑名單：這些 key 的 value 直接替換（不做 pattern 掃描）
_BLOCKED_FIELD_NAMES = frozenset({
    "password", "passwd", "pwd", "secret", "token", "api_key", "apikey",
    "private_key", "private_key_pem", "bot_token", "telegram_token",
    "hmac_key", "jwt", "authorization", "cookie", "session",
})

# LLM raw input/output 欄位名稱（只記 hash）
_LLM_RAW_FIELDS = frozenset({
    "raw_input", "raw_output", "llm_input", "llm_output",
    "prompt", "completion", "system_prompt",
})


# ─────────────────────────────────────────────────────────────────────────────
# Sanitization pipeline
# ─────────────────────────────────────────────────────────────────────────────

def _redact_string(value: str) -> str:
    """對字串套用所有 redaction patterns"""
    for pattern, tag in _REDACTION_PATTERNS:
        value = pattern.sub(f"[REDACTED:{tag}]", value)
    return value


def sanitize(details: dict[str, Any]) -> dict[str, Any]:
    """
    遞迴處理 details dict，套用所有 redaction 規則。

    規則優先序：
    1. key 在 _BLOCKED_FIELD_NAMES → value 替換為 [REDACTED:BLOCKED_FIELD]
    2. key 在 _LLM_RAW_FIELDS → value 替換為 sha256(str(value))（只記 hash）
    3. string value → pattern redaction
    4. nested dict/list → 遞迴處理
    """
    return _sanitize_value(details, depth=0)


def _sanitize_value(value: Any, depth: int = 0) -> Any:
    if depth > 10:
        return "[REDACTED:MAX_DEPTH]"

    if isinstance(value, dict):
        return {k: _sanitize_dict_entry(k, v, depth) for k, v in value.items()}
    if isinstance(value, list):
        return [_sanitize_value(item, depth + 1) for item in value]
    if isinstance(value, str):
        return _redact_string(value)
    return value


def _sanitize_dict_entry(key: str, value: Any, depth: int) -> Any:
    key_lower = key.lower()

    if key_lower in _BLOCKED_FIELD_NAMES:
        return "[REDACTED:BLOCKED_FIELD]"

    if key_lower in _LLM_RAW_FIELDS:
        # 只記 sha256 hash，不記原始內容
        raw_str = json.dumps(value, ensure_ascii=False) if not isinstance(value, str) else value
        return f"[LLM_RAW_HASH:{hashlib.sha256(raw_str.encode()).hexdigest()[:16]}]"

    return _sanitize_value(value, depth + 1)


# ─────────────────────────────────────────────────────────────────────────────
# Audit write
# ─────────────────────────────────────────────────────────────────────────────

async def write_audit(
    *,
    project_id: str,
    action: str,
    resource_type: str,
    resource_id: str,
    details: dict[str, Any] | None = None,
    run_id: str | None = None,
    trace_id: str | None = None,
) -> None:
    """
    統一 audit log 寫入入口（Phase 4+ 所有 service 必須透過此方法）。

    1. sanitize details（PII / secret redaction）
    2. 附加 run_id / trace_id（可觀測性）
    3. INSERT audit_logs（非阻擋 background task）
    """
    import asyncio

    asyncio.create_task(
        _write_audit_impl(
            project_id=project_id,
            action=action,
            resource_type=resource_type,
            resource_id=resource_id,
            details=details,
            run_id=run_id,
            trace_id=trace_id,
        ),
        name="audit_sink_write",
    )


async def _write_audit_impl(
    *,
    project_id: str,
    action: str,
    resource_type: str,
    resource_id: str,
    details: dict[str, Any] | None,
    run_id: str | None,
    trace_id: str | None,
) -> None:
    try:
        from sqlalchemy import text as sa_text
        from src.db.base import get_db_context

        clean_details: dict[str, Any] = sanitize(details or {})
        if run_id:
            clean_details["_run_id"] = run_id
        if trace_id:
            clean_details["_trace_id"] = trace_id

        async with get_db_context(project_id) as db:
            await db.execute(
                sa_text("""
                    INSERT INTO audit_logs
                        (project_id, action, resource_type, resource_id, details)
                    VALUES
                        (:project_id, :action, :resource_type, :resource_id, :details::jsonb)
                """),
                {
                    "project_id": project_id,
                    "action": action,
                    "resource_type": resource_type,
                    "resource_id": resource_id,
                    "details": json.dumps(clean_details),
                },
            )
    except Exception as exc:
        logger.warning(
            "audit_sink_write_failed",
            action=action,
            resource_id=resource_id,
            error=str(exc),
        )


# ─────────────────────────────────────────────────────────────────────────────
# Convenience：可在測試中驗證 sanitization 結果
# ─────────────────────────────────────────────────────────────────────────────

def sanitize_for_test(details: dict[str, Any]) -> dict[str, Any]:
    """同步 sanitize，供測試使用"""
    return sanitize(details)