awoooi/apps/api/src/services/proactive_inspector.py

"""
AWOOOI AIOps Phase 4 — Proactive Inspector（主動巡檢）
======================================================
職責：每 5 分鐘主動掃描異常，聚合動態偵測信號，產生 ProactiveAlert

協調三大 Phase 4 感官：
1. DynamicBaselineService  — 即時異常偵測（3σ 偏離）
2. LogAnomalyDetector      — K8s Pod 新 log pattern
3. TrendPredictor          — 4h 內閾值突破預警

設計原則：
- Shadow Mode（AIOPS_P4_SHADOW_MODE=True）：所有偵測只記錄，不觸發 Alert
- 熔斷：任一子感官失敗 → 繼續其他感官，不中斷整個巡檢
- 去重：同一 metric/cluster 在 30 分鐘內只上報一次（Redis TTL）
- 訓練調度：每次巡檢順便觸發 Holt-Winters 重訓（async，背景執行）

ADR-084: Phase 4 動態異常偵測源頭升級
2026-04-15 ogt + Claude Sonnet 4.6（亞太）: Phase 4 初始建立
"""

from __future__ import annotations

import asyncio
from dataclasses import dataclass, field
from typing import Any

import structlog

from src.utils.timezone import now_taipei

logger = structlog.get_logger(__name__)

# ── 常數 ────────────────────────────────────────────────────────────────────
INSPECTOR_INTERVAL_SEC = 300         # 每 5 分鐘巡檢一次
DEDUP_TTL_SEC = 1800                 # 同一異常 30 分鐘內去重
DEDUP_KEY_PREFIX = "proactive:dedup:"
K8S_NAMESPACE = "awoooi-prod"

# 需要監控的 metrics（Prometheus PromQL + 警戒閾值）
# 2026-04-24 ogt + Claude Sonnet 4.6: P0.6 修復 — 修正 PromQL labels 使其對應實際 Prometheus 資料
# - CPU/Memory: cadvisor 無 namespace label，改用 kube_pod_container_status_restarts_total 確認存在的 namespace 篩法
# - pod_restart_rate: 改用 sum() 聚合，避免回傳多 vector 使 _fetch_current_value 只取第一筆
# - db_connection_pool: datname 實際為 awoooi_prod（非 awoooi）
# - http_error_rate: cadvisor 無 http_requests_total，改用 probe_success 替代
# 2026-04-25 P0.6 修復 by Claude Engineer-B:
# - http_error_rate: Prometheus 實測確認 metric 名稱為 probe_success（非 blackbox_probe_success）
# - cpu_usage_awoooi_api: cadvisor up=0（停止），改用 node-exporter node_cpu_seconds_total（node level）
# - memory_usage_awoooi_api: cadvisor 停止，改用 node-exporter 記憶體使用率比例（0-1 scale）
# 2026-04-26 critic-H2/H4 hotfix by Claude Opus 4.7:
# - cpu/memory metric 改名 _node_188 — 語意從 container 變 node，避免污染既有 baseline
#   舊 baseline (container working set) 的數值範圍跟新 (node ratio) 完全不同，
#   保留同名會造成 baseline 學到的 σ 完全失真，5 分鐘內狂報假 anomaly。
# - http_error_rate 加 `or vector(1)` fallback：probe_success 全部不可達時不誤觸發
MONITORED_METRICS: list[dict[str, Any]] = [
    {
        "name": "http_error_rate",
        # probe_success：Blackbox Exporter 實際 metric 名稱（非 blackbox_probe_success）
        # 2026-04-26 H4 hotfix: avg() 在所有 target 缺值時回空 vector，搭配 fallback `or vector(1)`
        # 防 _fetch_current_value 把空當 0 → 1-0=1 → 5min 一次假告警洪水
        # 語意：探測全失敗時 fallback=1（視為「全部成功」），避免假告警；真正 down 由 BlackboxProbeFailure 抓
        "promql": '1 - avg(probe_success or on() vector(1))',
        "threshold": 0.05,   # > 5% probe 失敗 = 警戒
        "description": "HTTP Probe 失敗率（Blackbox Exporter）",
    },
    {
        # 2026-04-26 H2 hotfix: 改名 _node_188（原 _awoooi_api 語意是 container working set）
        "name": "cpu_usage_node_188",
        # cadvisor up=0（prod-docker-188 離線），改用 node-exporter node-level CPU
        # 實測確認：avg(rate(node_cpu_seconds_total{mode!="idle"}[5m])) → 有資料
        # threshold 0.85 = 85% CPU 使用率（node level，0-1 比例）
        "promql": 'avg(rate(node_cpu_seconds_total{host="188",mode!="idle"}[5m]))',
        "threshold": 0.85,   # > 85% node CPU（所有 core 平均）
        "description": "Node 188 CPU 使用率（node-exporter，cadvisor 停止時替代）",
    },
    {
        # 2026-05-05 ogt + Codex: 110/188 CPU 過載事故後補齊 110 動態基線。
        # Why: 事故主熱點在 110 Sentry ClickHouse/Snuba/Kafka，舊 inspector 只看未過濾的全域 CPU。
        "name": "cpu_usage_node_110",
        "promql": 'avg(rate(node_cpu_seconds_total{host="110",mode!="idle"}[5m]))',
        "threshold": 0.85,
        "description": "Node 110 CPU 使用率（node-exporter）",
    },
    {
        # 2026-04-26 H2 hotfix: 改名 _node_188
        "name": "memory_usage_node_188",
        # cadvisor 停止，改用 node-exporter 節點記憶體使用率比例（0-1）
        # 實測確認：188 機器 62.76 GiB，當前 ~30% 使用率
        # threshold 0.85 = 85% node memory usage
        "promql": '(node_memory_MemTotal_bytes{host="188"} - node_memory_MemAvailable_bytes{host="188"}) / node_memory_MemTotal_bytes{host="188"}',
        "threshold": 0.85,   # > 85% node memory（0-1 比例）
        "description": "Node 188 記憶體使用率（node-exporter，cadvisor 停止時替代）",
    },
    {
        # 2026-05-05 ogt + Codex: 110/188 CPU 過載事故後補齊 110 記憶體基線。
        "name": "memory_usage_node_110",
        "promql": '(node_memory_MemTotal_bytes{host="110"} - node_memory_MemAvailable_bytes{host="110"}) / node_memory_MemTotal_bytes{host="110"}',
        "threshold": 0.85,
        "description": "Node 110 記憶體使用率（node-exporter）",
    },
    {
        # 2026-05-05 ogt + Codex: 用 load/core 比例抓長時間過載，避免 CPU% 被 runnable queue 稀釋。
        "name": "load5_per_core_110",
        "promql": 'node_load5{host="110"} / count(count by(cpu) (node_cpu_seconds_total{host="110",mode="idle"}))',
        "threshold": 1.50,
        "description": "Node 110 load5 / CPU core 比例",
    },
    {
        "name": "load5_per_core_188",
        "promql": 'node_load5{host="188"} / count(count by(cpu) (node_cpu_seconds_total{host="188",mode="idle"}))',
        "threshold": 1.50,
        "description": "Node 188 load5 / CPU core 比例",
    },
    {
        # 2026-05-05 ogt + Codex: 補 Docker Compose 服務層基線，抓單容器長期吃超過 2 core。
        "name": "docker_max_container_cpu_cores",
        "promql": 'max(docker_container_cpu_cores) or max(rate(container_cpu_usage_seconds_total{name!="",id!="/"}[5m])) or on() vector(0)',
        "threshold": 2.0,
        "description": "Docker 單容器 CPU core 使用量上限",
    },
    {
        # 2026-05-05 ogt + Codex: cAdvisor v0.47 無 container restart metric，改吃 node-exporter textfile。
        "name": "docker_restart_spike_15m",
        "promql": (
            'sum(increase(docker_container_restart_count[15m])) '
            'or sum(increase(docker_container_inspect_restart_count[15m])) '
            'or on() vector(0)'
        ),
        "threshold": 5.0,
        "description": "Docker 容器 15 分鐘重啟增量",
    },
    {
        # 2026-05-05 ogt + Codex: 110 runner watchdog loop was invisible to Docker metrics.
        "name": "systemd_runner_restart_spike_15m",
        "promql": 'sum(increase(systemd_unit_restarts_total{unit=~"actions\\\\.runner\\\\..*"}[15m])) or on() vector(0)',
        "threshold": 2.0,
        "description": "Systemd GitHub/Gitea runner 15 分鐘重啟增量",
    },
    {
        # 2026-05-05 ogt + Codex: GitHub runner service should not use systemd WatchdogSec.
        "name": "systemd_runner_watchdog_enabled",
        "promql": 'max(systemd_unit_watchdog_seconds{unit=~"actions\\\\.runner\\\\..*"}) or on() vector(0)',
        "threshold": 0.0,
        "description": "Systemd runner WatchdogSec 是否啟用",
    },
    {
        "name": "pod_restart_rate",
        # kube-state-metrics: namespace=awoooi-prod，sum 聚合避免 multi-vector
        "promql": 'sum(increase(kube_pod_container_status_restarts_total{namespace="awoooi-prod"}[15m]))',
        "threshold": 2.0,    # 15 分鐘內 > 2 次重啟
        "description": "Pod 重啟次數（15分鐘窗口）",
    },
    {
        "name": "db_connection_pool",
        # datname 實際值為 awoooi_prod；sum 聚合所有 state
        # 實測確認：curl 查詢返回有效資料，datname=awoooi_prod 存在
        "promql": 'sum(pg_stat_activity_count{datname="awoooi_prod"})',
        "threshold": 80.0,   # > 80 個 DB 連線
        "description": "PostgreSQL 連線數（awoooi_prod）",
    },
]


# ─────────────────────────────────────────────────────────────────────────────
# Data Types
# ─────────────────────────────────────────────────────────────────────────────

@dataclass
class ProactiveAlert:
    """主動巡檢偵測到的預警事件"""
    alert_type: str          # "dynamic_anomaly" / "log_pattern" / "trend_breach"
    metric_name: str
    severity: str            # "warning" / "critical"
    description: str
    current_value: float = 0.0
    threshold: float = 0.0
    deviation_sigma: float = 0.0
    template: str = ""       # log pattern（log_pattern 類型）
    predicted_breach_hours: float | None = None
    shadow_mode: bool = True
    detected_at: str = ""
    source: str = "proactive_inspector"


@dataclass
class InspectionReport:
    """一次巡檢週期的完整報告"""
    started_at: str
    finished_at: str
    alerts: list[ProactiveAlert] = field(default_factory=list)
    baseline_anomalies: int = 0
    log_patterns_new: int = 0
    trend_breaches: int = 0
    errors: list[str] = field(default_factory=list)
    shadow_mode: bool = True


# ─────────────────────────────────────────────────────────────────────────────
# Main Service
# ─────────────────────────────────────────────────────────────────────────────

class ProactiveInspector:
    """
    主動巡檢協調器

    每 5 分鐘執行一輪：
    1. Prometheus metrics → DynamicBaselineService 異常偵測
    2. K8s Pod logs → LogAnomalyDetector 新 pattern
    3. 趨勢資料點 → TrendPredictor 4h 預警
    4. 背景觸發 Holt-Winters 重訓（每次巡檢）
    """

    def __init__(self) -> None:
        # Phase 4 ADR-084: 快取最後一次巡檢報告，供 PreDecisionInvestigator 8D 感官使用
        # 2026-04-15 ogt + Claude Sonnet 4.6（亞太）: Phase 4 8D 升級
        self._last_report: InspectionReport | None = None

    def get_last_report(self) -> InspectionReport | None:
        """取得最近一次巡檢報告（PreDecisionInvestigator 8D 感官用）。"""
        return self._last_report

    async def run_inspection(self) -> InspectionReport:
        """
        執行一次完整巡檢。

        Returns:
            InspectionReport（Shadow Mode 時只記錄，不觸發 Alert）
        """
        from src.core.feature_flags import aiops_flags

        if not aiops_flags.AIOPS_P4_PROACTIVE_INSPECTOR:
            return InspectionReport(
                started_at=now_taipei().isoformat(),
                finished_at=now_taipei().isoformat(),
            )

        shadow_mode = aiops_flags.AIOPS_P4_SHADOW_MODE
        started_at = now_taipei().isoformat()
        report = InspectionReport(
            started_at=started_at,
            finished_at="",
            shadow_mode=shadow_mode,
        )

        logger.info("proactive_inspection_started", shadow_mode=shadow_mode)

        # 三大感官並行執行（熔斷隔離）
        tasks = [
            self._inspect_dynamic_baseline(report),
            self._inspect_log_patterns(report),
        ]
        results = await asyncio.gather(*tasks, return_exceptions=True)
        for r in results:
            if isinstance(r, Exception):
                report.errors.append(str(r))

        # 趨勢預測（依賴 baseline 結果，稍後執行）
        try:
            await self._inspect_trends(report)
        except Exception as e:
            report.errors.append(f"trend_inspect_error: {e}")

        # 背景觸發基線重訓（不等待結果）
        asyncio.create_task(self._retrain_baselines_background())

        report.finished_at = now_taipei().isoformat()
        self._last_report = report  # 快取供 PreDecisionInvestigator 8D 感官讀取

        logger.info(
            "proactive_inspection_finished",
            baseline_anomalies=report.baseline_anomalies,
            log_patterns_new=report.log_patterns_new,
            trend_breaches=report.trend_breaches,
            errors=len(report.errors),
            shadow_mode=shadow_mode,
        )

        return report

    # ──────────────────────────────────────────────────────────────────────────
    # 感官 1: Dynamic Baseline 異常偵測
    # ──────────────────────────────────────────────────────────────────────────

    async def _inspect_dynamic_baseline(self, report: InspectionReport) -> None:
        """從 Prometheus 抓取當前值，與 Holt-Winters 基線比對。"""
        from src.core.feature_flags import aiops_flags
        from src.services.dynamic_baseline_service import get_dynamic_baseline_service

        if not aiops_flags.AIOPS_P4_DYNAMIC_BASELINE:
            return

        svc = get_dynamic_baseline_service()

        for metric_cfg in MONITORED_METRICS:
            metric_name = metric_cfg["name"]
            try:
                current = await self._fetch_current_value(metric_cfg["promql"])
                if current is None:
                    continue

                hour_of_day = now_taipei().hour
                result = await svc.is_anomaly(metric_name, current, hour_of_day=hour_of_day)

                if result.is_anomaly:
                    dedup_key = f"dynamic:{metric_name}"
                    if await self._is_dedup(dedup_key):
                        continue

                    severity = "critical" if result.deviation_sigma >= 5.0 else "warning"
                    alert = ProactiveAlert(
                        alert_type="dynamic_anomaly",
                        metric_name=metric_name,
                        severity=severity,
                        description=(
                            f"{metric_cfg['description']} 偏離基線 {result.deviation_sigma:.1f}σ "
                            f"（當前 {current:.4f}，期望 {result.expected_mean:.4f}）"
                        ),
                        current_value=current,
                        threshold=result.expected_mean + 3 * result.expected_std,
                        deviation_sigma=result.deviation_sigma,
                        shadow_mode=report.shadow_mode,
                        detected_at=now_taipei().isoformat(),
                    )
                    report.alerts.append(alert)
                    report.baseline_anomalies += 1
                    await self._mark_dedup(dedup_key)

            except Exception as e:
                logger.warning("baseline_inspect_metric_failed", metric=metric_name, error=str(e))

    # ──────────────────────────────────────────────────────────────────────────
    # 感官 2: Log Anomaly 新 Pattern
    # ──────────────────────────────────────────────────────────────────────────

    async def _inspect_log_patterns(self, report: InspectionReport) -> None:
        """掃描 K8s Pod 日誌，偵測新 log pattern。"""
        from src.core.feature_flags import aiops_flags
        from src.services.log_anomaly_detector import get_log_anomaly_detector

        if not aiops_flags.AIOPS_P4_LOG_ANOMALY:
            return

        detector = get_log_anomaly_detector()
        try:
            events = await detector.process_pod_logs(
                namespace=K8S_NAMESPACE,
                tail_lines=200,
            )
            for event in events:
                dedup_key = f"log:{event.cluster_id}"
                if await self._is_dedup(dedup_key):
                    continue

                alert = ProactiveAlert(
                    alert_type="log_pattern",
                    metric_name="log_anomaly",
                    severity="warning",
                    description=f"偵測到新 log pattern：{event.template[:200]}",
                    template=event.template,
                    shadow_mode=report.shadow_mode,
                    detected_at=event.detected_at,
                    source=f"k8s/{K8S_NAMESPACE}",
                )
                report.alerts.append(alert)
                report.log_patterns_new += 1
                await self._mark_dedup(dedup_key)

        except Exception as e:
            logger.warning("log_inspect_failed", error=str(e))

    # ──────────────────────────────────────────────────────────────────────────
    # 感官 3: Trend Predictor 4h 預警
    # ──────────────────────────────────────────────────────────────────────────

    async def _inspect_trends(self, report: InspectionReport) -> None:
        """對各 metric 做 4h 趨勢外推。"""
        from src.core.feature_flags import aiops_flags
        from src.services.trend_predictor import get_trend_predictor

        if not aiops_flags.AIOPS_P4_TREND_PREDICTOR:
            return

        predictor = get_trend_predictor()

        for metric_cfg in MONITORED_METRICS:
            metric_name = metric_cfg["name"]
            threshold = metric_cfg["threshold"]
            try:
                current = await self._fetch_current_value(metric_cfg["promql"])
                if current is None:
                    continue

                pred = await predictor.predict_breach(metric_name, current, threshold)
                pred.metric_name = metric_name  # TrendPredictor 內部未填 metric_name

                if pred.will_breach and pred.confidence in ("high", "medium"):
                    dedup_key = f"trend:{metric_name}"
                    if await self._is_dedup(dedup_key):
                        continue

                    hours_str = f"{pred.breach_in_hours:.1f}h" if pred.breach_in_hours is not None else "已超越"
                    alert = ProactiveAlert(
                        alert_type="trend_breach",
                        metric_name=metric_name,
                        severity="warning",
                        description=(
                            f"{metric_cfg['description']} 趨勢預警：預計 {hours_str} 後超越閾值 "
                            f"（當前 {current:.4f}，閾值 {threshold}，斜率 {pred.slope_per_hour:+.6f}/h，R²={pred.r_squared:.2f}）"
                        ),
                        current_value=current,
                        threshold=threshold,
                        predicted_breach_hours=pred.breach_in_hours,
                        shadow_mode=report.shadow_mode,
                        detected_at=now_taipei().isoformat(),
                    )
                    report.alerts.append(alert)
                    report.trend_breaches += 1
                    await self._mark_dedup(dedup_key)

            except Exception as e:
                logger.warning("trend_inspect_metric_failed", metric=metric_name, error=str(e))

    # ──────────────────────────────────────────────────────────────────────────
    # 背景重訓
    # ──────────────────────────────────────────────────────────────────────────

    async def _retrain_baselines_background(self) -> None:
        """背景重訓所有 Holt-Winters 基線（不阻塞巡檢）。"""
        from src.core.feature_flags import aiops_flags
        from src.services.dynamic_baseline_service import get_dynamic_baseline_service

        if not aiops_flags.AIOPS_P4_DYNAMIC_BASELINE:
            return

        svc = get_dynamic_baseline_service()
        for metric_cfg in MONITORED_METRICS:
            try:
                await svc.train_baseline(
                    metric_name=metric_cfg["name"],
                    promql=metric_cfg["promql"],
                )
            except Exception as e:
                logger.warning(
                    "baseline_retrain_background_failed",
                    metric=metric_cfg["name"],
                    error=str(e),
                )

    # ──────────────────────────────────────────────────────────────────────────
    # 工具方法
    # ──────────────────────────────────────────────────────────────────────────

    async def _fetch_current_value(self, promql: str) -> float | None:
        """從 Prometheus 抓取當前值（instant query）。"""
        import httpx

        from src.core.config import settings

        try:
            async with httpx.AsyncClient(timeout=10.0) as client:
                resp = await client.get(
                    f"{settings.PROMETHEUS_URL}/api/v1/query",
                    params={"query": promql},
                )
                resp.raise_for_status()
                data = resp.json()

            results = data.get("data", {}).get("result", [])
            if not results:
                return None

            value_str = results[0].get("value", [None, None])[1]
            if value_str is None or value_str == "NaN":
                return None
            return float(value_str)

        except Exception as e:
            logger.warning("prometheus_instant_query_failed", promql=promql[:80], error=str(e))
            return None

    async def _is_dedup(self, key: str) -> bool:
        """檢查是否在去重窗口內（30 分鐘）。"""
        try:
            from src.core.redis_client import get_redis
            r = get_redis()
            return bool(await r.exists(f"{DEDUP_KEY_PREFIX}{key}"))
        except Exception:
            return False

    async def _mark_dedup(self, key: str) -> None:
        """標記去重，TTL = 30 分鐘。"""
        try:
            from src.core.redis_client import get_redis
            r = get_redis()
            await r.set(f"{DEDUP_KEY_PREFIX}{key}", "1", ex=DEDUP_TTL_SEC)
        except Exception:
            pass


# ─────────────────────────────────────────────────────────────────────────────
# Background Loop（供 main.py lifespan 呼叫）
# ─────────────────────────────────────────────────────────────────────────────

async def run_proactive_inspector_loop() -> None:
    """
    永久迴圈：每 INSPECTOR_INTERVAL_SEC 秒執行一次巡檢。

    由 main.py lifespan 透過 asyncio.create_task() 啟動。
    Loop 內部有熔斷：任一輪失敗不會終止整個迴圈。
    """
    inspector = get_proactive_inspector()
    logger.info("proactive_inspector_loop_started", interval_sec=INSPECTOR_INTERVAL_SEC)

    while True:
        try:
            await inspector.run_inspection()
        except Exception as e:
            logger.warning("proactive_inspector_loop_error", error=str(e))

        await asyncio.sleep(INSPECTOR_INTERVAL_SEC)


# ─────────────────────────────────────────────────────────────────────────────
# Singleton
# ─────────────────────────────────────────────────────────────────────────────

_inspector: ProactiveInspector | None = None


def get_proactive_inspector() -> ProactiveInspector:
    global _inspector
    if _inspector is None:
        _inspector = ProactiveInspector()
    return _inspector