476 lines
20 KiB
Python
476 lines
20 KiB
Python
"""
|
||
Flywheel Stats Service — ADR-074 M1 + ADR-073-C C1
|
||
|
||
飛輪健康度指標計算服務：
|
||
- 供 Prometheus Exporter（M1）抓取
|
||
- 供前端 /api/v1/stats/flywheel 即時顯示（C1）
|
||
|
||
Metrics:
|
||
awoooi_flywheel_playbook_count 目標 ≥ 20
|
||
awoooi_flywheel_execution_success_rate 目標 ≥ 0.3
|
||
awoooi_flywheel_km_unvectorized_count 目標 = 0
|
||
awoooi_flywheel_alertname_null_rate 目標 = 0
|
||
awoooi_flywheel_incidents_stuck 目標 = 0
|
||
|
||
2026-04-12 ogt (ADR-074 M1 + ADR-073-C C1)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from datetime import datetime, timedelta
|
||
from typing import Any
|
||
|
||
import structlog
|
||
from sqlalchemy import func, select, text
|
||
|
||
from src.core.redis_client import get_redis
|
||
from src.db.base import get_db_context
|
||
from src.db.models import IncidentRecord, KnowledgeEntryRecord
|
||
from src.models.incident import IncidentStatus
|
||
from src.utils.timezone import now_taipei
|
||
|
||
logger = structlog.get_logger(__name__)
|
||
|
||
# Redis key prefix（與 playbook_repository.py 一致）
|
||
_PLAYBOOK_KEY_PREFIX = "playbook:"
|
||
|
||
# 2026-05-02 ogt + Claude Sonnet 4.6 — Bug 2 修復（W-3 fresh deploy 假告警）
|
||
# execution_success_rate 需要最少樣本數才有統計意義；
|
||
# Redis 空（fresh deploy / restart）時 total_exec=0 → rate=0.0 → watchdog W-3 立即觸發假告警
|
||
# 修法：total_exec < FLYWHEEL_MIN_SAMPLE 時回 None，watchdog 判 None 跳過 W-3 檢查
|
||
# TODO: 未來移至 settings（目前 hardcode 以避免 config 改動超出本輪範圍）
|
||
FLYWHEEL_MIN_SAMPLE = 10
|
||
|
||
# 飛輪六節點名稱
|
||
FLYWHEEL_NODES = [
|
||
"monitoring",
|
||
"deduplication",
|
||
"diagnosis",
|
||
"reasoning",
|
||
"execution",
|
||
"learning",
|
||
]
|
||
|
||
|
||
# =============================================================================
|
||
# 核心指標資料結構
|
||
# =============================================================================
|
||
|
||
|
||
class FlywheelMetrics:
|
||
"""飛輪健康度指標快照"""
|
||
|
||
def __init__(
|
||
self,
|
||
playbook_count: int,
|
||
execution_success_rate: float | None,
|
||
km_unvectorized_count: int,
|
||
alertname_null_rate: float,
|
||
incidents_stuck: int,
|
||
today_processed: int,
|
||
flywheel_conversions_today: int,
|
||
km_vectorized_rate: float,
|
||
node_stats: dict[str, Any],
|
||
current_flow: list[dict[str, Any]],
|
||
computed_at: datetime,
|
||
) -> None:
|
||
# 2026-05-02 ogt + Claude Sonnet 4.6 — Bug 2 修復
|
||
# execution_success_rate 為 None 時表示樣本不足（< FLYWHEEL_MIN_SAMPLE），
|
||
# watchdog W-3 應跳過該檢查，避免 fresh deploy 假告警
|
||
self.playbook_count = playbook_count
|
||
self.execution_success_rate = execution_success_rate
|
||
self.km_unvectorized_count = km_unvectorized_count
|
||
self.alertname_null_rate = alertname_null_rate
|
||
self.incidents_stuck = incidents_stuck
|
||
self.today_processed = today_processed
|
||
self.flywheel_conversions_today = flywheel_conversions_today
|
||
self.km_vectorized_rate = km_vectorized_rate
|
||
self.node_stats = node_stats
|
||
self.current_flow = current_flow
|
||
self.computed_at = computed_at
|
||
self.type4_count: int = 0 # TYPE-4 incidents 數（ADR-073-C C2）
|
||
|
||
def to_prometheus_lines(self) -> str:
|
||
"""輸出 Prometheus text format"""
|
||
ts = int(self.computed_at.timestamp() * 1000)
|
||
# 2026-05-02 ogt + Claude Opus 4.7 — Bug 2 後續修復（critic P0-1 連鎖修復）
|
||
# sentinel 用 NaN 而非 -1.0：Prometheus 對 NaN 比較永遠回 false，
|
||
# 既有 alert rule `awoooi_flywheel_execution_success_rate < 0.1` 自然不會被
|
||
# sentinel 觸發；同時 Grafana 渲染為「無資料」gap，比 -1 spike 直觀。
|
||
# 前次嘗試 -1.0 會讓 ops/monitoring/alerts.yml:775 等 3 份 prom rule
|
||
# 在 fresh deploy 後 2h 必噴 FlywheelExecutionSuccessLow 假告警，跟 watchdog skip 自相矛盾。
|
||
rate_str = (
|
||
f"{self.execution_success_rate:.4f}"
|
||
if self.execution_success_rate is not None
|
||
else "NaN"
|
||
)
|
||
lines = [
|
||
"# HELP awoooi_flywheel_playbook_count Total approved playbooks in Redis",
|
||
"# TYPE awoooi_flywheel_playbook_count gauge",
|
||
f"awoooi_flywheel_playbook_count {self.playbook_count} {ts}",
|
||
"",
|
||
"# HELP awoooi_flywheel_execution_success_rate Auto-repair success rate (0-1), NaN=insufficient sample",
|
||
"# TYPE awoooi_flywheel_execution_success_rate gauge",
|
||
f"awoooi_flywheel_execution_success_rate {rate_str} {ts}",
|
||
"",
|
||
"# HELP awoooi_flywheel_km_unvectorized_count KM entries not yet vectorized",
|
||
"# TYPE awoooi_flywheel_km_unvectorized_count gauge",
|
||
f"awoooi_flywheel_km_unvectorized_count {self.km_unvectorized_count} {ts}",
|
||
"",
|
||
"# HELP awoooi_flywheel_alertname_null_rate Fraction of incidents with null alertname",
|
||
"# TYPE awoooi_flywheel_alertname_null_rate gauge",
|
||
f"awoooi_flywheel_alertname_null_rate {self.alertname_null_rate:.4f} {ts}",
|
||
"",
|
||
"# HELP awoooi_flywheel_incidents_stuck Incidents stuck in INVESTIGATING > 24h",
|
||
"# TYPE awoooi_flywheel_incidents_stuck gauge",
|
||
f"awoooi_flywheel_incidents_stuck {self.incidents_stuck} {ts}",
|
||
"",
|
||
"# HELP awoooi_flywheel_km_vectorized_rate Fraction of KM entries vectorized",
|
||
"# TYPE awoooi_flywheel_km_vectorized_rate gauge",
|
||
f"awoooi_flywheel_km_vectorized_rate {self.km_vectorized_rate:.4f} {ts}",
|
||
]
|
||
return "\n".join(lines) + "\n"
|
||
|
||
def to_flywheel_api_dict(self) -> dict[str, Any]:
|
||
"""輸出 /api/v1/stats/flywheel 格式"""
|
||
return {
|
||
"nodes": self.node_stats,
|
||
"current_flow": self.current_flow,
|
||
"type4_count": self.type4_count,
|
||
"computed_at": self.computed_at.isoformat(),
|
||
}
|
||
|
||
def to_summary_api_dict(self) -> dict[str, Any]:
|
||
"""輸出 /api/v1/stats/summary 格式"""
|
||
return {
|
||
"playbook_count": self.playbook_count,
|
||
"execution_success_rate": round(self.execution_success_rate, 4) if self.execution_success_rate is not None else None,
|
||
"today_processed": self.today_processed,
|
||
"flywheel_conversions_today": self.flywheel_conversions_today,
|
||
"km_vectorized_rate": round(self.km_vectorized_rate, 4),
|
||
"km_unvectorized_count": self.km_unvectorized_count,
|
||
"alertname_null_rate": round(self.alertname_null_rate, 4),
|
||
"incidents_stuck": self.incidents_stuck,
|
||
"computed_at": self.computed_at.isoformat(),
|
||
}
|
||
|
||
|
||
# =============================================================================
|
||
# FlywheelStatsService
|
||
# =============================================================================
|
||
|
||
|
||
class FlywheelStatsService:
|
||
"""
|
||
飛輪健康度指標計算服務
|
||
|
||
ADR-074 M1: 供 Prometheus /metrics/flywheel 抓取
|
||
ADR-073-C C1: 供前端 /api/v1/stats/flywheel 顯示
|
||
"""
|
||
|
||
async def compute(self) -> FlywheelMetrics:
|
||
"""計算所有飛輪指標（單次完整查詢）"""
|
||
now = now_taipei()
|
||
|
||
playbook_count, execution_success_rate = await self._playbook_stats()
|
||
(
|
||
km_unvectorized_count,
|
||
km_vectorized_rate,
|
||
flywheel_conversions_today,
|
||
) = await self._km_stats(now)
|
||
(
|
||
alertname_null_rate,
|
||
incidents_stuck,
|
||
today_processed,
|
||
node_stats,
|
||
current_flow,
|
||
type4_count,
|
||
) = await self._incident_stats(now)
|
||
|
||
metrics = FlywheelMetrics(
|
||
playbook_count=playbook_count,
|
||
execution_success_rate=execution_success_rate,
|
||
km_unvectorized_count=km_unvectorized_count,
|
||
alertname_null_rate=alertname_null_rate,
|
||
incidents_stuck=incidents_stuck,
|
||
today_processed=today_processed,
|
||
flywheel_conversions_today=flywheel_conversions_today,
|
||
km_vectorized_rate=km_vectorized_rate,
|
||
node_stats=node_stats,
|
||
current_flow=current_flow,
|
||
computed_at=now,
|
||
)
|
||
metrics.type4_count = type4_count
|
||
return metrics
|
||
|
||
# ------------------------------------------------------------------
|
||
# Internal helpers
|
||
# ------------------------------------------------------------------
|
||
|
||
async def _playbook_stats(self) -> tuple[int, float | None]:
|
||
"""Playbook 數量 + 執行成功率（從 Redis）
|
||
|
||
2026-05-02 ogt + Claude Sonnet 4.6 — Bug 2 修復（W-3 fresh deploy 假告警）
|
||
total_exec < FLYWHEEL_MIN_SAMPLE 時回 None，代表樣本不足，
|
||
watchdog W-3 判 None 跳過該檢查，避免每次 restart 觸發假告警。
|
||
"""
|
||
try:
|
||
redis = get_redis()
|
||
count = 0
|
||
total_exec = 0
|
||
total_success = 0
|
||
|
||
async for key in redis.scan_iter(match=f"{_PLAYBOOK_KEY_PREFIX}PB-*", count=200):
|
||
raw = await redis.get(key)
|
||
if not raw:
|
||
continue
|
||
try:
|
||
pb = json.loads(raw)
|
||
status = pb.get("status", "")
|
||
if status == "approved":
|
||
count += 1
|
||
success_count = pb.get("success_count", 0) or 0
|
||
failure_count = pb.get("failure_count", 0) or 0
|
||
total_exec += success_count + failure_count
|
||
total_success += success_count
|
||
except (json.JSONDecodeError, KeyError):
|
||
continue
|
||
|
||
# 2026-05-06 ogt + Codex:
|
||
# 執行成功率的 source of truth 是 auto_repair_executions。
|
||
# Redis playbook success_count/failure_count 會因回寫鏈路中斷而落後，
|
||
# 造成 governance / heartbeat 判定「飛輪沒有執行」。
|
||
# 2026-05-29 Codex:
|
||
# 24h 低流量不是資料管線斷流；若 24h 未達最小樣本，改用 7d
|
||
# 穩定窗口，避免 FlywheelExecutionRateMissing 長期誤報。
|
||
try:
|
||
async with get_db_context() as db:
|
||
row = await db.execute(
|
||
text("""
|
||
SELECT
|
||
COUNT(*) FILTER (WHERE success IS TRUE) AS success,
|
||
COUNT(*) AS total
|
||
FROM auto_repair_executions
|
||
WHERE created_at >= NOW() - interval '24 hours'
|
||
""")
|
||
)
|
||
repair_stats = row.one()
|
||
db_total_exec = int(repair_stats.total or 0)
|
||
if db_total_exec >= FLYWHEEL_MIN_SAMPLE:
|
||
db_total_success = int(repair_stats.success or 0)
|
||
return count, db_total_success / db_total_exec
|
||
|
||
fallback_row = await db.execute(
|
||
text("""
|
||
SELECT
|
||
COUNT(*) FILTER (WHERE success IS TRUE) AS success,
|
||
COUNT(*) AS total
|
||
FROM auto_repair_executions
|
||
WHERE created_at >= NOW() - interval '7 days'
|
||
""")
|
||
)
|
||
fallback_stats = fallback_row.one()
|
||
fallback_total = int(fallback_stats.total or 0)
|
||
if fallback_total >= FLYWHEEL_MIN_SAMPLE:
|
||
fallback_success = int(fallback_stats.success or 0)
|
||
return count, fallback_success / fallback_total
|
||
if db_total_exec > 0 or fallback_total > 0:
|
||
return count, None
|
||
except Exception:
|
||
logger.warning("flywheel_stats_auto_repair_execution_query_failed")
|
||
|
||
if total_exec < FLYWHEEL_MIN_SAMPLE:
|
||
# 樣本不足（含 Redis 空），回 None 通知呼叫方跳過 W-3 告警判斷
|
||
return count, None
|
||
rate = total_success / total_exec
|
||
return count, rate
|
||
|
||
except Exception:
|
||
logger.exception("flywheel_stats_playbook_error")
|
||
return 0, None
|
||
|
||
async def _km_stats(self, now: datetime) -> tuple[int, float, int]:
|
||
"""KM 向量化率 + 今日飛輪轉化數（從 PostgreSQL）"""
|
||
try:
|
||
today_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
||
|
||
async with get_db_context() as db:
|
||
# 未向量化數量 (embedding IS NULL = 未向量化)
|
||
# 2026-04-15 ogt: KnowledgeEntryRecord ORM 不宣告 embedding 欄位（pgvector），
|
||
# 改用 raw SQL 避免 AttributeError
|
||
unvectorized_q = await db.execute(
|
||
text("SELECT COUNT(*) FROM knowledge_entries WHERE embedding IS NULL")
|
||
)
|
||
unvectorized = unvectorized_q.scalar_one_or_none() or 0
|
||
|
||
# 總數
|
||
total_q = await db.execute(select(func.count(KnowledgeEntryRecord.id)))
|
||
total = total_q.scalar_one_or_none() or 0
|
||
|
||
vectorized_rate = (total - unvectorized) / total if total > 0 else 0.0
|
||
|
||
# 今日轉化數（今日建立的 KM）
|
||
conversions_q = await db.execute(
|
||
select(func.count()).where(
|
||
KnowledgeEntryRecord.created_at >= today_start
|
||
)
|
||
)
|
||
conversions_today = conversions_q.scalar_one_or_none() or 0
|
||
|
||
return unvectorized, vectorized_rate, conversions_today
|
||
|
||
except Exception:
|
||
logger.exception("flywheel_stats_km_error")
|
||
return 0, 0.0, 0
|
||
|
||
async def _incident_stats(
|
||
self, now: datetime
|
||
) -> tuple[float, int, int, dict[str, Any], list[dict[str, Any]]]:
|
||
"""Incident 相關指標（alertname NULL 率、卡住數、今日處理數、節點狀態、當前流）"""
|
||
try:
|
||
today_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
||
stuck_threshold = now - timedelta(hours=24)
|
||
recent_1h = now - timedelta(hours=1)
|
||
|
||
async with get_db_context() as db:
|
||
# alertname NULL 率
|
||
total_q = await db.execute(select(func.count(IncidentRecord.incident_id)))
|
||
total = total_q.scalar_one_or_none() or 0
|
||
|
||
null_q = await db.execute(
|
||
select(func.count()).where(IncidentRecord.alertname.is_(None))
|
||
)
|
||
null_count = null_q.scalar_one_or_none() or 0
|
||
alertname_null_rate = null_count / total if total > 0 else 0.0
|
||
|
||
# 卡住的 Incident（INVESTIGATING > 24h）
|
||
stuck_q = await db.execute(
|
||
select(func.count()).where(
|
||
IncidentRecord.status == IncidentStatus.INVESTIGATING,
|
||
IncidentRecord.created_at <= stuck_threshold,
|
||
)
|
||
)
|
||
incidents_stuck = stuck_q.scalar_one_or_none() or 0
|
||
|
||
# TYPE-4 Incident 數（ADR-073-C C2 — 供前端 hasType4 判斷）
|
||
# 2026-04-12 ogt
|
||
type4_q = await db.execute(
|
||
select(func.count()).where(
|
||
IncidentRecord.notification_type == "TYPE-4",
|
||
IncidentRecord.status == IncidentStatus.INVESTIGATING,
|
||
)
|
||
)
|
||
type4_count = type4_q.scalar_one_or_none() or 0
|
||
|
||
# 今日處理數
|
||
today_q = await db.execute(
|
||
select(func.count()).where(
|
||
IncidentRecord.created_at >= today_start
|
||
)
|
||
)
|
||
today_processed = today_q.scalar_one_or_none() or 0
|
||
|
||
# 節點狀態（監控/去重/執行）
|
||
recent_q = await db.execute(
|
||
select(func.count()).where(
|
||
IncidentRecord.created_at >= recent_1h
|
||
)
|
||
)
|
||
count_1h = recent_q.scalar_one_or_none() or 0
|
||
|
||
# 自動執行成功數（今日）
|
||
success_q = await db.execute(
|
||
text(
|
||
"SELECT COUNT(*) FROM incidents WHERE created_at >= :today"
|
||
" AND outcome::text LIKE '%execution_success%true%'"
|
||
),
|
||
{"today": today_start},
|
||
)
|
||
exec_success_today = success_q.scalar_one_or_none() or 0
|
||
|
||
# 當前流（最近 10 筆活躍 Incident）
|
||
active_q = await db.execute(
|
||
select(
|
||
IncidentRecord.incident_id,
|
||
IncidentRecord.alertname,
|
||
IncidentRecord.status,
|
||
IncidentRecord.created_at,
|
||
)
|
||
.where(
|
||
IncidentRecord.status.in_([
|
||
IncidentStatus.INVESTIGATING.value,
|
||
IncidentStatus.MITIGATING.value,
|
||
])
|
||
)
|
||
.order_by(IncidentRecord.created_at.desc())
|
||
.limit(10)
|
||
)
|
||
active_rows = active_q.fetchall()
|
||
|
||
current_flow = [
|
||
{
|
||
"incident_id": row.incident_id,
|
||
"alertname": row.alertname or "unknown",
|
||
"current_node": _status_to_node(row.status),
|
||
"ts": row.created_at.isoformat() if row.created_at else None,
|
||
}
|
||
for row in active_rows
|
||
]
|
||
|
||
node_stats = {
|
||
"monitoring": {
|
||
"status": "active" if count_1h > 0 else "idle",
|
||
"count_1h": count_1h,
|
||
},
|
||
"deduplication": {
|
||
"status": "active",
|
||
"dedup_window_min": 30,
|
||
},
|
||
"diagnosis": {
|
||
"status": "active",
|
||
"mcp_providers_used": ["k8s", "ssh", "prometheus"],
|
||
},
|
||
"reasoning": {
|
||
"status": "active",
|
||
"today_processed": today_processed,
|
||
},
|
||
"execution": {
|
||
"status": "active",
|
||
"success_today": exec_success_today,
|
||
},
|
||
"learning": {
|
||
"status": "active",
|
||
},
|
||
}
|
||
|
||
return alertname_null_rate, incidents_stuck, today_processed, node_stats, current_flow, type4_count
|
||
|
||
except Exception:
|
||
logger.exception("flywheel_stats_incident_error")
|
||
return 0.0, 0, 0, {n: {"status": "unknown"} for n in FLYWHEEL_NODES}, [], 0
|
||
|
||
|
||
def _status_to_node(status: str) -> str:
|
||
mapping = {
|
||
IncidentStatus.INVESTIGATING.value: "diagnosis",
|
||
IncidentStatus.MITIGATING.value: "execution",
|
||
IncidentStatus.RESOLVED.value: "learning",
|
||
IncidentStatus.CLOSED.value: "learning",
|
||
}
|
||
return mapping.get(status, "reasoning")
|
||
|
||
|
||
# =============================================================================
|
||
# DI 工廠
|
||
# =============================================================================
|
||
|
||
_instance: FlywheelStatsService | None = None
|
||
|
||
|
||
def get_flywheel_stats_service() -> FlywheelStatsService:
|
||
global _instance
|
||
if _instance is None:
|
||
_instance = FlywheelStatsService()
|
||
return _instance
|