4153 lines
174 KiB
Python
4153 lines
174 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
routes/admin_observability_routes.py
|
||
Operation Ollama-First v5.0 / Phase 27 — Admin Observability Dashboard
|
||
|
||
提供 admin 介面看戰役累積的觀測資料：
|
||
/observability/ai_calls — ai_calls 即時查詢（含篩選 / 圖表）
|
||
/observability/promotion_review — Phase 28 PromotionGate 待審核列表
|
||
/observability/quality_trend — Phase 25 caller 反饋趨勢
|
||
/observability/host_health — 三主機 Ollama + MCP 健康度
|
||
|
||
設計原則：
|
||
- 純讀（除了 promotion approve/reject 是 mutation）
|
||
- 失敗安全：DB 失敗回空清單 + 警告 banner
|
||
- 每頁 100 筆分頁，無限捲動
|
||
- 不暴露 secret / prompt 原文
|
||
"""
|
||
|
||
import logging
|
||
import re
|
||
import threading
|
||
import time
|
||
from datetime import datetime, timedelta
|
||
from flask import Blueprint, render_template, request, jsonify, send_file, url_for
|
||
from sqlalchemy import text as sa_text
|
||
|
||
from auth import login_required, get_current_user
|
||
from database.manager import get_session
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
admin_observability_bp = Blueprint(
|
||
'admin_observability',
|
||
__name__,
|
||
url_prefix='/observability',
|
||
)
|
||
|
||
_PPT_AIDER_HEAL_LOCK = threading.Lock()
|
||
_PPT_AIDER_HEAL_ACTIVE = {}
|
||
_HEALTH_INDICATOR_CACHE_LOCK = threading.Lock()
|
||
_HEALTH_INDICATOR_CACHE = {
|
||
'expires_at': 0.0,
|
||
'payload': None,
|
||
}
|
||
_HEALTH_INDICATOR_CACHE_TTL_SECONDS = 30
|
||
_PPT_PUBLIC_RUNTIME_ERROR = '視覺審核暫時無法完成；請先用線上預覽人工確認版面，稍後重新執行審核。'
|
||
_PPT_INTERNAL_ERROR_MARKERS = (
|
||
'all 3 hosts failed',
|
||
'httpconnectionpool',
|
||
'multimodal data provided',
|
||
'model does not support',
|
||
'/api/generate',
|
||
'connectionerror',
|
||
'readtimeout',
|
||
'traceback',
|
||
'gcp-a',
|
||
'gcp-b',
|
||
)
|
||
_PPT_PUBLIC_REPLACEMENTS = (
|
||
('AiderHeal', '修復流程'),
|
||
('RAG', '知識建議'),
|
||
('Ollama', 'AI 建議服務'),
|
||
('minicpm-v', '視覺模型'),
|
||
('LibreOffice', '轉檔服務'),
|
||
('runtime', '執行條件'),
|
||
('DB', '產出紀錄'),
|
||
('database', '產出紀錄'),
|
||
('filesystem', '檔案來源'),
|
||
('ppt_audit_results', '審核紀錄'),
|
||
('ppt_generation_runs', '產出紀錄'),
|
||
)
|
||
|
||
|
||
_GEMINI_BACKUP_CALLER_DISPLAY = {
|
||
# 專用備援 caller 落地前的舊資料仍會存在；顯示時標成備援，避免誤判 Gemini-first。
|
||
'code_review_openclaw': 'code_review_openclaw_gemini',
|
||
'openclaw_qa': 'openclaw_qa_gemini_fallback',
|
||
}
|
||
_GEMINI_BACKUP_CALLERS = {
|
||
'code_review_openclaw_gemini',
|
||
'openclaw_daily_gemini_fallback',
|
||
'openclaw_daily_insight_gemini_fallback',
|
||
'openclaw_weekly_gemini_fallback',
|
||
'openclaw_monthly_gemini_fallback',
|
||
'openclaw_meta_gemini_fallback',
|
||
'openclaw_qa_gemini_fallback',
|
||
'openclaw_bot_gemini',
|
||
'openclaw_bot_image_gemini',
|
||
}
|
||
|
||
|
||
def _list_ppt_aider_heal_active_jobs():
|
||
with _PPT_AIDER_HEAL_LOCK:
|
||
jobs = [dict(job) for job in _PPT_AIDER_HEAL_ACTIVE.values()]
|
||
for job in jobs:
|
||
job['diagnosis'] = _public_ppt_text(job.get('diagnosis'), max_chars=100)
|
||
return jobs
|
||
|
||
|
||
def _ppt_text_has_internal_detail(value) -> bool:
|
||
text = str(value or '')
|
||
if not text:
|
||
return False
|
||
lowered = text.lower()
|
||
if any(marker in lowered for marker in _PPT_INTERNAL_ERROR_MARKERS):
|
||
return True
|
||
return bool(re.search(r'\b(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?\b', text))
|
||
|
||
|
||
def _public_ppt_text(value, *, empty='', max_chars=180):
|
||
"""把 PPT 觀測台的內部錯誤轉成操作員可讀的處置文字。"""
|
||
text = str(value or '').strip()
|
||
if not text:
|
||
return empty
|
||
if _ppt_text_has_internal_detail(text):
|
||
return _PPT_PUBLIC_RUNTIME_ERROR
|
||
for raw, label in _PPT_PUBLIC_REPLACEMENTS:
|
||
text = text.replace(raw, label)
|
||
text = re.sub(r'\b(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?\b', '內部主機', text)
|
||
text = re.sub(r'\s+', ' ', text).strip()
|
||
if max_chars and len(text) > max_chars:
|
||
return text[:max_chars].rstrip() + '…'
|
||
return text
|
||
|
||
|
||
def _public_ppt_text_list(values, *, max_chars=120):
|
||
public_values = []
|
||
for value in values or []:
|
||
text = _public_ppt_text(value, max_chars=max_chars)
|
||
if text and text not in public_values:
|
||
public_values.append(text)
|
||
return public_values
|
||
|
||
|
||
def _public_ppt_source_label(source):
|
||
return {
|
||
'both': '檔案 + 產出紀錄',
|
||
'database': '產出紀錄',
|
||
'filesystem': '檔案來源',
|
||
}.get(str(source or '').strip(), '檔案來源')
|
||
|
||
|
||
def _public_ppt_vision_status(status):
|
||
status = dict(status or {})
|
||
status['summary'] = _public_ppt_text(
|
||
status.get('summary'),
|
||
empty='視覺檢查狀態待確認。',
|
||
max_chars=140,
|
||
)
|
||
status['status_label'] = _public_ppt_text(
|
||
status.get('status_label'),
|
||
empty='待確認',
|
||
max_chars=40,
|
||
)
|
||
status['model_label'] = '視覺模型' if status.get('model') else '未啟用'
|
||
status['converter_label'] = '轉檔服務' if status.get('converter') else '轉檔條件待確認'
|
||
status['blockers'] = _public_ppt_text_list(
|
||
status.get('blockers'),
|
||
max_chars=80,
|
||
) or ['視覺檢查條件待確認']
|
||
status['next_actions'] = _public_ppt_text_list(
|
||
status.get('next_actions'),
|
||
max_chars=90,
|
||
) or ['確認視覺檢查條件後重新整理此頁。']
|
||
sanitized_checks = []
|
||
for check in status.get('readiness_checks') or []:
|
||
if not isinstance(check, dict):
|
||
continue
|
||
item = dict(check)
|
||
item['label'] = _public_ppt_text(item.get('label'), empty='檢查項目', max_chars=50)
|
||
item['value'] = _public_ppt_text(item.get('value'), empty='待確認', max_chars=50)
|
||
item['detail'] = _public_ppt_text(item.get('detail'), empty='等待檢查結果', max_chars=90)
|
||
sanitized_checks.append(item)
|
||
status['readiness_checks'] = sanitized_checks
|
||
return status
|
||
|
||
|
||
def _public_ppt_vision_audit_status(status):
|
||
status = dict(status or {})
|
||
status['status_label'] = _public_ppt_text(status.get('status_label'), empty='待確認', max_chars=60)
|
||
status['message'] = _public_ppt_text(
|
||
status.get('message'),
|
||
empty='視覺檢查狀態待確認。',
|
||
max_chars=140,
|
||
)
|
||
return status
|
||
|
||
|
||
def _build_ai_call_recent_row(row):
|
||
"""整理最近 ai_calls 列表，讓 Ollama-first 備援語意可被辨識。"""
|
||
caller = row[2] or ''
|
||
provider = row[3] or ''
|
||
caller_display = caller
|
||
route_badges = []
|
||
|
||
if provider == 'gemini':
|
||
if caller in _GEMINI_BACKUP_CALLER_DISPLAY:
|
||
caller_display = _GEMINI_BACKUP_CALLER_DISPLAY[caller]
|
||
route_badges.append('雲端備援')
|
||
route_badges.append('舊 caller')
|
||
elif caller in _GEMINI_BACKUP_CALLERS or caller.endswith('_gemini'):
|
||
route_badges.append('雲端備援')
|
||
else:
|
||
route_badges.append('ADR-028 鎖定/升級')
|
||
|
||
return {
|
||
'id': row[0],
|
||
'called_at': row[1].strftime('%H:%M:%S'),
|
||
'caller': caller,
|
||
'caller_display': caller_display,
|
||
'provider': provider,
|
||
'model': row[4],
|
||
'in_tokens': int(row[5] or 0),
|
||
'out_tokens': int(row[6] or 0),
|
||
'duration_ms': int(row[7] or 0),
|
||
'status': row[8],
|
||
'cost': float(row[9] or 0),
|
||
'cache_hit': bool(row[10]),
|
||
'rag_hit': bool(row[11]),
|
||
'route_badges': route_badges,
|
||
}
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# /observability/overview — Phase 45 總覽（單頁聚合 6 項 KPI）
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
@admin_observability_bp.route('/')
|
||
@admin_observability_bp.route('/overview')
|
||
@login_required
|
||
def observability_overview():
|
||
"""Phase 45 — 觀測台總覽：一頁式聚合 6 個 sub-page 的關鍵 KPI。
|
||
|
||
對應 Phase 44 daily Telegram summary 的 web 版本，做為 sidebar 入口頁。
|
||
所有區塊失敗安全：個別 query 失敗只跳過該卡片，不擋整頁渲染。
|
||
"""
|
||
from datetime import datetime as _dt
|
||
today = _dt.now()
|
||
month_start = _dt(today.year, today.month, 1)
|
||
summary = {}
|
||
|
||
session = get_session()
|
||
try:
|
||
# 三主機 24h 在線率
|
||
try:
|
||
host_rows = session.execute(
|
||
sa_text("""
|
||
SELECT host_label, COUNT(*) AS total,
|
||
COUNT(*) FILTER (WHERE healthy) AS up,
|
||
COALESCE(AVG(response_ms) FILTER (WHERE healthy), 0) AS avg_ms
|
||
FROM host_health_probes
|
||
WHERE probed_at >= NOW() - INTERVAL '24 hours'
|
||
GROUP BY host_label ORDER BY host_label
|
||
"""),
|
||
).fetchall()
|
||
summary['hosts'] = [
|
||
{
|
||
'label': r[0],
|
||
'total': int(r[1] or 0),
|
||
'up': int(r[2] or 0),
|
||
'avg_ms': int(r[3] or 0),
|
||
'uptime_pct': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0,
|
||
}
|
||
for r in host_rows
|
||
]
|
||
except Exception:
|
||
summary['hosts'] = []
|
||
|
||
# AI 呼叫 24h
|
||
try:
|
||
ai = session.execute(
|
||
sa_text("""
|
||
SELECT COUNT(*), COALESCE(SUM(input_tokens + output_tokens), 0),
|
||
COALESCE(SUM(cost_usd), 0),
|
||
COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')),
|
||
COUNT(*) FILTER (WHERE rag_hit),
|
||
COUNT(*) FILTER (WHERE cache_hit)
|
||
FROM ai_calls
|
||
WHERE called_at >= NOW() - INTERVAL '24 hours'
|
||
"""),
|
||
).fetchone()
|
||
total = int(ai[0] or 0)
|
||
summary['ai_calls'] = {
|
||
'total': total,
|
||
'tokens': int(ai[1] or 0),
|
||
'cost_24h': float(ai[2] or 0),
|
||
'errors': int(ai[3] or 0),
|
||
'rag_hits': int(ai[4] or 0),
|
||
'cache_hits': int(ai[5] or 0),
|
||
'error_rate': (float(ai[3] or 0) / total * 100) if total else 0,
|
||
'rag_rate': (float(ai[4] or 0) / total * 100) if total else 0,
|
||
'cache_rate': (float(ai[5] or 0) / total * 100) if total else 0,
|
||
}
|
||
except Exception:
|
||
summary['ai_calls'] = {}
|
||
|
||
# 當月成本
|
||
try:
|
||
month_cost = session.execute(
|
||
sa_text("SELECT COALESCE(SUM(cost_usd), 0) FROM ai_calls WHERE called_at >= :ms"),
|
||
{'ms': month_start},
|
||
).fetchone()[0]
|
||
summary['month_cost'] = float(month_cost or 0)
|
||
except Exception:
|
||
summary['month_cost'] = 0
|
||
|
||
# 預算 over 80%
|
||
try:
|
||
budgets = session.execute(
|
||
sa_text("""
|
||
SELECT b.period, b.provider, b.budget_usd, b.alert_pct,
|
||
COALESCE((
|
||
SELECT SUM(cost_usd) FROM ai_calls
|
||
WHERE called_at >= :ms
|
||
AND (b.provider IS NULL OR provider = b.provider)
|
||
), 0) AS spent
|
||
FROM ai_call_budgets b
|
||
"""),
|
||
{'ms': month_start},
|
||
).fetchall()
|
||
over_threshold = []
|
||
for r in budgets:
|
||
budget = float(r[2] or 0)
|
||
spent = float(r[4] or 0)
|
||
ratio = spent / budget if budget > 0 else 0
|
||
threshold = float(r[3] or 80) / 100
|
||
if ratio >= threshold:
|
||
over_threshold.append({
|
||
'period': r[0], 'provider': r[1] or '(全部)',
|
||
'spent': spent, 'budget': budget, 'ratio': ratio,
|
||
})
|
||
summary['budget_alerts'] = over_threshold
|
||
except Exception:
|
||
summary['budget_alerts'] = []
|
||
|
||
# 待審 + 蒸餾池
|
||
try:
|
||
ep_pending = session.execute(
|
||
sa_text("SELECT COUNT(*) FROM learning_episodes WHERE promotion_status = 'awaiting_review' AND reviewed_at IS NULL"),
|
||
).fetchone()[0]
|
||
ep_total_30d = session.execute(
|
||
sa_text("SELECT COUNT(*) FROM learning_episodes WHERE created_at >= NOW() - INTERVAL '30 days'"),
|
||
).fetchone()[0]
|
||
ep_approved_30d = session.execute(
|
||
sa_text("SELECT COUNT(*) FROM learning_episodes WHERE created_at >= NOW() - INTERVAL '30 days' AND promotion_status = 'approved'"),
|
||
).fetchone()[0]
|
||
summary['episodes'] = {
|
||
'pending': int(ep_pending or 0),
|
||
'total_30d': int(ep_total_30d or 0),
|
||
'approved_30d': int(ep_approved_30d or 0),
|
||
'approval_rate': (float(ep_approved_30d or 0) / float(ep_total_30d) * 100) if ep_total_30d else 0,
|
||
}
|
||
except Exception:
|
||
summary['episodes'] = {}
|
||
|
||
# PPT 視覺審核 7d
|
||
try:
|
||
ppt = session.execute(
|
||
sa_text("""
|
||
SELECT COUNT(*),
|
||
COUNT(*) FILTER (WHERE audit_status='passed'),
|
||
COUNT(*) FILTER (WHERE audit_status='failed')
|
||
FROM ppt_audit_results
|
||
WHERE audited_at >= NOW() - INTERVAL '7 days'
|
||
"""),
|
||
).fetchone()
|
||
ppt_total = int(ppt[0] or 0)
|
||
summary['ppt'] = {
|
||
'total': ppt_total,
|
||
'passed': int(ppt[1] or 0),
|
||
'failed': int(ppt[2] or 0),
|
||
'pass_rate': (float(ppt[1] or 0) / ppt_total * 100) if ppt_total else 0,
|
||
}
|
||
except Exception:
|
||
summary['ppt'] = {}
|
||
|
||
# AIOps 7d
|
||
try:
|
||
inc = session.execute(
|
||
sa_text("""
|
||
SELECT COUNT(*),
|
||
COUNT(*) FILTER (WHERE status='open'),
|
||
COUNT(*) FILTER (WHERE severity IN ('P0','P1'))
|
||
FROM incidents
|
||
WHERE created_at >= NOW() - INTERVAL '7 days'
|
||
"""),
|
||
).fetchone()
|
||
heal = session.execute(
|
||
sa_text("""
|
||
SELECT COUNT(*),
|
||
COUNT(*) FILTER (WHERE result='success')
|
||
FROM heal_logs
|
||
WHERE created_at >= NOW() - INTERVAL '7 days'
|
||
"""),
|
||
).fetchone()
|
||
summary['aiops'] = {
|
||
'incidents_total': int(inc[0] or 0),
|
||
'incidents_open': int(inc[1] or 0),
|
||
'incidents_p0_p1': int(inc[2] or 0),
|
||
'heals_total': int(heal[0] or 0),
|
||
'heals_success': int(heal[1] or 0),
|
||
'heal_rate': (float(heal[1] or 0) / float(heal[0]) * 100) if heal[0] else 0,
|
||
}
|
||
except Exception:
|
||
summary['aiops'] = {}
|
||
|
||
# MCP 24h
|
||
try:
|
||
mcp = session.execute(
|
||
sa_text("""
|
||
SELECT COUNT(*), COUNT(DISTINCT server),
|
||
COUNT(*) FILTER (WHERE cache_hit),
|
||
COALESCE(SUM(cost_usd), 0)
|
||
FROM mcp_calls
|
||
WHERE called_at >= NOW() - INTERVAL '24 hours'
|
||
"""),
|
||
).fetchone()
|
||
mcp_total = int(mcp[0] or 0)
|
||
summary['mcp'] = {
|
||
'total': mcp_total,
|
||
'servers': int(mcp[1] or 0),
|
||
'cache_hits': int(mcp[2] or 0),
|
||
'cost': float(mcp[3] or 0),
|
||
'cache_rate': (float(mcp[2] or 0) / mcp_total * 100) if mcp_total else 0,
|
||
}
|
||
except Exception:
|
||
summary['mcp'] = {}
|
||
finally:
|
||
session.close()
|
||
|
||
# Phase 51 O-3: 24h 三主機健康 sparkline 資料（每小時 bucket × 3 host）
|
||
host_sparkline = {}
|
||
try:
|
||
s_sp = get_session()
|
||
try:
|
||
sp_rows = s_sp.execute(
|
||
sa_text("""
|
||
SELECT host_label,
|
||
date_trunc('hour', probed_at) AS hr,
|
||
COUNT(*) AS total,
|
||
COUNT(*) FILTER (WHERE healthy) AS up
|
||
FROM host_health_probes
|
||
WHERE probed_at >= NOW() - INTERVAL '24 hours'
|
||
GROUP BY host_label, hr
|
||
ORDER BY host_label, hr ASC
|
||
"""),
|
||
).fetchall()
|
||
for r in sp_rows:
|
||
label, hr, total, up = r[0], r[1], int(r[2] or 0), int(r[3] or 0)
|
||
if label not in host_sparkline:
|
||
host_sparkline[label] = {'hours': [], 'uptime_pct': []}
|
||
host_sparkline[label]['hours'].append(
|
||
hr.strftime('%H:00') if hr else ''
|
||
)
|
||
host_sparkline[label]['uptime_pct'].append(
|
||
(up / total * 100) if total else 0
|
||
)
|
||
finally:
|
||
s_sp.close()
|
||
except Exception:
|
||
pass
|
||
|
||
return render_template(
|
||
'admin/observability_overview.html',
|
||
active_page='obs_overview',
|
||
summary=summary,
|
||
host_sparkline=host_sparkline,
|
||
today=today.strftime('%Y-%m-%d'),
|
||
)
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# /observability/rag_queries — Phase 51 RAG 召回詳情
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
@admin_observability_bp.route('/rag_queries')
|
||
@login_required
|
||
def rag_queries_dashboard():
|
||
"""Phase 51 — RAG 召回詳情：每筆 query 的命中、saved_call、反饋。
|
||
|
||
補完 RAG 觀測深度：之前只看 caller 級命中率，現在看每筆查詢的真實內容。
|
||
"""
|
||
hours = int(request.args.get('hours', '24'))
|
||
caller_filter = request.args.get('caller', '').strip()
|
||
saved_only = request.args.get('saved_only', '').strip() == '1'
|
||
|
||
session = get_session()
|
||
try:
|
||
rag_query_log_exists = bool(session.execute(
|
||
sa_text("SELECT to_regclass('public.rag_query_log') IS NOT NULL")
|
||
).scalar())
|
||
if not rag_query_log_exists:
|
||
return render_template(
|
||
'admin/rag_queries.html',
|
||
active_page='obs_rag_queries',
|
||
hours=hours,
|
||
caller_filter=caller_filter,
|
||
saved_only=saved_only,
|
||
summary={},
|
||
callers=[],
|
||
by_caller=[],
|
||
queries=[],
|
||
error='rag_query_log 尚未建立，RAG 召回資料待接入。',
|
||
)
|
||
|
||
# 整體統計
|
||
summary_row = session.execute(
|
||
sa_text("""
|
||
SELECT COUNT(*) AS total,
|
||
COUNT(*) FILTER (WHERE saved_call) AS saved,
|
||
COUNT(*) FILTER (WHERE hit_count > 0) AS with_hits,
|
||
COALESCE(AVG(hit_count), 0) AS avg_hits,
|
||
COALESCE(AVG(feedback_score) FILTER (WHERE feedback_score IS NOT NULL), 0) AS avg_score,
|
||
COUNT(*) FILTER (WHERE feedback_score IS NOT NULL) AS feedback_count,
|
||
COUNT(DISTINCT caller) AS distinct_callers
|
||
FROM rag_query_log
|
||
WHERE queried_at >= NOW() - (:h * INTERVAL '1 hour')
|
||
"""),
|
||
{'h': hours},
|
||
).fetchone()
|
||
total = int(summary_row[0] or 0)
|
||
saved = int(summary_row[1] or 0)
|
||
with_hits = int(summary_row[2] or 0)
|
||
summary = {
|
||
'total': total,
|
||
'saved': saved,
|
||
'with_hits': with_hits,
|
||
'no_hits': total - with_hits,
|
||
'avg_hits': round(float(summary_row[3] or 0), 2),
|
||
'avg_score': round(float(summary_row[4] or 0), 2),
|
||
'feedback_count': int(summary_row[5] or 0),
|
||
'distinct_callers': int(summary_row[6] or 0),
|
||
'saved_rate': (float(saved) / total * 100) if total else 0,
|
||
'hit_rate': (float(with_hits) / total * 100) if total else 0,
|
||
}
|
||
|
||
# caller 列表（dropdown）
|
||
callers = session.execute(
|
||
sa_text("""
|
||
SELECT DISTINCT caller FROM rag_query_log
|
||
WHERE queried_at >= NOW() - (:h * INTERVAL '1 hour')
|
||
ORDER BY caller
|
||
"""),
|
||
{'h': hours},
|
||
).fetchall()
|
||
caller_list = [r[0] for r in callers]
|
||
|
||
# by caller 統計
|
||
by_caller = session.execute(
|
||
sa_text("""
|
||
SELECT caller,
|
||
COUNT(*) AS total,
|
||
COUNT(*) FILTER (WHERE saved_call) AS saved,
|
||
COUNT(*) FILTER (WHERE hit_count > 0) AS with_hits,
|
||
COALESCE(AVG(feedback_score) FILTER (WHERE feedback_score IS NOT NULL), 0) AS avg_score,
|
||
COUNT(*) FILTER (WHERE feedback_score IS NOT NULL) AS fb_count
|
||
FROM rag_query_log
|
||
WHERE queried_at >= NOW() - (:h * INTERVAL '1 hour')
|
||
GROUP BY caller
|
||
ORDER BY total DESC
|
||
"""),
|
||
{'h': hours},
|
||
).fetchall()
|
||
|
||
# 最近 50 筆查詢（套 caller filter + saved_only）
|
||
params = {'h': hours, 'caller_f': caller_filter}
|
||
recent_queries = session.execute(
|
||
sa_text(f"""
|
||
SELECT id, queried_at, caller, LEFT(query_text, 200) AS qtext,
|
||
top_k, threshold, hit_count, used_results, saved_call,
|
||
feedback_score, request_id
|
||
FROM rag_query_log
|
||
WHERE queried_at >= NOW() - (:h * INTERVAL '1 hour')
|
||
AND (:caller_f = '' OR caller = :caller_f)
|
||
{"AND saved_call = TRUE" if saved_only else ""}
|
||
ORDER BY queried_at DESC
|
||
LIMIT 50
|
||
"""),
|
||
params,
|
||
).fetchall()
|
||
queries = []
|
||
for r in recent_queries:
|
||
used_ids = list(r[7]) if r[7] else []
|
||
queries.append({
|
||
'id': int(r[0]),
|
||
'queried_at': r[1].strftime('%Y-%m-%d %H:%M:%S') if r[1] else '',
|
||
'caller': r[2],
|
||
'query_text': r[3] or '',
|
||
'take_count': int(r[4] or 0),
|
||
'threshold': round(float(r[5] or 0), 3),
|
||
'hit_count': int(r[6] or 0),
|
||
'used_results': used_ids,
|
||
'saved_call': bool(r[8]),
|
||
'feedback_score': int(r[9]) if r[9] is not None else None,
|
||
'request_id': r[10],
|
||
})
|
||
|
||
return render_template(
|
||
'admin/rag_queries.html',
|
||
active_page='obs_rag_queries',
|
||
hours=hours,
|
||
caller_filter=caller_filter,
|
||
saved_only=saved_only,
|
||
summary=summary,
|
||
callers=caller_list,
|
||
by_caller=[
|
||
{
|
||
'caller': r[0], 'total': int(r[1] or 0),
|
||
'saved': int(r[2] or 0), 'with_hits': int(r[3] or 0),
|
||
'avg_score': round(float(r[4] or 0), 2),
|
||
'fb_count': int(r[5] or 0),
|
||
'saved_rate': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0,
|
||
'hit_rate': (float(r[3] or 0) / float(r[1]) * 100) if r[1] else 0,
|
||
}
|
||
for r in by_caller
|
||
],
|
||
queries=queries,
|
||
error=None,
|
||
)
|
||
except Exception as e:
|
||
return render_template(
|
||
'admin/rag_queries.html',
|
||
active_page='obs_rag_queries', hours=hours,
|
||
caller_filter=caller_filter, saved_only=saved_only,
|
||
summary={}, callers=[], by_caller=[], queries=[],
|
||
error='RAG 召回資料暫時不可用，已切換安全空狀態。',
|
||
)
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
@admin_observability_bp.route('/rag_queries/<int:query_id>/hits', methods=['GET'])
|
||
@login_required
|
||
def rag_query_hits(query_id: int):
|
||
"""Phase 51 — JSON API：回傳單筆 query 的 hits 詳細內容（給 modal 展開）。"""
|
||
try:
|
||
session = get_session()
|
||
try:
|
||
row = session.execute(
|
||
sa_text("""
|
||
SELECT id, query_text, used_results, hit_count, threshold
|
||
FROM rag_query_log WHERE id = :id
|
||
"""),
|
||
{'id': query_id},
|
||
).fetchone()
|
||
if not row:
|
||
return jsonify({'ok': False, 'error': 'not found'}), 404
|
||
|
||
used_ids = list(row[2]) if row[2] else []
|
||
hits = []
|
||
if used_ids:
|
||
rows = session.execute(
|
||
sa_text("""
|
||
SELECT id, insight_type, period, product_sku,
|
||
LEFT(content, 300) AS preview, created_at
|
||
FROM ai_insights
|
||
WHERE id = ANY(:ids)
|
||
ORDER BY created_at DESC
|
||
"""),
|
||
{'ids': used_ids},
|
||
).fetchall()
|
||
hits = [
|
||
{
|
||
'id': int(h[0]),
|
||
'insight_type': h[1],
|
||
'period': h[2],
|
||
'product_sku': h[3],
|
||
'content': h[4] or '',
|
||
'created_at': h[5].strftime('%Y-%m-%d') if h[5] else '',
|
||
}
|
||
for h in rows
|
||
]
|
||
return jsonify({
|
||
'ok': True,
|
||
'query_id': query_id,
|
||
'query_text': row[1],
|
||
'hit_count': int(row[3] or 0),
|
||
'threshold': round(float(row[4] or 0), 3),
|
||
'hits': hits,
|
||
})
|
||
finally:
|
||
session.close()
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# /observability/business_intel — Phase 48 商業面 × AI 編排
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
@admin_observability_bp.route('/business_intel')
|
||
@login_required
|
||
def business_intel_dashboard():
|
||
"""Phase 48 — 商業面 × AI 編排：把 AI 觀測台延伸到商業層級。
|
||
|
||
展現「AI 在做什麼生意」：
|
||
- ai_price_recommendations × competitor_prices: AI 看到什麼定價機會
|
||
- action_plans × action_outcomes: 計畫到 verdict 的閉環
|
||
- competitor_match_attempts: 競品比對失敗追蹤
|
||
"""
|
||
days = int(request.args.get('days', '7'))
|
||
session = get_session()
|
||
try:
|
||
# 1. ai_price_recommendations 30d 總覽
|
||
rec_summary = session.execute(
|
||
sa_text(f"""
|
||
SELECT strategy, COUNT(*) AS cnt,
|
||
COALESCE(AVG(confidence), 0) AS avg_conf,
|
||
COALESCE(AVG(gap_pct), 0) AS avg_gap_pct,
|
||
COALESCE(AVG(sales_7d_delta), 0) AS avg_sales_delta
|
||
FROM ai_price_recommendations
|
||
WHERE created_at >= NOW() - INTERVAL '{int(days)} days'
|
||
GROUP BY strategy ORDER BY cnt DESC
|
||
"""),
|
||
).fetchall()
|
||
rec_by_strategy = [
|
||
{
|
||
'strategy': r[0], 'count': int(r[1] or 0),
|
||
'avg_confidence': round(float(r[2] or 0), 3),
|
||
'avg_gap_pct': round(float(r[3] or 0), 2),
|
||
'avg_sales_delta': round(float(r[4] or 0), 2),
|
||
}
|
||
for r in rec_summary
|
||
]
|
||
|
||
# 2. ai_price_recommendations 最近 20 筆詳細
|
||
latest_recs = session.execute(
|
||
sa_text("""
|
||
SELECT id, sku, LEFT(name, 50), strategy, confidence,
|
||
momo_price, pchome_price, gap_pct, sales_7d_delta,
|
||
LEFT(reason, 120), created_at
|
||
FROM ai_price_recommendations
|
||
ORDER BY created_at DESC LIMIT 20
|
||
"""),
|
||
).fetchall()
|
||
latest_recommendations = [
|
||
{
|
||
'id': r[0], 'sku': r[1], 'name': r[2], 'strategy': r[3],
|
||
'confidence': round(float(r[4] or 0), 3),
|
||
'momo_price': float(r[5] or 0),
|
||
'pchome_price': float(r[6] or 0) if r[6] else None,
|
||
'gap_pct': round(float(r[7] or 0), 2),
|
||
'sales_delta': round(float(r[8] or 0), 2) if r[8] is not None else None,
|
||
'reason': r[9] or '',
|
||
'created_at': r[10].strftime('%m-%d %H:%M') if r[10] else '',
|
||
}
|
||
for r in latest_recs
|
||
]
|
||
|
||
# 3. action_plans × action_outcomes 閉環（30d）
|
||
closed_loops = session.execute(
|
||
sa_text(f"""
|
||
SELECT p.id, p.sku, p.plan_type, p.status,
|
||
p.created_by, p.created_at, p.executed_at,
|
||
o.verdict, o.metric_type, o.before_val, o.after_val
|
||
FROM action_plans p
|
||
LEFT JOIN action_outcomes o ON o.plan_id = p.id
|
||
WHERE p.created_at >= NOW() - INTERVAL '{int(days)} days'
|
||
ORDER BY p.created_at DESC LIMIT 25
|
||
"""),
|
||
).fetchall()
|
||
loop_records = []
|
||
for r in closed_loops:
|
||
before = float(r[9]) if r[9] is not None else None
|
||
after = float(r[10]) if r[10] is not None else None
|
||
change_pct = None
|
||
if before and before != 0 and after is not None:
|
||
change_pct = (after - before) / abs(before) * 100
|
||
loop_records.append({
|
||
'plan_id': r[0], 'sku': r[1], 'plan_type': r[2],
|
||
'status': r[3], 'created_by': r[4],
|
||
'created_at': r[5].strftime('%m-%d %H:%M') if r[5] else '',
|
||
'executed_at': r[6].strftime('%m-%d %H:%M') if r[6] else None,
|
||
'verdict': r[7], 'metric_type': r[8],
|
||
'before': before, 'after': after, 'change_pct': change_pct,
|
||
})
|
||
|
||
# 4. action_outcomes verdict 統計
|
||
verdict_summary = session.execute(
|
||
sa_text(f"""
|
||
SELECT verdict, COUNT(*) AS cnt,
|
||
AVG(after_val - before_val) AS avg_delta
|
||
FROM action_outcomes
|
||
WHERE created_at >= NOW() - INTERVAL '{int(days)} days'
|
||
AND before_val IS NOT NULL AND after_val IS NOT NULL
|
||
GROUP BY verdict ORDER BY cnt DESC
|
||
"""),
|
||
).fetchall()
|
||
verdict_stats = [
|
||
{
|
||
'verdict': r[0] or 'unknown', 'count': int(r[1] or 0),
|
||
'avg_delta': round(float(r[2] or 0), 2),
|
||
}
|
||
for r in verdict_summary
|
||
]
|
||
|
||
# 5. competitor_match_attempts 失敗統計（30d）
|
||
match_attempts = session.execute(
|
||
sa_text(f"""
|
||
SELECT attempt_status, COUNT(*) AS cnt,
|
||
COALESCE(AVG(candidate_count), 0) AS avg_candidates,
|
||
COALESCE(AVG(best_match_score), 0) AS avg_score
|
||
FROM competitor_match_attempts
|
||
WHERE attempted_at >= NOW() - INTERVAL '{int(days)} days'
|
||
GROUP BY attempt_status ORDER BY cnt DESC
|
||
"""),
|
||
).fetchall()
|
||
match_stats = [
|
||
{
|
||
'status': r[0], 'count': int(r[1] or 0),
|
||
'avg_candidates': round(float(r[2] or 0), 1),
|
||
'avg_score': round(float(r[3] or 0), 3),
|
||
}
|
||
for r in match_attempts
|
||
]
|
||
|
||
# 6. competitor_prices 24h 變動 TOP 10
|
||
recent_competitor = session.execute(
|
||
sa_text("""
|
||
SELECT cph.sku, cph.competitor_product_name, cph.price,
|
||
cph.momo_price, cph.discount_pct, cph.match_score,
|
||
cph.crawled_at, cph.source
|
||
FROM competitor_price_history cph
|
||
WHERE cph.crawled_at >= NOW() - INTERVAL '24 hours'
|
||
AND cph.match_score >= 0.7
|
||
AND COALESCE(cph.tags, '[]'::jsonb) ? 'identity_v2'
|
||
ORDER BY cph.crawled_at DESC LIMIT 12
|
||
"""),
|
||
).fetchall()
|
||
recent_competitor_prices = [
|
||
{
|
||
'sku': r[0],
|
||
'product_name': (r[1] or '')[:50],
|
||
'pchome_price': float(r[2] or 0),
|
||
'momo_price': float(r[3] or 0) if r[3] else None,
|
||
'discount_pct': int(r[4]) if r[4] else None,
|
||
'match_score': round(float(r[5] or 0), 3),
|
||
'gap': (float(r[3]) - float(r[2])) if (r[2] and r[3]) else None,
|
||
'crawled_at': r[6].strftime('%m-%d %H:%M') if r[6] else '',
|
||
'source': r[7] or '外部電商',
|
||
}
|
||
for r in recent_competitor
|
||
]
|
||
|
||
promo_watch_rows = []
|
||
for item in recent_competitor_prices:
|
||
discount_pct = float(item.get('discount_pct') or 0)
|
||
gap = item.get('gap')
|
||
gap_value = float(gap) if gap is not None else 0.0
|
||
is_external_pressure = gap_value < 0
|
||
is_discount_signal = discount_pct >= 5
|
||
if not (is_external_pressure or is_discount_signal):
|
||
continue
|
||
if is_external_pressure:
|
||
pressure_label = '外部低價壓力'
|
||
recommended_action = '檢查 PChome 售價、折扣券、組合包與商品頁主賣點'
|
||
else:
|
||
pressure_label = '外部促銷訊號'
|
||
recommended_action = '比對活動條件後，安排 PChome 主推曝光或會員回饋'
|
||
promo_watch_rows.append({
|
||
**item,
|
||
'pressure_label': pressure_label,
|
||
'recommended_action': recommended_action,
|
||
'gap_abs': abs(gap_value),
|
||
})
|
||
if len(promo_watch_rows) >= 8:
|
||
break
|
||
|
||
# 7. 高 confidence 但未 follow-through (recommendation 沒對應 action_plan)
|
||
unfollowed = session.execute(
|
||
sa_text(f"""
|
||
SELECT COUNT(*)
|
||
FROM ai_price_recommendations r
|
||
WHERE r.created_at >= NOW() - INTERVAL '{int(days)} days'
|
||
AND r.confidence >= 0.7
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM action_plans p
|
||
WHERE p.sku = r.sku
|
||
AND p.created_at >= r.created_at
|
||
AND p.created_at < r.created_at + INTERVAL '7 days'
|
||
)
|
||
"""),
|
||
).fetchone()
|
||
unfollowed_count = int(unfollowed[0] or 0) if unfollowed else 0
|
||
|
||
return render_template(
|
||
'admin/business_intel.html',
|
||
active_page='obs_business_intel',
|
||
days=days,
|
||
rec_by_strategy=rec_by_strategy,
|
||
latest_recommendations=latest_recommendations,
|
||
loop_records=loop_records,
|
||
verdict_stats=verdict_stats,
|
||
match_stats=match_stats,
|
||
recent_competitor_prices=recent_competitor_prices,
|
||
promo_watch_rows=promo_watch_rows,
|
||
unfollowed_count=unfollowed_count,
|
||
error=None,
|
||
)
|
||
except Exception as e:
|
||
return render_template(
|
||
'admin/business_intel.html',
|
||
active_page='obs_business_intel', days=days,
|
||
rec_by_strategy=[], latest_recommendations=[], loop_records=[],
|
||
verdict_stats=[], match_stats=[], recent_competitor_prices=[],
|
||
promo_watch_rows=[],
|
||
unfollowed_count=0,
|
||
error='商業 AI 資料暫時不可用，已切換安全空狀態。',
|
||
)
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# /observability/agent_orchestration — Phase 46 編排矩陣
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
# caller → agent 歸類規則（同 services/* 各 agent 真實 caller 值）
|
||
_AGENT_CALLER_GROUPS = {
|
||
'openclaw': [
|
||
'openclaw_qa', 'openclaw_qa_gemini_fallback', 'openclaw_qa_nim',
|
||
'openclaw_daily', 'openclaw_daily_gemini_fallback', 'openclaw_daily_nim',
|
||
'openclaw_daily_insight', 'openclaw_daily_insight_gemini_fallback',
|
||
'openclaw_daily_insight_nim',
|
||
'openclaw_meta', 'openclaw_meta_gemini_fallback', 'openclaw_meta_nim',
|
||
'openclaw_monthly', 'openclaw_monthly_gemini_fallback', 'openclaw_monthly_nim',
|
||
'openclaw_weekly', 'openclaw_weekly_gemini_fallback', 'openclaw_weekly_nim',
|
||
'openclaw_bot_main', 'openclaw_bot_gemini', 'openclaw_bot_nim',
|
||
'sales_copy', 'code_review_openclaw', 'code_review_openclaw_gemini',
|
||
],
|
||
'hermes': [
|
||
'hermes_analyst', 'hermes_intent', 'code_review_hermes',
|
||
],
|
||
'nemotron': [
|
||
'nemotron_dispatch',
|
||
],
|
||
'elephant_alpha': [
|
||
'ea_engine', 'code_review_elephant',
|
||
],
|
||
}
|
||
_AGENT_LABELS = {
|
||
'openclaw': ('🤖 OpenClaw', '主編排者 / Bot 對話 / 報告生成'),
|
||
'hermes': ('🔍 Hermes', '價格/程式碼分析師'),
|
||
'nemotron': ('🧬 NemoTron', '任務 dispatcher'),
|
||
'elephant_alpha': ('🐘 ElephantAlpha', '自主決策引擎'),
|
||
}
|
||
|
||
# Provider → 類別歸類
|
||
_PROVIDER_TIER = {
|
||
'gcp_ollama': 'ollama_local',
|
||
'ollama_secondary': 'ollama_local',
|
||
'ollama_111': 'ollama_local',
|
||
'ollama_other': 'ollama_local',
|
||
'gemini': 'paid_external',
|
||
'claude': 'paid_external',
|
||
'nim': 'paid_external',
|
||
'nim_via_elephant': 'paid_external',
|
||
'openrouter': 'paid_external',
|
||
}
|
||
|
||
|
||
@admin_observability_bp.route('/agent_orchestration')
|
||
@login_required
|
||
def agent_orchestration_dashboard():
|
||
"""Phase 46 — 4 Agent × Models × MCP × RAG 編排矩陣
|
||
|
||
展現「組合發揮」：每個 agent 在 24h 內如何調用 Ollama/Gemini，
|
||
搭配 MCP tool（外部 + 內部 mcp_collector），與 RAG 知識庫的協作。
|
||
|
||
資料來源：ai_calls × mcp_calls × rag_query_log 三表跨 JOIN + caller 分組。
|
||
"""
|
||
hours = int(request.args.get('hours', '24'))
|
||
session = get_session()
|
||
try:
|
||
# 1. 整體統計
|
||
overall = session.execute(
|
||
sa_text("""
|
||
SELECT COUNT(*),
|
||
COALESCE(SUM(cost_usd), 0),
|
||
COUNT(*) FILTER (WHERE provider IN ('gemini','claude','nim','openrouter','nim_via_elephant')),
|
||
COUNT(*) FILTER (WHERE provider IN ('gcp_ollama','ollama_secondary','ollama_111','ollama_other')),
|
||
COUNT(*) FILTER (WHERE rag_hit),
|
||
COALESCE(SUM(input_tokens + output_tokens), 0)
|
||
FROM ai_calls
|
||
WHERE called_at >= NOW() - (:h * INTERVAL '1 hour')
|
||
"""),
|
||
{'h': hours},
|
||
).fetchone()
|
||
total_calls = int(overall[0] or 0)
|
||
total_cost = float(overall[1] or 0)
|
||
paid_calls = int(overall[2] or 0)
|
||
local_calls = int(overall[3] or 0)
|
||
rag_hits = int(overall[4] or 0)
|
||
total_tokens = int(overall[5] or 0)
|
||
mcp_calls_table_exists = bool(session.execute(
|
||
sa_text("SELECT to_regclass('public.mcp_calls') IS NOT NULL")
|
||
).scalar())
|
||
|
||
# 2. 每個 agent group 的細節
|
||
agent_matrix = []
|
||
for agent_key, callers in _AGENT_CALLER_GROUPS.items():
|
||
ag_row = session.execute(
|
||
sa_text("""
|
||
SELECT COUNT(*) AS calls,
|
||
COALESCE(SUM(input_tokens + output_tokens), 0) AS tokens,
|
||
COALESCE(SUM(cost_usd), 0) AS cost,
|
||
COUNT(*) FILTER (WHERE rag_hit) AS rag_hits,
|
||
COUNT(*) FILTER (WHERE provider IN ('gcp_ollama','ollama_secondary','ollama_111','ollama_other')) AS ollama,
|
||
COUNT(*) FILTER (WHERE provider = 'gcp_ollama') AS ollama_gcp_a,
|
||
COUNT(*) FILTER (WHERE provider = 'ollama_secondary') AS ollama_gcp_b,
|
||
COUNT(*) FILTER (WHERE provider = 'ollama_111') AS ollama_111,
|
||
COUNT(*) FILTER (WHERE provider = 'gemini') AS gemini,
|
||
COUNT(*) FILTER (WHERE provider IN ('claude','nim','openrouter','nim_via_elephant')) AS other_paid,
|
||
COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')) AS errors,
|
||
COALESCE(AVG(duration_ms), 0) AS avg_ms
|
||
FROM ai_calls
|
||
WHERE called_at >= NOW() - (:h * INTERVAL '1 hour')
|
||
AND caller = ANY(:callers)
|
||
"""),
|
||
{'h': hours, 'callers': callers},
|
||
).fetchone()
|
||
calls = int(ag_row[0] or 0)
|
||
if calls == 0:
|
||
# 沒呼叫也佔位顯示
|
||
agent_matrix.append({
|
||
'key': agent_key, 'label': _AGENT_LABELS[agent_key][0],
|
||
'desc': _AGENT_LABELS[agent_key][1],
|
||
'calls': 0, 'tokens': 0, 'cost': 0,
|
||
'rag_hits': 0, 'rag_rate': 0,
|
||
'ollama_pct': 0, 'gemini_pct': 0, 'paid_pct': 0,
|
||
'ollama_gcp_a': 0, 'ollama_gcp_b': 0, 'ollama_111': 0,
|
||
'gemini': 0, 'other_paid': 0,
|
||
'errors': 0, 'error_rate': 0,
|
||
'avg_ms': 0, 'mcp_calls': 0, 'mcp_rate': 0,
|
||
'callers_in_group': callers,
|
||
})
|
||
continue
|
||
|
||
# MCP 編排率（透過 request_id 串接）。mcp_calls 尚未 migration 時安全降級為 0。
|
||
if mcp_calls_table_exists:
|
||
mcp_count = session.execute(
|
||
sa_text("""
|
||
SELECT COUNT(DISTINCT a.request_id)
|
||
FROM ai_calls a
|
||
INNER JOIN mcp_calls m ON m.request_id = a.request_id
|
||
WHERE a.called_at >= NOW() - (:h * INTERVAL '1 hour')
|
||
AND a.caller = ANY(:callers)
|
||
AND a.request_id IS NOT NULL
|
||
"""),
|
||
{'h': hours, 'callers': callers},
|
||
).fetchone()[0] or 0
|
||
else:
|
||
mcp_count = 0
|
||
|
||
errors = int(ag_row[10] or 0)
|
||
ollama = int(ag_row[4] or 0)
|
||
gemini = int(ag_row[8] or 0)
|
||
other_paid = int(ag_row[9] or 0)
|
||
|
||
agent_matrix.append({
|
||
'key': agent_key,
|
||
'label': _AGENT_LABELS[agent_key][0],
|
||
'desc': _AGENT_LABELS[agent_key][1],
|
||
'calls': calls,
|
||
'tokens': int(ag_row[1] or 0),
|
||
'cost': float(ag_row[2] or 0),
|
||
'rag_hits': int(ag_row[3] or 0),
|
||
'rag_rate': (float(ag_row[3] or 0) / calls * 100) if calls else 0,
|
||
'ollama': ollama, 'ollama_pct': (ollama / calls * 100) if calls else 0,
|
||
'ollama_gcp_a': int(ag_row[5] or 0),
|
||
'ollama_gcp_b': int(ag_row[6] or 0),
|
||
'ollama_111': int(ag_row[7] or 0),
|
||
'gemini': gemini, 'gemini_pct': (gemini / calls * 100) if calls else 0,
|
||
'other_paid': other_paid,
|
||
'paid_pct': ((gemini + other_paid) / calls * 100) if calls else 0,
|
||
'errors': errors, 'error_rate': (errors / calls * 100) if calls else 0,
|
||
'avg_ms': int(ag_row[11] or 0),
|
||
'mcp_calls': int(mcp_count),
|
||
'mcp_rate': (float(mcp_count) / calls * 100) if calls else 0,
|
||
'callers_in_group': callers,
|
||
})
|
||
|
||
# 3. MCP server 24h 工作量（同 host_health 邏輯）
|
||
if mcp_calls_table_exists:
|
||
mcp_servers = session.execute(
|
||
sa_text("""
|
||
SELECT server, caller, COUNT(*) AS calls,
|
||
COUNT(*) FILTER (WHERE cache_hit) AS cache_hits,
|
||
COALESCE(SUM(cost_usd), 0) AS cost
|
||
FROM mcp_calls
|
||
WHERE called_at >= NOW() - (:h * INTERVAL '1 hour')
|
||
GROUP BY server, caller
|
||
ORDER BY calls DESC
|
||
LIMIT 30
|
||
"""),
|
||
{'h': hours},
|
||
).fetchall()
|
||
mcp_matrix = [
|
||
{
|
||
'server': r[0], 'caller': r[1],
|
||
'calls': int(r[2] or 0),
|
||
'cache_hits': int(r[3] or 0),
|
||
'cost': float(r[4] or 0),
|
||
'cache_rate': (float(r[3] or 0) / float(r[2]) * 100) if r[2] else 0,
|
||
}
|
||
for r in mcp_servers
|
||
]
|
||
else:
|
||
mcp_matrix = []
|
||
|
||
# 4. 自動編排建議（rule-based 提案）
|
||
recommendations = []
|
||
if not mcp_calls_table_exists:
|
||
recommendations.append({
|
||
'severity': 'med', 'agent': 'MCP 觀測',
|
||
'finding': 'mcp_calls 尚未建立，MCP 編排率目前以 0 顯示',
|
||
'suggestion': '執行 Phase 10.7 migration 後，本頁會自動接回 MCP server/caller 矩陣',
|
||
})
|
||
for ag in agent_matrix:
|
||
if ag['calls'] == 0:
|
||
continue
|
||
# 規則 1：付費比例 > 50% 且 ollama 比例 < 20% → 建議切 Hermes-first
|
||
if ag['paid_pct'] > 50 and ag['ollama_pct'] < 20:
|
||
recommendations.append({
|
||
'severity': 'high', 'agent': ag['label'],
|
||
'finding': f"付費 LLM 比例 {ag['paid_pct']:.0f}%（cost ${ag['cost']:.2f}）",
|
||
'suggestion': '改用 Hermes-first 短路機制：先試 Ollama 三主機 5s timeout，0 hits 才 escalate Gemini',
|
||
})
|
||
# 規則 2：錯誤率 > 10% → 建議跑 code review
|
||
if ag['error_rate'] > 10:
|
||
recommendations.append({
|
||
'severity': 'high', 'agent': ag['label'],
|
||
'finding': f"錯誤率 {ag['error_rate']:.1f}%（{ag['errors']}/{ag['calls']}）",
|
||
'suggestion': '觸發程式碼審查流程找回歸問題（可由 AI 流量控制台執行）',
|
||
})
|
||
# 規則 3：MCP 編排率 < 5% 但 calls 多 → 建議擴大 MCP 使用
|
||
if mcp_calls_table_exists and ag['mcp_rate'] < 5 and ag['calls'] > 50:
|
||
recommendations.append({
|
||
'severity': 'med', 'agent': ag['label'],
|
||
'finding': f"MCP 編排率僅 {ag['mcp_rate']:.1f}%，未善用外部工具",
|
||
'suggestion': '考慮加 MCP omnisearch / firecrawl 補強事實查證鏈',
|
||
})
|
||
# 規則 4：RAG 命中率高（≥40%）但有 saved_call=False 的多 → 提醒 feedback
|
||
if ag['rag_rate'] >= 40 and ag['rag_hits'] >= 20:
|
||
recommendations.append({
|
||
'severity': 'low', 'agent': ag['label'],
|
||
'finding': f"RAG 命中率 {ag['rag_rate']:.1f}%（{ag['rag_hits']} hits）— 知識庫貢獻度高",
|
||
'suggestion': '推 Telegram inline button 收集 feedback_score 強化 promotion gate',
|
||
})
|
||
# 規則 5：111 fallback 比例 > 20% → 警示
|
||
if ag['calls'] > 0 and ag['ollama_111'] / max(ag['calls'], 1) > 0.20:
|
||
fb_pct = ag['ollama_111'] / ag['calls'] * 100
|
||
recommendations.append({
|
||
'severity': 'med', 'agent': ag['label'],
|
||
'finding': f"111 fallback 比例 {fb_pct:.0f}%（GCP 兩台不可達？）",
|
||
'suggestion': '檢查 mo.wooo.work/observability/host_health AIOps incidents',
|
||
})
|
||
|
||
return render_template(
|
||
'admin/agent_orchestration.html',
|
||
active_page='obs_agent_orchestration',
|
||
hours=hours,
|
||
agent_matrix=agent_matrix,
|
||
mcp_matrix=mcp_matrix,
|
||
recommendations=recommendations,
|
||
overall={
|
||
'total_calls': total_calls,
|
||
'total_cost': total_cost,
|
||
'total_tokens': total_tokens,
|
||
'paid_calls': paid_calls,
|
||
'local_calls': local_calls,
|
||
'rag_hits': rag_hits,
|
||
'paid_pct': (paid_calls / total_calls * 100) if total_calls else 0,
|
||
'local_pct': (local_calls / total_calls * 100) if total_calls else 0,
|
||
'rag_rate': (rag_hits / total_calls * 100) if total_calls else 0,
|
||
},
|
||
error=None,
|
||
)
|
||
except Exception as e:
|
||
return render_template(
|
||
'admin/agent_orchestration.html',
|
||
active_page='obs_agent_orchestration', hours=hours,
|
||
agent_matrix=[], mcp_matrix=[], recommendations=[], overall={},
|
||
error='資料查詢暫時不可用，已切換安全空狀態。',
|
||
)
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# /observability/ai_calls — Phase 27 主入口
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
@admin_observability_bp.route('/ai_calls')
|
||
@login_required
|
||
def ai_calls_dashboard():
|
||
"""ai_calls 表觀測 dashboard（24h 預設視窗）"""
|
||
hours = int(request.args.get('hours', '24'))
|
||
caller_filter = request.args.get('caller', '').strip()
|
||
provider_filter = request.args.get('provider', '').strip()
|
||
|
||
since = datetime.now() - timedelta(hours=hours)
|
||
session = get_session()
|
||
try:
|
||
# 1. 總覽
|
||
summary = session.execute(
|
||
sa_text("""
|
||
SELECT
|
||
COUNT(*) AS total_calls,
|
||
COALESCE(SUM(input_tokens + output_tokens), 0) AS total_tokens,
|
||
COALESCE(SUM(cost_usd), 0) AS total_cost,
|
||
COALESCE(AVG(duration_ms), 0) AS avg_duration,
|
||
COUNT(*) FILTER (WHERE status = 'ok') AS ok_calls,
|
||
COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')) AS error_calls,
|
||
COUNT(*) FILTER (WHERE rag_hit) AS rag_hits,
|
||
COUNT(*) FILTER (WHERE cache_hit) AS cache_hits
|
||
FROM ai_calls
|
||
WHERE called_at >= :since
|
||
"""),
|
||
{'since': since},
|
||
).fetchone()
|
||
|
||
# 2. by provider
|
||
by_provider = session.execute(
|
||
sa_text("""
|
||
SELECT provider, COUNT(*) AS calls,
|
||
COALESCE(SUM(input_tokens + output_tokens), 0) AS tokens,
|
||
COALESCE(SUM(cost_usd), 0) AS cost
|
||
FROM ai_calls
|
||
WHERE called_at >= :since
|
||
GROUP BY provider
|
||
ORDER BY tokens DESC
|
||
"""),
|
||
{'since': since},
|
||
).fetchall()
|
||
|
||
# 3. TOP 100 calls — Phase 33 Critic HIGH #2 修補：
|
||
# 改用固定 SQL + 全綁參數，移除 f-string 動態 WHERE 拼接（防後人不慎注入）
|
||
recent = session.execute(
|
||
sa_text("""
|
||
SELECT id, called_at, caller, provider, model,
|
||
input_tokens, output_tokens, duration_ms, status,
|
||
cost_usd, cache_hit, rag_hit
|
||
FROM ai_calls
|
||
WHERE called_at >= :since
|
||
AND (:caller_f = '' OR caller = :caller_f)
|
||
AND (:provider_f = '' OR provider = :provider_f)
|
||
ORDER BY called_at DESC
|
||
LIMIT 100
|
||
"""),
|
||
{
|
||
'since': since,
|
||
'caller_f': caller_filter,
|
||
'provider_f': provider_filter,
|
||
},
|
||
).fetchall()
|
||
|
||
# 4. caller 列表（給篩選 dropdown）
|
||
callers = session.execute(
|
||
sa_text("""
|
||
SELECT DISTINCT caller FROM ai_calls
|
||
WHERE called_at >= :since ORDER BY caller
|
||
"""),
|
||
{'since': since},
|
||
).fetchall()
|
||
|
||
# 5b. Phase 47 K-2: by model 細分（不只 provider，到實際 model）
|
||
by_model = session.execute(
|
||
sa_text("""
|
||
SELECT model, provider, COUNT(*) AS calls,
|
||
COALESCE(SUM(input_tokens + output_tokens), 0) AS tokens,
|
||
COALESCE(SUM(cost_usd), 0) AS cost,
|
||
COALESCE(AVG(duration_ms), 0) AS avg_ms,
|
||
COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')) AS errors
|
||
FROM ai_calls
|
||
WHERE called_at >= :since
|
||
AND model IS NOT NULL AND model != ''
|
||
GROUP BY model, provider
|
||
ORDER BY calls DESC
|
||
LIMIT 15
|
||
"""),
|
||
{'since': since},
|
||
).fetchall()
|
||
|
||
# 5c. Phase 47 K-2: hourly 呼叫量趨勢（24 個 bucket）
|
||
hourly_trend = session.execute(
|
||
sa_text("""
|
||
SELECT date_trunc('hour', called_at) AS hr,
|
||
COUNT(*) AS calls,
|
||
COALESCE(SUM(cost_usd), 0) AS cost,
|
||
COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')) AS errors
|
||
FROM ai_calls
|
||
WHERE called_at >= NOW() - INTERVAL '24 hours'
|
||
GROUP BY hr ORDER BY hr ASC
|
||
"""),
|
||
).fetchall()
|
||
|
||
# 5d. Phase 47 K-2: agent_context 最近 10 筆（OpenClaw/Hermes 對話上下文）
|
||
recent_contexts = session.execute(
|
||
sa_text("""
|
||
SELECT created_at, agent_name, context_key, ttl_minutes,
|
||
LEFT(context_val, 120) AS preview
|
||
FROM agent_context
|
||
ORDER BY created_at DESC LIMIT 10
|
||
"""),
|
||
).fetchall()
|
||
|
||
# 5. Phase 39 D-3: caller × RAG 命中率 × MCP 編排率（跨表 JOIN）
|
||
# mcp_calls / rag_query_log 尚未 migration 時安全降級，不曝露 DB exception。
|
||
mcp_calls_table_exists = bool(session.execute(
|
||
sa_text("SELECT to_regclass('public.mcp_calls') IS NOT NULL")
|
||
).scalar())
|
||
rag_query_log_exists = bool(session.execute(
|
||
sa_text("SELECT to_regclass('public.rag_query_log') IS NOT NULL")
|
||
).scalar())
|
||
if mcp_calls_table_exists and rag_query_log_exists:
|
||
caller_richness = session.execute(
|
||
sa_text("""
|
||
SELECT a.caller,
|
||
COUNT(*) AS total_calls,
|
||
COUNT(*) FILTER (WHERE a.rag_hit) AS rag_hits,
|
||
COUNT(DISTINCT m.request_id) AS mcp_orchestrated,
|
||
COALESCE(AVG(rl.feedback_score) FILTER (WHERE rl.feedback_score IS NOT NULL), 0)
|
||
AS avg_rag_feedback,
|
||
COUNT(rl.feedback_score) AS feedback_count
|
||
FROM ai_calls a
|
||
LEFT JOIN mcp_calls m
|
||
ON m.request_id = a.request_id
|
||
AND m.called_at >= :since
|
||
LEFT JOIN rag_query_log rl
|
||
ON rl.caller = a.caller
|
||
AND rl.queried_at >= :since
|
||
WHERE a.called_at >= :since
|
||
GROUP BY a.caller
|
||
HAVING COUNT(*) >= 5
|
||
ORDER BY total_calls DESC
|
||
LIMIT 12
|
||
"""),
|
||
{'since': since},
|
||
).fetchall()
|
||
else:
|
||
caller_richness = session.execute(
|
||
sa_text("""
|
||
SELECT caller,
|
||
COUNT(*) AS total_calls,
|
||
COUNT(*) FILTER (WHERE rag_hit) AS rag_hits
|
||
FROM ai_calls
|
||
WHERE called_at >= :since
|
||
GROUP BY caller
|
||
HAVING COUNT(*) >= 5
|
||
ORDER BY total_calls DESC
|
||
LIMIT 12
|
||
"""),
|
||
{'since': since},
|
||
).fetchall()
|
||
|
||
return render_template(
|
||
'admin/ai_calls_dashboard.html',
|
||
active_page='obs_ai_calls',
|
||
hours=hours,
|
||
caller_filter=caller_filter,
|
||
provider_filter=provider_filter,
|
||
summary={
|
||
'total_calls': int(summary[0] or 0),
|
||
'total_tokens': int(summary[1] or 0),
|
||
'total_cost': float(summary[2] or 0),
|
||
'avg_duration': int(summary[3] or 0),
|
||
'ok_calls': int(summary[4] or 0),
|
||
'error_calls': int(summary[5] or 0),
|
||
'rag_hits': int(summary[6] or 0),
|
||
'cache_hits': int(summary[7] or 0),
|
||
},
|
||
by_provider=[
|
||
{'provider': r[0], 'calls': int(r[1] or 0),
|
||
'tokens': int(r[2] or 0), 'cost': float(r[3] or 0)}
|
||
for r in by_provider
|
||
],
|
||
recent=[_build_ai_call_recent_row(r) for r in recent],
|
||
callers=[r[0] for r in callers],
|
||
by_model=[
|
||
{
|
||
'model': r[0], 'provider': r[1],
|
||
'calls': int(r[2] or 0), 'tokens': int(r[3] or 0),
|
||
'cost': float(r[4] or 0), 'avg_ms': int(r[5] or 0),
|
||
'errors': int(r[6] or 0),
|
||
}
|
||
for r in by_model
|
||
],
|
||
hourly_trend=[
|
||
{
|
||
'hour': r[0].strftime('%H:%M') if r[0] else '',
|
||
'calls': int(r[1] or 0),
|
||
'cost': float(r[2] or 0),
|
||
'errors': int(r[3] or 0),
|
||
}
|
||
for r in hourly_trend
|
||
],
|
||
recent_contexts=[
|
||
{
|
||
'created_at': r[0].strftime('%Y-%m-%d %H:%M') if r[0] else '',
|
||
'agent_name': r[1], 'context_key': r[2],
|
||
'ttl_minutes': int(r[3] or 0),
|
||
'preview': r[4] or '',
|
||
}
|
||
for r in recent_contexts
|
||
],
|
||
caller_richness=[
|
||
{
|
||
'caller': r[0],
|
||
'total_calls': int(r[1] or 0),
|
||
'rag_hits': int(r[2] or 0),
|
||
'mcp_orchestrated': int(r[3] or 0) if mcp_calls_table_exists and rag_query_log_exists else 0,
|
||
'avg_rag_feedback': round(float(r[4] or 0), 2) if mcp_calls_table_exists and rag_query_log_exists else 0,
|
||
'feedback_count': int(r[5] or 0) if mcp_calls_table_exists and rag_query_log_exists else 0,
|
||
'rag_hit_rate': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0,
|
||
'mcp_rate': (float(r[3] or 0) / float(r[1]) * 100) if mcp_calls_table_exists and rag_query_log_exists and r[1] else 0,
|
||
}
|
||
for r in caller_richness
|
||
],
|
||
error=None,
|
||
)
|
||
except Exception as e:
|
||
return render_template(
|
||
'admin/ai_calls_dashboard.html',
|
||
active_page='obs_ai_calls',
|
||
hours=hours, caller_filter=caller_filter,
|
||
provider_filter=provider_filter,
|
||
summary={}, by_provider=[], recent=[], callers=[], caller_richness=[],
|
||
by_model=[], hourly_trend=[], recent_contexts=[],
|
||
error='AI 呼叫資料暫時不可用，已切換安全空狀態。',
|
||
)
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# /observability/promotion_review — Phase 28 PromotionGate 待審核列表
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
@admin_observability_bp.route('/promotion_review')
|
||
@login_required
|
||
def promotion_review_list():
|
||
"""awaiting_review episodes 列表（24h 內 reviewed_at IS NULL）
|
||
|
||
Phase 39（D-1）：每筆 episode 自動跑 RAG 找 Top 3 相似已晉升 ai_insights，
|
||
輔助人工判斷晉升價值。RAG fail-safe：失敗則 similar_insights=[]，不擋頁面。
|
||
"""
|
||
session = get_session()
|
||
try:
|
||
rows = session.execute(
|
||
sa_text("""
|
||
SELECT id, created_at, episode_type, source_table, source_id,
|
||
distilled_text, quality_score, weight, promotion_status
|
||
FROM learning_episodes
|
||
WHERE promotion_status = 'awaiting_review'
|
||
AND reviewed_at IS NULL
|
||
ORDER BY weight DESC, created_at ASC
|
||
LIMIT 50
|
||
"""),
|
||
).fetchall()
|
||
|
||
# ai_insights 全表大小（給「晉升後 KB 增長」對照）
|
||
kb_size = 0
|
||
try:
|
||
kb_row = session.execute(
|
||
sa_text("SELECT COUNT(*) FROM ai_insights"),
|
||
).fetchone()
|
||
kb_size = int(kb_row[0] or 0)
|
||
except Exception:
|
||
pass
|
||
|
||
episodes = [
|
||
{'id': r[0], 'created_at': r[1].strftime('%Y-%m-%d %H:%M'),
|
||
'episode_type': r[2], 'source_table': r[3], 'source_id': r[4],
|
||
'distilled_text': (r[5] or '')[:600],
|
||
'quality_score': float(r[6] or 0),
|
||
'weight': float(r[7] or 0),
|
||
'status': r[8],
|
||
'similar_insights': []}
|
||
for r in rows
|
||
]
|
||
|
||
# Phase 39 D-1：對每筆 episode 跑 RAG 找 Top 3 相似已晉升
|
||
try:
|
||
from services.rag_service import rag_service
|
||
for ep in episodes:
|
||
try:
|
||
rag_result = rag_service.query(
|
||
text=ep['distilled_text'][:500],
|
||
caller='admin_promotion_review',
|
||
top_k=3,
|
||
threshold=0.7,
|
||
)
|
||
ep['similar_insights'] = [
|
||
{
|
||
'id': h.get('id'),
|
||
'insight_type': h.get('insight_type'),
|
||
'content': (h.get('content') or '')[:180],
|
||
'similarity': round(float(h.get('similarity', 0)), 3),
|
||
'created_at': h.get('created_at').strftime('%Y-%m-%d')
|
||
if h.get('created_at') else '',
|
||
}
|
||
for h in rag_result.hits[:3]
|
||
]
|
||
except Exception:
|
||
logger.debug(
|
||
"Promotion review similar-insight lookup failed for episode_id=%s",
|
||
ep.get('id'),
|
||
exc_info=True,
|
||
)
|
||
except Exception:
|
||
logger.debug("Promotion review RAG service unavailable; skipping similar insights", exc_info=True)
|
||
|
||
# Phase 47 K-4: 蒸餾池 status 分布（30d）
|
||
ep_distribution = session.execute(
|
||
sa_text("""
|
||
SELECT promotion_status, COUNT(*) AS cnt
|
||
FROM learning_episodes
|
||
WHERE created_at >= NOW() - INTERVAL '30 days'
|
||
GROUP BY promotion_status ORDER BY cnt DESC
|
||
"""),
|
||
).fetchall()
|
||
episode_distribution_30d = {r[0]: int(r[1] or 0) for r in ep_distribution}
|
||
|
||
# Phase 47 K-4: ai_insights 最近 10 筆已晉升（type/created_at 視覺）
|
||
latest_insights = session.execute(
|
||
sa_text("""
|
||
SELECT id, insight_type, period, product_sku, created_at,
|
||
LEFT(content, 160) AS preview
|
||
FROM ai_insights
|
||
ORDER BY created_at DESC LIMIT 10
|
||
"""),
|
||
).fetchall()
|
||
|
||
# Phase 47 K-4: agent_strategy_weights TOP 12（OpenClaw 學習權重）
|
||
strategy_weights = session.execute(
|
||
sa_text("""
|
||
SELECT strategy_key, weight, success_cnt, fail_cnt, updated_at
|
||
FROM agent_strategy_weights
|
||
ORDER BY (success_cnt + fail_cnt) DESC
|
||
LIMIT 12
|
||
"""),
|
||
).fetchall()
|
||
|
||
return render_template(
|
||
'admin/promotion_review.html',
|
||
active_page='obs_promotion_review',
|
||
episodes=episodes,
|
||
kb_size=kb_size,
|
||
episode_distribution_30d=episode_distribution_30d,
|
||
latest_insights=[
|
||
{
|
||
'id': r[0], 'insight_type': r[1], 'period': r[2],
|
||
'product_sku': r[3],
|
||
'created_at': r[4].strftime('%Y-%m-%d %H:%M') if r[4] else '',
|
||
'preview': r[5] or '',
|
||
}
|
||
for r in latest_insights
|
||
],
|
||
strategy_weights=[
|
||
{
|
||
'strategy_key': r[0], 'weight': float(r[1] or 0),
|
||
'success': int(r[2] or 0), 'fail': int(r[3] or 0),
|
||
'updated_at': r[4].strftime('%Y-%m-%d') if r[4] else '',
|
||
'success_rate': (
|
||
float(r[2] or 0) / float((r[2] or 0) + (r[3] or 0)) * 100
|
||
) if ((r[2] or 0) + (r[3] or 0)) > 0 else 0,
|
||
}
|
||
for r in strategy_weights
|
||
],
|
||
error=None,
|
||
)
|
||
except Exception as e:
|
||
return render_template(
|
||
'admin/promotion_review.html',
|
||
active_page='obs_promotion_review',
|
||
episodes=[],
|
||
kb_size=0,
|
||
episode_distribution_30d={},
|
||
latest_insights=[],
|
||
strategy_weights=[],
|
||
error='RAG 晉升資料暫時不可用，已切換安全空狀態。',
|
||
)
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
@admin_observability_bp.route('/promotion_review/approve/<int:episode_id>', methods=['POST'])
|
||
@login_required
|
||
def promotion_review_approve(episode_id: int):
|
||
"""Web 介面「通過」按鈕 — 等同於 Telegram pg_ok callback"""
|
||
try:
|
||
from services.learning_pipeline import promotion_gate, hash_human_approver
|
||
# 從 Flask session 取（已過 @login_required）— 不信任 client header
|
||
user = get_current_user() or {}
|
||
username = user.get('username', 'web_admin')
|
||
approver_hash = hash_human_approver(username)
|
||
insight_id = promotion_gate.promote(
|
||
episode_id,
|
||
human_approver=approver_hash,
|
||
)
|
||
if insight_id:
|
||
return jsonify({'ok': True, 'insight_id': insight_id, 'approver': approver_hash})
|
||
return jsonify({'ok': False, 'error': 'promote failed'}), 500
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': str(e)[:200]}), 500
|
||
|
||
|
||
@admin_observability_bp.route('/promotion_review/reject/<int:episode_id>', methods=['POST'])
|
||
@login_required
|
||
def promotion_review_reject(episode_id: int):
|
||
"""Web 介面「拒絕」按鈕"""
|
||
try:
|
||
from services.learning_pipeline import promotion_gate, hash_human_approver
|
||
user = get_current_user() or {}
|
||
username = user.get('username', 'web_admin')
|
||
approver_hash = hash_human_approver(username)
|
||
ok = promotion_gate.reject(
|
||
episode_id,
|
||
'rejected_human',
|
||
detail='web admin reject',
|
||
human_approver=approver_hash,
|
||
)
|
||
return jsonify({'ok': ok})
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': str(e)[:200]}), 500
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# /observability/quality_trend — Phase 25 caller 反饋趨勢視覺化
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
@admin_observability_bp.route('/quality_trend')
|
||
@login_required
|
||
def quality_trend_dashboard():
|
||
"""caller × feedback 趨勢（30 日窗格）"""
|
||
days = int(request.args.get('days', '30'))
|
||
try:
|
||
from services.feedback_quality_tracker import (
|
||
compute_caller_quality_trend, get_caller_recommendations,
|
||
)
|
||
trends = compute_caller_quality_trend(days=days)
|
||
recommendations = get_caller_recommendations(days=days)
|
||
|
||
# 排序：avg_score 升序（最差先看）
|
||
sorted_trends = sorted(
|
||
trends.items(),
|
||
key=lambda kv: kv[1].get('avg_score', 5),
|
||
)
|
||
|
||
# Phase 40 D-6: learning_episodes 各 status 分布（蒸餾池飽和度）
|
||
episode_distribution = {}
|
||
try:
|
||
session = get_session()
|
||
try:
|
||
rows = session.execute(
|
||
sa_text("""
|
||
SELECT promotion_status, COUNT(*) AS cnt
|
||
FROM learning_episodes
|
||
WHERE created_at >= NOW() - INTERVAL ':days days'
|
||
GROUP BY promotion_status
|
||
""").bindparams(days=days),
|
||
).fetchall() if False else session.execute(
|
||
sa_text(f"""
|
||
SELECT promotion_status, COUNT(*) AS cnt
|
||
FROM learning_episodes
|
||
WHERE created_at >= NOW() - INTERVAL '{int(days)} days'
|
||
GROUP BY promotion_status
|
||
"""),
|
||
).fetchall()
|
||
episode_distribution = {r[0]: int(r[1] or 0) for r in rows}
|
||
finally:
|
||
session.close()
|
||
except Exception:
|
||
pass
|
||
|
||
# Phase 40 D-6: 對最差 3 名 caller 跑 RAG 找根因建議
|
||
rag_root_causes = []
|
||
try:
|
||
from services.rag_service import rag_service
|
||
worst_3 = sorted_trends[:3] if len(sorted_trends) >= 3 else sorted_trends
|
||
for caller, info in worst_3:
|
||
if info.get('avg_score', 5) < 3.0 and info.get('total_feedback', 0) >= 3:
|
||
try:
|
||
q = (
|
||
f"caller {caller} 反饋分數低 平均 "
|
||
f"{info.get('avg_score', 0):.1f}/5 應採取什麼根因排查"
|
||
)
|
||
rag_result = rag_service.query(
|
||
text=q,
|
||
caller='admin_quality_trend',
|
||
top_k=2,
|
||
threshold=0.6,
|
||
)
|
||
if rag_result.hits:
|
||
rag_root_causes.append({
|
||
'caller': caller,
|
||
'avg_score': info.get('avg_score', 0),
|
||
'feedback_n': info.get('total_feedback', 0),
|
||
'hits': [
|
||
{
|
||
'id': h.get('id'),
|
||
'insight_type': h.get('insight_type'),
|
||
'content': (h.get('content') or '')[:200],
|
||
'similarity': round(float(h.get('similarity', 0)), 3),
|
||
}
|
||
for h in rag_result.hits[:2]
|
||
],
|
||
})
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
|
||
# Phase 47 K-5: action_outcomes verdict 統計（ADR-012 閉環學習結果）
|
||
action_outcomes_stats = []
|
||
action_plans_status = []
|
||
rag_overall_dist = []
|
||
try:
|
||
session = get_session()
|
||
try:
|
||
# action_outcomes verdict 分布（30d）
|
||
ao_rows = session.execute(
|
||
sa_text(f"""
|
||
SELECT verdict, COUNT(*) AS cnt
|
||
FROM action_outcomes
|
||
WHERE created_at >= NOW() - INTERVAL '{int(days)} days'
|
||
GROUP BY verdict ORDER BY cnt DESC
|
||
"""),
|
||
).fetchall()
|
||
action_outcomes_stats = [{'verdict': r[0] or 'unknown', 'count': int(r[1] or 0)} for r in ao_rows]
|
||
|
||
# action_plans status 分布（30d）
|
||
ap_rows = session.execute(
|
||
sa_text(f"""
|
||
SELECT status, plan_type, COUNT(*) AS cnt
|
||
FROM action_plans
|
||
WHERE created_at >= NOW() - INTERVAL '{int(days)} days'
|
||
GROUP BY status, plan_type ORDER BY cnt DESC
|
||
"""),
|
||
).fetchall()
|
||
action_plans_status = [
|
||
{'status': r[0], 'plan_type': r[1] or 'misc', 'count': int(r[2] or 0)}
|
||
for r in ap_rows
|
||
]
|
||
|
||
# rag_query_log 整體 feedback 分布（不只 caller-level，整體）
|
||
rag_dist_rows = session.execute(
|
||
sa_text(f"""
|
||
SELECT feedback_score, COUNT(*) AS cnt
|
||
FROM rag_query_log
|
||
WHERE queried_at >= NOW() - INTERVAL '{int(days)} days'
|
||
AND feedback_score IS NOT NULL
|
||
GROUP BY feedback_score ORDER BY feedback_score
|
||
"""),
|
||
).fetchall()
|
||
rag_overall_dist = [{'score': int(r[0] or 0), 'count': int(r[1] or 0)} for r in rag_dist_rows]
|
||
finally:
|
||
session.close()
|
||
except Exception:
|
||
pass
|
||
|
||
return render_template(
|
||
'admin/quality_trend.html',
|
||
active_page='obs_quality_trend',
|
||
days=days,
|
||
trends=[(c, info) for c, info in sorted_trends],
|
||
recommendations=recommendations,
|
||
episode_distribution=episode_distribution,
|
||
rag_root_causes=rag_root_causes,
|
||
action_outcomes_stats=action_outcomes_stats,
|
||
action_plans_status=action_plans_status,
|
||
rag_overall_dist=rag_overall_dist,
|
||
error=None,
|
||
)
|
||
except Exception as e:
|
||
return render_template(
|
||
'admin/quality_trend.html',
|
||
active_page='obs_quality_trend',
|
||
days=days, trends=[], recommendations=[],
|
||
episode_distribution={}, rag_root_causes=[],
|
||
action_outcomes_stats=[], action_plans_status=[], rag_overall_dist=[],
|
||
error='AI 品質資料暫時不可用，已切換安全空狀態。',
|
||
)
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# /observability/budget — Phase 29 預算管理 + 手動 throttle
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
@admin_observability_bp.route('/budget')
|
||
@login_required
|
||
def budget_dashboard():
|
||
"""ai_call_budgets 編輯 + 當月 spent 即時對比"""
|
||
from datetime import datetime as _dt
|
||
today = _dt.now()
|
||
month_start = _dt(today.year, today.month, 1)
|
||
|
||
session = get_session()
|
||
try:
|
||
ai_call_budgets_exists = bool(session.execute(
|
||
sa_text("SELECT to_regclass('public.ai_call_budgets') IS NOT NULL")
|
||
).scalar())
|
||
if ai_call_budgets_exists:
|
||
budgets = session.execute(
|
||
sa_text("""
|
||
SELECT id, period, provider, budget_usd, alert_pct, updated_at
|
||
FROM ai_call_budgets
|
||
ORDER BY period, provider NULLS FIRST
|
||
"""),
|
||
).fetchall()
|
||
else:
|
||
budgets = []
|
||
|
||
spent_rows = session.execute(
|
||
sa_text("""
|
||
SELECT provider, COALESCE(SUM(cost_usd), 0) AS spent
|
||
FROM ai_calls
|
||
WHERE called_at >= :ms
|
||
GROUP BY provider
|
||
"""),
|
||
{'ms': month_start},
|
||
).fetchall()
|
||
spent_map = {r[0]: float(r[1] or 0) for r in spent_rows}
|
||
|
||
# throttle 狀態
|
||
throttle_state = {}
|
||
try:
|
||
from services.cost_throttle_service import get_throttle_state
|
||
throttle_state = get_throttle_state()
|
||
except Exception:
|
||
pass
|
||
|
||
rows = []
|
||
for b in budgets:
|
||
provider = b[2] # 可能 None（全供應商總額）
|
||
spent = spent_map.get(provider, 0.0) if provider else sum(spent_map.values())
|
||
budget_usd = float(b[3] or 0)
|
||
ratio = (spent / budget_usd) if budget_usd > 0 else 0
|
||
rows.append({
|
||
'id': b[0], 'period': b[1], 'provider': provider or '(all)',
|
||
'budget_usd': budget_usd, 'alert_pct': int(b[4] or 80),
|
||
'spent': spent, 'ratio': ratio,
|
||
'throttled': throttle_state.get(provider, {}).get('throttled', False) if provider else False,
|
||
'updated_at': b[5].strftime('%Y-%m-%d %H:%M') if b[5] else '-',
|
||
})
|
||
|
||
# Phase 47 K-3: 30d daily cost trend by provider
|
||
cost_30d = session.execute(
|
||
sa_text("""
|
||
SELECT date_trunc('day', called_at)::date AS d,
|
||
provider, COALESCE(SUM(cost_usd), 0) AS cost
|
||
FROM ai_calls
|
||
WHERE called_at >= NOW() - INTERVAL '30 days'
|
||
GROUP BY d, provider
|
||
ORDER BY d DESC, cost DESC
|
||
"""),
|
||
).fetchall()
|
||
cost_trend_30d = []
|
||
for r in cost_30d:
|
||
cost_trend_30d.append({
|
||
'date': r[0].strftime('%m-%d') if r[0] else '',
|
||
'provider': r[1],
|
||
'cost': float(r[2] or 0),
|
||
})
|
||
|
||
# Phase 55 S-3: 當月各 provider cost 分布（給圓餅圖用）
|
||
provider_cost_month = session.execute(
|
||
sa_text("""
|
||
SELECT provider, COALESCE(SUM(cost_usd), 0) AS cost
|
||
FROM ai_calls
|
||
WHERE called_at >= :ms AND cost_usd > 0
|
||
GROUP BY provider ORDER BY cost DESC
|
||
"""),
|
||
{'ms': month_start},
|
||
).fetchall()
|
||
|
||
# Phase 47 K-3: top 5 cost-burning caller (當月)
|
||
top_cost_callers = session.execute(
|
||
sa_text("""
|
||
SELECT caller, COUNT(*) AS calls,
|
||
COALESCE(SUM(cost_usd), 0) AS cost,
|
||
COALESCE(SUM(input_tokens + output_tokens), 0) AS tokens
|
||
FROM ai_calls
|
||
WHERE called_at >= :ms
|
||
AND cost_usd > 0
|
||
GROUP BY caller
|
||
ORDER BY cost DESC LIMIT 5
|
||
"""),
|
||
{'ms': month_start},
|
||
).fetchall()
|
||
|
||
# Phase 47 K-3: ai_price_recommendations 7d 統計
|
||
price_rec_7d = session.execute(
|
||
sa_text("""
|
||
SELECT strategy, COUNT(*) AS cnt,
|
||
COALESCE(AVG(confidence), 0) AS avg_conf
|
||
FROM ai_price_recommendations
|
||
WHERE created_at >= NOW() - INTERVAL '7 days'
|
||
GROUP BY strategy ORDER BY cnt DESC
|
||
"""),
|
||
).fetchall()
|
||
|
||
# Phase 39 D-4: RAG 自動建議策略（針對超 80% 的 row）
|
||
budget_strategies = []
|
||
over_threshold_rows = [r for r in rows if r.get('ratio', 0) >= 0.8]
|
||
if over_threshold_rows:
|
||
try:
|
||
from services.rag_service import rag_service
|
||
top_breach = max(over_threshold_rows, key=lambda r: r.get('ratio', 0))
|
||
query_text = (
|
||
f"預算超出 alert_pct provider={top_breach['provider']} "
|
||
f"ratio={int(top_breach['ratio']*100)}% 應採取什麼節流策略"
|
||
)
|
||
rag_result = rag_service.query(
|
||
text=query_text,
|
||
caller='admin_budget_dashboard',
|
||
top_k=3,
|
||
threshold=0.65,
|
||
)
|
||
budget_strategies = [
|
||
{
|
||
'id': h.get('id'),
|
||
'insight_type': h.get('insight_type'),
|
||
'content': (h.get('content') or '')[:240],
|
||
'similarity': round(float(h.get('similarity', 0)), 3),
|
||
}
|
||
for h in rag_result.hits[:3]
|
||
]
|
||
except Exception:
|
||
pass
|
||
|
||
return render_template(
|
||
'admin/budget.html',
|
||
active_page='obs_budget',
|
||
rows=rows,
|
||
budget_strategies=budget_strategies,
|
||
cost_trend_30d=cost_trend_30d,
|
||
top_cost_callers=[
|
||
{
|
||
'caller': r[0], 'calls': int(r[1] or 0),
|
||
'cost': float(r[2] or 0), 'tokens': int(r[3] or 0),
|
||
}
|
||
for r in top_cost_callers
|
||
],
|
||
provider_cost_month=[
|
||
{'provider': r[0], 'cost': float(r[1] or 0)}
|
||
for r in provider_cost_month
|
||
],
|
||
price_rec_7d=[
|
||
{
|
||
'strategy': r[0], 'count': int(r[1] or 0),
|
||
'avg_confidence': round(float(r[2] or 0), 3),
|
||
}
|
||
for r in price_rec_7d
|
||
],
|
||
error=None,
|
||
)
|
||
except Exception as e:
|
||
return render_template('admin/budget.html', active_page='obs_budget', rows=[],
|
||
budget_strategies=[], cost_trend_30d=[],
|
||
top_cost_callers=[], price_rec_7d=[],
|
||
provider_cost_month=[],
|
||
error='預算資料暫時不可用，已切換安全空狀態。')
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
@admin_observability_bp.route('/ai_calls/trigger_code_review', methods=['POST'])
|
||
@login_required
|
||
def ai_calls_trigger_code_review():
|
||
"""Phase 40 D-7 (L2 自動化)：對高錯誤率時段觸發程式碼審查流程。
|
||
|
||
用途：admin 在觀測台看到某 caller 錯誤率飆高時，一鍵觸發 5-step
|
||
pipeline (read→hermes_scan→openclaw_summary→ea_decision→nemoton_act)
|
||
在 daemon thread 自動審查最近 commit 變更檔案，找出可能的回歸問題。
|
||
"""
|
||
try:
|
||
import subprocess
|
||
import threading
|
||
from services.code_review_pipeline_service import CodeReviewPipeline
|
||
|
||
# 取最新 commit + 變更檔案
|
||
commit_sha = subprocess.check_output(
|
||
['git', 'rev-parse', 'HEAD'], stderr=subprocess.DEVNULL,
|
||
).decode().strip()
|
||
changed = subprocess.check_output(
|
||
['git', 'diff-tree', '--no-commit-id', '--name-only', '-r', '-m', commit_sha],
|
||
stderr=subprocess.DEVNULL,
|
||
).decode().strip().split('\n')
|
||
changed = list(dict.fromkeys(f for f in changed if f))
|
||
|
||
if not changed:
|
||
return jsonify({'ok': False, 'error': '最新 commit 無變更檔案'}), 503
|
||
|
||
pipeline = CodeReviewPipeline(
|
||
commit_sha=commit_sha,
|
||
changed_files=changed,
|
||
branch='main',
|
||
deploy_type='manual_observability',
|
||
)
|
||
threading.Thread(target=pipeline.run, daemon=True).start()
|
||
return jsonify({
|
||
'ok': True,
|
||
'pipeline_id': pipeline.pipeline_id,
|
||
'commit_sha': commit_sha[:8],
|
||
'changed_files_count': len(changed),
|
||
'message': f'已觸發程式碼審查流程（流程編號：{pipeline.pipeline_id}）在背景執行，'
|
||
f'5 個步驟完成後會推 Telegram 通知。',
|
||
})
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500
|
||
|
||
|
||
@admin_observability_bp.route('/ppt_audit/trigger_aider_heal', methods=['POST'])
|
||
@login_required
|
||
def ppt_audit_trigger_aider_heal():
|
||
"""Phase 40 D-8 (L2 自動化)：對失敗 PPT audit 觸發 AiderHeal 修 generator。
|
||
|
||
用途：admin 在觀測台看到 PPT vision audit 連續失敗時，一鍵觸發 AiderHeal
|
||
自動修 services/ppt_generator.py（或對應 template generator），
|
||
結果會 git push 到 main 觸發 CD 自動部署。
|
||
"""
|
||
try:
|
||
from services import aider_heal_executor
|
||
|
||
data = request.get_json(silent=True) or {}
|
||
error_msg = (data.get('error_msg') or '').strip()
|
||
issue_summary = (data.get('issue_summary') or '').strip()
|
||
pptx_filename = (data.get('pptx_filename') or '').strip()
|
||
diagnosis = error_msg or issue_summary
|
||
if not diagnosis:
|
||
return jsonify({'ok': False, 'error': '需提供 error_msg 或 issue_summary'}), 400
|
||
|
||
error_message = diagnosis[:500]
|
||
if pptx_filename:
|
||
error_message = f"PPT 視覺審核失敗：{pptx_filename}\n診斷：{error_message}"
|
||
|
||
# 構造 context 給 AiderHeal
|
||
context = {
|
||
'error_type': 'ppt_vision_audit_failure',
|
||
'error_message': error_message[:500],
|
||
'target_file': 'services/ppt_generator.py',
|
||
'pptx_filename': pptx_filename,
|
||
'triggered_by': 'admin_observability',
|
||
'issue_summary': issue_summary[:500],
|
||
}
|
||
heal_key = pptx_filename or diagnosis[:160] or 'manual'
|
||
queued_at = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||
active_job = {
|
||
'key': heal_key,
|
||
'pptx_filename': pptx_filename,
|
||
'target_file': 'services/ppt_generator.py',
|
||
'queued_at': queued_at,
|
||
'diagnosis': _public_ppt_text(diagnosis, max_chars=120),
|
||
}
|
||
with _PPT_AIDER_HEAL_LOCK:
|
||
if heal_key in _PPT_AIDER_HEAL_ACTIVE:
|
||
existing_job = dict(_PPT_AIDER_HEAL_ACTIVE.get(heal_key) or active_job)
|
||
existing_job['diagnosis'] = _public_ppt_text(existing_job.get('diagnosis'), max_chars=120)
|
||
return jsonify({
|
||
'ok': True,
|
||
'status': 'already_running',
|
||
'action': 'CODE_FIX',
|
||
'message': '這份簡報的修復流程已在背景執行中，請等通知結果回報。',
|
||
'target_file': 'services/ppt_generator.py',
|
||
'active_count': len(_PPT_AIDER_HEAL_ACTIVE),
|
||
'job': existing_job,
|
||
}), 202
|
||
_PPT_AIDER_HEAL_ACTIVE[heal_key] = active_job
|
||
|
||
def _heal_worker():
|
||
try:
|
||
result = aider_heal_executor.execute_code_fix(
|
||
error_type='ppt_vision_audit_failure',
|
||
error_message=error_message,
|
||
target_file='services/ppt_generator.py',
|
||
context=context,
|
||
)
|
||
logger.info(
|
||
"[PPTAudit] AiderHeal 背景任務完成 | file=%s | ok=%s | message=%s",
|
||
pptx_filename or '-',
|
||
bool(result.get('success')),
|
||
(result.get('message') or '')[:160],
|
||
)
|
||
except Exception:
|
||
logger.exception(
|
||
"[PPTAudit] AiderHeal 背景任務失敗 | file=%s",
|
||
pptx_filename or '-',
|
||
)
|
||
finally:
|
||
with _PPT_AIDER_HEAL_LOCK:
|
||
_PPT_AIDER_HEAL_ACTIVE.pop(heal_key, None)
|
||
|
||
thread_key = ''.join(ch for ch in pptx_filename if ch.isalnum())[:24] or 'manual'
|
||
threading.Thread(
|
||
target=_heal_worker,
|
||
daemon=True,
|
||
name=f"ppt-aider-heal-{thread_key}",
|
||
).start()
|
||
|
||
return jsonify({
|
||
'ok': True,
|
||
'status': 'queued',
|
||
'action': 'CODE_FIX',
|
||
'message': '修復流程已排入背景執行；完成後會由通知結果回報。',
|
||
'target_file': 'services/ppt_generator.py',
|
||
'active_count': len(_list_ppt_aider_heal_active_jobs()),
|
||
'job': active_job,
|
||
}), 202
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500
|
||
|
||
|
||
@admin_observability_bp.route('/ppt_audit/aider_heal_status')
|
||
@login_required
|
||
def ppt_audit_aider_heal_status():
|
||
jobs = _list_ppt_aider_heal_active_jobs()
|
||
return jsonify({
|
||
'ok': True,
|
||
'active_count': len(jobs),
|
||
'jobs': jobs,
|
||
})
|
||
|
||
|
||
@admin_observability_bp.route('/ppt_audit/generate_missing', methods=['POST'])
|
||
@login_required
|
||
def ppt_audit_generate_missing():
|
||
"""補齊 PPT audit 頁定義中的簡報產出。
|
||
|
||
這是非阻塞入口：Web 頁面只負責排入背景 thread，真正的產生流程共用
|
||
Telegram/OpenClaw 既有 generator 與 cache key。
|
||
"""
|
||
try:
|
||
from services.ppt_auto_generation_service import start_defined_ppt_generation_background
|
||
|
||
data = request.get_json(silent=True) or {}
|
||
report_types = data.get('report_types') or None
|
||
force = bool(data.get('force'))
|
||
result = start_defined_ppt_generation_background(
|
||
report_types=report_types,
|
||
schedule_kind='manual',
|
||
force=force,
|
||
)
|
||
return jsonify(result), 202 if result.get('status') == 'queued' else 200
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500
|
||
|
||
|
||
@admin_observability_bp.route('/ppt_audit/run_vision', methods=['POST'])
|
||
@login_required
|
||
def ppt_audit_run_vision():
|
||
"""Queue a non-blocking visual QA run for selected generated PPT files."""
|
||
try:
|
||
from services.ppt_vision_service import start_ppt_vision_audit_background
|
||
|
||
data = request.get_json(silent=True) or {}
|
||
filenames = data.get('filenames') or []
|
||
if isinstance(filenames, str):
|
||
filenames = [filenames]
|
||
filenames = [str(name) for name in filenames if str(name).lower().endswith('.pptx')]
|
||
max_files = data.get('max_files') or (len(filenames) if filenames else 10)
|
||
try:
|
||
max_files = max(1, min(int(max_files), 20))
|
||
except Exception:
|
||
max_files = 10
|
||
|
||
result = start_ppt_vision_audit_background(
|
||
reports_dir=None,
|
||
filenames=filenames,
|
||
max_files=max_files,
|
||
hours=24,
|
||
)
|
||
return jsonify(result), 202 if result.get('status') == 'queued' else 200
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500
|
||
|
||
|
||
@admin_observability_bp.route('/ppt_audit/vision_status')
|
||
@login_required
|
||
def ppt_audit_vision_status():
|
||
"""Expose current/last background PPT vision audit status for the admin UI."""
|
||
try:
|
||
from services.ppt_vision_service import get_ppt_vision_audit_status
|
||
|
||
return jsonify(get_ppt_vision_audit_status())
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500
|
||
|
||
|
||
def _resolve_ppt_report_path(filename: str):
|
||
"""在 REPORTS_DIR 內解析簡報檔名，並阻擋路徑逃逸。"""
|
||
import os
|
||
from utils.security import safe_join
|
||
|
||
reports_dir = os.environ.get('REPORTS_DIR', '/app/data/reports')
|
||
safe_path = safe_join(reports_dir, filename)
|
||
if not safe_path.exists() or not safe_path.is_file():
|
||
return None, ('檔案不存在', 404)
|
||
if safe_path.suffix.lower() != '.pptx':
|
||
return None, ('不支援的檔案格式', 400)
|
||
return safe_path, None
|
||
|
||
|
||
def _validate_pptx_for_preview(safe_path):
|
||
import zipfile
|
||
|
||
try:
|
||
with zipfile.ZipFile(safe_path, 'r') as zf:
|
||
bad = zf.testzip()
|
||
if bad is not None:
|
||
return f'PPT 檔案損毀，無法預覽（損毀區段：{bad}）'
|
||
except zipfile.BadZipFile:
|
||
return 'PPT 檔案損毀，無法預覽（非有效 zip）'
|
||
except Exception as e:
|
||
return f'預覽檢查失敗：{type(e).__name__}'
|
||
return None
|
||
|
||
|
||
@admin_observability_bp.route('/ppt_audit_file/<path:filename>/prewarm', methods=['POST'])
|
||
@login_required
|
||
def ppt_audit_file_prewarm(filename: str):
|
||
"""建立單一 PPT 的 PDF 預覽快取，並回傳 JSON 狀態。"""
|
||
try:
|
||
safe_path, error_response = _resolve_ppt_report_path(filename)
|
||
if error_response:
|
||
message, status_code = error_response
|
||
return jsonify({'ok': False, 'error': message}), status_code
|
||
|
||
validation_error = _validate_pptx_for_preview(safe_path)
|
||
if validation_error:
|
||
return jsonify({'ok': False, 'error': validation_error}), 409
|
||
|
||
from services.ppt_preview_service import build_ppt_preview
|
||
|
||
preview = build_ppt_preview(safe_path)
|
||
if not preview.ok or not preview.pdf_path:
|
||
return jsonify({'ok': False, 'error': preview.error or '無法產生預覽'}), 409
|
||
|
||
return jsonify({
|
||
'ok': True,
|
||
'filename': safe_path.name,
|
||
'cache_hit': bool(preview.cache_hit),
|
||
'converter': preview.converter,
|
||
'message': 'PDF 預覽快取已建立' if not preview.cache_hit else 'PDF 預覽快取已存在',
|
||
})
|
||
except ValueError:
|
||
return jsonify({'ok': False, 'error': '非法路徑'}), 400
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500
|
||
|
||
|
||
@admin_observability_bp.route('/ppt_audit_file/<path:filename>')
|
||
@login_required
|
||
def ppt_audit_file(filename: str):
|
||
"""提供觀測台簡報檔案預覽/下載。
|
||
|
||
- action=view 開啟預覽（預設）
|
||
- action=pdf 產生/回傳線上預覽 PDF
|
||
- action=download 直接下載
|
||
"""
|
||
action = (request.args.get('action', 'view') or 'view').strip().lower()
|
||
try:
|
||
safe_path, error_response = _resolve_ppt_report_path(filename)
|
||
if error_response:
|
||
message, status_code = error_response
|
||
return message, status_code
|
||
|
||
if action in ('view', 'pdf'):
|
||
validation_error = _validate_pptx_for_preview(safe_path)
|
||
if validation_error:
|
||
return validation_error, 409
|
||
|
||
if action in ('view', 'pdf'):
|
||
from services.ppt_preview_service import build_ppt_preview
|
||
|
||
preview = build_ppt_preview(safe_path)
|
||
if action == 'pdf':
|
||
if not preview.ok or not preview.pdf_path:
|
||
return preview.error or '無法產生預覽', 409
|
||
return send_file(
|
||
preview.pdf_path,
|
||
mimetype='application/pdf',
|
||
as_attachment=False,
|
||
download_name=f'{safe_path.stem}.pdf',
|
||
)
|
||
|
||
return render_template(
|
||
'admin/ppt_audit_preview.html',
|
||
active_page='obs_ppt_audit',
|
||
filename=safe_path.name,
|
||
file_size_kb=round(safe_path.stat().st_size / 1024, 1),
|
||
file_mtime=datetime.fromtimestamp(safe_path.stat().st_mtime).strftime('%Y-%m-%d %H:%M'),
|
||
preview=preview,
|
||
pdf_url=url_for('admin_observability.ppt_audit_file', filename=safe_path.name, action='pdf'),
|
||
download_url=url_for('admin_observability.ppt_audit_file', filename=safe_path.name, action='download'),
|
||
back_url=url_for('admin_observability.ppt_audit_history'),
|
||
)
|
||
|
||
return send_file(
|
||
str(safe_path),
|
||
mimetype='application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||
as_attachment=(action == 'download'),
|
||
download_name=safe_path.name,
|
||
)
|
||
except ValueError:
|
||
return '非法路徑', 400
|
||
except Exception as e:
|
||
return f'{type(e).__name__}: {str(e)[:200]}', 500
|
||
|
||
|
||
@admin_observability_bp.route('/api/health_indicator')
|
||
@login_required
|
||
def health_indicator_api():
|
||
"""Phase 52 P-1：給 topbar 觀測台 indicator 用的輕量 JSON API。
|
||
|
||
回傳當前是否有「需要關注」的事件：
|
||
- 三主機掛掉
|
||
- 待審 episode > 0
|
||
- 過去 1h 錯誤率 ≥ 30%
|
||
- 預算 ≥ 90%
|
||
"""
|
||
try:
|
||
now_ts = time.time()
|
||
with _HEALTH_INDICATOR_CACHE_LOCK:
|
||
cached_payload = _HEALTH_INDICATOR_CACHE.get('payload')
|
||
if cached_payload and now_ts < float(_HEALTH_INDICATOR_CACHE.get('expires_at') or 0):
|
||
return jsonify(dict(cached_payload))
|
||
|
||
session = get_session()
|
||
try:
|
||
# 三主機最新狀態
|
||
host_unhealthy = 0
|
||
try:
|
||
rows = session.execute(
|
||
sa_text("""
|
||
WITH latest AS (
|
||
SELECT host_label,
|
||
FIRST_VALUE(healthy) OVER (
|
||
PARTITION BY host_label ORDER BY probed_at DESC
|
||
) AS healthy
|
||
FROM host_health_probes
|
||
WHERE probed_at >= NOW() - INTERVAL '1 hour'
|
||
)
|
||
SELECT host_label, BOOL_AND(NOT healthy) AS down
|
||
FROM latest
|
||
GROUP BY host_label
|
||
"""),
|
||
).fetchall()
|
||
host_unhealthy = sum(1 for r in rows if r[1])
|
||
except Exception:
|
||
pass
|
||
|
||
# 待審 episode
|
||
ep_pending = 0
|
||
try:
|
||
ep_pending = int(session.execute(
|
||
sa_text("SELECT COUNT(*) FROM learning_episodes WHERE promotion_status = 'awaiting_review' AND reviewed_at IS NULL"),
|
||
).fetchone()[0] or 0)
|
||
except Exception:
|
||
pass
|
||
|
||
# 1h 錯誤率
|
||
error_rate = 0
|
||
try:
|
||
row = session.execute(
|
||
sa_text("""
|
||
SELECT COUNT(*),
|
||
COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only'))
|
||
FROM ai_calls WHERE called_at >= NOW() - INTERVAL '1 hour'
|
||
"""),
|
||
).fetchone()
|
||
total = int(row[0] or 0)
|
||
errs = int(row[1] or 0)
|
||
error_rate = (errs / total * 100) if total > 20 else 0
|
||
except Exception:
|
||
pass
|
||
|
||
# 預算告警（任一 ≥ 90%）
|
||
budget_alert = False
|
||
try:
|
||
from datetime import datetime as _dt
|
||
today = _dt.now()
|
||
ms = _dt(today.year, today.month, 1)
|
||
bgs = session.execute(
|
||
sa_text("""
|
||
SELECT b.budget_usd,
|
||
COALESCE((SELECT SUM(cost_usd) FROM ai_calls
|
||
WHERE called_at >= :ms
|
||
AND (b.provider IS NULL OR provider = b.provider)), 0) AS spent
|
||
FROM ai_call_budgets b
|
||
"""),
|
||
{'ms': ms},
|
||
).fetchall()
|
||
for budget, spent in bgs:
|
||
if budget and float(budget) > 0 and float(spent) / float(budget) >= 0.9:
|
||
budget_alert = True
|
||
break
|
||
except Exception:
|
||
pass
|
||
|
||
alert_count = (
|
||
host_unhealthy
|
||
+ (1 if ep_pending > 0 else 0)
|
||
+ (1 if error_rate >= 30 else 0)
|
||
+ (1 if budget_alert else 0)
|
||
)
|
||
payload = {
|
||
'ok': True,
|
||
'alert_count': alert_count,
|
||
'host_unhealthy': host_unhealthy,
|
||
'ep_pending': ep_pending,
|
||
'error_rate_high': error_rate >= 30,
|
||
'budget_alert': budget_alert,
|
||
'tooltip': _build_indicator_tooltip(host_unhealthy, ep_pending, error_rate, budget_alert),
|
||
}
|
||
with _HEALTH_INDICATOR_CACHE_LOCK:
|
||
_HEALTH_INDICATOR_CACHE['payload'] = dict(payload)
|
||
_HEALTH_INDICATOR_CACHE['expires_at'] = time.time() + _HEALTH_INDICATOR_CACHE_TTL_SECONDS
|
||
return jsonify(payload)
|
||
finally:
|
||
session.close()
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500
|
||
|
||
|
||
def _build_indicator_tooltip(host_unhealthy, ep_pending, error_rate, budget_alert) -> str:
|
||
parts = []
|
||
if host_unhealthy:
|
||
parts.append(f"{host_unhealthy} 主機異常")
|
||
if ep_pending > 0:
|
||
parts.append(f"{ep_pending} 待審")
|
||
if error_rate >= 30:
|
||
parts.append(f"錯誤率 {error_rate:.0f}%")
|
||
if budget_alert:
|
||
parts.append("預算 ≥ 90%")
|
||
if not parts:
|
||
return "AI 觀測台（一切正常）"
|
||
return "AI 觀測台 — " + " / ".join(parts)
|
||
|
||
|
||
def _latest_host_probe_unhealthy(host_label: str, window_minutes: int = 30) -> bool:
|
||
"""查 DB 最新 host_health_probe，作為 AutoHeal 按鈕的真實狀態來源。
|
||
|
||
`_is_unhealthy()` 只代表 Ollama client 在 30 秒 TTL 內的記憶體標記；
|
||
scheduler / 頁面 probe 寫入的是 `host_health_probes`。L2 AutoHeal 入口
|
||
必須接受 DB 最新探針異常，避免 Telegram 或 Web 顯示主機已掛、按鈕卻拒絕執行。
|
||
"""
|
||
if not host_label:
|
||
return False
|
||
session = get_session()
|
||
try:
|
||
row = session.execute(
|
||
sa_text("""
|
||
SELECT healthy
|
||
FROM host_health_probes
|
||
WHERE host_label = :label
|
||
AND probed_at >= NOW() - (:minutes || ' minutes')::interval
|
||
ORDER BY probed_at DESC
|
||
LIMIT 1
|
||
"""),
|
||
{'label': host_label, 'minutes': int(window_minutes)},
|
||
).fetchone()
|
||
return bool(row is not None and row[0] is False)
|
||
except Exception:
|
||
return False
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
@admin_observability_bp.route('/playbooks/toggle/<int:playbook_id>', methods=['POST'])
|
||
@login_required
|
||
def playbook_toggle(playbook_id: int):
|
||
"""Phase 50 N-3：一鍵啟用/停用 playbook（is_active 翻轉）。
|
||
|
||
用途：在 host_health 觀測台直接管理 AutoHeal playbook，
|
||
不需 SSH 188 改 DB。
|
||
"""
|
||
try:
|
||
session = get_session()
|
||
try:
|
||
row = session.execute(
|
||
sa_text("SELECT id, name, is_active FROM playbooks WHERE id = :id"),
|
||
{'id': playbook_id},
|
||
).fetchone()
|
||
if not row:
|
||
return jsonify({'ok': False, 'error': f'playbook #{playbook_id} 不存在'}), 404
|
||
new_active = not bool(row[2])
|
||
session.execute(
|
||
sa_text("UPDATE playbooks SET is_active = :a, updated_at = NOW() WHERE id = :id"),
|
||
{'a': new_active, 'id': playbook_id},
|
||
)
|
||
session.commit()
|
||
return jsonify({
|
||
'ok': True,
|
||
'playbook_id': playbook_id,
|
||
'name': row[1],
|
||
'is_active': new_active,
|
||
'message': f'Playbook 「{row[1]}」已{"啟用" if new_active else "停用"}',
|
||
})
|
||
finally:
|
||
session.close()
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500
|
||
|
||
|
||
@admin_observability_bp.route('/host_health/trigger_autoheal', methods=['POST'])
|
||
@login_required
|
||
def host_health_trigger_autoheal():
|
||
"""Phase 40 D-9 (L2 自動化)：對掛掉的主機觸發 AutoHeal playbook。
|
||
|
||
用途：admin 看到某台 Ollama 主機標記 unhealthy 時一鍵觸發 AutoHeal
|
||
(ADR-013) 跑對應 playbook（DOCKER_RESTART / SSH_CMD / ALERT_ONLY）。
|
||
|
||
安全：只能對已標記 unhealthy 的 host 觸發；不接受任意 host URL（防 SSRF）。
|
||
"""
|
||
try:
|
||
data = request.json or {}
|
||
host_label = (data.get('host_label') or '').strip()
|
||
from services.auto_heal_service import auto_heal_service
|
||
from services.ollama_service import _is_unhealthy, OLLAMA_HOST_PRIMARY, OLLAMA_HOST_SECONDARY, OLLAMA_HOST_FALLBACK
|
||
|
||
# 白名單對應
|
||
host_map = {
|
||
'Primary (GCP)': OLLAMA_HOST_PRIMARY,
|
||
'Secondary (GCP)': OLLAMA_HOST_SECONDARY,
|
||
'Fallback (111)': OLLAMA_HOST_FALLBACK,
|
||
}
|
||
host_url = host_map.get(host_label)
|
||
if not host_url:
|
||
return jsonify({'ok': False, 'error': f'未知 host_label: {host_label}'}), 400
|
||
|
||
if not (_is_unhealthy(host_url) or _latest_host_probe_unhealthy(host_label)):
|
||
return jsonify({
|
||
'ok': False,
|
||
'error': f'{host_label} 目前未標記異常，無需 AutoHeal',
|
||
}), 400
|
||
|
||
result = auto_heal_service.handle_exception(
|
||
error_type='ollama_unhealthy',
|
||
context={
|
||
'host_label': host_label,
|
||
'host_url': host_url,
|
||
'error_message': f'Ollama host {host_label} ({host_url}) marked unhealthy',
|
||
'triggered_by': 'admin_observability',
|
||
},
|
||
)
|
||
return jsonify({
|
||
'ok': bool(getattr(result, 'success', False)),
|
||
'action': getattr(result, 'action', None),
|
||
'message': getattr(result, 'message', '') or 'AutoHeal 已派遣',
|
||
})
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500
|
||
|
||
|
||
@admin_observability_bp.route('/budget/force_throttle', methods=['POST'])
|
||
@login_required
|
||
def budget_force_throttle():
|
||
"""Phase 39 D-4 (L2 自動化)：立即強制執行 cost_throttle evaluate（不等 hourly cron）。
|
||
|
||
用途：admin 在觀測台看到 ratio 飆超 110% 時不需等下次 cron，
|
||
直接點按鈕強制 re-evaluate 三主機 throttle 狀態（claude→gemini fallback 立即生效）。
|
||
"""
|
||
try:
|
||
from services.cost_throttle_service import (
|
||
evaluate_throttle_status, is_cost_throttle_enabled,
|
||
)
|
||
if not is_cost_throttle_enabled():
|
||
return jsonify({
|
||
'ok': False,
|
||
'error': 'COST_THROTTLE_ENABLED=false（先設環境變數）',
|
||
}), 400
|
||
new_state = evaluate_throttle_status()
|
||
throttled = [p for p, s in new_state.items() if s.get('throttled')]
|
||
return jsonify({
|
||
'ok': True,
|
||
'throttled_providers': throttled,
|
||
'state': new_state,
|
||
'message': f'已立即重算 throttle 狀態，被節流的 provider：{throttled or "（無）"}',
|
||
})
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500
|
||
|
||
|
||
@admin_observability_bp.route('/budget/update/<int:budget_id>', methods=['POST'])
|
||
@login_required
|
||
def budget_update(budget_id: int):
|
||
"""更新 budget_usd / alert_pct"""
|
||
try:
|
||
new_budget = float(request.json.get('budget_usd'))
|
||
new_alert = int(request.json.get('alert_pct', 80))
|
||
if new_budget <= 0 or not (1 <= new_alert <= 100):
|
||
return jsonify({'ok': False, 'error': 'invalid range'}), 400
|
||
|
||
session = get_session()
|
||
try:
|
||
session.execute(
|
||
sa_text("""
|
||
UPDATE ai_call_budgets
|
||
SET budget_usd = :b, alert_pct = :a, updated_at = NOW()
|
||
WHERE id = :id
|
||
"""),
|
||
{'b': new_budget, 'a': new_alert, 'id': budget_id},
|
||
)
|
||
session.commit()
|
||
return jsonify({'ok': True})
|
||
finally:
|
||
session.close()
|
||
except Exception as e:
|
||
return jsonify({'ok': False, 'error': str(e)[:200]}), 500
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# /observability/ppt_audit_history — Phase 29 PPT 視覺審核歷史
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
def _guess_ppt_report_type_from_filename(filename: str) -> str:
|
||
"""從產出的 PPT 檔名推回報表類型，供 QA 失敗重跑與 triage 使用。"""
|
||
name = str(filename or "")
|
||
if not name:
|
||
return ""
|
||
try:
|
||
from services.ppt_auto_generation_service import REPORT_PREFIXES
|
||
|
||
for report_type, prefix in REPORT_PREFIXES.items():
|
||
if prefix and name.startswith(prefix):
|
||
return report_type
|
||
except Exception:
|
||
pass
|
||
|
||
fallback_prefixes = {
|
||
"ocbot_daily_": "daily",
|
||
"ocbot_weekly_": "weekly",
|
||
"ocbot_monthly_": "monthly",
|
||
"ocbot_quarterly_": "quarterly",
|
||
"ocbot_half_yearly_": "half_yearly",
|
||
"ocbot_annual_": "annual",
|
||
"ocbot_ttm_": "ttm",
|
||
"ocbot_strategy_": "strategy",
|
||
"ocbot_competitor_v4_": "competitor_v4",
|
||
"ocbot_competitor_": "competitor",
|
||
"ocbot_promo_compare_": "promo_compare",
|
||
"ocbot_promo_": "promo",
|
||
"ocbot_forecast_pre_event_": "forecast_pre_event",
|
||
"ocbot_vendor_": "vendor",
|
||
"ocbot_category_": "category",
|
||
"ocbot_customer_": "customer",
|
||
"ocbot_new_product_": "new_product",
|
||
"ocbot_market_intel_": "market_intel",
|
||
"ocbot_price_elasticity_": "price_elasticity",
|
||
}
|
||
for prefix, report_type in fallback_prefixes.items():
|
||
if name.startswith(prefix):
|
||
return report_type
|
||
return ""
|
||
|
||
|
||
def _ppt_filename_matches_month(filename: str, *, year: int, month: int) -> bool:
|
||
"""判斷檔名是否明確帶有指定月份，用於補足歷史檔案 mtime 漂移。"""
|
||
name = str(filename or "")
|
||
for match in re.finditer(r"(?<!\d)(20\d{2})(0[1-9]|1[0-2])([0-3]\d)(?!\d)", name):
|
||
try:
|
||
parsed = datetime(int(match.group(1)), int(match.group(2)), int(match.group(3)))
|
||
except ValueError:
|
||
continue
|
||
if parsed.year == year and parsed.month == month:
|
||
return True
|
||
for match in re.finditer(r"(?<!\d)(20\d{2})(0[1-9]|1[0-2])(?!\d)", name):
|
||
if int(match.group(1)) == year and int(match.group(2)) == month:
|
||
return True
|
||
return False
|
||
|
||
|
||
def _build_ppt_pipeline_view(files, auto_generation, audit_stats, generation_runs, vision_status, audit_records=None):
|
||
"""Compose page-level PPT pipeline health so the template stays declarative."""
|
||
files = files or []
|
||
auto_generation = auto_generation or {}
|
||
audit_stats = audit_stats or {}
|
||
generation_runs = generation_runs or []
|
||
vision_status = vision_status or {}
|
||
audit_records = audit_records or []
|
||
|
||
def _as_int(value):
|
||
try:
|
||
return int(value or 0)
|
||
except Exception:
|
||
return 0
|
||
|
||
def _as_float(value):
|
||
try:
|
||
return float(value or 0)
|
||
except Exception:
|
||
return 0.0
|
||
|
||
ready_count = _as_int(auto_generation.get('ready_count'))
|
||
total_count = _as_int(auto_generation.get('total'))
|
||
missing_count = _as_int(auto_generation.get('missing_count'))
|
||
coverage_pct = round((ready_count / total_count * 100), 1) if total_count else 0
|
||
valid_preview_count = sum(1 for item in files if item.get('file_exists') and item.get('is_valid_ppt'))
|
||
cached_preview_count = sum(
|
||
1 for item in files
|
||
if item.get('file_exists') and item.get('is_valid_ppt') and item.get('preview_cache_ready')
|
||
)
|
||
uncached_preview_count = max(valid_preview_count - cached_preview_count, 0)
|
||
broken_file_count = sum(1 for item in files if item.get('file_exists') and not item.get('is_valid_ppt'))
|
||
db_backed_count = sum(1 for item in files if item.get('source') in ('database', 'both'))
|
||
run_error_count = sum(1 for item in generation_runs if item.get('status') == 'error')
|
||
run_ready_count = sum(1 for item in generation_runs if item.get('status') == 'ready')
|
||
audit_total = _as_int(audit_stats.get('total'))
|
||
audit_issues = _as_int(audit_stats.get('total_issues'))
|
||
pass_rate = round(_as_float(audit_stats.get('pass_rate')), 1)
|
||
latest_run = generation_runs[0] if generation_runs else {}
|
||
latest_file = files[0] if files else {}
|
||
|
||
if not vision_status.get('ready'):
|
||
health_status = 'partial'
|
||
health_title = '視覺審核環境待確認'
|
||
health_message = 'PPT 可產出與預覽，但視覺檢查與轉檔條件仍需維持就緒。'
|
||
elif run_error_count or broken_file_count:
|
||
health_status = 'error'
|
||
health_title = '產線有異常待處理'
|
||
health_message = f'目前有 {run_error_count} 筆產出失敗、{broken_file_count} 份檔案不可預覽，應先處理最近錯誤。'
|
||
elif missing_count > 0:
|
||
health_status = 'partial'
|
||
health_title = '定義簡報尚未全數補齊'
|
||
health_message = f'本月已完成 {ready_count}/{total_count} 類，仍缺 {missing_count} 類，可等排程或手動補齊。'
|
||
elif audit_total and pass_rate < 80:
|
||
health_status = 'partial'
|
||
health_title = '審核通過率偏低'
|
||
health_message = f'本月視覺 QA 通過率 {pass_rate:.1f}%，需優先檢查失敗熱點與修復建議。'
|
||
elif total_count:
|
||
health_status = 'ready'
|
||
health_title = '產線覆蓋完整'
|
||
health_message = '定義簡報、產出紀錄、線上預覽與視覺 QA 都已具備可追蹤入口。'
|
||
else:
|
||
health_status = 'planned'
|
||
health_title = '產線等待資料'
|
||
health_message = '目前尚未讀到定義簡報覆蓋資料，頁面會保留安全空狀態。'
|
||
|
||
if audit_total:
|
||
qa_value = f'{pass_rate:.0f}%'
|
||
qa_meta = f'{audit_total} 筆審核 / {audit_issues} 個問題'
|
||
qa_status = 'ready' if pass_rate >= 80 and audit_issues == 0 else 'partial'
|
||
else:
|
||
qa_value = '待審核'
|
||
qa_meta = '可立即補跑，或等待 22:00 排程'
|
||
qa_status = 'planned'
|
||
|
||
stages = [
|
||
{
|
||
'key': 'schedule',
|
||
'icon': 'calendar-check',
|
||
'label': '排程節奏',
|
||
'value': '6 條',
|
||
'meta': '每日 / 每週 / 每月 / 每季 / 半年 / 年度',
|
||
'detail': auto_generation.get('cadence_summary') or '等待排程設定',
|
||
'status': 'ready' if auto_generation.get('enabled') else 'partial',
|
||
},
|
||
{
|
||
'key': 'coverage',
|
||
'icon': 'diagram-project',
|
||
'label': '定義覆蓋',
|
||
'value': f'{ready_count}/{total_count}' if total_count else '—',
|
||
'meta': f'{coverage_pct:.1f}% 完成',
|
||
'detail': f'缺漏 {missing_count} 類' if missing_count else '當期目標完整',
|
||
'status': 'ready' if total_count and missing_count == 0 else 'partial',
|
||
},
|
||
{
|
||
'key': 'records',
|
||
'icon': 'database',
|
||
'label': '產出紀錄',
|
||
'value': f'{len(generation_runs)} 筆',
|
||
'meta': f'{run_ready_count} 成功 / {run_error_count} 失敗',
|
||
'detail': latest_run.get('started_at') or '尚無本月產出紀錄',
|
||
'status': 'error' if run_error_count else ('ready' if generation_runs else 'planned'),
|
||
},
|
||
{
|
||
'key': 'preview',
|
||
'icon': 'desktop',
|
||
'label': '線上預覽',
|
||
'value': f'{valid_preview_count} 份',
|
||
'meta': f'{cached_preview_count} 份 PDF 快取',
|
||
'detail': latest_file.get('name') or '尚無可預覽檔案',
|
||
'status': 'error' if broken_file_count else ('ready' if valid_preview_count else 'planned'),
|
||
},
|
||
{
|
||
'key': 'qa',
|
||
'icon': 'eye',
|
||
'label': '視覺 QA',
|
||
'value': qa_value,
|
||
'meta': qa_meta,
|
||
'detail': '視覺檢查 + 修復建議 + 派工',
|
||
'status': qa_status,
|
||
},
|
||
]
|
||
missing_items = [
|
||
item for item in auto_generation.get('items', [])
|
||
if not item.get('ready')
|
||
]
|
||
preview_items = [
|
||
item for item in files
|
||
if item.get('file_exists') and item.get('is_valid_ppt')
|
||
]
|
||
audit_attention = [
|
||
item for item in audit_records
|
||
if item.get('audit_status') in ('failed', 'error')
|
||
]
|
||
run_failures = [
|
||
item for item in generation_runs
|
||
if item.get('status') == 'error'
|
||
]
|
||
broken_files = [
|
||
item for item in files
|
||
if item.get('file_exists') and not item.get('is_valid_ppt')
|
||
]
|
||
triage_entries = []
|
||
for item in run_failures[:3]:
|
||
triage_entries.append({
|
||
'title': item.get('report_label') or item.get('report_type') or '未知簡報',
|
||
'meta': item.get('started_at') or '時間未知',
|
||
'detail': _public_ppt_text(
|
||
item.get('error_msg'),
|
||
empty=f"{item.get('schedule_label') or '手動'} · {item.get('target_label') or '最新資料'}",
|
||
),
|
||
'status_label': '產出失敗',
|
||
'filename': item.get('file_name') or '',
|
||
'report_type': item.get('report_type') or '',
|
||
'can_regenerate': bool(item.get('report_type')),
|
||
})
|
||
for item in broken_files[:3]:
|
||
triage_entries.append({
|
||
'title': item.get('name') or '未命名檔案',
|
||
'meta': item.get('mtime') or '時間未知',
|
||
'detail': _public_ppt_text(
|
||
item.get('file_error'),
|
||
empty='PPTX 檔案不可預覽，建議重新產生。',
|
||
),
|
||
'status_label': '檔案異常',
|
||
'filename': item.get('name') or '',
|
||
'report_type': item.get('report_type') or '',
|
||
'can_regenerate': bool(item.get('report_type')),
|
||
})
|
||
for item in audit_attention[:3]:
|
||
filename = item.get('pptx_filename') or ''
|
||
inferred_report_type = item.get('report_type') or _guess_ppt_report_type_from_filename(filename)
|
||
triage_entries.append({
|
||
'title': filename or '未命名檔案',
|
||
'meta': item.get('audited_at') or '時間未知',
|
||
'detail': _public_ppt_text(
|
||
item.get('issue_summary') or item.get('error_msg'),
|
||
empty=f"問題 {item.get('issues_count', 0)} 個",
|
||
),
|
||
'status_label': '視覺 QA',
|
||
'filename': filename,
|
||
'report_type': inferred_report_type,
|
||
'can_regenerate': bool(inferred_report_type),
|
||
})
|
||
|
||
action_lanes = [
|
||
{
|
||
'key': 'triage',
|
||
'label': '異常優先',
|
||
'status': 'error' if triage_entries else 'ready',
|
||
'count': len(triage_entries),
|
||
'empty_text': '目前沒有產出失敗、檔案異常或視覺 QA 失敗。',
|
||
'entries': triage_entries[:4],
|
||
},
|
||
{
|
||
'key': 'missing',
|
||
'label': '待補齊',
|
||
'status': 'partial' if missing_items else 'ready',
|
||
'count': len(missing_items),
|
||
'empty_text': '目前定義簡報都已對齊本期目標。',
|
||
'entries': [
|
||
{
|
||
'title': item.get('label') or item.get('key') or '未命名簡報',
|
||
'meta': item.get('target_label') or '最新資料',
|
||
'detail': item.get('status_hint') or item.get('status_label') or '等待排程補齊',
|
||
'status_label': item.get('status_label') or '待補齊',
|
||
'report_type': item.get('key') or '',
|
||
'can_regenerate': bool(item.get('key')),
|
||
}
|
||
for item in missing_items[:4]
|
||
],
|
||
},
|
||
{
|
||
'key': 'preview',
|
||
'label': '可預覽',
|
||
'status': 'ready' if preview_items else 'planned',
|
||
'count': len(preview_items),
|
||
'empty_text': '目前沒有可線上預覽的 PPTX 檔案。',
|
||
'entries': [
|
||
{
|
||
'title': item.get('name') or '未命名檔案',
|
||
'meta': item.get('mtime') or '時間未知',
|
||
'detail': (
|
||
f"{item.get('size_kb') if item.get('size_kb') is not None else '—'} KB · "
|
||
f"{item.get('source_label') or _public_ppt_source_label(item.get('source'))} · "
|
||
f"{'PDF 已快取' if item.get('preview_cache_ready') else '開啟時轉檔'}"
|
||
),
|
||
'status_label': 'PDF 快取' if item.get('preview_cache_ready') else '線上預覽',
|
||
'filename': item.get('name'),
|
||
}
|
||
for item in preview_items[:4]
|
||
],
|
||
},
|
||
{
|
||
'key': 'audit',
|
||
'label': '視覺 QA',
|
||
'status': 'error' if audit_attention else ('ready' if audit_total else 'planned'),
|
||
'count': len(audit_attention) if audit_attention else audit_total,
|
||
'empty_text': '目前沒有需要處理的視覺 QA 失敗紀錄。',
|
||
'entries': [
|
||
{
|
||
'title': item.get('pptx_filename') or '未命名檔案',
|
||
'meta': item.get('audited_at') or '時間未知',
|
||
'detail': _public_ppt_text(
|
||
item.get('issue_summary') or item.get('error_msg'),
|
||
empty=f"問題 {item.get('issues_count', 0)} 個，信心 {item.get('confidence', 0):.2f}",
|
||
),
|
||
'status_label': '需修復' if item.get('audit_status') == 'failed' else '需排查',
|
||
'filename': item.get('pptx_filename'),
|
||
'report_type': item.get('report_type') or _guess_ppt_report_type_from_filename(item.get('pptx_filename') or ''),
|
||
'can_regenerate': bool(item.get('report_type') or _guess_ppt_report_type_from_filename(item.get('pptx_filename') or '')),
|
||
}
|
||
for item in audit_attention[:4]
|
||
],
|
||
},
|
||
{
|
||
'key': 'records',
|
||
'label': '產出紀錄',
|
||
'status': 'error' if run_error_count else ('ready' if generation_runs else 'planned'),
|
||
'count': len(generation_runs),
|
||
'empty_text': '本月尚未看到簡報產出紀錄。',
|
||
'entries': [
|
||
{
|
||
'title': item.get('report_label') or item.get('report_type') or '未知簡報',
|
||
'meta': item.get('started_at') or '時間未知',
|
||
'detail': f"{item.get('schedule_label') or '手動'} · {item.get('target_label') or '最新資料'}",
|
||
'status_label': item.get('status_label') or item.get('status') or '未知',
|
||
'filename': item.get('file_name') or '',
|
||
}
|
||
for item in generation_runs[:4]
|
||
],
|
||
},
|
||
]
|
||
|
||
return {
|
||
'status': health_status,
|
||
'title': health_title,
|
||
'message': health_message,
|
||
'ready_count': ready_count,
|
||
'total_count': total_count,
|
||
'missing_count': missing_count,
|
||
'coverage_pct': coverage_pct,
|
||
'valid_preview_count': valid_preview_count,
|
||
'cached_preview_count': cached_preview_count,
|
||
'uncached_preview_count': uncached_preview_count,
|
||
'broken_file_count': broken_file_count,
|
||
'db_backed_count': db_backed_count,
|
||
'run_error_count': run_error_count,
|
||
'pass_rate': pass_rate,
|
||
'audit_total': audit_total,
|
||
'latest_run': latest_run,
|
||
'latest_file': latest_file,
|
||
'stages': stages,
|
||
'action_lanes': action_lanes,
|
||
}
|
||
|
||
|
||
def _build_ppt_operator_summary(files, auto_generation, pipeline_view, vision_status, audit_stats, generation_runs):
|
||
"""Build first-screen operator copy that prioritizes deck work over raw pipeline states."""
|
||
files = files or []
|
||
auto_generation = auto_generation or {}
|
||
pipeline_view = pipeline_view or {}
|
||
vision_status = vision_status or {}
|
||
audit_stats = audit_stats or {}
|
||
generation_runs = generation_runs or []
|
||
|
||
latest_preview = next(
|
||
(
|
||
item for item in files
|
||
if item.get('file_exists') and item.get('is_valid_ppt') and item.get('name')
|
||
),
|
||
None,
|
||
)
|
||
issue_count = int(audit_stats.get('total_issues') or 0) if audit_stats else 0
|
||
missing_count = int(auto_generation.get('missing_count') or 0)
|
||
valid_preview_count = int(pipeline_view.get('valid_preview_count') or 0)
|
||
cached_preview_count = int(pipeline_view.get('cached_preview_count') or 0)
|
||
audit_total = int(pipeline_view.get('audit_total') or 0)
|
||
run_error_count = int(pipeline_view.get('run_error_count') or 0)
|
||
broken_file_count = int(pipeline_view.get('broken_file_count') or 0)
|
||
blockers = vision_status.get('blockers') or []
|
||
|
||
if run_error_count or broken_file_count:
|
||
status = 'error'
|
||
headline = '先處理異常，再放行簡報'
|
||
message = f'目前有 {run_error_count} 筆產出失敗、{broken_file_count} 份檔案不可預覽，建議先看 Action Queue。'
|
||
primary_action = '查看待處理'
|
||
primary_anchor = '#ppt-action-queue'
|
||
elif missing_count:
|
||
status = 'partial'
|
||
headline = '定期簡報尚未全數補齊'
|
||
message = f'本期還有 {missing_count} 類定義簡報缺漏，可手動補齊或等待排程寫入產出紀錄。'
|
||
primary_action = '補齊缺漏'
|
||
primary_anchor = '#ppt-production-center'
|
||
elif not vision_status.get('ready'):
|
||
status = 'partial'
|
||
headline = '簡報可管理，視覺 QA 待啟用'
|
||
message = 'PPT 產出與預覽入口仍可用；視覺檢查與轉檔條件補齊後才會自動審核。'
|
||
primary_action = '查看就緒檢查'
|
||
primary_anchor = '#ppt-runtime-diagnostic'
|
||
elif issue_count:
|
||
status = 'partial'
|
||
headline = '有視覺問題待回放'
|
||
message = f'本期視覺 QA 發現 {issue_count} 個問題，請從問題追蹤或審核歷史回放檢查。'
|
||
primary_action = '查看問題'
|
||
primary_anchor = '#ppt-issue-board'
|
||
else:
|
||
status = 'ready' if valid_preview_count else 'planned'
|
||
headline = '簡報工作台待命'
|
||
message = '最新簡報、PDF 預覽、產出紀錄與視覺 QA 都集中在同一頁追蹤。'
|
||
primary_action = '查看簡報'
|
||
primary_anchor = '#ppt-deck-workbench'
|
||
|
||
latest_run = generation_runs[0] if generation_runs else {}
|
||
latest_deck_label = latest_preview.get('name') if latest_preview else '尚無可預覽 PPTX'
|
||
latest_deck_meta = (
|
||
f"{latest_preview.get('mtime') or '時間未知'} · "
|
||
f"{latest_preview.get('size_kb') if latest_preview.get('size_kb') is not None else '—'} KB · "
|
||
f"{'PDF 已快取' if latest_preview and latest_preview.get('preview_cache_ready') else '首次開啟轉檔'}"
|
||
if latest_preview else
|
||
'請先補齊本期簡報或切換月份 / 報表類型'
|
||
)
|
||
|
||
return {
|
||
'status': status,
|
||
'headline': headline,
|
||
'message': message,
|
||
'primary_action': primary_action,
|
||
'primary_anchor': primary_anchor,
|
||
'latest_deck': latest_preview or {},
|
||
'latest_deck_label': latest_deck_label,
|
||
'latest_deck_meta': latest_deck_meta,
|
||
'latest_run_label': latest_run.get('report_label') or latest_run.get('report_type') or '尚無產出紀錄',
|
||
'latest_run_meta': latest_run.get('started_at') or '等待下一次排程寫入',
|
||
'blocker_text': '；'.join(_public_ppt_text_list(blockers[:2])) if blockers else '',
|
||
'signals': [
|
||
{
|
||
'label': '可預覽簡報',
|
||
'value': valid_preview_count,
|
||
'meta': f'{cached_preview_count} 份 PDF 快取',
|
||
'status': 'ready' if valid_preview_count else 'planned',
|
||
},
|
||
{
|
||
'label': '待補齊定義',
|
||
'value': missing_count,
|
||
'meta': f"{auto_generation.get('ready_count', 0)}/{auto_generation.get('total', 0)} 已覆蓋",
|
||
'status': 'ready' if missing_count == 0 and auto_generation.get('total') else 'partial',
|
||
},
|
||
{
|
||
'label': '視覺 QA',
|
||
'value': audit_total if audit_total else '待跑',
|
||
'meta': '已就緒' if vision_status.get('ready') else '執行條件待確認',
|
||
'status': 'ready' if vision_status.get('ready') and not issue_count else 'partial',
|
||
},
|
||
{
|
||
'label': '視覺問題',
|
||
'value': issue_count,
|
||
'meta': '需回放' if issue_count else '目前無待處理',
|
||
'status': 'partial' if issue_count else 'ready',
|
||
},
|
||
],
|
||
}
|
||
|
||
|
||
def _enrich_ppt_coverage_items(auto_generation_items, files, generation_runs, audit_records):
|
||
"""Join coverage rows with file, DB run, preview and QA state for the UI matrix."""
|
||
files = files or []
|
||
generation_runs = generation_runs or []
|
||
audit_records = audit_records or []
|
||
|
||
file_by_name = {item.get('name'): item for item in files if item.get('name')}
|
||
latest_file_by_type = {}
|
||
for item in files:
|
||
report_type = item.get('report_type')
|
||
if report_type and report_type not in latest_file_by_type:
|
||
latest_file_by_type[report_type] = item
|
||
|
||
latest_run_by_type = {}
|
||
for item in generation_runs:
|
||
report_type = item.get('report_type')
|
||
if report_type and report_type not in latest_run_by_type:
|
||
latest_run_by_type[report_type] = item
|
||
|
||
latest_audit_by_file = {}
|
||
for item in audit_records:
|
||
filename = item.get('pptx_filename')
|
||
if filename and filename not in latest_audit_by_file:
|
||
latest_audit_by_file[filename] = item
|
||
|
||
enriched = []
|
||
for raw_item in auto_generation_items or []:
|
||
item = dict(raw_item)
|
||
report_type = item.get('key') or ''
|
||
latest_run = latest_run_by_type.get(report_type, {})
|
||
candidate_file = latest_file_by_type.get(report_type, {})
|
||
file_name = (
|
||
item.get('latest_file_name')
|
||
or latest_run.get('file_name')
|
||
or candidate_file.get('name')
|
||
or ''
|
||
)
|
||
file_item = file_by_name.get(file_name) or candidate_file or {}
|
||
audit = latest_audit_by_file.get(file_name, {})
|
||
sources = set(item.get('sources') or [])
|
||
if file_item.get('source'):
|
||
sources.add(file_item.get('source'))
|
||
|
||
file_exists = bool(file_item.get('file_exists') or ('filesystem' in sources and file_name))
|
||
valid_ppt = bool(file_exists and (file_item.get('is_valid_ppt') is not False))
|
||
db_backed = bool(
|
||
latest_run
|
||
or 'database' in sources
|
||
or file_item.get('source') in ('database', 'both')
|
||
)
|
||
preview_cached = bool(valid_ppt and file_item.get('preview_cache_ready'))
|
||
audit_status = audit.get('audit_status') or ''
|
||
run_status = latest_run.get('status') or ''
|
||
|
||
if latest_run and run_status == 'error':
|
||
db_status, db_label = 'error', '紀錄失敗'
|
||
elif db_backed:
|
||
db_status, db_label = 'ready', '已記錄'
|
||
else:
|
||
db_status, db_label = 'planned', '待紀錄'
|
||
|
||
if valid_ppt and preview_cached:
|
||
preview_status, preview_label = 'ready', 'PDF 快取'
|
||
elif valid_ppt:
|
||
preview_status, preview_label = 'partial', '可預覽'
|
||
elif file_name:
|
||
preview_status, preview_label = 'error', '不可預覽'
|
||
else:
|
||
preview_status, preview_label = 'planned', '待產檔'
|
||
|
||
if audit_status == 'passed':
|
||
qa_status, qa_label = 'ready', 'QA 通過'
|
||
elif audit_status == 'failed':
|
||
qa_status, qa_label = 'error', 'QA 有問題'
|
||
elif audit_status == 'error':
|
||
qa_status, qa_label = 'error', 'QA 錯誤'
|
||
elif audit_status == 'skipped':
|
||
qa_status, qa_label = 'partial', 'QA 跳過'
|
||
elif valid_ppt:
|
||
qa_status, qa_label = 'planned', '待 QA'
|
||
else:
|
||
qa_status, qa_label = 'planned', '待產檔'
|
||
|
||
if not item.get('ready'):
|
||
delivery_status, delivery_label = 'missing', '待產出'
|
||
elif latest_run and run_status == 'error':
|
||
delivery_status, delivery_label = 'error', '產出失敗'
|
||
elif file_name and not valid_ppt:
|
||
delivery_status, delivery_label = 'error', '檔案異常'
|
||
elif qa_status == 'error':
|
||
delivery_status, delivery_label = 'error', '需修復'
|
||
elif valid_ppt and db_backed and audit_status == 'passed':
|
||
delivery_status, delivery_label = 'ready', '可交付'
|
||
elif valid_ppt:
|
||
delivery_status, delivery_label = 'partial', '待驗收'
|
||
else:
|
||
delivery_status, delivery_label = item.get('status') or 'planned', item.get('status_label') or '待確認'
|
||
|
||
item.update({
|
||
'latest_file_name': file_name,
|
||
'latest_file_mtime': file_item.get('mtime') or item.get('latest_generated_at') or '',
|
||
'latest_file_size_kb': file_item.get('size_kb'),
|
||
'file_exists': file_exists,
|
||
'is_valid_ppt': valid_ppt,
|
||
'preview_cache_ready': preview_cached,
|
||
'db_status': db_status,
|
||
'db_label': db_label,
|
||
'preview_status': preview_status,
|
||
'preview_label': preview_label,
|
||
'qa_status': qa_status,
|
||
'qa_label': qa_label,
|
||
'delivery_status': delivery_status,
|
||
'delivery_label': delivery_label,
|
||
'audit_summary': _public_ppt_text(
|
||
audit.get('issue_summary') or audit.get('error_msg'),
|
||
max_chars=160,
|
||
),
|
||
'can_preview': valid_ppt and bool(file_name),
|
||
'can_prewarm': valid_ppt and bool(file_name) and not preview_cached,
|
||
'can_regenerate': bool(report_type),
|
||
})
|
||
enriched.append(item)
|
||
|
||
return enriched
|
||
|
||
|
||
@admin_observability_bp.route('/ppt_audit_history')
|
||
@login_required
|
||
def ppt_audit_history():
|
||
"""掃 reports/ 目錄列指定月份 daily 報表 + 從 ppt_audit_results 讀審核歷史（Phase 38）"""
|
||
import os
|
||
import zipfile
|
||
reports_dir = os.environ.get('REPORTS_DIR', '/app/data/reports')
|
||
files = []
|
||
audit_records = []
|
||
error = None
|
||
month_arg = request.args.get('month', '').strip()
|
||
report_type = request.args.get('report_type', 'daily').strip().lower() or 'daily'
|
||
try:
|
||
from services.ppt_auto_generation_service import get_report_type_options
|
||
report_type_options = get_report_type_options()
|
||
except Exception:
|
||
report_type_options = [
|
||
{'key': 'daily', 'label': '每日日報', 'prefix': 'ocbot_daily_'},
|
||
{'key': 'weekly', 'label': '週報', 'prefix': 'ocbot_weekly_'},
|
||
{'key': 'monthly', 'label': '月報', 'prefix': 'ocbot_monthly_'},
|
||
{'key': 'strategy', 'label': '策略', 'prefix': 'ocbot_strategy_'},
|
||
{'key': 'competitor', 'label': '競品', 'prefix': 'ocbot_competitor_'},
|
||
{'key': 'promo', 'label': '促銷', 'prefix': 'ocbot_promo_'},
|
||
{'key': 'all', 'label': '全部', 'prefix': 'all'},
|
||
]
|
||
report_type_map = {opt['key']: opt for opt in report_type_options}
|
||
if report_type not in report_type_map:
|
||
report_type = 'daily'
|
||
selected_report_type = report_type_map[report_type]
|
||
report_prefix = selected_report_type['prefix']
|
||
|
||
now = datetime.now()
|
||
target_year = now.year
|
||
target_month = now.month
|
||
if month_arg:
|
||
sep = '-' if '-' in month_arg else '/' if '/' in month_arg else None
|
||
parts = month_arg.split(sep) if sep else [month_arg]
|
||
try:
|
||
if len(parts) == 2:
|
||
target_year = int(parts[0])
|
||
target_month = int(parts[1])
|
||
elif len(parts) == 1 and 1 <= len(parts[0]) <= 2:
|
||
target_month = int(parts[0])
|
||
else:
|
||
raise ValueError
|
||
if not (1 <= target_month <= 12):
|
||
raise ValueError
|
||
except Exception:
|
||
target_year = now.year
|
||
target_month = now.month
|
||
|
||
month_start = datetime(target_year, target_month, 1)
|
||
month_end = datetime(target_year + 1, 1, 1) if target_month == 12 else datetime(target_year, target_month + 1, 1)
|
||
month_start_ts = int(month_start.timestamp())
|
||
month_end_ts = int(month_end.timestamp())
|
||
month_label = month_start.strftime('%Y-%m')
|
||
prev_month = target_month - 1
|
||
prev_year = target_year
|
||
if prev_month == 0:
|
||
prev_month = 12
|
||
prev_year -= 1
|
||
next_month = target_month + 1
|
||
next_year = target_year
|
||
if next_month == 13:
|
||
next_month = 1
|
||
next_year += 1
|
||
prev_month_label = f"{prev_year:04d}-{prev_month:02d}"
|
||
next_month_label = f"{next_year:04d}-{next_month:02d}"
|
||
show_next_month = (next_year < now.year) or (next_year == now.year and next_month <= now.month)
|
||
|
||
def _inspect_ppt_file(file_path: str):
|
||
try:
|
||
with zipfile.ZipFile(file_path, 'r') as zf:
|
||
bad = zf.testzip()
|
||
return (bad is None, None if bad is None else f'壓縮檔異常：{bad}')
|
||
except zipfile.BadZipFile:
|
||
return False, 'PPTX 檔案損毀（非有效 zip）'
|
||
except Exception as e:
|
||
return False, f'檢查失敗：{str(e)[:60]}'
|
||
|
||
def _guess_report_type(filename: str) -> str:
|
||
for opt in report_type_options:
|
||
prefix = opt.get('prefix')
|
||
if prefix and prefix != 'all' and filename.startswith(prefix):
|
||
return opt.get('key') or ''
|
||
return report_type if report_type != 'all' else ''
|
||
|
||
def _preview_cache_payload(file_path: str):
|
||
payload = {
|
||
'preview_cache_ready': False,
|
||
'preview_cache_size_kb': None,
|
||
'preview_cache_mtime': '',
|
||
}
|
||
try:
|
||
from services.ppt_preview_service import get_ppt_preview_cache_info
|
||
|
||
info = get_ppt_preview_cache_info(file_path)
|
||
payload['preview_cache_ready'] = bool(info.cache_exists)
|
||
payload['preview_cache_size_kb'] = info.cache_size_kb
|
||
if info.cache_mtime_ts:
|
||
payload['preview_cache_mtime'] = datetime.fromtimestamp(info.cache_mtime_ts).strftime('%Y-%m-%d %H:%M')
|
||
except Exception:
|
||
logger.debug("PPT preview cache state unavailable", exc_info=True)
|
||
return payload
|
||
|
||
try:
|
||
if not os.path.isdir(reports_dir):
|
||
error = f'{reports_dir} 目錄不存在'
|
||
else:
|
||
files_by_name = {}
|
||
for f in os.listdir(reports_dir):
|
||
if not f.lower().endswith('.pptx'):
|
||
continue
|
||
if report_prefix != 'all' and not f.startswith(report_prefix):
|
||
continue
|
||
full = os.path.join(reports_dir, f)
|
||
# symlink 防護：reports/ 內不接受 symlink，避免目錄逃逸（Critic MEDIUM #2）
|
||
if os.path.islink(full):
|
||
continue
|
||
try:
|
||
mtime = os.path.getmtime(full)
|
||
matches_selected_month = (
|
||
month_start_ts <= mtime < month_end_ts
|
||
or _ppt_filename_matches_month(f, year=target_year, month=target_month)
|
||
)
|
||
if matches_selected_month:
|
||
is_valid, check_msg = _inspect_ppt_file(full)
|
||
files_by_name[f] = {
|
||
'source': 'filesystem',
|
||
'name': f,
|
||
'size_kb': round(os.path.getsize(full) / 1024, 1),
|
||
'mtime': datetime.fromtimestamp(mtime).strftime('%Y-%m-%d %H:%M'),
|
||
'mtime_ts': mtime,
|
||
'file_exists': True,
|
||
'file_path': full,
|
||
'report_type': _guess_report_type(f),
|
||
'is_valid_ppt': is_valid,
|
||
'file_error': check_msg,
|
||
**_preview_cache_payload(full),
|
||
}
|
||
except OSError:
|
||
continue
|
||
|
||
# 補充：若 188 主機僅保留 DB 快取紀錄或掃描過程漏掉，仍可回補當月報表清單
|
||
try:
|
||
session = get_session()
|
||
try:
|
||
sql = """
|
||
SELECT report_type, file_path, file_size, generated_at
|
||
FROM ppt_reports
|
||
WHERE generated_at >= :month_start
|
||
AND generated_at < :month_end
|
||
"""
|
||
params = {'month_start': month_start, 'month_end': month_end}
|
||
if report_type != 'all':
|
||
sql += " AND report_type = :report_type"
|
||
params['report_type'] = report_type
|
||
rows = session.execute(sa_text(sql), params).fetchall()
|
||
|
||
for rpt_type, file_path, file_size, generated_at in rows:
|
||
if not file_path:
|
||
continue
|
||
name = os.path.basename(file_path)
|
||
if not name.lower().endswith('.pptx'):
|
||
continue
|
||
if report_prefix != 'all' and not name.startswith(report_prefix):
|
||
continue
|
||
if name in files_by_name:
|
||
files_by_name[name]['source'] = 'both'
|
||
continue
|
||
if not generated_at:
|
||
continue
|
||
mtime = generated_at.timestamp()
|
||
if not (month_start_ts <= mtime < month_end_ts):
|
||
continue
|
||
candidate_path = os.path.join(reports_dir, name)
|
||
exists = os.path.isfile(candidate_path)
|
||
is_valid = False
|
||
check_msg = '檔案未落盤'
|
||
if exists:
|
||
is_valid, check_msg = _inspect_ppt_file(candidate_path)
|
||
files_by_name[name] = {
|
||
'source': 'database',
|
||
'name': name,
|
||
'size_kb': round((file_size or 0) / 1024, 1),
|
||
'mtime': generated_at.strftime('%Y-%m-%d %H:%M'),
|
||
'mtime_ts': mtime,
|
||
'file_exists': exists,
|
||
'file_path': candidate_path if exists else file_path,
|
||
'report_type': rpt_type,
|
||
'is_valid_ppt': is_valid,
|
||
'file_error': None if exists else check_msg,
|
||
**(_preview_cache_payload(candidate_path) if exists else {}),
|
||
}
|
||
finally:
|
||
session.close()
|
||
except Exception:
|
||
pass
|
||
|
||
files = list(files_by_name.values())
|
||
files.sort(key=lambda x: x['mtime_ts'], reverse=True)
|
||
for item in files:
|
||
item['source_label'] = _public_ppt_source_label(item.get('source'))
|
||
item['file_error'] = _public_ppt_text(item.get('file_error'), max_chars=120)
|
||
except Exception as e:
|
||
error = _public_ppt_text(f'{type(e).__name__}: {str(e)[:200]}', empty='簡報清單讀取異常')
|
||
|
||
audit_filter_sql = ""
|
||
audit_params = {'month_start': month_start, 'month_end': month_end}
|
||
if report_prefix != 'all':
|
||
audit_filter_sql = " AND pptx_filename LIKE :audit_prefix"
|
||
audit_params['audit_prefix'] = f"{report_prefix}%"
|
||
|
||
def _summarize_ppt_issues(raw_issues) -> str:
|
||
"""把 ppt_audit_results.issues_found 壓成表格可讀的診斷摘要。"""
|
||
if not raw_issues:
|
||
return ''
|
||
try:
|
||
import json as _json
|
||
issues_payload = _json.loads(raw_issues) if isinstance(raw_issues, str) else raw_issues
|
||
except Exception:
|
||
return ''
|
||
if not isinstance(issues_payload, list):
|
||
return ''
|
||
snippets = []
|
||
for item in issues_payload:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
slide = item.get('slide')
|
||
for issue in item.get('issues') or []:
|
||
text = _public_ppt_text(issue, max_chars=120)
|
||
if not text:
|
||
continue
|
||
prefix = f"S{slide}: " if slide else ""
|
||
snippets.append(f"{prefix}{text}")
|
||
if len(snippets) >= 3:
|
||
return '；'.join(snippets)
|
||
return '；'.join(snippets)
|
||
|
||
def _load_ppt_issues(raw_issues):
|
||
if not raw_issues:
|
||
return []
|
||
try:
|
||
import json as _json
|
||
issues_payload = _json.loads(raw_issues) if isinstance(raw_issues, str) else raw_issues
|
||
except Exception:
|
||
return []
|
||
return issues_payload if isinstance(issues_payload, list) else []
|
||
|
||
def _classify_ppt_issue(issue_text: str):
|
||
text = issue_text or ''
|
||
if _ppt_text_has_internal_detail(text):
|
||
return '審核執行', 'warn'
|
||
if any(k in text for k in ['圖表', '切掉', '截斷', '超出', '溢出']):
|
||
return '版面越界', 'error'
|
||
if any(k in text for k in ['空白', '未填', '缺少', '無資料']):
|
||
return '內容缺漏', 'warn'
|
||
if any(k in text for k in ['低對比', '顏色', '字體', '字型', '閱讀']):
|
||
return '可讀性', 'warn'
|
||
return '視覺問題', 'warn'
|
||
|
||
def _extract_ppt_issue_items(raw_issues, *, pptx_filename: str, audited_at: str):
|
||
issue_items = []
|
||
report_type_for_file = _guess_report_type(pptx_filename)
|
||
for slide_item in _load_ppt_issues(raw_issues):
|
||
if not isinstance(slide_item, dict):
|
||
continue
|
||
slide = slide_item.get('slide')
|
||
slide_label = f"S{slide}" if slide else 'S?'
|
||
for raw_issue in slide_item.get('issues') or []:
|
||
category, status = _classify_ppt_issue(str(raw_issue or ''))
|
||
issue_text = _public_ppt_text(raw_issue, max_chars=140)
|
||
if not issue_text:
|
||
continue
|
||
issue_items.append({
|
||
'pptx_filename': pptx_filename,
|
||
'report_type': report_type_for_file,
|
||
'audited_at': audited_at,
|
||
'slide_label': slide_label,
|
||
'category': category,
|
||
'status': status,
|
||
'text': issue_text,
|
||
})
|
||
return issue_items
|
||
|
||
# Phase 38+：讀指定月份 / 指定簡報類型 audit 歷史
|
||
try:
|
||
session = get_session()
|
||
try:
|
||
audit_rows = session.execute(
|
||
sa_text(f"""
|
||
SELECT audited_at, pptx_filename, audit_status,
|
||
issues_count, confidence, duration_ms, error_msg,
|
||
issues_found
|
||
FROM ppt_audit_results
|
||
WHERE audited_at >= :month_start
|
||
AND audited_at < :month_end
|
||
{audit_filter_sql}
|
||
ORDER BY audited_at DESC
|
||
LIMIT 1000
|
||
"""),
|
||
audit_params,
|
||
).fetchall()
|
||
audit_records = []
|
||
for r in audit_rows:
|
||
audited_at = r[0].strftime('%Y-%m-%d %H:%M')
|
||
pptx_filename = r[1]
|
||
raw_issues = r[7]
|
||
audit_records.append({
|
||
'audited_at': audited_at,
|
||
'pptx_filename': pptx_filename,
|
||
'report_type': _guess_report_type(pptx_filename),
|
||
'audit_status': r[2],
|
||
'issues_count': int(r[3] or 0),
|
||
'confidence': float(r[4] or 0),
|
||
'duration_ms': int(r[5] or 0),
|
||
'error_msg': _public_ppt_text(r[6], max_chars=160),
|
||
'issue_summary': _summarize_ppt_issues(raw_issues),
|
||
'issue_items': _extract_ppt_issue_items(
|
||
raw_issues,
|
||
pptx_filename=pptx_filename,
|
||
audited_at=audited_at,
|
||
),
|
||
})
|
||
finally:
|
||
session.close()
|
||
except Exception:
|
||
logger.debug("PPT audit history table unavailable; rendering empty audit history", exc_info=True)
|
||
|
||
issue_items = [
|
||
issue
|
||
for record in audit_records
|
||
for issue in record.get('issue_items', [])
|
||
]
|
||
issue_files = {issue.get('pptx_filename') for issue in issue_items if issue.get('pptx_filename')}
|
||
issue_digest = {
|
||
'total': len(issue_items),
|
||
'files': len(issue_files),
|
||
'error_count': sum(1 for issue in issue_items if issue.get('status') == 'error'),
|
||
'warn_count': sum(1 for issue in issue_items if issue.get('status') == 'warn'),
|
||
'latest_audit': issue_items[0].get('audited_at') if issue_items else '',
|
||
}
|
||
|
||
# PPT vision 啟用狀態
|
||
vision_status = {
|
||
'enabled': False,
|
||
'ready': False,
|
||
'blockers': ['視覺狀態讀取失敗'],
|
||
'ready_count': 0,
|
||
'check_count': 0,
|
||
'summary': '視覺 QA runtime 狀態讀取失敗。',
|
||
'readiness_checks': [],
|
||
'next_actions': ['確認 ppt_vision_service import 與 runtime 設定後重新整理此頁。'],
|
||
}
|
||
try:
|
||
from services.ppt_vision_service import get_ppt_vision_audit_status, get_ppt_vision_runtime_status
|
||
vision_status = get_ppt_vision_runtime_status()
|
||
vision_enabled = bool(vision_status.get('enabled'))
|
||
vision_audit_status = get_ppt_vision_audit_status()
|
||
except Exception:
|
||
vision_enabled = False
|
||
vision_audit_status = {
|
||
'ok': False,
|
||
'running': False,
|
||
'status': 'unknown',
|
||
'status_label': '讀取失敗',
|
||
'message': '最近視覺 QA 狀態讀取失敗。',
|
||
'last_run': None,
|
||
}
|
||
vision_status = _public_ppt_vision_status(vision_status)
|
||
vision_audit_status = _public_ppt_vision_audit_status(vision_audit_status)
|
||
|
||
# Phase 47 K-6: 月報表統計 + top failure files
|
||
audit_30d_stats = {}
|
||
top_failure_files = []
|
||
try:
|
||
s_ppt = get_session()
|
||
try:
|
||
stat_row = s_ppt.execute(
|
||
sa_text(f"""
|
||
SELECT COUNT(*),
|
||
COUNT(*) FILTER (WHERE audit_status = 'passed'),
|
||
COUNT(*) FILTER (WHERE audit_status = 'failed'),
|
||
COUNT(*) FILTER (WHERE audit_status = 'skipped'),
|
||
COUNT(*) FILTER (WHERE audit_status = 'error'),
|
||
COALESCE(AVG(confidence) FILTER (WHERE audit_status = 'passed'), 0),
|
||
COALESCE(SUM(issues_count), 0)
|
||
FROM ppt_audit_results
|
||
WHERE audited_at >= :month_start
|
||
AND audited_at < :month_end
|
||
{audit_filter_sql}
|
||
"""),
|
||
audit_params,
|
||
).fetchone()
|
||
total_30d = int(stat_row[0] or 0)
|
||
audit_30d_stats = {
|
||
'total': total_30d,
|
||
'passed': int(stat_row[1] or 0),
|
||
'failed': int(stat_row[2] or 0),
|
||
'skipped': int(stat_row[3] or 0),
|
||
'error': int(stat_row[4] or 0),
|
||
'avg_confidence': round(float(stat_row[5] or 0), 3),
|
||
'total_issues': int(stat_row[6] or 0),
|
||
'pass_rate': (float(stat_row[1] or 0) / total_30d * 100) if total_30d else 0,
|
||
}
|
||
|
||
top_fail_rows = s_ppt.execute(
|
||
sa_text(f"""
|
||
SELECT pptx_filename, COUNT(*) AS attempts,
|
||
SUM(issues_count) AS total_issues,
|
||
MAX(audited_at) AS last_audit
|
||
FROM ppt_audit_results
|
||
WHERE audit_status IN ('failed', 'error')
|
||
AND audited_at >= :month_start
|
||
AND audited_at < :month_end
|
||
{audit_filter_sql}
|
||
GROUP BY pptx_filename
|
||
ORDER BY attempts DESC, total_issues DESC LIMIT 10
|
||
"""),
|
||
audit_params,
|
||
).fetchall()
|
||
top_failure_files = [
|
||
{
|
||
'filename': r[0], 'attempts': int(r[1] or 0),
|
||
'total_issues': int(r[2] or 0),
|
||
'last_audit': r[3].strftime('%Y-%m-%d %H:%M') if r[3] else '',
|
||
}
|
||
for r in top_fail_rows
|
||
]
|
||
finally:
|
||
s_ppt.close()
|
||
except Exception:
|
||
pass
|
||
|
||
# Phase 41 E-2: 對最近 3 筆 failed audit 跑 RAG 找相似修法
|
||
rag_fixes = []
|
||
failed_records = [r for r in audit_records if r.get('audit_status') in ('failed', 'error')][:3]
|
||
if failed_records:
|
||
try:
|
||
from services.rag_service import rag_service
|
||
for fr in failed_records:
|
||
try:
|
||
err_text = fr.get('error_msg') or 'PPT vision audit failed'
|
||
rag_result = rag_service.query(
|
||
text=f"PPT 視覺審核失敗 {err_text[:200]} 怎麼修",
|
||
caller='admin_ppt_audit',
|
||
top_k=2,
|
||
threshold=0.6,
|
||
)
|
||
if rag_result.hits:
|
||
rag_fixes.append({
|
||
'pptx_filename': fr.get('pptx_filename'),
|
||
'audited_at': fr.get('audited_at'),
|
||
'error_msg': _public_ppt_text(err_text, max_chars=160),
|
||
'hits': [
|
||
{
|
||
'id': h.get('id'),
|
||
'insight_type': h.get('insight_type'),
|
||
'content': _public_ppt_text(h.get('content'), max_chars=180),
|
||
'similarity': round(float(h.get('similarity', 0)), 3),
|
||
}
|
||
for h in rag_result.hits[:2]
|
||
],
|
||
})
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
|
||
auto_generation = {
|
||
'enabled': False,
|
||
'items': [],
|
||
'missing_report_types': [],
|
||
'missing_count': 0,
|
||
'ready_count': 0,
|
||
'total': 0,
|
||
'last_run': None,
|
||
'can_auto_start': False,
|
||
'cadences': [],
|
||
'cadence_summary': '',
|
||
}
|
||
generation_runs = []
|
||
try:
|
||
from services.ppt_auto_generation_service import (
|
||
get_defined_report_coverage,
|
||
get_generation_run_history,
|
||
get_schedule_cadence_status,
|
||
)
|
||
|
||
auto_generation = get_defined_report_coverage(
|
||
month_start=month_start,
|
||
month_end=month_end,
|
||
reports_dir=reports_dir,
|
||
)
|
||
auto_generation.setdefault('cadences', get_schedule_cadence_status(auto_generation.get('items', [])))
|
||
auto_generation.setdefault(
|
||
'cadence_summary',
|
||
'、'.join(c.get('schedule_text', '') for c in auto_generation.get('cadences', []) if c.get('schedule_text')),
|
||
)
|
||
auto_generation['can_auto_start'] = (
|
||
bool(auto_generation.get('enabled'))
|
||
and int(auto_generation.get('missing_count') or 0) > 0
|
||
and month_label == datetime.now().strftime('%Y-%m')
|
||
)
|
||
generation_runs = get_generation_run_history(
|
||
month_start=month_start,
|
||
month_end=month_end,
|
||
limit=24,
|
||
)
|
||
generation_runs = [
|
||
{
|
||
**dict(item),
|
||
'error_msg': _public_ppt_text(item.get('error_msg'), max_chars=160),
|
||
'status_label': _public_ppt_text(item.get('status_label'), max_chars=60) or item.get('status_label'),
|
||
}
|
||
for item in generation_runs
|
||
]
|
||
except Exception:
|
||
logger.debug("PPT auto-generation coverage unavailable", exc_info=True)
|
||
|
||
pipeline_view = _build_ppt_pipeline_view(
|
||
files=files,
|
||
auto_generation=auto_generation,
|
||
audit_stats=audit_30d_stats,
|
||
generation_runs=generation_runs,
|
||
vision_status=vision_status,
|
||
audit_records=audit_records,
|
||
)
|
||
operator_summary = _build_ppt_operator_summary(
|
||
files=files,
|
||
auto_generation=auto_generation,
|
||
pipeline_view=pipeline_view,
|
||
vision_status=vision_status,
|
||
audit_stats=audit_30d_stats,
|
||
generation_runs=generation_runs,
|
||
)
|
||
vision_audit_filenames = [
|
||
item.get('name')
|
||
for item in files
|
||
if item.get('file_exists') and item.get('is_valid_ppt') and item.get('name')
|
||
][:10]
|
||
aider_heal_active_jobs = _list_ppt_aider_heal_active_jobs()
|
||
auto_generation_items = _enrich_ppt_coverage_items(
|
||
auto_generation.get('items', []),
|
||
files,
|
||
generation_runs,
|
||
audit_records,
|
||
)
|
||
|
||
return render_template(
|
||
'admin/ppt_audit_history.html',
|
||
active_page='obs_ppt_audit',
|
||
report_month=month_label,
|
||
report_type=report_type,
|
||
report_type_options=report_type_options,
|
||
selected_report_type=selected_report_type,
|
||
prev_month_label=prev_month_label,
|
||
next_month_label=next_month_label,
|
||
show_next_month=show_next_month,
|
||
files=files,
|
||
audit_records=audit_records,
|
||
rag_fixes=rag_fixes,
|
||
audit_30d_stats=audit_30d_stats,
|
||
top_failure_files=top_failure_files,
|
||
vision_enabled=vision_enabled,
|
||
vision_status=vision_status,
|
||
vision_audit_status=vision_audit_status,
|
||
auto_generation=auto_generation,
|
||
auto_generation_items=auto_generation_items,
|
||
auto_generation_missing_report_types=auto_generation.get('missing_report_types', []),
|
||
generation_runs=generation_runs,
|
||
pipeline_view=pipeline_view,
|
||
operator_summary=operator_summary,
|
||
vision_audit_filenames=vision_audit_filenames,
|
||
issue_items=issue_items,
|
||
issue_digest=issue_digest,
|
||
aider_heal_active_jobs=aider_heal_active_jobs,
|
||
aider_heal_active_count=len(aider_heal_active_jobs),
|
||
error=error,
|
||
)
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# /observability/host_health — 三主機 + MCP 健康度
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
@admin_observability_bp.route('/host_health')
|
||
@login_required
|
||
def host_health_dashboard():
|
||
"""三主機 Ollama + 4 個 MCP server 即時健康（同時寫入 host_health_probes 留歷史）"""
|
||
import time as _time
|
||
|
||
def _session_uses_sqlite(session) -> bool:
|
||
try:
|
||
bind = session.get_bind()
|
||
return getattr(getattr(bind, 'dialect', None), 'name', None) == 'sqlite'
|
||
except Exception:
|
||
return False
|
||
|
||
ollama_hosts = []
|
||
probe_records = [] # 收集本次 probe 結果以批次寫 DB
|
||
try:
|
||
from services.ollama_service import (
|
||
OLLAMA_HOST_PRIMARY, OLLAMA_HOST_SECONDARY, OLLAMA_HOST_FALLBACK,
|
||
_is_unhealthy, _unhealthy_marks,
|
||
)
|
||
from services.ollama_health_probe import (
|
||
host_health_model_probe_enabled,
|
||
probe_ollama_embedding_runtime,
|
||
)
|
||
import requests as _r
|
||
for label, host in [
|
||
('Primary (GCP)', OLLAMA_HOST_PRIMARY),
|
||
('Secondary (GCP)', OLLAMA_HOST_SECONDARY),
|
||
('Fallback (111)', OLLAMA_HOST_FALLBACK),
|
||
]:
|
||
entry = {'label': label, 'host': host, 'healthy': False,
|
||
'unhealthy_mark': _is_unhealthy(host), 'models': [], 'error': None}
|
||
t0 = _time.monotonic()
|
||
err = None
|
||
try:
|
||
resp = _r.get(f"{host.rstrip('/')}/api/tags", timeout=3)
|
||
if resp.status_code == 200:
|
||
entry['healthy'] = True
|
||
entry['models'] = [
|
||
m.get('name', '') for m in resp.json().get('models', [])
|
||
][:15]
|
||
if host_health_model_probe_enabled(label):
|
||
model_ok, model_err = probe_ollama_embedding_runtime(_r, host)
|
||
if not model_ok:
|
||
entry['healthy'] = False
|
||
err = model_err
|
||
else:
|
||
err = f"HTTP {resp.status_code}"
|
||
except Exception as e:
|
||
err = f"{type(e).__name__}: {str(e)[:200]}"
|
||
entry['error'] = err
|
||
response_ms = int((_time.monotonic() - t0) * 1000)
|
||
probe_records.append({
|
||
'host_label': label, 'host_url': host, 'healthy': entry['healthy'],
|
||
'unhealthy_mark': entry['unhealthy_mark'],
|
||
'models_count': len(entry['models']), 'response_ms': response_ms,
|
||
'error_msg': err,
|
||
})
|
||
ollama_hosts.append(entry)
|
||
except Exception:
|
||
pass
|
||
|
||
# Phase 38：寫入 host_health_probes 留歷史（失敗安全，不擋頁面渲染）
|
||
if probe_records:
|
||
try:
|
||
_session = get_session()
|
||
try:
|
||
if _session_uses_sqlite(_session):
|
||
logger.debug("Skipping host health probe persistence on SQLite")
|
||
else:
|
||
for rec in probe_records:
|
||
_session.execute(
|
||
sa_text("""
|
||
INSERT INTO host_health_probes
|
||
(host_label, host_url, healthy, unhealthy_mark,
|
||
models_count, response_ms, error_msg)
|
||
VALUES
|
||
(:host_label, :host_url, :healthy, :unhealthy_mark,
|
||
:models_count, :response_ms, :error_msg)
|
||
"""),
|
||
rec,
|
||
)
|
||
_session.commit()
|
||
finally:
|
||
_session.close()
|
||
except Exception:
|
||
logger.warning("Failed to persist host health probe records", exc_info=True)
|
||
|
||
# MCP server 健康
|
||
mcp_status = {}
|
||
try:
|
||
from services.mcp_router import mcp_router
|
||
mcp_status = mcp_router.health_check()
|
||
except Exception:
|
||
pass
|
||
|
||
# cost throttle 狀態
|
||
throttle_state = {}
|
||
try:
|
||
from services.cost_throttle_service import get_throttle_state
|
||
throttle_state = get_throttle_state()
|
||
except Exception:
|
||
pass
|
||
|
||
# Phase 38：讀過去 24h 三主機健康歷史（給趨勢卡片）
|
||
health_history = []
|
||
mcp_24h = [] # Phase 39 D-2: MCP 24h 各 server 工作量
|
||
aiops_summary = {} # Phase 39 D-5: incidents + heal_logs 7d 統計
|
||
try:
|
||
_session2 = get_session()
|
||
try:
|
||
history_rows = _session2.execute(
|
||
sa_text("""
|
||
SELECT host_label,
|
||
COUNT(*) FILTER (WHERE healthy) AS up_count,
|
||
COUNT(*) FILTER (WHERE NOT healthy) AS down_count,
|
||
COALESCE(AVG(response_ms) FILTER (WHERE healthy), 0) AS avg_ms,
|
||
COUNT(*) AS total
|
||
FROM host_health_probes
|
||
WHERE probed_at >= NOW() - INTERVAL '24 hours'
|
||
GROUP BY host_label
|
||
ORDER BY host_label
|
||
"""),
|
||
).fetchall()
|
||
health_history = [
|
||
{
|
||
'host_label': r[0],
|
||
'up_count': int(r[1] or 0),
|
||
'down_count': int(r[2] or 0),
|
||
'avg_ms': int(r[3] or 0),
|
||
'total': int(r[4] or 0),
|
||
'uptime_pct': (float(r[1] or 0) / float(r[4]) * 100) if r[4] else 0,
|
||
}
|
||
for r in history_rows
|
||
]
|
||
|
||
# Phase 39 D-5：incidents + heal_logs 過去 7d 統計
|
||
try:
|
||
inc_rows = _session2.execute(
|
||
sa_text("""
|
||
SELECT
|
||
COUNT(*) AS total_incidents,
|
||
COUNT(*) FILTER (WHERE status = 'open') AS open_count,
|
||
COUNT(*) FILTER (WHERE status = 'resolved') AS resolved_count,
|
||
COUNT(*) FILTER (WHERE severity = 'P0') AS p0_count,
|
||
COUNT(*) FILTER (WHERE severity = 'P1') AS p1_count
|
||
FROM incidents
|
||
WHERE created_at >= NOW() - INTERVAL '7 days'
|
||
"""),
|
||
).fetchone()
|
||
heal_rows = _session2.execute(
|
||
sa_text("""
|
||
SELECT
|
||
COUNT(*) AS total_heals,
|
||
COUNT(*) FILTER (WHERE result = 'success') AS heal_success,
|
||
COUNT(*) FILTER (WHERE result = 'failed') AS heal_failed,
|
||
COALESCE(AVG(duration_ms) FILTER (WHERE result = 'success'), 0) AS avg_ms
|
||
FROM heal_logs
|
||
WHERE created_at >= NOW() - INTERVAL '7 days'
|
||
"""),
|
||
).fetchone()
|
||
aiops_summary = {
|
||
'incidents_total': int(inc_rows[0] or 0),
|
||
'incidents_open': int(inc_rows[1] or 0),
|
||
'incidents_resolved': int(inc_rows[2] or 0),
|
||
'incidents_p0': int(inc_rows[3] or 0),
|
||
'incidents_p1': int(inc_rows[4] or 0),
|
||
'heals_total': int(heal_rows[0] or 0),
|
||
'heals_success': int(heal_rows[1] or 0),
|
||
'heals_failed': int(heal_rows[2] or 0),
|
||
'heals_avg_ms': int(heal_rows[3] or 0),
|
||
'heal_success_rate': (
|
||
float(heal_rows[1] or 0) / float(heal_rows[0]) * 100
|
||
) if heal_rows[0] else 0,
|
||
}
|
||
|
||
# Phase 54 R-3: heal 7d daily success rate sparkline
|
||
heal_daily = _session2.execute(
|
||
sa_text("""
|
||
SELECT date_trunc('day', created_at)::date AS d,
|
||
COUNT(*) AS total,
|
||
COUNT(*) FILTER (WHERE result = 'success') AS ok
|
||
FROM heal_logs
|
||
WHERE created_at >= NOW() - INTERVAL '7 days'
|
||
GROUP BY d ORDER BY d ASC
|
||
"""),
|
||
).fetchall()
|
||
aiops_summary['heal_sparkline'] = [
|
||
{
|
||
'date': r[0].strftime('%m-%d') if r[0] else '',
|
||
'total': int(r[1] or 0),
|
||
'ok': int(r[2] or 0),
|
||
'rate': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0,
|
||
}
|
||
for r in heal_daily
|
||
]
|
||
except Exception:
|
||
aiops_summary = {}
|
||
|
||
# Phase 39 D-2：MCP 24h 工作量（每個 server）
|
||
mcp_rows = _session2.execute(
|
||
sa_text("""
|
||
SELECT server,
|
||
COUNT(*) AS total_calls,
|
||
COUNT(*) FILTER (WHERE status = 'ok') AS ok_calls,
|
||
COUNT(*) FILTER (WHERE cache_hit) AS cache_hits,
|
||
COALESCE(SUM(cost_usd), 0) AS total_cost,
|
||
COALESCE(AVG(duration_ms), 0) AS avg_ms,
|
||
COUNT(DISTINCT tool) AS tools_used
|
||
FROM mcp_calls
|
||
WHERE called_at >= NOW() - INTERVAL '24 hours'
|
||
GROUP BY server
|
||
ORDER BY total_calls DESC
|
||
"""),
|
||
).fetchall()
|
||
mcp_24h = [
|
||
{
|
||
'server': r[0],
|
||
'total_calls': int(r[1] or 0),
|
||
'ok_calls': int(r[2] or 0),
|
||
'cache_hits': int(r[3] or 0),
|
||
'total_cost': float(r[4] or 0),
|
||
'avg_ms': int(r[5] or 0),
|
||
'tools_used': int(r[6] or 0),
|
||
'success_rate': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0,
|
||
'cache_rate': (float(r[3] or 0) / float(r[1]) * 100) if r[1] else 0,
|
||
}
|
||
for r in mcp_rows
|
||
]
|
||
finally:
|
||
_session2.close()
|
||
except Exception:
|
||
logger.debug("MCP calls table unavailable; rendering empty MCP 24h summary", exc_info=True)
|
||
|
||
# Phase 47 K-1: incidents + heal_logs 詳細列表 + playbooks 排行 + backup + embed queue
|
||
recent_incidents = []
|
||
recent_heals = []
|
||
playbook_ranking = []
|
||
backup_history = []
|
||
embed_queue_pending = 0
|
||
embed_queue_failed = 0
|
||
try:
|
||
s3 = get_session()
|
||
try:
|
||
inc_rows = s3.execute(
|
||
sa_text("""
|
||
SELECT id, created_at, task_name, error_type, severity,
|
||
status, error_message, retry_count, resolved_at
|
||
FROM incidents
|
||
ORDER BY created_at DESC LIMIT 10
|
||
"""),
|
||
).fetchall()
|
||
recent_incidents = [
|
||
{
|
||
'id': r[0], 'created_at': r[1].strftime('%Y-%m-%d %H:%M'),
|
||
'task_name': r[2], 'error_type': r[3], 'severity': r[4],
|
||
'status': r[5], 'error_message': (r[6] or '')[:200],
|
||
'retry_count': int(r[7] or 0),
|
||
'resolved_at': r[8].strftime('%Y-%m-%d %H:%M') if r[8] else None,
|
||
}
|
||
for r in inc_rows
|
||
]
|
||
heal_rows = s3.execute(
|
||
sa_text("""
|
||
SELECT h.id, h.created_at, h.action_type, h.result,
|
||
h.duration_ms, h.action_detail, h.incident_id,
|
||
i.error_type
|
||
FROM heal_logs h
|
||
LEFT JOIN incidents i ON i.id = h.incident_id
|
||
ORDER BY h.created_at DESC LIMIT 10
|
||
"""),
|
||
).fetchall()
|
||
recent_heals = [
|
||
{
|
||
'id': r[0], 'created_at': r[1].strftime('%Y-%m-%d %H:%M'),
|
||
'action_type': r[2], 'result': r[3],
|
||
'duration_ms': int(r[4] or 0),
|
||
'action_detail': (r[5] or '')[:160],
|
||
'incident_id': r[6], 'error_type': r[7],
|
||
}
|
||
for r in heal_rows
|
||
]
|
||
|
||
# playbooks 庫排行（success_count + fail_count + 是否 active）
|
||
pb_rows = s3.execute(
|
||
sa_text("""
|
||
SELECT id, name, error_type, action_type, severity_min,
|
||
success_count, fail_count, is_active, cooldown_min
|
||
FROM playbooks
|
||
ORDER BY (success_count + fail_count) DESC, success_count DESC
|
||
LIMIT 12
|
||
"""),
|
||
).fetchall()
|
||
playbook_ranking = [
|
||
{
|
||
'id': int(r[0]),
|
||
'name': r[1], 'error_type': r[2], 'action_type': r[3],
|
||
'severity': r[4], 'success': int(r[5] or 0),
|
||
'fail': int(r[6] or 0), 'is_active': bool(r[7]),
|
||
'cooldown_min': int(r[8] or 0),
|
||
'success_rate': (
|
||
float(r[5] or 0) / float((r[5] or 0) + (r[6] or 0)) * 100
|
||
) if ((r[5] or 0) + (r[6] or 0)) > 0 else 0,
|
||
}
|
||
for r in pb_rows
|
||
]
|
||
|
||
# backup_log 7d 歷史
|
||
bk_rows = s3.execute(
|
||
sa_text("""
|
||
SELECT created_at, backup_type, status, file_size_bytes,
|
||
duration_seconds, error_message
|
||
FROM backup_log
|
||
WHERE created_at >= NOW() - INTERVAL '7 days'
|
||
ORDER BY created_at DESC LIMIT 10
|
||
"""),
|
||
).fetchall()
|
||
backup_history = [
|
||
{
|
||
'created_at': r[0].strftime('%Y-%m-%d %H:%M'),
|
||
'backup_type': r[1], 'status': r[2],
|
||
'size_mb': round(float(r[3] or 0) / (1024 * 1024), 1),
|
||
'duration_s': round(float(r[4] or 0), 1),
|
||
'error': (r[5] or '')[:120],
|
||
}
|
||
for r in bk_rows
|
||
]
|
||
|
||
# embedding_retry_queue pending / failed
|
||
embed_q = s3.execute(
|
||
sa_text("""
|
||
SELECT
|
||
COUNT(*) FILTER (WHERE status = 'pending'),
|
||
COUNT(*) FILTER (WHERE status = 'failed')
|
||
FROM embedding_retry_queue
|
||
"""),
|
||
).fetchone()
|
||
embed_queue_pending = int(embed_q[0] or 0)
|
||
embed_queue_failed = int(embed_q[1] or 0)
|
||
finally:
|
||
s3.close()
|
||
except Exception:
|
||
pass
|
||
|
||
return render_template(
|
||
'admin/host_health.html',
|
||
active_page='obs_host_health',
|
||
ollama_hosts=ollama_hosts,
|
||
mcp_status=mcp_status,
|
||
throttle_state=throttle_state,
|
||
health_history=health_history,
|
||
mcp_24h=mcp_24h,
|
||
aiops_summary=aiops_summary,
|
||
recent_incidents=recent_incidents,
|
||
recent_heals=recent_heals,
|
||
playbook_ranking=playbook_ranking,
|
||
backup_history=backup_history,
|
||
embed_queue_pending=embed_queue_pending,
|
||
embed_queue_failed=embed_queue_failed,
|
||
)
|