From e0a8d87c2cc6cd721e0aee7d8c6b835b8adcf528 Mon Sep 17 00:00:00 2001 From: OoO Date: Mon, 4 May 2026 20:09:28 +0800 Subject: [PATCH] =?UTF-8?q?feat(p51):=20RAG=20=E5=8F=AC=E5=9B=9E=E8=A9=B3?= =?UTF-8?q?=E6=83=85=E6=96=B0=E9=A0=81=20+=20overview=20=E4=B8=89=E4=B8=BB?= =?UTF-8?q?=E6=A9=9F=2024h=20sparkline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新頁 /observability/rag_queries：補完 RAG 觀測深度 之前只看 caller 級命中率，現在能看每筆查詢的真實內容。 O-1: route + template - 篩選：時段（1/6/24/72/168h）/ caller / saved_only flag - 整體 KPI 4 卡：總查詢 / 命中率 / saved_call 率 / 反饋平均分 - by caller 表：每個 caller 的查詢/命中/saved/反饋細節 - 最近 50 筆查詢詳情表 - 「查 hits」按鈕 → 彈 modal 載入 ai_insights JOIN 內容預覽 （新 endpoint /observability/rag_queries//hits 回傳 JSON） O-2: 入口 - sidebar AI 觀測 group 加「RAG 召回詳情」（11b） - /observability/overview 入口卡升級為 9 項 O-3: overview 三主機 24h sparkline - 每張主機卡片下方加 60px 高 chart.js sparkline - 折線：每小時 uptime % bucket（0-100% Y 軸隱藏，純視覺） - routes/admin_observability_routes.py::observability_overview 新加 host_sparkline 查詢（GROUP BY host_label, hour） - 三主機卡片視覺化升級：原本只有「100%」字，現在加趨勢線 Phase 38→51 累計 16 commits / 10 觀測頁。 觀測台戰役從「raw stats」到「視覺方格 UI 完整體」。 Co-Authored-By: Claude Opus 4.7 (1M context) --- routes/admin_observability_routes.py | 231 +++++++++++++++++ templates/admin/observability_overview.html | 56 +++- templates/admin/rag_queries.html | 267 ++++++++++++++++++++ templates/components/_ewoooc_shell.html | 5 + 4 files changed, 555 insertions(+), 4 deletions(-) create mode 100644 templates/admin/rag_queries.html diff --git a/routes/admin_observability_routes.py b/routes/admin_observability_routes.py index 9f5979a..b37bcef 100644 --- a/routes/admin_observability_routes.py +++ b/routes/admin_observability_routes.py @@ -239,14 +239,245 @@ def observability_overview(): finally: session.close() + # Phase 51 O-3: 24h 三主機健康 sparkline 資料（每小時 bucket × 3 host） + host_sparkline = {} + try: + s_sp = get_session() + try: + sp_rows = s_sp.execute( + sa_text(""" + SELECT host_label, + date_trunc('hour', probed_at) AS hr, + COUNT(*) AS total, + COUNT(*) FILTER (WHERE healthy) AS up + FROM host_health_probes + WHERE probed_at >= NOW() - INTERVAL '24 hours' + GROUP BY host_label, hr + ORDER BY host_label, hr ASC + """), + ).fetchall() + for r in sp_rows: + label, hr, total, up = r[0], r[1], int(r[2] or 0), int(r[3] or 0) + if label not in host_sparkline: + host_sparkline[label] = {'hours': [], 'uptime_pct': []} + host_sparkline[label]['hours'].append( + hr.strftime('%H:00') if hr else '' + ) + host_sparkline[label]['uptime_pct'].append( + (up / total * 100) if total else 0 + ) + finally: + s_sp.close() + except Exception: + pass + return render_template( 'admin/observability_overview.html', active_page='obs_overview', summary=summary, + host_sparkline=host_sparkline, today=today.strftime('%Y-%m-%d'), ) +# ───────────────────────────────────────────────────────────────────────────── +# /observability/rag_queries — Phase 51 RAG 召回詳情 +# ───────────────────────────────────────────────────────────────────────────── + +@admin_observability_bp.route('/rag_queries') +@login_required +def rag_queries_dashboard(): + """Phase 51 — RAG 召回詳情：每筆 query 的命中、saved_call、反饋。 + + 補完 RAG 觀測深度：之前只看 caller 級命中率，現在看每筆查詢的真實內容。 + """ + hours = int(request.args.get('hours', '24')) + caller_filter = request.args.get('caller', '').strip() + saved_only = request.args.get('saved_only', '').strip() == '1' + + session = get_session() + try: + # 整體統計 + summary_row = session.execute( + sa_text(""" + SELECT COUNT(*) AS total, + COUNT(*) FILTER (WHERE saved_call) AS saved, + COUNT(*) FILTER (WHERE hit_count > 0) AS with_hits, + COALESCE(AVG(hit_count), 0) AS avg_hits, + COALESCE(AVG(feedback_score) FILTER (WHERE feedback_score IS NOT NULL), 0) AS avg_score, + COUNT(*) FILTER (WHERE feedback_score IS NOT NULL) AS feedback_count, + COUNT(DISTINCT caller) AS distinct_callers + FROM rag_query_log + WHERE queried_at >= NOW() - (:h * INTERVAL '1 hour') + """), + {'h': hours}, + ).fetchone() + total = int(summary_row[0] or 0) + saved = int(summary_row[1] or 0) + with_hits = int(summary_row[2] or 0) + summary = { + 'total': total, + 'saved': saved, + 'with_hits': with_hits, + 'no_hits': total - with_hits, + 'avg_hits': round(float(summary_row[3] or 0), 2), + 'avg_score': round(float(summary_row[4] or 0), 2), + 'feedback_count': int(summary_row[5] or 0), + 'distinct_callers': int(summary_row[6] or 0), + 'saved_rate': (float(saved) / total * 100) if total else 0, + 'hit_rate': (float(with_hits) / total * 100) if total else 0, + } + + # caller 列表（dropdown） + callers = session.execute( + sa_text(""" + SELECT DISTINCT caller FROM rag_query_log + WHERE queried_at >= NOW() - (:h * INTERVAL '1 hour') + ORDER BY caller + """), + {'h': hours}, + ).fetchall() + caller_list = [r[0] for r in callers] + + # by caller 統計 + by_caller = session.execute( + sa_text(""" + SELECT caller, + COUNT(*) AS total, + COUNT(*) FILTER (WHERE saved_call) AS saved, + COUNT(*) FILTER (WHERE hit_count > 0) AS with_hits, + COALESCE(AVG(feedback_score) FILTER (WHERE feedback_score IS NOT NULL), 0) AS avg_score, + COUNT(*) FILTER (WHERE feedback_score IS NOT NULL) AS fb_count + FROM rag_query_log + WHERE queried_at >= NOW() - (:h * INTERVAL '1 hour') + GROUP BY caller + ORDER BY total DESC + """), + {'h': hours}, + ).fetchall() + + # 最近 50 筆查詢（套 caller filter + saved_only） + params = {'h': hours, 'caller_f': caller_filter} + recent_queries = session.execute( + sa_text(f""" + SELECT id, queried_at, caller, LEFT(query_text, 200) AS qtext, + top_k, threshold, hit_count, used_results, saved_call, + feedback_score, request_id + FROM rag_query_log + WHERE queried_at >= NOW() - (:h * INTERVAL '1 hour') + AND (:caller_f = '' OR caller = :caller_f) + {"AND saved_call = TRUE" if saved_only else ""} + ORDER BY queried_at DESC + LIMIT 50 + """), + params, + ).fetchall() + queries = [] + for r in recent_queries: + used_ids = list(r[7]) if r[7] else [] + queries.append({ + 'id': int(r[0]), + 'queried_at': r[1].strftime('%Y-%m-%d %H:%M:%S') if r[1] else '', + 'caller': r[2], + 'query_text': r[3] or '', + 'top_k': int(r[4] or 0), + 'threshold': round(float(r[5] or 0), 3), + 'hit_count': int(r[6] or 0), + 'used_results': used_ids, + 'saved_call': bool(r[8]), + 'feedback_score': int(r[9]) if r[9] is not None else None, + 'request_id': r[10], + }) + + return render_template( + 'admin/rag_queries.html', + active_page='obs_rag_queries', + hours=hours, + caller_filter=caller_filter, + saved_only=saved_only, + summary=summary, + callers=caller_list, + by_caller=[ + { + 'caller': r[0], 'total': int(r[1] or 0), + 'saved': int(r[2] or 0), 'with_hits': int(r[3] or 0), + 'avg_score': round(float(r[4] or 0), 2), + 'fb_count': int(r[5] or 0), + 'saved_rate': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0, + 'hit_rate': (float(r[3] or 0) / float(r[1]) * 100) if r[1] else 0, + } + for r in by_caller + ], + queries=queries, + error=None, + ) + except Exception as e: + return render_template( + 'admin/rag_queries.html', + active_page='obs_rag_queries', hours=hours, + caller_filter=caller_filter, saved_only=saved_only, + summary={}, callers=[], by_caller=[], queries=[], + error=f'查詢失敗: {type(e).__name__}: {str(e)[:200]}', + ) + finally: + session.close() + + +@admin_observability_bp.route('/rag_queries//hits', methods=['GET']) +@login_required +def rag_query_hits(query_id: int): + """Phase 51 — JSON API：回傳單筆 query 的 hits 詳細內容（給 modal 展開）。""" + try: + session = get_session() + try: + row = session.execute( + sa_text(""" + SELECT id, query_text, used_results, hit_count, threshold + FROM rag_query_log WHERE id = :id + """), + {'id': query_id}, + ).fetchone() + if not row: + return jsonify({'ok': False, 'error': 'not found'}), 404 + + used_ids = list(row[2]) if row[2] else [] + hits = [] + if used_ids: + rows = session.execute( + sa_text(""" + SELECT id, insight_type, period, product_sku, + LEFT(content, 300) AS preview, created_at + FROM ai_insights + WHERE id = ANY(:ids) + ORDER BY created_at DESC + """), + {'ids': used_ids}, + ).fetchall() + hits = [ + { + 'id': int(h[0]), + 'insight_type': h[1], + 'period': h[2], + 'product_sku': h[3], + 'content': h[4] or '', + 'created_at': h[5].strftime('%Y-%m-%d') if h[5] else '', + } + for h in rows + ] + return jsonify({ + 'ok': True, + 'query_id': query_id, + 'query_text': row[1], + 'hit_count': int(row[3] or 0), + 'threshold': round(float(row[4] or 0), 3), + 'hits': hits, + }) + finally: + session.close() + except Exception as e: + return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500 + + # ───────────────────────────────────────────────────────────────────────────── # /observability/business_intel — Phase 48 商業面 × AI 編排 # ───────────────────────────────────────────────────────────────────────────── diff --git a/templates/admin/observability_overview.html b/templates/admin/observability_overview.html index 1a59ad5..9e0f89d 100644 --- a/templates/admin/observability_overview.html +++ b/templates/admin/observability_overview.html @@ -8,7 +8,7 @@ {{ today }} · 全景一頁看（資料來源 8 表跨 JOIN） - +
{% if summary.hosts %} {% for h in summary.hosts %} @@ -29,6 +29,11 @@
{{ h.avg_ms }} ms
+ {% if host_sparkline.get(h.label) %} +
+ +
+ {% endif %} @@ -219,9 +224,9 @@ {% endif %} - +
-
8 大子頁入口
+
9 大子頁入口
@@ -260,6 +265,12 @@ 待審 episode + RAG Top 3 相似已晉升輔助
+
+ + RAG 召回詳情 + 每筆 query 的 hits / saved_call / 反饋追蹤 + +
Caller 反饋趨勢 @@ -277,9 +288,46 @@

- Operation Ollama-First v5.0 / Phase 45 — AI 觀測台總覽 + Operation Ollama-First v5.0 / Phase 51 — AI 觀測台總覽（含 24h sparkline） （資料來源：host_health_probes / ai_calls / ai_call_budgets / learning_episodes / ai_insights / rag_query_log / mcp_calls / incidents / heal_logs / ppt_audit_results）

+ + + {% endblock %} diff --git a/templates/admin/rag_queries.html b/templates/admin/rag_queries.html new file mode 100644 index 0000000..901fdb8 --- /dev/null +++ b/templates/admin/rag_queries.html @@ -0,0 +1,267 @@ +{% extends "ewoooc_base.html" %} + +{% block title %}RAG 召回詳情{% endblock %} + +{% block ewooo_content %} +
+

RAG 召回詳情 + 過去 {{ hours }} 小時 · 每筆 query 的 hits / saved_call / 反饋 +

+ + {% if error %} +
{{ error }}
+ {% endif %} + + +
+
+ +
+
+ +
+
+
+ + +
+
+
+ + + {% if summary and summary.total > 0 %} +
+
+
+
+ 總查詢 +

{{ "{:,}".format(summary.total) }}

+ {{ summary.distinct_callers }} 個呼叫端 +
+
+
+
+
+
+ 命中率 +

{{ "%.1f"|format(summary.hit_rate) }}%

+ {{ summary.with_hits }} hit · {{ summary.no_hits }} 未命中 +
+
+
+
+
+
+ saved_call 率 +

{{ "%.1f"|format(summary.saved_rate) }}%

+ {{ summary.saved }} 次省下 LLM +
+
+
+
+
+
+ 反饋平均分 +

{{ "%.2f"|format(summary.avg_score) }}/5

+ {{ summary.feedback_count }} 筆反饋 · 平均 {{ summary.avg_hits }} hits +
+
+
+
+ {% endif %} + + + {% if by_caller %} +
+
各呼叫端 RAG 表現 + 資料來源：rag_query_log GROUP BY caller +
+
+ + + + + + + + + + + + + + + {% for c in by_caller %} + + + + + + + + + + + {% endfor %} + +
呼叫端查詢命中命中率savedsaved 率反饋平均分
{{ c.caller }}{{ "{:,}".format(c.total) }}{{ c.with_hits }} + + {{ "%.1f"|format(c.hit_rate) }}% + + {{ c.saved }} + + {{ "%.1f"|format(c.saved_rate) }}% + + {{ c.fb_count }} + {% if c.fb_count > 0 %} + + {{ "%.2f"|format(c.avg_score) }} + + {% else %}—{% endif %} +
+
+
+ {% endif %} + + + {% if queries %} +
+
最近 50 筆查詢詳情 + 點「查 hits」展開命中的 ai_insights 內容 +
+
+ + + + + + + + + + + + + {% for q in queries %} + + + + + + + + + + + + {% endfor %} + +
時間呼叫端查詢top_k門檻命中saved反饋動作
{{ q.queried_at }}{{ q.caller }}{{ q.query_text }}{% if q.query_text|length >= 200 %}…{% endif %}{{ q.top_k }}{{ q.threshold }} + {% if q.hit_count > 0 %}{{ q.hit_count }} + {% else %}0{% endif %} + + {% if q.saved_call %}saved + {% else %}—{% endif %} + + {% if q.feedback_score is not none %} + {% if q.feedback_score >= 4 %}{{ q.feedback_score }}/5 + {% elif q.feedback_score >= 3 %}{{ q.feedback_score }}/5 + {% else %}{{ q.feedback_score }}/5{% endif %} + {% else %}—{% endif %} + + {% if q.hit_count > 0 %} + + {% endif %} +
+
+
+ {% else %} +
+ 過去 {{ hours }} 小時無符合條件的 RAG 查詢紀錄。 +
+ {% endif %} + +

+ Operation Ollama-First v5.0 / Phase 51 — RAG 召回詳情 + （3 表跨 JOIN：rag_query_log × ai_insights × ai_calls.request_id） +

+
+ + + + + +{% endblock %} diff --git a/templates/components/_ewoooc_shell.html b/templates/components/_ewoooc_shell.html index ec364c2..230ede4 100644 --- a/templates/components/_ewoooc_shell.html +++ b/templates/components/_ewoooc_shell.html @@ -111,6 +111,11 @@ RAG 晉升審核 11 + + + RAG 召回詳情 + 11b + 反饋趨勢