feat(p24): ROI 月報生成器 — 每月 1 日 09:00 推 Telegram
All checks were successful
CD Pipeline / deploy (push) Successful in 2m49s
All checks were successful
CD Pipeline / deploy (push) Successful in 2m49s
Operation Ollama-First v5.0 / Phase 24 — 戰役 ROI 自動量化 services/roi_report_service.py (200+ 行) - BASELINE 戰前估算（gemini 50M/$20、nim 5.8M、ollama 30M、total $25/月） - query_last_month_stats() — SQL 查上月 ai_calls + mcp_calls + rag_query_log - render_roi_report(stats) — HTML 訊息（含 5 區塊：成本攔截/provider 分布/RAG/MCP+Cache/KPI） - generate_and_send_roi_report() — 主入口，推 Telegram + 寫 ai_insights 長期記錄 - 達標標記：Gemini -23.5% ✅ / RAG 命中 ≥25% ✅ Telegram 訊息範例: 📊 ROI 月報 2026年04月 💰 成本攔截 Gemini: 35,000,000 tokens / $14.00 vs 戰前: 50,000,000 / $20.00 ✅ 攔截: 15,000,000 tokens / $6.00 (30.0%) 🤖 Provider 分布 🧠 RAG 自主學習（含 saved_call / 反饋分數） 🔧 MCP + Cache 📈 戰役 v5.0 KPI ✅ 達標 run_scheduler.py — 每日 09:00 跑（內部判斷月初第 1 日才送） - run_roi_monthly_report_if_new_month task wrapper - 失敗 swallow log，不影響其他排程 tests/test_roi_report_service.py (7 tests 全綠) - BASELINE 必要欄位 / 月份範圍計算 (1月→去年12月) - 達標訊息含 ✅ + 攔截數字 - 未達標訊息含 ⚠️ - 空 stats 容錯 / DB fail 回空 dict 不 raise Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
OoO
@@ -127,6 +127,10 @@ def _register_schedules():
|
||||
schedule.every().day.at("00:05").do(run_cost_throttle_reset_if_new_month)
|
||||
logger.info("📅 每日 00:05：cost_throttle_reset_if_new_month")
|
||||
|
||||
# Phase 24: ROI 月報（每日 09:00 跑，內部判斷是否月初第 1 日）
|
||||
schedule.every().day.at("09:00").do(run_roi_monthly_report_if_new_month)
|
||||
logger.info("📅 每日 09:00：roi_monthly_report（月初第 1 日才送）")
|
||||
|
||||
schedule.every().day.at("03:00").do(run_db_backup_task)
|
||||
logger.info("📅 每日 03:00：db_backup")
|
||||
|
||||
@@ -270,6 +274,24 @@ def run_cost_throttle_reset_if_new_month():
|
||||
logger.error(f"[CostThrottle] reset failed: {e}", exc_info=True)
|
||||
|
||||
|
||||
def run_roi_monthly_report_if_new_month():
|
||||
"""每日 09:00 — Phase 24 ROI 月報（內部判斷月初第 1 日才送）
|
||||
|
||||
對比上月 ai_calls 統計 vs 戰前 baseline，推 Telegram「節省 X tokens / $Y」。
|
||||
寫入 ai_insights (type='roi_monthly_report') 作長期記錄。
|
||||
"""
|
||||
try:
|
||||
from datetime import datetime
|
||||
if datetime.now().day != 1:
|
||||
return # 非月初第 1 日 skip
|
||||
from services.roi_report_service import generate_and_send_roi_report
|
||||
result = generate_and_send_roi_report()
|
||||
logger.info("[ROIReport] sent=%s period=%s",
|
||||
result.get('sent'), result.get('period', '?'))
|
||||
except Exception as e:
|
||||
logger.error(f"[ROIReport] task failed: {e}", exc_info=True)
|
||||
|
||||
|
||||
def run_embed_consistency_check():
|
||||
"""每週日 04:30 — BGE-M3 跨主機一致性驗證（ADR-033 護欄 #3）。
|
||||
|
||||
|
||||
231
services/roi_report_service.py
Normal file
231
services/roi_report_service.py
Normal file
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
services/roi_report_service.py
|
||||
Operation Ollama-First v5.0 / Phase 24 — ROI 月報生成器
|
||||
|
||||
設計原則：
|
||||
- 每月 1 日 09:00 跑（與 daily_report 同時段）
|
||||
- SQL 統計上月 ai_calls × mcp_calls × rag_query_log
|
||||
- 對比戰前 baseline（hardcode 戰前估算數字）
|
||||
- 推 Telegram「上月節省 X tokens / $Y / 攔截 Z 次 LLM 呼叫」
|
||||
- 寫進 ai_insights（type='roi_monthly_report'）作長期記錄
|
||||
|
||||
戰前 baseline（戰役 v5.0 啟動前估算）:
|
||||
- Gemini 月 ~50M tokens / $20
|
||||
- NIM 月 ~5.8M tokens
|
||||
- Ollama 月 ~30M tokens（自架免費）
|
||||
- Total ~$25/月（含 OpenRouter）
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from calendar import monthrange
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 戰前 baseline（v5.0 啟動前估算，後續可從 ai_call_budgets 表動態算）
|
||||
BASELINE = {
|
||||
'gemini_monthly_tokens': 50_000_000,
|
||||
'gemini_monthly_cost_usd': 20.0,
|
||||
'nim_monthly_tokens': 5_800_000,
|
||||
'ollama_monthly_tokens': 30_000_000,
|
||||
'total_monthly_cost_usd': 25.0,
|
||||
}
|
||||
|
||||
|
||||
def _last_month_range(today: Optional[datetime] = None) -> tuple:
|
||||
"""取上月 1 號 00:00 到 本月 1 號 00:00"""
|
||||
today = today or datetime.now()
|
||||
this_month_start = datetime(today.year, today.month, 1)
|
||||
if today.month == 1:
|
||||
last_month_start = datetime(today.year - 1, 12, 1)
|
||||
else:
|
||||
last_month_start = datetime(today.year, today.month - 1, 1)
|
||||
return last_month_start, this_month_start
|
||||
|
||||
|
||||
def query_last_month_stats() -> Dict[str, Any]:
|
||||
"""SQL 查上月 ai_calls / mcp_calls / rag_query_log 統計"""
|
||||
try:
|
||||
from sqlalchemy import text as sa_text
|
||||
from database.manager import get_session
|
||||
except Exception as exc:
|
||||
logger.warning('[ROI] DB import failed: %s', exc)
|
||||
return {}
|
||||
|
||||
last_start, this_start = _last_month_range()
|
||||
period_label = last_start.strftime('%Y年%m月')
|
||||
|
||||
session = get_session()
|
||||
try:
|
||||
# 1. ai_calls 統計
|
||||
ai = session.execute(
|
||||
sa_text("""
|
||||
SELECT
|
||||
COALESCE(SUM(input_tokens + output_tokens), 0) AS total_tokens,
|
||||
COALESCE(SUM(cost_usd), 0) AS total_cost,
|
||||
COUNT(*) AS total_calls,
|
||||
COUNT(*) FILTER (WHERE provider IN ('gcp_ollama','ollama_secondary','ollama_111')) AS ollama_calls,
|
||||
COUNT(*) FILTER (WHERE provider = 'gemini') AS gemini_calls,
|
||||
COUNT(*) FILTER (WHERE provider = 'claude') AS claude_calls,
|
||||
COUNT(*) FILTER (WHERE provider IN ('nim','nim_via_elephant')) AS nim_calls,
|
||||
COUNT(*) FILTER (WHERE rag_hit) AS rag_hit_calls,
|
||||
COUNT(*) FILTER (WHERE cache_hit) AS cache_hit_calls,
|
||||
COALESCE(SUM(input_tokens + output_tokens) FILTER (WHERE provider = 'gemini'), 0) AS gemini_tokens,
|
||||
COALESCE(SUM(cost_usd) FILTER (WHERE provider = 'gemini'), 0) AS gemini_cost,
|
||||
COALESCE(SUM(cost_usd) FILTER (WHERE provider = 'claude'), 0) AS claude_cost
|
||||
FROM ai_calls
|
||||
WHERE called_at >= :start AND called_at < :end
|
||||
"""),
|
||||
{'start': last_start, 'end': this_start},
|
||||
).fetchone()
|
||||
|
||||
# 2. mcp_calls 統計
|
||||
try:
|
||||
mcp = session.execute(
|
||||
sa_text("""
|
||||
SELECT COUNT(*) AS total, COUNT(*) FILTER (WHERE cache_hit) AS cache_hits
|
||||
FROM mcp_calls
|
||||
WHERE called_at >= :start AND called_at < :end
|
||||
"""),
|
||||
{'start': last_start, 'end': this_start},
|
||||
).fetchone()
|
||||
except Exception:
|
||||
mcp = None
|
||||
|
||||
# 3. rag_query_log 統計
|
||||
try:
|
||||
rag = session.execute(
|
||||
sa_text("""
|
||||
SELECT
|
||||
COUNT(*) AS total_queries,
|
||||
COUNT(*) FILTER (WHERE saved_call) AS saved_calls,
|
||||
AVG(feedback_score) FILTER (WHERE feedback_score IS NOT NULL) AS avg_feedback
|
||||
FROM rag_query_log
|
||||
WHERE queried_at >= :start AND queried_at < :end
|
||||
"""),
|
||||
{'start': last_start, 'end': this_start},
|
||||
).fetchone()
|
||||
except Exception:
|
||||
rag = None
|
||||
|
||||
return {
|
||||
'period_label': period_label,
|
||||
'period_start': last_start,
|
||||
'period_end': this_start,
|
||||
'ai_total_tokens': int(ai[0] or 0) if ai else 0,
|
||||
'ai_total_cost': float(ai[1] or 0) if ai else 0.0,
|
||||
'ai_total_calls': int(ai[2] or 0) if ai else 0,
|
||||
'ollama_calls': int(ai[3] or 0) if ai else 0,
|
||||
'gemini_calls': int(ai[4] or 0) if ai else 0,
|
||||
'claude_calls': int(ai[5] or 0) if ai else 0,
|
||||
'nim_calls': int(ai[6] or 0) if ai else 0,
|
||||
'rag_hit_calls': int(ai[7] or 0) if ai else 0,
|
||||
'cache_hit_calls': int(ai[8] or 0) if ai else 0,
|
||||
'gemini_tokens': int(ai[9] or 0) if ai else 0,
|
||||
'gemini_cost': float(ai[10] or 0) if ai else 0.0,
|
||||
'claude_cost': float(ai[11] or 0) if ai else 0.0,
|
||||
'mcp_total': int(mcp[0] or 0) if mcp else 0,
|
||||
'mcp_cache_hits': int(mcp[1] or 0) if mcp else 0,
|
||||
'rag_total': int(rag[0] or 0) if rag else 0,
|
||||
'rag_saved': int(rag[1] or 0) if rag else 0,
|
||||
'rag_avg_feedback': float(rag[2] or 0) if rag and rag[2] else 0.0,
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.error('[ROI] query failed: %s', exc)
|
||||
return {}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def render_roi_report(stats: Dict[str, Any]) -> str:
|
||||
"""組 Telegram 訊息（HTML format）"""
|
||||
if not stats:
|
||||
return "⚠️ ROI 月報資料查詢失敗，請查 logs"
|
||||
|
||||
period = stats['period_label']
|
||||
gemini_saved_tokens = max(0, BASELINE['gemini_monthly_tokens'] - stats['gemini_tokens'])
|
||||
gemini_saved_cost = max(0.0, BASELINE['gemini_monthly_cost_usd'] - stats['gemini_cost'])
|
||||
saved_pct = (
|
||||
gemini_saved_tokens / BASELINE['gemini_monthly_tokens'] * 100
|
||||
if BASELINE['gemini_monthly_tokens'] else 0
|
||||
)
|
||||
|
||||
return (
|
||||
f"📊 <b>ROI 月報 {period}</b>\n"
|
||||
f"━━━━━━━━━━━━━━━━━━━━\n"
|
||||
f"💰 <b>成本攔截</b>\n"
|
||||
f" Gemini: {stats['gemini_tokens']:,} tokens / ${stats['gemini_cost']:.2f}\n"
|
||||
f" vs 戰前: {BASELINE['gemini_monthly_tokens']:,} / ${BASELINE['gemini_monthly_cost_usd']:.2f}\n"
|
||||
f" ✅ 攔截: <b>{gemini_saved_tokens:,} tokens / ${gemini_saved_cost:.2f} ({saved_pct:.1f}%)</b>\n"
|
||||
f"\n"
|
||||
f"🤖 <b>Provider 分布</b>\n"
|
||||
f" Ollama (自架免費): {stats['ollama_calls']:,} calls\n"
|
||||
f" Gemini: {stats['gemini_calls']:,} calls\n"
|
||||
f" Claude: {stats['claude_calls']:,} calls / ${stats['claude_cost']:.2f}\n"
|
||||
f" NIM: {stats['nim_calls']:,} calls\n"
|
||||
f"\n"
|
||||
f"🧠 <b>RAG 自主學習</b>\n"
|
||||
f" 查詢: {stats['rag_total']:,} 次\n"
|
||||
f" 攔截 LLM: {stats['rag_saved']:,} 次（saved_call=true）\n"
|
||||
f" 反饋分: {stats['rag_avg_feedback']:.2f} / 5\n"
|
||||
f"\n"
|
||||
f"🔧 <b>MCP + Cache</b>\n"
|
||||
f" MCP 呼叫: {stats['mcp_total']:,}\n"
|
||||
f" Cache 命中: {stats['cache_hit_calls']:,} ai_calls + {stats['mcp_cache_hits']:,} mcp_calls\n"
|
||||
f"\n"
|
||||
f"📈 <b>戰役 v5.0 KPI</b>\n"
|
||||
f" Gemini -23.5% 目標：{'✅ 達標' if saved_pct >= 23 else f'⚠️ {saved_pct:.1f}%'}\n"
|
||||
f" RAG 命中 ≥25% 目標：{'✅' if stats['rag_total'] > 0 and stats['rag_saved']/max(stats['rag_total'],1) >= 0.25 else '⏳'}"
|
||||
)
|
||||
|
||||
|
||||
def generate_and_send_roi_report() -> Dict[str, Any]:
|
||||
"""每月 1 日 09:00 cron 呼叫主入口"""
|
||||
stats = query_last_month_stats()
|
||||
if not stats:
|
||||
logger.warning('[ROI] no stats; skip')
|
||||
return {'sent': False, 'reason': 'no_stats'}
|
||||
|
||||
msg = render_roi_report(stats)
|
||||
|
||||
# 推 Telegram
|
||||
try:
|
||||
from services.telegram_templates import _send_telegram_raw
|
||||
_send_telegram_raw(msg)
|
||||
logger.info('[ROI] %s 月報已推 Telegram', stats['period_label'])
|
||||
except Exception as exc:
|
||||
logger.warning('[ROI] telegram push failed: %s', exc)
|
||||
|
||||
# 寫 ai_insights 長期記錄
|
||||
try:
|
||||
from sqlalchemy import text as sa_text
|
||||
from database.manager import get_session
|
||||
session = get_session()
|
||||
try:
|
||||
session.execute(
|
||||
sa_text("""
|
||||
INSERT INTO ai_insights (insight_type, content, status, created_by, confidence)
|
||||
VALUES ('roi_monthly_report', :content, 'approved', 'roi_report_service', 0.95)
|
||||
"""),
|
||||
{'content': msg},
|
||||
)
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
except Exception as exc:
|
||||
logger.warning('[ROI] persist failed: %s', exc)
|
||||
|
||||
return {'sent': True, 'period': stats['period_label'], 'msg_chars': len(msg)}
|
||||
|
||||
|
||||
__all__ = [
|
||||
'query_last_month_stats',
|
||||
'render_roi_report',
|
||||
'generate_and_send_roi_report',
|
||||
'BASELINE',
|
||||
]
|
||||
120
tests/test_roi_report_service.py
Normal file
120
tests/test_roi_report_service.py
Normal file
@@ -0,0 +1,120 @@
|
||||
"""
|
||||
tests/test_roi_report_service.py
|
||||
─────────────────────────────────────────────────────────────────
|
||||
Operation Ollama-First v5.0 / Phase 24 — ROI 月報生成器驗證
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_baseline_constants():
|
||||
"""戰前 baseline 應有 5 個必要欄位"""
|
||||
from services.roi_report_service import BASELINE
|
||||
|
||||
required = {'gemini_monthly_tokens', 'gemini_monthly_cost_usd',
|
||||
'nim_monthly_tokens', 'ollama_monthly_tokens',
|
||||
'total_monthly_cost_usd'}
|
||||
assert required.issubset(BASELINE.keys())
|
||||
assert BASELINE['gemini_monthly_tokens'] == 50_000_000
|
||||
assert BASELINE['gemini_monthly_cost_usd'] == 20.0
|
||||
|
||||
|
||||
def test_last_month_range_january():
|
||||
"""1 月跑時上月應為去年 12 月"""
|
||||
from services.roi_report_service import _last_month_range
|
||||
|
||||
today = datetime(2026, 1, 5)
|
||||
last_start, this_start = _last_month_range(today)
|
||||
assert last_start == datetime(2025, 12, 1)
|
||||
assert this_start == datetime(2026, 1, 1)
|
||||
|
||||
|
||||
def test_last_month_range_normal_month():
|
||||
from services.roi_report_service import _last_month_range
|
||||
|
||||
today = datetime(2026, 5, 15)
|
||||
last_start, this_start = _last_month_range(today)
|
||||
assert last_start == datetime(2026, 4, 1)
|
||||
assert this_start == datetime(2026, 5, 1)
|
||||
|
||||
|
||||
def test_render_roi_report_with_savings():
|
||||
"""戰役後成功攔截 Gemini → 訊息含「攔截」+ 達標 ✅"""
|
||||
from services.roi_report_service import render_roi_report
|
||||
|
||||
stats = {
|
||||
'period_label': '2026年04月',
|
||||
'period_start': datetime(2026, 4, 1),
|
||||
'period_end': datetime(2026, 5, 1),
|
||||
'ai_total_tokens': 80_000_000,
|
||||
'ai_total_cost': 12.0,
|
||||
'ai_total_calls': 5000,
|
||||
'ollama_calls': 4000,
|
||||
'gemini_calls': 800,
|
||||
'claude_calls': 100,
|
||||
'nim_calls': 100,
|
||||
'rag_hit_calls': 200,
|
||||
'cache_hit_calls': 150,
|
||||
'gemini_tokens': 35_000_000, # 戰前 50M → 省 15M (30%)
|
||||
'gemini_cost': 14.0, # 戰前 $20 → 省 $6
|
||||
'claude_cost': 5.0,
|
||||
'mcp_total': 180,
|
||||
'mcp_cache_hits': 50,
|
||||
'rag_total': 800,
|
||||
'rag_saved': 250, # 31% RAG 攔截率
|
||||
'rag_avg_feedback': 4.2,
|
||||
}
|
||||
msg = render_roi_report(stats)
|
||||
|
||||
assert '2026年04月' in msg
|
||||
assert '攔截' in msg
|
||||
assert '15,000,000' in msg or '15M' in msg or '15,000' in msg
|
||||
assert '$6' in msg or '6.00' in msg
|
||||
assert '✅' in msg # Gemini -23.5% 目標達標
|
||||
|
||||
|
||||
def test_render_roi_report_below_target():
|
||||
"""未達 -23.5% 目標 → 訊息含 ⚠️"""
|
||||
from services.roi_report_service import render_roi_report
|
||||
|
||||
stats = {
|
||||
'period_label': '2026年04月',
|
||||
'period_start': datetime(2026, 4, 1),
|
||||
'period_end': datetime(2026, 5, 1),
|
||||
'ai_total_tokens': 80_000_000, 'ai_total_cost': 18.0, 'ai_total_calls': 5000,
|
||||
'ollama_calls': 1000, 'gemini_calls': 3000, 'claude_calls': 500, 'nim_calls': 500,
|
||||
'rag_hit_calls': 50, 'cache_hit_calls': 30,
|
||||
'gemini_tokens': 45_000_000, # 戰前 50M → 只省 10% < 23%
|
||||
'gemini_cost': 18.0, 'claude_cost': 5.0,
|
||||
'mcp_total': 100, 'mcp_cache_hits': 10,
|
||||
'rag_total': 200, 'rag_saved': 30, 'rag_avg_feedback': 3.5,
|
||||
}
|
||||
msg = render_roi_report(stats)
|
||||
|
||||
assert '⚠️' in msg # 未達標標記
|
||||
|
||||
|
||||
def test_render_empty_stats():
|
||||
"""空 stats → 預設失敗訊息"""
|
||||
from services.roi_report_service import render_roi_report
|
||||
|
||||
assert '⚠️' in render_roi_report({})
|
||||
|
||||
|
||||
def test_query_last_month_stats_db_fail_returns_empty(monkeypatch):
|
||||
"""DB 失敗應回 {} 不 raise"""
|
||||
from services.roi_report_service import query_last_month_stats
|
||||
|
||||
class _BrokenSession:
|
||||
def execute(self, *a, **kw):
|
||||
raise RuntimeError('DB connection lost')
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
import services.roi_report_service as ros
|
||||
monkeypatch.setattr('database.manager.get_session', lambda: _BrokenSession())
|
||||
result = ros.query_last_month_stats()
|
||||
assert result == {}
|
||||
Reference in New Issue
Block a user