diff --git a/run_scheduler.py b/run_scheduler.py index 4b4b273..178c61c 100644 --- a/run_scheduler.py +++ b/run_scheduler.py @@ -127,6 +127,10 @@ def _register_schedules(): schedule.every().day.at("00:05").do(run_cost_throttle_reset_if_new_month) logger.info("📅 每日 00:05:cost_throttle_reset_if_new_month") + # Phase 24: ROI 月報(每日 09:00 跑,內部判斷是否月初第 1 日) + schedule.every().day.at("09:00").do(run_roi_monthly_report_if_new_month) + logger.info("📅 每日 09:00:roi_monthly_report(月初第 1 日才送)") + schedule.every().day.at("03:00").do(run_db_backup_task) logger.info("📅 每日 03:00:db_backup") @@ -270,6 +274,24 @@ def run_cost_throttle_reset_if_new_month(): logger.error(f"[CostThrottle] reset failed: {e}", exc_info=True) +def run_roi_monthly_report_if_new_month(): + """每日 09:00 — Phase 24 ROI 月報(內部判斷月初第 1 日才送) + + 對比上月 ai_calls 統計 vs 戰前 baseline,推 Telegram「節省 X tokens / $Y」。 + 寫入 ai_insights (type='roi_monthly_report') 作長期記錄。 + """ + try: + from datetime import datetime + if datetime.now().day != 1: + return # 非月初第 1 日 skip + from services.roi_report_service import generate_and_send_roi_report + result = generate_and_send_roi_report() + logger.info("[ROIReport] sent=%s period=%s", + result.get('sent'), result.get('period', '?')) + except Exception as e: + logger.error(f"[ROIReport] task failed: {e}", exc_info=True) + + def run_embed_consistency_check(): """每週日 04:30 — BGE-M3 跨主機一致性驗證(ADR-033 護欄 #3)。 diff --git a/services/roi_report_service.py b/services/roi_report_service.py new file mode 100644 index 0000000..63be39a --- /dev/null +++ b/services/roi_report_service.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +services/roi_report_service.py +Operation Ollama-First v5.0 / Phase 24 — ROI 月報生成器 + +設計原則: +- 每月 1 日 09:00 跑(與 daily_report 同時段) +- SQL 統計上月 ai_calls × mcp_calls × rag_query_log +- 對比戰前 baseline(hardcode 戰前估算數字) +- 推 Telegram「上月節省 X tokens / $Y / 攔截 Z 次 LLM 呼叫」 +- 寫進 ai_insights(type='roi_monthly_report')作長期記錄 + +戰前 baseline(戰役 v5.0 啟動前估算): +- Gemini 月 ~50M tokens / $20 +- NIM 月 ~5.8M tokens +- Ollama 月 ~30M tokens(自架免費) +- Total ~$25/月(含 OpenRouter) +""" + +from __future__ import annotations +import os +import logging +from datetime import datetime, timedelta +from calendar import monthrange +from typing import Dict, Any, Optional + +logger = logging.getLogger(__name__) + +# 戰前 baseline(v5.0 啟動前估算,後續可從 ai_call_budgets 表動態算) +BASELINE = { + 'gemini_monthly_tokens': 50_000_000, + 'gemini_monthly_cost_usd': 20.0, + 'nim_monthly_tokens': 5_800_000, + 'ollama_monthly_tokens': 30_000_000, + 'total_monthly_cost_usd': 25.0, +} + + +def _last_month_range(today: Optional[datetime] = None) -> tuple: + """取上月 1 號 00:00 到 本月 1 號 00:00""" + today = today or datetime.now() + this_month_start = datetime(today.year, today.month, 1) + if today.month == 1: + last_month_start = datetime(today.year - 1, 12, 1) + else: + last_month_start = datetime(today.year, today.month - 1, 1) + return last_month_start, this_month_start + + +def query_last_month_stats() -> Dict[str, Any]: + """SQL 查上月 ai_calls / mcp_calls / rag_query_log 統計""" + try: + from sqlalchemy import text as sa_text + from database.manager import get_session + except Exception as exc: + logger.warning('[ROI] DB import failed: %s', exc) + return {} + + last_start, this_start = _last_month_range() + period_label = last_start.strftime('%Y年%m月') + + session = get_session() + try: + # 1. ai_calls 統計 + ai = session.execute( + sa_text(""" + SELECT + COALESCE(SUM(input_tokens + output_tokens), 0) AS total_tokens, + COALESCE(SUM(cost_usd), 0) AS total_cost, + COUNT(*) AS total_calls, + COUNT(*) FILTER (WHERE provider IN ('gcp_ollama','ollama_secondary','ollama_111')) AS ollama_calls, + COUNT(*) FILTER (WHERE provider = 'gemini') AS gemini_calls, + COUNT(*) FILTER (WHERE provider = 'claude') AS claude_calls, + COUNT(*) FILTER (WHERE provider IN ('nim','nim_via_elephant')) AS nim_calls, + COUNT(*) FILTER (WHERE rag_hit) AS rag_hit_calls, + COUNT(*) FILTER (WHERE cache_hit) AS cache_hit_calls, + COALESCE(SUM(input_tokens + output_tokens) FILTER (WHERE provider = 'gemini'), 0) AS gemini_tokens, + COALESCE(SUM(cost_usd) FILTER (WHERE provider = 'gemini'), 0) AS gemini_cost, + COALESCE(SUM(cost_usd) FILTER (WHERE provider = 'claude'), 0) AS claude_cost + FROM ai_calls + WHERE called_at >= :start AND called_at < :end + """), + {'start': last_start, 'end': this_start}, + ).fetchone() + + # 2. mcp_calls 統計 + try: + mcp = session.execute( + sa_text(""" + SELECT COUNT(*) AS total, COUNT(*) FILTER (WHERE cache_hit) AS cache_hits + FROM mcp_calls + WHERE called_at >= :start AND called_at < :end + """), + {'start': last_start, 'end': this_start}, + ).fetchone() + except Exception: + mcp = None + + # 3. rag_query_log 統計 + try: + rag = session.execute( + sa_text(""" + SELECT + COUNT(*) AS total_queries, + COUNT(*) FILTER (WHERE saved_call) AS saved_calls, + AVG(feedback_score) FILTER (WHERE feedback_score IS NOT NULL) AS avg_feedback + FROM rag_query_log + WHERE queried_at >= :start AND queried_at < :end + """), + {'start': last_start, 'end': this_start}, + ).fetchone() + except Exception: + rag = None + + return { + 'period_label': period_label, + 'period_start': last_start, + 'period_end': this_start, + 'ai_total_tokens': int(ai[0] or 0) if ai else 0, + 'ai_total_cost': float(ai[1] or 0) if ai else 0.0, + 'ai_total_calls': int(ai[2] or 0) if ai else 0, + 'ollama_calls': int(ai[3] or 0) if ai else 0, + 'gemini_calls': int(ai[4] or 0) if ai else 0, + 'claude_calls': int(ai[5] or 0) if ai else 0, + 'nim_calls': int(ai[6] or 0) if ai else 0, + 'rag_hit_calls': int(ai[7] or 0) if ai else 0, + 'cache_hit_calls': int(ai[8] or 0) if ai else 0, + 'gemini_tokens': int(ai[9] or 0) if ai else 0, + 'gemini_cost': float(ai[10] or 0) if ai else 0.0, + 'claude_cost': float(ai[11] or 0) if ai else 0.0, + 'mcp_total': int(mcp[0] or 0) if mcp else 0, + 'mcp_cache_hits': int(mcp[1] or 0) if mcp else 0, + 'rag_total': int(rag[0] or 0) if rag else 0, + 'rag_saved': int(rag[1] or 0) if rag else 0, + 'rag_avg_feedback': float(rag[2] or 0) if rag and rag[2] else 0.0, + } + except Exception as exc: + logger.error('[ROI] query failed: %s', exc) + return {} + finally: + session.close() + + +def render_roi_report(stats: Dict[str, Any]) -> str: + """組 Telegram 訊息(HTML format)""" + if not stats: + return "⚠️ ROI 月報資料查詢失敗,請查 logs" + + period = stats['period_label'] + gemini_saved_tokens = max(0, BASELINE['gemini_monthly_tokens'] - stats['gemini_tokens']) + gemini_saved_cost = max(0.0, BASELINE['gemini_monthly_cost_usd'] - stats['gemini_cost']) + saved_pct = ( + gemini_saved_tokens / BASELINE['gemini_monthly_tokens'] * 100 + if BASELINE['gemini_monthly_tokens'] else 0 + ) + + return ( + f"📊 ROI 月報 {period}\n" + f"━━━━━━━━━━━━━━━━━━━━\n" + f"💰 成本攔截\n" + f" Gemini: {stats['gemini_tokens']:,} tokens / ${stats['gemini_cost']:.2f}\n" + f" vs 戰前: {BASELINE['gemini_monthly_tokens']:,} / ${BASELINE['gemini_monthly_cost_usd']:.2f}\n" + f" ✅ 攔截: {gemini_saved_tokens:,} tokens / ${gemini_saved_cost:.2f} ({saved_pct:.1f}%)\n" + f"\n" + f"🤖 Provider 分布\n" + f" Ollama (自架免費): {stats['ollama_calls']:,} calls\n" + f" Gemini: {stats['gemini_calls']:,} calls\n" + f" Claude: {stats['claude_calls']:,} calls / ${stats['claude_cost']:.2f}\n" + f" NIM: {stats['nim_calls']:,} calls\n" + f"\n" + f"🧠 RAG 自主學習\n" + f" 查詢: {stats['rag_total']:,} 次\n" + f" 攔截 LLM: {stats['rag_saved']:,} 次(saved_call=true)\n" + f" 反饋分: {stats['rag_avg_feedback']:.2f} / 5\n" + f"\n" + f"🔧 MCP + Cache\n" + f" MCP 呼叫: {stats['mcp_total']:,}\n" + f" Cache 命中: {stats['cache_hit_calls']:,} ai_calls + {stats['mcp_cache_hits']:,} mcp_calls\n" + f"\n" + f"📈 戰役 v5.0 KPI\n" + f" Gemini -23.5% 目標:{'✅ 達標' if saved_pct >= 23 else f'⚠️ {saved_pct:.1f}%'}\n" + f" RAG 命中 ≥25% 目標:{'✅' if stats['rag_total'] > 0 and stats['rag_saved']/max(stats['rag_total'],1) >= 0.25 else '⏳'}" + ) + + +def generate_and_send_roi_report() -> Dict[str, Any]: + """每月 1 日 09:00 cron 呼叫主入口""" + stats = query_last_month_stats() + if not stats: + logger.warning('[ROI] no stats; skip') + return {'sent': False, 'reason': 'no_stats'} + + msg = render_roi_report(stats) + + # 推 Telegram + try: + from services.telegram_templates import _send_telegram_raw + _send_telegram_raw(msg) + logger.info('[ROI] %s 月報已推 Telegram', stats['period_label']) + except Exception as exc: + logger.warning('[ROI] telegram push failed: %s', exc) + + # 寫 ai_insights 長期記錄 + try: + from sqlalchemy import text as sa_text + from database.manager import get_session + session = get_session() + try: + session.execute( + sa_text(""" + INSERT INTO ai_insights (insight_type, content, status, created_by, confidence) + VALUES ('roi_monthly_report', :content, 'approved', 'roi_report_service', 0.95) + """), + {'content': msg}, + ) + session.commit() + finally: + session.close() + except Exception as exc: + logger.warning('[ROI] persist failed: %s', exc) + + return {'sent': True, 'period': stats['period_label'], 'msg_chars': len(msg)} + + +__all__ = [ + 'query_last_month_stats', + 'render_roi_report', + 'generate_and_send_roi_report', + 'BASELINE', +] diff --git a/tests/test_roi_report_service.py b/tests/test_roi_report_service.py new file mode 100644 index 0000000..99a100f --- /dev/null +++ b/tests/test_roi_report_service.py @@ -0,0 +1,120 @@ +""" +tests/test_roi_report_service.py +───────────────────────────────────────────────────────────────── +Operation Ollama-First v5.0 / Phase 24 — ROI 月報生成器驗證 +""" + +from datetime import datetime +from unittest.mock import patch + +import pytest + + +def test_baseline_constants(): + """戰前 baseline 應有 5 個必要欄位""" + from services.roi_report_service import BASELINE + + required = {'gemini_monthly_tokens', 'gemini_monthly_cost_usd', + 'nim_monthly_tokens', 'ollama_monthly_tokens', + 'total_monthly_cost_usd'} + assert required.issubset(BASELINE.keys()) + assert BASELINE['gemini_monthly_tokens'] == 50_000_000 + assert BASELINE['gemini_monthly_cost_usd'] == 20.0 + + +def test_last_month_range_january(): + """1 月跑時上月應為去年 12 月""" + from services.roi_report_service import _last_month_range + + today = datetime(2026, 1, 5) + last_start, this_start = _last_month_range(today) + assert last_start == datetime(2025, 12, 1) + assert this_start == datetime(2026, 1, 1) + + +def test_last_month_range_normal_month(): + from services.roi_report_service import _last_month_range + + today = datetime(2026, 5, 15) + last_start, this_start = _last_month_range(today) + assert last_start == datetime(2026, 4, 1) + assert this_start == datetime(2026, 5, 1) + + +def test_render_roi_report_with_savings(): + """戰役後成功攔截 Gemini → 訊息含「攔截」+ 達標 ✅""" + from services.roi_report_service import render_roi_report + + stats = { + 'period_label': '2026年04月', + 'period_start': datetime(2026, 4, 1), + 'period_end': datetime(2026, 5, 1), + 'ai_total_tokens': 80_000_000, + 'ai_total_cost': 12.0, + 'ai_total_calls': 5000, + 'ollama_calls': 4000, + 'gemini_calls': 800, + 'claude_calls': 100, + 'nim_calls': 100, + 'rag_hit_calls': 200, + 'cache_hit_calls': 150, + 'gemini_tokens': 35_000_000, # 戰前 50M → 省 15M (30%) + 'gemini_cost': 14.0, # 戰前 $20 → 省 $6 + 'claude_cost': 5.0, + 'mcp_total': 180, + 'mcp_cache_hits': 50, + 'rag_total': 800, + 'rag_saved': 250, # 31% RAG 攔截率 + 'rag_avg_feedback': 4.2, + } + msg = render_roi_report(stats) + + assert '2026年04月' in msg + assert '攔截' in msg + assert '15,000,000' in msg or '15M' in msg or '15,000' in msg + assert '$6' in msg or '6.00' in msg + assert '✅' in msg # Gemini -23.5% 目標達標 + + +def test_render_roi_report_below_target(): + """未達 -23.5% 目標 → 訊息含 ⚠️""" + from services.roi_report_service import render_roi_report + + stats = { + 'period_label': '2026年04月', + 'period_start': datetime(2026, 4, 1), + 'period_end': datetime(2026, 5, 1), + 'ai_total_tokens': 80_000_000, 'ai_total_cost': 18.0, 'ai_total_calls': 5000, + 'ollama_calls': 1000, 'gemini_calls': 3000, 'claude_calls': 500, 'nim_calls': 500, + 'rag_hit_calls': 50, 'cache_hit_calls': 30, + 'gemini_tokens': 45_000_000, # 戰前 50M → 只省 10% < 23% + 'gemini_cost': 18.0, 'claude_cost': 5.0, + 'mcp_total': 100, 'mcp_cache_hits': 10, + 'rag_total': 200, 'rag_saved': 30, 'rag_avg_feedback': 3.5, + } + msg = render_roi_report(stats) + + assert '⚠️' in msg # 未達標標記 + + +def test_render_empty_stats(): + """空 stats → 預設失敗訊息""" + from services.roi_report_service import render_roi_report + + assert '⚠️' in render_roi_report({}) + + +def test_query_last_month_stats_db_fail_returns_empty(monkeypatch): + """DB 失敗應回 {} 不 raise""" + from services.roi_report_service import query_last_month_stats + + class _BrokenSession: + def execute(self, *a, **kw): + raise RuntimeError('DB connection lost') + def close(self): + pass + + import services.roi_report_service as ros + monkeypatch.setattr('database.manager.get_session', lambda: _BrokenSession()) + result = ros.query_last_month_stats() + assert result == {}