diff --git a/config.py b/config.py index 3f926c6..94b83d4 100644 --- a/config.py +++ b/config.py @@ -325,7 +325,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.424" +SYSTEM_VERSION = "V10.425" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 1fb32dd..c023d44 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-05-24 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯，Gemini 備援預設關閉 -> **適用版本**: V10.424 +> **適用版本**: V10.425 --- @@ -31,6 +31,7 @@ - Gemini 不可被任何狀態面板或 router 推薦為主提供者：`AIProviderService._get_recommended_provider()` 不得回傳 `gemini`，只能顯示為 fallback 狀態；`llm_model_router` 的 `ea_engine` 若收到 `gemini-*` default 必須改回 `hermes3:latest`，需要深推理時才升本地 `deepseek-r1:14b`。 - ElephantAlpha prompt / agent registry 不得再把 OpenClaw 描述為 Gemini 主模型；OpenClaw 是 `qwen2.5-coder:7b` / `qwen3:14b` Ollama-first 策略師，Gemini 僅能在 guard 顯式解鎖後作 emergency fallback。 - 111 `192.168.0.111` 只是最後一道 Mac fallback，不承接 7B+、vision、long-context 模型長駐；`OllamaService.generate()` 落到 111 時會將 `qwen3`、`deepseek-r1`、`hermes3`、`qwen2.5*`、`gemma3`、`llava`、`minicpm-v` 與 7B+ 模型依 `OLLAMA_111_MODEL_DOWNGRADE_PATTERNS` 降級到 `OLLAMA_111_MODEL_FALLBACK=llama3.2:latest`，並以 `OLLAMA_111_KEEP_ALIVE=5m`、`OLLAMA_111_MAX_TIMEOUT=20`、`OLLAMA_111_NUM_CTX=4096`、`OLLAMA_111_NUM_PREDICT=512` 封頂。OpenClaw 報告型路徑的業務 keep-alive 預設 `5m`；Code Review 以 `CODE_REVIEW_ALLOW_111_FALLBACK=false`、Hermes 以 `HERMES_ALLOW_111_FALLBACK=false` 預設跳過 111，避免 16GB RAM 主機與 GCP-B 被長駐 runner、長輸出與 24h keep-alive 壓到高 load。 +- Scheduler 每 15 分鐘執行 `run_ollama_111_usage_guard_check()`，只讀 `ai_calls` 統計最近視窗的 GCP-A / GCP-B / 111 呼叫量；預設 60 分鐘內 Ollama 呼叫至少 20 次、111 至少 3 次且占比 >= 5% 才推 Telegram。這是觀測護欄，不改路由、不寫 DB、不自動重啟服務。 - 111 的 LAN 入口必須經 `scripts/ops/ollama111_allow_proxy.py` allowlist proxy：真實 Ollama 綁 `127.0.0.1:11434`，proxy 綁 `192.168.0.111:11434`，預設只允許 111 本機與 188 生產宿主；110 / 121 / 其他 LAN client 不能直接打 111，避免跨專案 CI 或 VM 繞過 momo-pro router 載入 7B+ runner。111 上以 `scripts/ops/install_ollama111_allow_proxy.sh` 安裝 user LaunchAgent，安裝器會把 proxy script 複製到 `~/.local/share/momo-pro-system/ollama111_allow_proxy.py`，讓 LaunchAgent 不依賴 iCloud repo 掛載路徑，並讓 proxy 與 `OLLAMA_HOST=127.0.0.1:11434` 在登入/重啟後自動恢復。 - ElephantAlpha 的 `price_drop_alert` / `market_opportunity` Telegram HITL 告警必須把同款證據獨立呈現，至少包含 `match_type`、`price_basis`、`alert_tier` 與 `match_score`；沒有高信心同款與總價可比證據時，不得把 PChome/MOMO 價差寫成可直接跟價建議。 diff --git a/docs/memory/history_logs.md b/docs/memory/history_logs.md index 80c3171..8e5014f 100644 --- a/docs/memory/history_logs.md +++ b/docs/memory/history_logs.md @@ -13,6 +13,7 @@ ## 📅 詳細更新日誌 (考古存檔) ### 2026-05-24：PChome 近門檻身份回收第二輪 +- **V10.425 111 fallback 使用率護欄**: Scheduler 每 15 分鐘只讀 `ai_calls` 檢查 111 Ollama fallback 使用率，預設 60 分鐘內 Ollama 呼叫 >=20、111 呼叫 >=3 且占比 >=5% 才推 Telegram，並列出 111 caller Top 5；此護欄只觀測與告警，不改路由、不寫 DB、不重啟服務，讓 111 被異常承接高負載時可即早發現。 - **V10.424 111 proxy LaunchAgent 安裝路徑穩定化**: `install_ollama111_allow_proxy.sh` 會把 proxy script 複製到 `~/.local/share/momo-pro-system/ollama111_allow_proxy.py` 後再寫入 LaunchAgent，避免 111 重啟或 iCloud repo 路徑未掛載時代理失效；同時清空舊 stderr log，讓安裝後狀態更容易判讀。 - **V10.423 12 Agent 決策信封**: `triaged_alert()` 支援 `decision_envelope` 結構化區塊，讓 Hermes / NemoTron / OpenClaw / ElephantAlpha 與後續 12 角色決策統一輸出 `severity`、`evidence`、`recommended_action`、`expected_impact`、`confidence`、`guardrails` 與 `trace`；缺證據時必須明確標記資料品質與 HITL 邊界，避免再出現空泛效益預測或不可追溯告警。 - **V10.422 111 proxy LaunchAgent 持久化**: 新增 `scripts/ops/install_ollama111_allow_proxy.sh`，在 111 以 user LaunchAgent 安裝 `com.momo.ollama111-allow-proxy`，啟動時設定 `OLLAMA_HOST=127.0.0.1:11434`、重啟 Ollama、載入 allowlist proxy，避免重開機或重新登入後 111 又回到 LAN 全開狀態。 diff --git a/run_scheduler.py b/run_scheduler.py index 8915102..7e861e0 100644 --- a/run_scheduler.py +++ b/run_scheduler.py @@ -53,6 +53,7 @@ logging.basicConfig( logger = logging.getLogger(__name__) _AI_CALLS_ERROR_SPIKE_LAST_PUSH_TS = 0.0 +_OLLAMA_111_USAGE_LAST_PUSH_TS = 0.0 def _env_flag(name: str, default: bool = False) -> bool: @@ -204,6 +205,10 @@ def _register_schedules(): schedule.every(30).minutes.do(run_ai_calls_error_spike_check) logger.info("📅 每 30 分鐘：ai_calls_error_spike_check（錯誤率 ≥ 30% 推 Telegram）") + # Phase 57: 111 Ollama 使用率護欄，避免 final fallback 默默承接高負載 + schedule.every(15).minutes.do(run_ollama_111_usage_guard_check) + logger.info("📅 每 15 分鐘：ollama_111_usage_guard_check（111 fallback 使用率告警）") + # Phase 44: 觀測台每日 09:30 健康摘要推送 schedule.every().day.at("09:30").do(run_observability_daily_summary) logger.info("📅 每日 09:30：observability_daily_summary（早晨報三主機/AI/Cost/PPT）") @@ -724,6 +729,126 @@ def run_ai_calls_error_spike_check(): ) +def run_ollama_111_usage_guard_check(): + """Phase 57 — final fallback 111 使用率告警。 + + 111 是最後防線；這個 guard 只觀測 ai_calls，不改路由。 + 預設條件：最近 60 分鐘 Ollama 呼叫 >= 20、111 呼叫 >= 3、111 占比 >= 5%。 + """ + if not _env_flag("OLLAMA_111_USAGE_ALERT_ENABLED", True): + return + + try: + from sqlalchemy import text as _sa + from database.manager import DatabaseManager + + window_minutes = int(os.getenv("OLLAMA_111_USAGE_ALERT_WINDOW_MINUTES", "60")) + threshold_pct = float(os.getenv("OLLAMA_111_USAGE_ALERT_PCT", "5")) + min_total = int(os.getenv("OLLAMA_111_USAGE_ALERT_MIN_TOTAL", "20")) + min_111 = int(os.getenv("OLLAMA_111_USAGE_ALERT_MIN_111", "3")) + dedup_sec = int(os.getenv("OLLAMA_111_USAGE_ALERT_DEDUP_SEC", "3600")) + + session = DatabaseManager().get_session() + try: + row = session.execute( + _sa(""" + SELECT + COUNT(*) FILTER ( + WHERE provider IN ('gcp_ollama','ollama_secondary','ollama_111') + ) AS total_ollama, + COUNT(*) FILTER (WHERE provider = 'gcp_ollama') AS gcp_a, + COUNT(*) FILTER (WHERE provider = 'ollama_secondary') AS gcp_b, + COUNT(*) FILTER (WHERE provider = 'ollama_111') AS host_111 + FROM ai_calls + WHERE called_at >= NOW() - (:window_minutes || ' minutes')::interval + """), + {"window_minutes": window_minutes}, + ).fetchone() + + total_ollama = int(row[0] or 0) + gcp_a = int(row[1] or 0) + gcp_b = int(row[2] or 0) + host_111 = int(row[3] or 0) + + if total_ollama < min_total or host_111 < min_111: + return + + rate_pct = (host_111 / total_ollama * 100.0) if total_ollama else 0.0 + if rate_pct < threshold_pct: + return + + top_callers = session.execute( + _sa(""" + SELECT caller, + COALESCE(model, '') AS model, + COUNT(*) AS calls, + COALESCE(SUM(input_tokens + output_tokens), 0) AS tokens, + COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')) AS errors + FROM ai_calls + WHERE called_at >= NOW() - (:window_minutes || ' minutes')::interval + AND provider = 'ollama_111' + GROUP BY caller, model + ORDER BY calls DESC, tokens DESC + LIMIT 5 + """), + {"window_minutes": window_minutes}, + ).fetchall() + finally: + session.close() + + global _OLLAMA_111_USAGE_LAST_PUSH_TS + now_ts = time.time() + if now_ts - _OLLAMA_111_USAGE_LAST_PUSH_TS < dedup_sec: + logger.info("[Ollama111Guard] skip duplicate alert within %ss window", dedup_sec) + return + + from services.telegram_templates import send_telegram_with_result + + lines = [ + "⚠️ 111 Ollama 使用率偏高", + "", + f"過去 {window_minutes} 分鐘 Ollama 呼叫：{total_ollama} 次", + f"111 fallback：{host_111} 次（{rate_pct:.1f}%）", + f"GCP-A：{gcp_a} 次 · GCP-B：{gcp_b} 次", + "", + ] + if top_callers: + lines.append("111 caller Top 5：") + for caller, model, calls, tokens, errors in top_callers: + model_part = f" / {model}" if model else "" + err_part = f" · err {errors}" if int(errors or 0) else "" + lines.append( + f"• {caller}{model_part}：{calls} 次 · {int(tokens or 0):,} tokens{err_part}" + ) + lines.append("") + lines.extend([ + "建議先看 GCP-A/GCP-B health probe 與近期 unhealthy mark；", + "若 GCP 正常，檢查是否有 fallback flag 或重任務意外打到 111。", + ]) + + reply_markup = { + "inline_keyboard": [ + [{"text": "🏥 主機健康", "callback_data": "cmd:obs_health"}, + {"text": "📊 AI 呼叫", "callback_data": "cmd:obs_ai_calls"}], + ], + } + send_telegram_with_result("\n".join(lines), reply_markup=reply_markup, parse_mode="HTML") + _OLLAMA_111_USAGE_LAST_PUSH_TS = now_ts + logger.warning( + "[Ollama111Guard] alert pushed: total=%s gcp_a=%s gcp_b=%s host_111=%s rate=%.1f%%", + total_ollama, gcp_a, gcp_b, host_111, rate_pct, + ) + except Exception as e: + logger.error(f"[Ollama111Guard] failed: {e}", exc_info=True) + _notify_scheduler_failure( + "run_ollama_111_usage_guard_check", + e, + source="Scheduler.Ollama111Guard", + event_type="ollama_111_usage_guard_failure", + title="111 Ollama 使用率護欄失敗", + ) + + def run_observability_daily_summary(): """Phase 44 — 每日 09:30 推送觀測台健康摘要（早晨報）。 diff --git a/scripts/ops/install_ollama111_allow_proxy.sh b/scripts/ops/install_ollama111_allow_proxy.sh index f5192fe..2726638 100755 --- a/scripts/ops/install_ollama111_allow_proxy.sh +++ b/scripts/ops/install_ollama111_allow_proxy.sh @@ -16,7 +16,7 @@ INSTALL_SCRIPT_PATH="${INSTALL_DIR}/ollama111_allow_proxy.py" PYTHON_BIN="${PYTHON_BIN:-/usr/bin/python3}" OLLAMA_APP="${OLLAMA_APP:-/Applications/Ollama.app}" OLLAMA_HOST_VALUE="${OLLAMA_HOST_VALUE:-127.0.0.1:11434}" -ALLOWED_CIDRS="${OLLAMA111_PROXY_ALLOWED_CIDRS:-127.0.0.1/32,192.168.0.80/32,192.168.0.111/32,192.168.0.188/32}" +ALLOWED_CIDRS="${OLLAMA111_PROXY_ALLOWED_CIDRS:-127.0.0.1/32,192.168.0.111/32,192.168.0.188/32}" GUI_DOMAIN="gui/$(id -u)" if [[ ! -f "${PROJECT_DIR}/scripts/ops/ollama111_allow_proxy.py" ]]; then diff --git a/scripts/ops/ollama111_allow_proxy.py b/scripts/ops/ollama111_allow_proxy.py index 62d6b1c..9fa4f32 100755 --- a/scripts/ops/ollama111_allow_proxy.py +++ b/scripts/ops/ollama111_allow_proxy.py @@ -15,7 +15,7 @@ import ipaddress import logging import os import signal -from typing import Iterable +import sys LISTEN_HOST = os.getenv("OLLAMA111_PROXY_LISTEN_HOST", "192.168.0.111") @@ -26,7 +26,7 @@ ALLOWED_CIDRS = tuple( item.strip() for item in os.getenv( "OLLAMA111_PROXY_ALLOWED_CIDRS", - "127.0.0.1/32,192.168.0.80/32,192.168.0.111/32,192.168.0.188/32", + "127.0.0.1/32,192.168.0.111/32,192.168.0.188/32", ).split(",") if item.strip() ) @@ -93,6 +93,7 @@ async def _main() -> None: logging.basicConfig( level=os.getenv("OLLAMA111_PROXY_LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s %(message)s", + stream=sys.stdout, ) server = await asyncio.start_server(_handle_client, LISTEN_HOST, LISTEN_PORT) sockets = ", ".join(str(sock.getsockname()) for sock in (server.sockets or [])) diff --git a/tests/test_ollama111_proxy_contract.py b/tests/test_ollama111_proxy_contract.py new file mode 100644 index 0000000..effc970 --- /dev/null +++ b/tests/test_ollama111_proxy_contract.py @@ -0,0 +1,23 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_ollama111_proxy_default_allowlist_stays_production_only(): + proxy_source = (ROOT / "scripts/ops/ollama111_allow_proxy.py").read_text() + installer_source = (ROOT / "scripts/ops/install_ollama111_allow_proxy.sh").read_text() + + assert "192.168.0.188/32" in proxy_source + assert "192.168.0.188/32" in installer_source + assert "192.168.0.111/32" in proxy_source + assert "192.168.0.111/32" in installer_source + assert "192.168.0.80/32" not in proxy_source + assert "192.168.0.80/32" not in installer_source + + +def test_ollama111_proxy_logs_to_stdout_for_launchagent_collection(): + proxy_source = (ROOT / "scripts/ops/ollama111_allow_proxy.py").read_text() + + assert "import sys" in proxy_source + assert "stream=sys.stdout" in proxy_source diff --git a/tests/test_run_scheduler_embed_consistency.py b/tests/test_run_scheduler_embed_consistency.py index 0d1e9e1..9e67216 100644 --- a/tests/test_run_scheduler_embed_consistency.py +++ b/tests/test_run_scheduler_embed_consistency.py @@ -146,6 +146,7 @@ def test_v2_cron_blind_spot_list_has_failure_notifications(monkeypatch): "run_cost_throttle_reset_if_new_month", "run_ppt_vision_audit", "run_embed_consistency_check", + "run_ollama_111_usage_guard_check", ]: source = inspect.getsource(getattr(run_scheduler, fn_name)) assert "_notify_scheduler_failure(" in source @@ -161,6 +162,20 @@ def test_roi_ai_smoke_and_daily_report_schedules_stay_staggered(): assert 'schedule.every().day.at("09:05").do(run_roi_monthly_report_if_new_month)' in source assert 'schedule.every().day.at("09:10").do(run_ai_smoke_daily_summary_task)' in source assert "schedule.every(6).hours.do(run_action_plan_hygiene_task)" in source + assert "schedule.every(15).minutes.do(run_ollama_111_usage_guard_check)" in source + + +def test_ollama_111_usage_guard_stays_observational(monkeypatch): + run_scheduler = _load_run_scheduler(monkeypatch) + source = inspect.getsource(run_scheduler.run_ollama_111_usage_guard_check) + + assert "OLLAMA_111_USAGE_ALERT_ENABLED" in source + assert "provider = 'ollama_111'" in source + assert "send_telegram_with_result" in source + assert "_notify_scheduler_failure(" in source + assert "只觀測 ai_calls，不改路由" in source + assert "UPDATE" not in source + assert "DELETE" not in source def test_legacy_edm_and_seasonal_promo_schedules_are_opt_in(monkeypatch):