diff --git a/.env.example b/.env.example index d59eeb9..237d9bb 100644 --- a/.env.example +++ b/.env.example @@ -428,6 +428,8 @@ OLLAMA_HOST= OLLAMA_HOST_PRIMARY=http://34.143.170.20:11434 OLLAMA_HOST_SECONDARY=http://34.21.145.224:11434 OLLAMA_HOST_FALLBACK=http://192.168.0.111:11434 +OLLAMA_HOST_PRIMARY_PROXY=http://192.168.0.110:11435 +OLLAMA_HOST_SECONDARY_PROXY=http://192.168.0.110:11436 OLLAMA_MODEL=gemma3:4b OLLAMA_TIMEOUT=120 OLLAMA_COPY_TIMEOUT=180 diff --git a/config.py b/config.py index e0065d2..e3e626c 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.625" +SYSTEM_VERSION = "V10.626" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 024109d..d725378 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,8 +1,8 @@ # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth > **最後更新**: 2026-06-18 (台北時間) -> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化與 GCP embedding 熔斷延後處理已建立 -> **適用版本**: V10.625 +> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、GCP embedding 熔斷延後處理與 110 proxy rescue 已建立 +> **適用版本**: V10.626 --- @@ -10,6 +10,7 @@ - 所有 AI Agent、LLM 推理與 embedding 預設必須走 Ollama 三主機級聯：GCP-A `34.143.170.20:11434` → GCP-B `34.21.145.224:11434` → 111 `192.168.0.111:11434`。 - `services/ollama_service.resolve_ollama_host()` 是主機解析契約；`OLLAMA_HOST`、`HERMES_URL`、`EMBEDDING_HOST`、`OLLAMA_API_BASE` 只接受 GCP-A / GCP-B / 111 或 110 的核准轉發端口。 +- 188 直連 GCP-A / GCP-B timeout 時，resolver 可先使用同順位 110 proxy rescue：GCP-A direct → `192.168.0.110:11435` → GCP-B direct → `192.168.0.110:11436` → 111。proxy rescue 只是同一順位的可用入口，不代表 GCP direct host 已恢復。 - `config.OLLAMA_HOST`、`config.HERMES_URL`、`config.EMBEDDING_HOST` 只保留為舊 caller 相容常數；import-time 不得 probe network，也不得因 GCP-A/GCP-B 短暫不可用而 freeze 到 111。需要即時路由時一律呼叫 `get_ollama_host()`、`get_hermes_url()`、`get_embedding_host()` 或 `OllamaService`。 - Gemini 只能作為 Ollama 主路徑失敗後的備援；MCP Grounding、PPT/vision、週/月報、Code Review、EA HITL、複雜 SKU 升級等舊鎖定場景也必須先走 GCP-A → GCP-B → 111。 - 188 `192.168.0.188` 僅是 App / DB / scheduler / Telegram bot 容器宿主與 AutoHeal target，不可作為 Ollama 節點。 diff --git a/docs/memory/current_execution_queue_20260524.md b/docs/memory/current_execution_queue_20260524.md index 62c801f..d29272f 100644 --- a/docs/memory/current_execution_queue_20260524.md +++ b/docs/memory/current_execution_queue_20260524.md @@ -317,3 +317,10 @@ - V10.625 將 GCP embedding failure circuit 狀態公開為 `is_embedding_gcp_circuit_open()` / `embedding_gcp_circuit_remaining_seconds()`，讓 worker 可用明確狀態判斷，不再猜測空向量原因。 - `OpenClawLearningService` worker 在熔斷中不 claim 新任務；若處理中開啟熔斷，當筆與同批剩餘任務會退回 `pending` 並寫入延後原因，不扣 `attempts`、不刷成 `failed`。 - 背景 embedding 仍維持 GCP-A → GCP-B，不落 111；111 不承接 `bge-m3` 背景批次的治理規則不變。 + +## 29. 2026-06-18 V10.626 GCP-A direct timeout 改走 110 proxy rescue + +- 正式診斷腳本顯示：188 直連 GCP-A `34.143.170.20:11434` `/api/version` timeout，但 GCP-B direct、111、110 `11435` primary proxy、110 `11436` secondary proxy 都可用；GCP-B `bge-m3` embed 實測約 2.9 秒。 +- V10.626 新增 `OLLAMA_HOST_PRIMARY_PROXY` / `OLLAMA_HOST_SECONDARY_PROXY`，預設為 `http://192.168.0.110:11435` / `http://192.168.0.110:11436`。 +- `resolve_ollama_host()` 順序調整為 GCP-A direct → GCP-A via 110 proxy → GCP-B direct → GCP-B via 110 proxy → 111；proxy rescue 是同順位入口救援，不代表 direct GCP host 已恢復。 +- 近 24 小時 `ai_calls` 只有 `ollama_secondary=51`、`gcp_ollama=3`、`nim=1`，沒有 Gemini provider；Gemini hard disabled / fallback disabled 的紅線仍有效。 diff --git a/services/ollama_service.py b/services/ollama_service.py index be99aa2..b69c15f 100644 --- a/services/ollama_service.py +++ b/services/ollama_service.py @@ -51,6 +51,8 @@ def approved_ollama_env(name: str, default: str = '') -> str: OLLAMA_HOST_PRIMARY = approved_ollama_env('OLLAMA_HOST_PRIMARY', 'http://34.143.170.20:11434') OLLAMA_HOST_SECONDARY = approved_ollama_env('OLLAMA_HOST_SECONDARY', 'http://34.21.145.224:11434') OLLAMA_HOST_FALLBACK = approved_ollama_env('OLLAMA_HOST_FALLBACK', 'http://192.168.0.111:11434') +OLLAMA_HOST_PRIMARY_PROXY = approved_ollama_env('OLLAMA_HOST_PRIMARY_PROXY', 'http://192.168.0.110:11435') +OLLAMA_HOST_SECONDARY_PROXY = approved_ollama_env('OLLAMA_HOST_SECONDARY_PROXY', 'http://192.168.0.110:11436') # 舊 OLLAMA_HOST 只接受核准主機；否則回到 primary，由 resolve_ollama_host() 管控級聯 OLLAMA_HOST = approved_ollama_env('OLLAMA_HOST', OLLAMA_HOST_PRIMARY) DEFAULT_MODEL = os.getenv('OLLAMA_MODEL', 'llama3.1:8b') # 較快速的模型 @@ -407,6 +409,36 @@ def _canonical_host_chain() -> List[str]: return chain +def _proxy_rescue_for_primary(primary: str) -> str: + """188 直連 GCP-A 不通時，允許先走 110 primary proxy 救援同一順位。""" + clean_primary = _normalize_host(primary) + if clean_primary != _normalize_host(OLLAMA_HOST_PRIMARY): + return '' + proxy = _normalize_host(OLLAMA_HOST_PRIMARY_PROXY) + if proxy and proxy != clean_primary: + return proxy + return '' + + +def _proxy_rescue_for_secondary(secondary: str) -> str: + """GCP-B direct 不通時，允許走 110 secondary proxy 再進 111。""" + clean_secondary = _normalize_host(secondary) + if clean_secondary != _normalize_host(OLLAMA_HOST_SECONDARY): + return '' + proxy = _normalize_host(OLLAMA_HOST_SECONDARY_PROXY) + if proxy and proxy != clean_secondary: + return proxy + return '' + + +def _is_proxy_rescue_host(host: str) -> bool: + clean_host = _normalize_host(host) + return clean_host in { + _normalize_host(OLLAMA_HOST_PRIMARY_PROXY), + _normalize_host(OLLAMA_HOST_SECONDARY_PROXY), + } + + def _is_unhealthy(host: str) -> bool: """檢查 host 是否在 unhealthy TTL 內""" import time @@ -473,13 +505,28 @@ def resolve_ollama_host(primary: str = OLLAMA_HOST_PRIMARY, except Exception: return False - # B4: primary 若被標 unhealthy，嘗試 secondary + primary_proxy = _proxy_rescue_for_primary(primary) + secondary_proxy = _proxy_rescue_for_secondary(secondary) + + # B4: primary 若被標 unhealthy，先嘗試同順位 110 proxy，再嘗試 secondary if not _is_unhealthy(primary) and _is_reachable(primary): selected = primary logger.info(f"[OllamaHost] Primary 主機可用: {primary}") + elif primary_proxy and not _is_unhealthy(primary_proxy) and _is_reachable(primary_proxy): + selected = primary_proxy + logger.warning( + "[OllamaHost] Primary direct 不可用，使用 110 primary proxy: %s", + primary_proxy, + ) elif not _is_unhealthy(secondary) and _is_reachable(secondary): selected = secondary logger.info(f"[OllamaHost] Primary 不可用，使用 Secondary: {secondary}") + elif secondary_proxy and not _is_unhealthy(secondary_proxy) and _is_reachable(secondary_proxy): + selected = secondary_proxy + logger.warning( + "[OllamaHost] Secondary direct 不可用，使用 110 secondary proxy: %s", + secondary_proxy, + ) else: selected = fallback logger.warning(f"[OllamaHost] Primary 與 Secondary 皆無法連線，切換 Fallback: {fallback}") @@ -668,7 +715,10 @@ class OllamaService: # unhealthy TTL(30s) 時第三輪又 resolve 回 primary，導致 111 # final fallback 永遠沒被打到。 next_host = None - if self._explicit_host is None and current_host in allowed_hosts: + if ( + self._explicit_host is None + and (current_host in allowed_hosts or _is_proxy_rescue_host(current_host)) + ): next_host = next((host for host in allowed_hosts if host not in attempted_hosts), None) if not next_host: # 非標準 host 或 explicit host 維持原行為：跳出避免無限迴圈。 @@ -1250,7 +1300,7 @@ class OllamaService: visited_hosts = attempted_hosts + skipped_hosts if target_host in visited_hosts: next_host = None - if target_host in allowed_hosts: + if not host and (target_host in allowed_hosts or _is_proxy_rescue_host(target_host)): next_host = next((candidate for candidate in allowed_hosts if candidate not in visited_hosts), None) if not next_host: break # cache 還沒過期或同主機，避免無限迴圈 diff --git a/tests/test_ollama_resolve.py b/tests/test_ollama_resolve.py index 785b349..2a17f92 100644 --- a/tests/test_ollama_resolve.py +++ b/tests/test_ollama_resolve.py @@ -75,6 +75,31 @@ def test_resolve_falls_back_on_request_exception(): assert host == 'http://fallback.example:11434' +def test_resolve_uses_primary_proxy_rescue_before_secondary(): + """正式主機直連 GCP-A 不通時，先走 110 primary proxy，再考慮 GCP-B。""" + from services import ollama_service as oss + + fake_ok = MagicMock(status_code=200) + seen_urls = [] + + def fake_get(url, timeout=None): + seen_urls.append(url) + if url == f"{oss.OLLAMA_HOST_PRIMARY}/api/version": + raise Exception("primary direct timeout") + if url == f"{oss.OLLAMA_HOST_PRIMARY_PROXY}/api/version": + return fake_ok + raise AssertionError(f"should not reach {url}") + + with patch('services.ollama_service.requests.get', side_effect=fake_get): + host = oss.resolve_ollama_host() + + assert host == oss.OLLAMA_HOST_PRIMARY_PROXY + assert seen_urls == [ + f"{oss.OLLAMA_HOST_PRIMARY}/api/version", + f"{oss.OLLAMA_HOST_PRIMARY_PROXY}/api/version", + ] + + # ═══════════════════════════════════════════════════════════════════════════ # B4 — mark_unhealthy 行為 # ═══════════════════════════════════════════════════════════════════════════