fix: rescue gcp ollama through proxy
All checks were successful
CD Pipeline / deploy (push) Successful in 1m9s
All checks were successful
CD Pipeline / deploy (push) Successful in 1m9s
This commit is contained in:
OoO
@@ -428,6 +428,8 @@ OLLAMA_HOST=
|
||||
OLLAMA_HOST_PRIMARY=http://34.143.170.20:11434
|
||||
OLLAMA_HOST_SECONDARY=http://34.21.145.224:11434
|
||||
OLLAMA_HOST_FALLBACK=http://192.168.0.111:11434
|
||||
OLLAMA_HOST_PRIMARY_PROXY=http://192.168.0.110:11435
|
||||
OLLAMA_HOST_SECONDARY_PROXY=http://192.168.0.110:11436
|
||||
OLLAMA_MODEL=gemma3:4b
|
||||
OLLAMA_TIMEOUT=120
|
||||
OLLAMA_COPY_TIMEOUT=180
|
||||
|
||||
@@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '')
|
||||
# ==========================================
|
||||
# 系統版本與路徑
|
||||
# ==========================================
|
||||
SYSTEM_VERSION = "V10.625"
|
||||
SYSTEM_VERSION = "V10.626"
|
||||
LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log')
|
||||
public_url = PUBLIC_URL # 用於模板顯示
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth
|
||||
|
||||
> **最後更新**: 2026-06-18 (台北時間)
|
||||
> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化與 GCP embedding 熔斷延後處理已建立
|
||||
> **適用版本**: V10.625
|
||||
> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、GCP embedding 熔斷延後處理與 110 proxy rescue 已建立
|
||||
> **適用版本**: V10.626
|
||||
|
||||
---
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
|
||||
- 所有 AI Agent、LLM 推理與 embedding 預設必須走 Ollama 三主機級聯：GCP-A `34.143.170.20:11434` → GCP-B `34.21.145.224:11434` → 111 `192.168.0.111:11434`。
|
||||
- `services/ollama_service.resolve_ollama_host()` 是主機解析契約；`OLLAMA_HOST`、`HERMES_URL`、`EMBEDDING_HOST`、`OLLAMA_API_BASE` 只接受 GCP-A / GCP-B / 111 或 110 的核准轉發端口。
|
||||
- 188 直連 GCP-A / GCP-B timeout 時，resolver 可先使用同順位 110 proxy rescue：GCP-A direct → `192.168.0.110:11435` → GCP-B direct → `192.168.0.110:11436` → 111。proxy rescue 只是同一順位的可用入口，不代表 GCP direct host 已恢復。
|
||||
- `config.OLLAMA_HOST`、`config.HERMES_URL`、`config.EMBEDDING_HOST` 只保留為舊 caller 相容常數；import-time 不得 probe network，也不得因 GCP-A/GCP-B 短暫不可用而 freeze 到 111。需要即時路由時一律呼叫 `get_ollama_host()`、`get_hermes_url()`、`get_embedding_host()` 或 `OllamaService`。
|
||||
- Gemini 只能作為 Ollama 主路徑失敗後的備援；MCP Grounding、PPT/vision、週/月報、Code Review、EA HITL、複雜 SKU 升級等舊鎖定場景也必須先走 GCP-A → GCP-B → 111。
|
||||
- 188 `192.168.0.188` 僅是 App / DB / scheduler / Telegram bot 容器宿主與 AutoHeal target，不可作為 Ollama 節點。
|
||||
|
||||
@@ -317,3 +317,10 @@
|
||||
- V10.625 將 GCP embedding failure circuit 狀態公開為 `is_embedding_gcp_circuit_open()` / `embedding_gcp_circuit_remaining_seconds()`，讓 worker 可用明確狀態判斷，不再猜測空向量原因。
|
||||
- `OpenClawLearningService` worker 在熔斷中不 claim 新任務；若處理中開啟熔斷，當筆與同批剩餘任務會退回 `pending` 並寫入延後原因，不扣 `attempts`、不刷成 `failed`。
|
||||
- 背景 embedding 仍維持 GCP-A → GCP-B，不落 111；111 不承接 `bge-m3` 背景批次的治理規則不變。
|
||||
|
||||
## 29. 2026-06-18 V10.626 GCP-A direct timeout 改走 110 proxy rescue
|
||||
|
||||
- 正式診斷腳本顯示：188 直連 GCP-A `34.143.170.20:11434` `/api/version` timeout，但 GCP-B direct、111、110 `11435` primary proxy、110 `11436` secondary proxy 都可用；GCP-B `bge-m3` embed 實測約 2.9 秒。
|
||||
- V10.626 新增 `OLLAMA_HOST_PRIMARY_PROXY` / `OLLAMA_HOST_SECONDARY_PROXY`，預設為 `http://192.168.0.110:11435` / `http://192.168.0.110:11436`。
|
||||
- `resolve_ollama_host()` 順序調整為 GCP-A direct → GCP-A via 110 proxy → GCP-B direct → GCP-B via 110 proxy → 111；proxy rescue 是同順位入口救援，不代表 direct GCP host 已恢復。
|
||||
- 近 24 小時 `ai_calls` 只有 `ollama_secondary=51`、`gcp_ollama=3`、`nim=1`，沒有 Gemini provider；Gemini hard disabled / fallback disabled 的紅線仍有效。
|
||||
|
||||
@@ -51,6 +51,8 @@ def approved_ollama_env(name: str, default: str = '') -> str:
|
||||
OLLAMA_HOST_PRIMARY = approved_ollama_env('OLLAMA_HOST_PRIMARY', 'http://34.143.170.20:11434')
|
||||
OLLAMA_HOST_SECONDARY = approved_ollama_env('OLLAMA_HOST_SECONDARY', 'http://34.21.145.224:11434')
|
||||
OLLAMA_HOST_FALLBACK = approved_ollama_env('OLLAMA_HOST_FALLBACK', 'http://192.168.0.111:11434')
|
||||
OLLAMA_HOST_PRIMARY_PROXY = approved_ollama_env('OLLAMA_HOST_PRIMARY_PROXY', 'http://192.168.0.110:11435')
|
||||
OLLAMA_HOST_SECONDARY_PROXY = approved_ollama_env('OLLAMA_HOST_SECONDARY_PROXY', 'http://192.168.0.110:11436')
|
||||
# 舊 OLLAMA_HOST 只接受核准主機；否則回到 primary，由 resolve_ollama_host() 管控級聯
|
||||
OLLAMA_HOST = approved_ollama_env('OLLAMA_HOST', OLLAMA_HOST_PRIMARY)
|
||||
DEFAULT_MODEL = os.getenv('OLLAMA_MODEL', 'llama3.1:8b') # 較快速的模型
|
||||
@@ -407,6 +409,36 @@ def _canonical_host_chain() -> List[str]:
|
||||
return chain
|
||||
|
||||
|
||||
def _proxy_rescue_for_primary(primary: str) -> str:
|
||||
"""188 直連 GCP-A 不通時，允許先走 110 primary proxy 救援同一順位。"""
|
||||
clean_primary = _normalize_host(primary)
|
||||
if clean_primary != _normalize_host(OLLAMA_HOST_PRIMARY):
|
||||
return ''
|
||||
proxy = _normalize_host(OLLAMA_HOST_PRIMARY_PROXY)
|
||||
if proxy and proxy != clean_primary:
|
||||
return proxy
|
||||
return ''
|
||||
|
||||
|
||||
def _proxy_rescue_for_secondary(secondary: str) -> str:
|
||||
"""GCP-B direct 不通時，允許走 110 secondary proxy 再進 111。"""
|
||||
clean_secondary = _normalize_host(secondary)
|
||||
if clean_secondary != _normalize_host(OLLAMA_HOST_SECONDARY):
|
||||
return ''
|
||||
proxy = _normalize_host(OLLAMA_HOST_SECONDARY_PROXY)
|
||||
if proxy and proxy != clean_secondary:
|
||||
return proxy
|
||||
return ''
|
||||
|
||||
|
||||
def _is_proxy_rescue_host(host: str) -> bool:
|
||||
clean_host = _normalize_host(host)
|
||||
return clean_host in {
|
||||
_normalize_host(OLLAMA_HOST_PRIMARY_PROXY),
|
||||
_normalize_host(OLLAMA_HOST_SECONDARY_PROXY),
|
||||
}
|
||||
|
||||
|
||||
def _is_unhealthy(host: str) -> bool:
|
||||
"""檢查 host 是否在 unhealthy TTL 內"""
|
||||
import time
|
||||
@@ -473,13 +505,28 @@ def resolve_ollama_host(primary: str = OLLAMA_HOST_PRIMARY,
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# B4: primary 若被標 unhealthy，嘗試 secondary
|
||||
primary_proxy = _proxy_rescue_for_primary(primary)
|
||||
secondary_proxy = _proxy_rescue_for_secondary(secondary)
|
||||
|
||||
# B4: primary 若被標 unhealthy，先嘗試同順位 110 proxy，再嘗試 secondary
|
||||
if not _is_unhealthy(primary) and _is_reachable(primary):
|
||||
selected = primary
|
||||
logger.info(f"[OllamaHost] Primary 主機可用: {primary}")
|
||||
elif primary_proxy and not _is_unhealthy(primary_proxy) and _is_reachable(primary_proxy):
|
||||
selected = primary_proxy
|
||||
logger.warning(
|
||||
"[OllamaHost] Primary direct 不可用，使用 110 primary proxy: %s",
|
||||
primary_proxy,
|
||||
)
|
||||
elif not _is_unhealthy(secondary) and _is_reachable(secondary):
|
||||
selected = secondary
|
||||
logger.info(f"[OllamaHost] Primary 不可用，使用 Secondary: {secondary}")
|
||||
elif secondary_proxy and not _is_unhealthy(secondary_proxy) and _is_reachable(secondary_proxy):
|
||||
selected = secondary_proxy
|
||||
logger.warning(
|
||||
"[OllamaHost] Secondary direct 不可用，使用 110 secondary proxy: %s",
|
||||
secondary_proxy,
|
||||
)
|
||||
else:
|
||||
selected = fallback
|
||||
logger.warning(f"[OllamaHost] Primary 與 Secondary 皆無法連線，切換 Fallback: {fallback}")
|
||||
@@ -668,7 +715,10 @@ class OllamaService:
|
||||
# unhealthy TTL(30s) 時第三輪又 resolve 回 primary，導致 111
|
||||
# final fallback 永遠沒被打到。
|
||||
next_host = None
|
||||
if self._explicit_host is None and current_host in allowed_hosts:
|
||||
if (
|
||||
self._explicit_host is None
|
||||
and (current_host in allowed_hosts or _is_proxy_rescue_host(current_host))
|
||||
):
|
||||
next_host = next((host for host in allowed_hosts if host not in attempted_hosts), None)
|
||||
if not next_host:
|
||||
# 非標準 host 或 explicit host 維持原行為：跳出避免無限迴圈。
|
||||
@@ -1250,7 +1300,7 @@ class OllamaService:
|
||||
visited_hosts = attempted_hosts + skipped_hosts
|
||||
if target_host in visited_hosts:
|
||||
next_host = None
|
||||
if target_host in allowed_hosts:
|
||||
if not host and (target_host in allowed_hosts or _is_proxy_rescue_host(target_host)):
|
||||
next_host = next((candidate for candidate in allowed_hosts if candidate not in visited_hosts), None)
|
||||
if not next_host:
|
||||
break # cache 還沒過期或同主機，避免無限迴圈
|
||||
|
||||
@@ -75,6 +75,31 @@ def test_resolve_falls_back_on_request_exception():
|
||||
assert host == 'http://fallback.example:11434'
|
||||
|
||||
|
||||
def test_resolve_uses_primary_proxy_rescue_before_secondary():
|
||||
"""正式主機直連 GCP-A 不通時，先走 110 primary proxy，再考慮 GCP-B。"""
|
||||
from services import ollama_service as oss
|
||||
|
||||
fake_ok = MagicMock(status_code=200)
|
||||
seen_urls = []
|
||||
|
||||
def fake_get(url, timeout=None):
|
||||
seen_urls.append(url)
|
||||
if url == f"{oss.OLLAMA_HOST_PRIMARY}/api/version":
|
||||
raise Exception("primary direct timeout")
|
||||
if url == f"{oss.OLLAMA_HOST_PRIMARY_PROXY}/api/version":
|
||||
return fake_ok
|
||||
raise AssertionError(f"should not reach {url}")
|
||||
|
||||
with patch('services.ollama_service.requests.get', side_effect=fake_get):
|
||||
host = oss.resolve_ollama_host()
|
||||
|
||||
assert host == oss.OLLAMA_HOST_PRIMARY_PROXY
|
||||
assert seen_urls == [
|
||||
f"{oss.OLLAMA_HOST_PRIMARY}/api/version",
|
||||
f"{oss.OLLAMA_HOST_PRIMARY_PROXY}/api/version",
|
||||
]
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# B4 — mark_unhealthy 行為
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
Reference in New Issue
Block a user