fix: rescue gcp ollama through proxy
All checks were successful
CD Pipeline / deploy (push) Successful in 1m9s

This commit is contained in:
OoO
2026-06-18 12:13:38 +08:00
parent 9bafa73ffc
commit ee50e440ce
6 changed files with 91 additions and 6 deletions

View File

@@ -51,6 +51,8 @@ def approved_ollama_env(name: str, default: str = '') -> str:
OLLAMA_HOST_PRIMARY = approved_ollama_env('OLLAMA_HOST_PRIMARY', 'http://34.143.170.20:11434')
OLLAMA_HOST_SECONDARY = approved_ollama_env('OLLAMA_HOST_SECONDARY', 'http://34.21.145.224:11434')
OLLAMA_HOST_FALLBACK = approved_ollama_env('OLLAMA_HOST_FALLBACK', 'http://192.168.0.111:11434')
OLLAMA_HOST_PRIMARY_PROXY = approved_ollama_env('OLLAMA_HOST_PRIMARY_PROXY', 'http://192.168.0.110:11435')
OLLAMA_HOST_SECONDARY_PROXY = approved_ollama_env('OLLAMA_HOST_SECONDARY_PROXY', 'http://192.168.0.110:11436')
# 舊 OLLAMA_HOST 只接受核准主機;否則回到 primary由 resolve_ollama_host() 管控級聯
OLLAMA_HOST = approved_ollama_env('OLLAMA_HOST', OLLAMA_HOST_PRIMARY)
DEFAULT_MODEL = os.getenv('OLLAMA_MODEL', 'llama3.1:8b') # 較快速的模型
@@ -407,6 +409,36 @@ def _canonical_host_chain() -> List[str]:
return chain
def _proxy_rescue_for_primary(primary: str) -> str:
"""188 直連 GCP-A 不通時,允許先走 110 primary proxy 救援同一順位。"""
clean_primary = _normalize_host(primary)
if clean_primary != _normalize_host(OLLAMA_HOST_PRIMARY):
return ''
proxy = _normalize_host(OLLAMA_HOST_PRIMARY_PROXY)
if proxy and proxy != clean_primary:
return proxy
return ''
def _proxy_rescue_for_secondary(secondary: str) -> str:
"""GCP-B direct 不通時,允許走 110 secondary proxy 再進 111。"""
clean_secondary = _normalize_host(secondary)
if clean_secondary != _normalize_host(OLLAMA_HOST_SECONDARY):
return ''
proxy = _normalize_host(OLLAMA_HOST_SECONDARY_PROXY)
if proxy and proxy != clean_secondary:
return proxy
return ''
def _is_proxy_rescue_host(host: str) -> bool:
clean_host = _normalize_host(host)
return clean_host in {
_normalize_host(OLLAMA_HOST_PRIMARY_PROXY),
_normalize_host(OLLAMA_HOST_SECONDARY_PROXY),
}
def _is_unhealthy(host: str) -> bool:
"""檢查 host 是否在 unhealthy TTL 內"""
import time
@@ -473,13 +505,28 @@ def resolve_ollama_host(primary: str = OLLAMA_HOST_PRIMARY,
except Exception:
return False
# B4: primary 若被標 unhealthy嘗試 secondary
primary_proxy = _proxy_rescue_for_primary(primary)
secondary_proxy = _proxy_rescue_for_secondary(secondary)
# B4: primary 若被標 unhealthy先嘗試同順位 110 proxy再嘗試 secondary
if not _is_unhealthy(primary) and _is_reachable(primary):
selected = primary
logger.info(f"[OllamaHost] Primary 主機可用: {primary}")
elif primary_proxy and not _is_unhealthy(primary_proxy) and _is_reachable(primary_proxy):
selected = primary_proxy
logger.warning(
"[OllamaHost] Primary direct 不可用,使用 110 primary proxy: %s",
primary_proxy,
)
elif not _is_unhealthy(secondary) and _is_reachable(secondary):
selected = secondary
logger.info(f"[OllamaHost] Primary 不可用,使用 Secondary: {secondary}")
elif secondary_proxy and not _is_unhealthy(secondary_proxy) and _is_reachable(secondary_proxy):
selected = secondary_proxy
logger.warning(
"[OllamaHost] Secondary direct 不可用,使用 110 secondary proxy: %s",
secondary_proxy,
)
else:
selected = fallback
logger.warning(f"[OllamaHost] Primary 與 Secondary 皆無法連線,切換 Fallback: {fallback}")
@@ -668,7 +715,10 @@ class OllamaService:
# unhealthy TTL(30s) 時第三輪又 resolve 回 primary導致 111
# final fallback 永遠沒被打到。
next_host = None
if self._explicit_host is None and current_host in allowed_hosts:
if (
self._explicit_host is None
and (current_host in allowed_hosts or _is_proxy_rescue_host(current_host))
):
next_host = next((host for host in allowed_hosts if host not in attempted_hosts), None)
if not next_host:
# 非標準 host 或 explicit host 維持原行為:跳出避免無限迴圈。
@@ -1250,7 +1300,7 @@ class OllamaService:
visited_hosts = attempted_hosts + skipped_hosts
if target_host in visited_hosts:
next_host = None
if target_host in allowed_hosts:
if not host and (target_host in allowed_hosts or _is_proxy_rescue_host(target_host)):
next_host = next((candidate for candidate in allowed_hosts if candidate not in visited_hosts), None)
if not next_host:
break # cache 還沒過期或同主機,避免無限迴圈