fix: rescue gcp ollama through proxy
All checks were successful
CD Pipeline / deploy (push) Successful in 1m9s
All checks were successful
CD Pipeline / deploy (push) Successful in 1m9s
This commit is contained in:
@@ -51,6 +51,8 @@ def approved_ollama_env(name: str, default: str = '') -> str:
|
||||
OLLAMA_HOST_PRIMARY = approved_ollama_env('OLLAMA_HOST_PRIMARY', 'http://34.143.170.20:11434')
|
||||
OLLAMA_HOST_SECONDARY = approved_ollama_env('OLLAMA_HOST_SECONDARY', 'http://34.21.145.224:11434')
|
||||
OLLAMA_HOST_FALLBACK = approved_ollama_env('OLLAMA_HOST_FALLBACK', 'http://192.168.0.111:11434')
|
||||
OLLAMA_HOST_PRIMARY_PROXY = approved_ollama_env('OLLAMA_HOST_PRIMARY_PROXY', 'http://192.168.0.110:11435')
|
||||
OLLAMA_HOST_SECONDARY_PROXY = approved_ollama_env('OLLAMA_HOST_SECONDARY_PROXY', 'http://192.168.0.110:11436')
|
||||
# 舊 OLLAMA_HOST 只接受核准主機;否則回到 primary,由 resolve_ollama_host() 管控級聯
|
||||
OLLAMA_HOST = approved_ollama_env('OLLAMA_HOST', OLLAMA_HOST_PRIMARY)
|
||||
DEFAULT_MODEL = os.getenv('OLLAMA_MODEL', 'llama3.1:8b') # 較快速的模型
|
||||
@@ -407,6 +409,36 @@ def _canonical_host_chain() -> List[str]:
|
||||
return chain
|
||||
|
||||
|
||||
def _proxy_rescue_for_primary(primary: str) -> str:
|
||||
"""188 直連 GCP-A 不通時,允許先走 110 primary proxy 救援同一順位。"""
|
||||
clean_primary = _normalize_host(primary)
|
||||
if clean_primary != _normalize_host(OLLAMA_HOST_PRIMARY):
|
||||
return ''
|
||||
proxy = _normalize_host(OLLAMA_HOST_PRIMARY_PROXY)
|
||||
if proxy and proxy != clean_primary:
|
||||
return proxy
|
||||
return ''
|
||||
|
||||
|
||||
def _proxy_rescue_for_secondary(secondary: str) -> str:
|
||||
"""GCP-B direct 不通時,允許走 110 secondary proxy 再進 111。"""
|
||||
clean_secondary = _normalize_host(secondary)
|
||||
if clean_secondary != _normalize_host(OLLAMA_HOST_SECONDARY):
|
||||
return ''
|
||||
proxy = _normalize_host(OLLAMA_HOST_SECONDARY_PROXY)
|
||||
if proxy and proxy != clean_secondary:
|
||||
return proxy
|
||||
return ''
|
||||
|
||||
|
||||
def _is_proxy_rescue_host(host: str) -> bool:
|
||||
clean_host = _normalize_host(host)
|
||||
return clean_host in {
|
||||
_normalize_host(OLLAMA_HOST_PRIMARY_PROXY),
|
||||
_normalize_host(OLLAMA_HOST_SECONDARY_PROXY),
|
||||
}
|
||||
|
||||
|
||||
def _is_unhealthy(host: str) -> bool:
|
||||
"""檢查 host 是否在 unhealthy TTL 內"""
|
||||
import time
|
||||
@@ -473,13 +505,28 @@ def resolve_ollama_host(primary: str = OLLAMA_HOST_PRIMARY,
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# B4: primary 若被標 unhealthy,嘗試 secondary
|
||||
primary_proxy = _proxy_rescue_for_primary(primary)
|
||||
secondary_proxy = _proxy_rescue_for_secondary(secondary)
|
||||
|
||||
# B4: primary 若被標 unhealthy,先嘗試同順位 110 proxy,再嘗試 secondary
|
||||
if not _is_unhealthy(primary) and _is_reachable(primary):
|
||||
selected = primary
|
||||
logger.info(f"[OllamaHost] Primary 主機可用: {primary}")
|
||||
elif primary_proxy and not _is_unhealthy(primary_proxy) and _is_reachable(primary_proxy):
|
||||
selected = primary_proxy
|
||||
logger.warning(
|
||||
"[OllamaHost] Primary direct 不可用,使用 110 primary proxy: %s",
|
||||
primary_proxy,
|
||||
)
|
||||
elif not _is_unhealthy(secondary) and _is_reachable(secondary):
|
||||
selected = secondary
|
||||
logger.info(f"[OllamaHost] Primary 不可用,使用 Secondary: {secondary}")
|
||||
elif secondary_proxy and not _is_unhealthy(secondary_proxy) and _is_reachable(secondary_proxy):
|
||||
selected = secondary_proxy
|
||||
logger.warning(
|
||||
"[OllamaHost] Secondary direct 不可用,使用 110 secondary proxy: %s",
|
||||
secondary_proxy,
|
||||
)
|
||||
else:
|
||||
selected = fallback
|
||||
logger.warning(f"[OllamaHost] Primary 與 Secondary 皆無法連線,切換 Fallback: {fallback}")
|
||||
@@ -668,7 +715,10 @@ class OllamaService:
|
||||
# unhealthy TTL(30s) 時第三輪又 resolve 回 primary,導致 111
|
||||
# final fallback 永遠沒被打到。
|
||||
next_host = None
|
||||
if self._explicit_host is None and current_host in allowed_hosts:
|
||||
if (
|
||||
self._explicit_host is None
|
||||
and (current_host in allowed_hosts or _is_proxy_rescue_host(current_host))
|
||||
):
|
||||
next_host = next((host for host in allowed_hosts if host not in attempted_hosts), None)
|
||||
if not next_host:
|
||||
# 非標準 host 或 explicit host 維持原行為:跳出避免無限迴圈。
|
||||
@@ -1250,7 +1300,7 @@ class OllamaService:
|
||||
visited_hosts = attempted_hosts + skipped_hosts
|
||||
if target_host in visited_hosts:
|
||||
next_host = None
|
||||
if target_host in allowed_hosts:
|
||||
if not host and (target_host in allowed_hosts or _is_proxy_rescue_host(target_host)):
|
||||
next_host = next((candidate for candidate in allowed_hosts if candidate not in visited_hosts), None)
|
||||
if not next_host:
|
||||
break # cache 還沒過期或同主機,避免無限迴圈
|
||||
|
||||
Reference in New Issue
Block a user