feat(ai): add bounded model-aware decision fallback
Some checks failed
CD Pipeline / deploy (push) Has been cancelled
Some checks failed
CD Pipeline / deploy (push) Has been cancelled
This commit is contained in:
ogt
@@ -460,7 +460,8 @@ OLLAMA_HOST_HEALTH_MODEL_PROBE_INCLUDE_111=false
|
||||
OLLAMA_HOST_HEALTH_EMBED_MODEL=bge-m3:latest
|
||||
OLLAMA_HOST_HEALTH_EMBED_TIMEOUT=30
|
||||
OLLAMA_HOST_HEALTH_EMBED_KEEP_ALIVE=1m
|
||||
# 111 是 Mac final fallback，不承接 7B+ / vision / long-context / 長輸出任務；落到 111 時自動降級與縮短常駐。
|
||||
# 111 的一般 OllamaService fallback 不承接 7B+ / vision / long-context / 長輸出任務。
|
||||
# NemoTron 是唯一受控特例：只允許下方 exact-digest qwen3:8b，並強制總 deadline、num_ctx 4096、num_predict 512。
|
||||
OLLAMA_111_MODEL_FALLBACK=llama3.2:latest
|
||||
OLLAMA_111_MODEL_DOWNGRADE_PATTERNS=qwen3:*,deepseek-r1:*,hermes3:*,llama3.1:*,qwen2.5:*,qwen2.5-coder:*,gemma3:*,minicpm-v:*,llava:*,*:7b*,*:8b*,*:14b*,*:32b*,*:70b*
|
||||
OLLAMA_111_KEEP_ALIVE=5m
|
||||
@@ -480,6 +481,11 @@ NEMOTRON_OLLAMA_FIRST=true
|
||||
NEMOTRON_OLLAMA_MODEL=qwen3:14b
|
||||
NEMOTRON_OLLAMA_TIMEOUT=180
|
||||
NEMOTRON_OLLAMA_EXPECTED_DIGEST=bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8
|
||||
NEMOTRON_OLLAMA_FALLBACK_MODEL=qwen3:8b
|
||||
NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST=500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41
|
||||
NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC=60
|
||||
NEMOTRON_111_ATTEMPT_TIMEOUT_SEC=45
|
||||
NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC=180
|
||||
NEMOTRON_DECISION_CANARY_TIMEOUT_SEC=300
|
||||
NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC=10
|
||||
NEMOTRON_DECISION_CANARY_MAX_AGE_HOURS=26
|
||||
|
||||
@@ -414,7 +414,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '')
|
||||
# ==========================================
|
||||
# 系統版本與路徑
|
||||
# ==========================================
|
||||
SYSTEM_VERSION = "V10.815"
|
||||
SYSTEM_VERSION = "V10.816"
|
||||
LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log')
|
||||
public_url = PUBLIC_URL # 用於模板顯示
|
||||
|
||||
|
||||
@@ -107,6 +107,12 @@ services:
|
||||
- RAG_EMBED_EXPECTED_DIGEST=${RAG_EMBED_EXPECTED_DIGEST:-7907646426070047a77226ac3e684fbbe8410524f7b4a74d02837e43f2146bab}
|
||||
- EMBED_CONSISTENCY_TIMEOUT_SEC=${EMBED_CONSISTENCY_TIMEOUT_SEC:-150}
|
||||
- NEMOTRON_OLLAMA_EXPECTED_DIGEST=${NEMOTRON_OLLAMA_EXPECTED_DIGEST:-bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8}
|
||||
- NEMOTRON_OLLAMA_FALLBACK_MODEL=${NEMOTRON_OLLAMA_FALLBACK_MODEL:-qwen3:8b}
|
||||
- NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST=${NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST:-500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41}
|
||||
- NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC=${NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC:-60}
|
||||
- NEMOTRON_111_ATTEMPT_TIMEOUT_SEC=${NEMOTRON_111_ATTEMPT_TIMEOUT_SEC:-45}
|
||||
- NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC=${NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC:-180}
|
||||
- NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC=${NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC:-10}
|
||||
- NEMOTRON_DECISION_CANARY_TIMEOUT_SEC=${NEMOTRON_DECISION_CANARY_TIMEOUT_SEC:-300}
|
||||
# Gemini 只能作緊急備援；即使 .env 有 API key，預設也不得產生付費出站。
|
||||
- GEMINI_API_HARD_DISABLED=${GEMINI_API_HARD_DISABLED:-true}
|
||||
@@ -250,6 +256,12 @@ services:
|
||||
- RAG_EMBED_EXPECTED_DIGEST=${RAG_EMBED_EXPECTED_DIGEST:-7907646426070047a77226ac3e684fbbe8410524f7b4a74d02837e43f2146bab}
|
||||
- EMBED_CONSISTENCY_TIMEOUT_SEC=${EMBED_CONSISTENCY_TIMEOUT_SEC:-150}
|
||||
- NEMOTRON_OLLAMA_EXPECTED_DIGEST=${NEMOTRON_OLLAMA_EXPECTED_DIGEST:-bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8}
|
||||
- NEMOTRON_OLLAMA_FALLBACK_MODEL=${NEMOTRON_OLLAMA_FALLBACK_MODEL:-qwen3:8b}
|
||||
- NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST=${NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST:-500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41}
|
||||
- NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC=${NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC:-60}
|
||||
- NEMOTRON_111_ATTEMPT_TIMEOUT_SEC=${NEMOTRON_111_ATTEMPT_TIMEOUT_SEC:-45}
|
||||
- NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC=${NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC:-180}
|
||||
- NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC=${NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC:-10}
|
||||
- NEMOTRON_DECISION_CANARY_TIMEOUT_SEC=${NEMOTRON_DECISION_CANARY_TIMEOUT_SEC:-300}
|
||||
# Gemini 只能作緊急備援；即使 .env 有 API key，預設也不得產生付費出站。
|
||||
- GEMINI_API_HARD_DISABLED=${GEMINI_API_HARD_DISABLED:-true}
|
||||
@@ -322,6 +334,12 @@ services:
|
||||
- RAG_EMBED_EXPECTED_DIGEST=${RAG_EMBED_EXPECTED_DIGEST:-7907646426070047a77226ac3e684fbbe8410524f7b4a74d02837e43f2146bab}
|
||||
- EMBED_CONSISTENCY_TIMEOUT_SEC=${EMBED_CONSISTENCY_TIMEOUT_SEC:-150}
|
||||
- NEMOTRON_OLLAMA_EXPECTED_DIGEST=${NEMOTRON_OLLAMA_EXPECTED_DIGEST:-bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8}
|
||||
- NEMOTRON_OLLAMA_FALLBACK_MODEL=${NEMOTRON_OLLAMA_FALLBACK_MODEL:-qwen3:8b}
|
||||
- NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST=${NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST:-500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41}
|
||||
- NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC=${NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC:-60}
|
||||
- NEMOTRON_111_ATTEMPT_TIMEOUT_SEC=${NEMOTRON_111_ATTEMPT_TIMEOUT_SEC:-45}
|
||||
- NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC=${NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC:-180}
|
||||
- NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC=${NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC:-10}
|
||||
- NEMOTRON_DECISION_CANARY_TIMEOUT_SEC=${NEMOTRON_DECISION_CANARY_TIMEOUT_SEC:-300}
|
||||
# Gemini 只能作緊急備援；即使 .env 有 API key，預設也不得產生付費出站。
|
||||
- GEMINI_API_HARD_DISABLED=${GEMINI_API_HARD_DISABLED:-true}
|
||||
|
||||
@@ -1,19 +1,21 @@
|
||||
# PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth
|
||||
|
||||
> **最後更新**: 2026-07-17 (台北時間)
|
||||
> **狀態**: 🟠 Partial。Production runtime 仍是 V10.813；V10.814 source 已修正 BGE-M3 冷啟動假陰性並加入 NemoTron decision-only canary，但部署與 production canary 尚未在本段 source truth 中宣稱完成。Windows 99 audit run `534c6208-2088-428c-ba6f-f7a63a67f422` 證明舊 project `astral-gateway-484913-d7` 已為 `DELETE_REQUESTED`、GCP-A `22/11434` 不可達；GCP-B Ollama `0.22.1` 可達，`bge-m3:latest` digest `7907646426070047a77226ac3e684fbbe8410524f7b4a74d02837e43f2146bab` 的 1024 維 no-write probe 成功但冷啟動需 `116.475s`，`qwen3:14b` digest 為 `bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8`。四 Agent runtime、MCP/RAG flags 與 telemetry 仍沿用 V10.813 production readback；程式、測試、tags 或 digest 可見都不能覆蓋尚未完成的 production closure。
|
||||
> **適用版本**: V10.813 production runtime；V10.814 source candidate；AI Agent/MCP/RAG full product integration remains partial
|
||||
> **最後更新**: 2026-07-22 (台北時間)
|
||||
> **狀態**: 🟠 Partial。Production runtime 是 V10.815、exact Gitea object `283c8c80c631f5d97315708885413b62ee5a34ea`；Windows 99 controlled run `736e6de1-fbc3-4c9b-bcfe-9b6ef488b932` 已完成 `36/36` source hash、`12/12` runtime env、三應用健康與 `momo-db` identity unchanged 的獨立讀回。PixelRAG receipt -> BGE-M3 -> pgvector no-write canary 已通過，但 `RAG_ENABLED=false` 與 embedding redundancy degraded 仍阻擋 activation。GCP-A 仍不可達；GCP-B exact-digest `qwen3:14b` chat 逾時，較小的 `qwen2.5:7b-instruct` 亦逾時，證明問題包含共享 CPU/queue，而非只靠換小模型即可解決。111 exact-digest `qwen3:8b` decision-only probe 於 `19.053s` 通過。V10.816 source candidate 已把這份實證接成 model-aware、bounded、exact-digest 的 GCP-A -> GCP-B -> 111 fallback；尚未完成本版 production deploy/readback 前，不得宣稱正式 runtime 已採用。
|
||||
> **適用版本**: V10.815 production runtime；V10.816 source candidate；AI Agent/MCP/RAG full product integration remains partial
|
||||
|
||||
---
|
||||
|
||||
## 零之負五、AI Agent 產品整合 truth 與 internal RAG/NemoTron canary（V10.814 source）
|
||||
## 零之負五、AI Agent 產品整合 truth 與 internal RAG/NemoTron canary（V10.815 production / V10.816 source）
|
||||
|
||||
- `/api/ai-automation/agent-product-integration` 與 `scripts/ops/report_ai_agent_product_integration.py` 分開輸出四 Agent source/scheduler wiring、七日 `ai_calls` 實際呼叫與錯誤率、MCP/RAG telemetry、action plan/outcome、AutoHeal incident retry，以及九階段 closure。只有四 Agent 全部有健康 runtime、MCP/RAG 已啟用且有 telemetry、internal RAG canary 已通過、受控執行/驗證/重試/學習都有實證時才可回 `fully_integrated`。
|
||||
- `/api/ai-automation/internal-rag-candidate-canary` 只有 GET，永遠是 no-model/no-DB-write readback；production execute 由 `momo-scheduler` 每日 04:45 自動跑一次。V10.814 起 consistency probe 會先由 Ollama `/api/tags` 驗證 manifest digest，再以 canary 專用 `150s` timeout 容納 GCP-B 的實測冷啟動；一般 embedding 呼叫仍保留既有短 timeout cap，不把所有 request 放大成 150 秒。至少一台核准 GCP host 必須同時通過 digest 與 1024 維 embedding，111 不可形成 quorum；單一 GCP 可進 degraded read-only pgvector shadow probe，但 `rag_embedding_redundancy_degraded` 仍阻擋正式 activation。
|
||||
- `/api/ai-automation/internal-rag-candidate-canary` 只有 GET，永遠是 no-model/no-DB-write readback；production execute 由 `momo-scheduler` 每日 04:45 自動跑一次。V10.815 production 已使用先由 Ollama `/api/tags` 驗證 manifest digest、再以 canary 專用 `150s` timeout 容納 GCP-B 冷啟動的契約；一般 embedding 呼叫仍保留既有短 timeout cap，不把所有 request 放大成 150 秒。至少一台核准 GCP host 必須同時通過 digest 與 1024 維 embedding，111 不可形成 quorum；單一 GCP 可進 degraded read-only pgvector shadow probe，但 `rag_embedding_redundancy_degraded` 仍阻擋正式 activation。
|
||||
- RAG canary 不 INSERT/UPDATE `ai_insights`、`competitor_prices`、`external_offers` 或任何正式價格表；artifact 與 scheduler receipt 共用 `trace_id/run_id/work_item_id`，明確輸出 `transaction_read_only`、similarity、embedding signature、expected/observed digest、GCP reachability、Telegram acknowledgement 與 zero-write/rollback terminal。digest drift、零核准 GCP host、向量維度錯誤、pgvector probe 失敗或 semantic threshold 未達都必須 fail closed。
|
||||
- `bge-m3:latest` 只可在 expected digest gate 下使用：source contract 固定 digest，runtime 每次 canary 由 `/api/tags` 比對，漂移時在 embedding/DB 前阻擋。Ollama 官方 `/api/tags` 明確提供模型 `digest`，因此 immutable contract 使用「固定 expected digest + runtime verifier」，不是只信任 floating tag。MCP runtime 仍受 localhost-only、read-only tool contract 與 required secret presence preflight 約束，不可因 registry 已存在就宣稱上線。
|
||||
- V10.814 新增 `/api/ai-automation/nemotron-decision-canary`、`scripts/ops/run_nemotron_decision_canary.py` 與每日 05:00 scheduler lane。execute 只用 production 共用 qwen3 tool-calling payload產生 synthetic decision，驗證 expected model digest、allowed tool、synthetic SKU 與 post-call digest，固定 `tool_execution_count=0`、`database_call_performed=false`、`writes_price_tables=false`、`writes_ai_insights=false`、`telegram_sent=false`；scheduler 只在 model canary 後發生命週期 acknowledgement 並原子回寫 receipt。
|
||||
- `AI Agent product integration truth` 會讀取 fresh NemoTron decision-only receipt；這份 shadow canary 能證明模型決策路徑真的執行，但不能冒充正式商品 action、agent outcome 或完整 Controlled Apply。舊的 NemoTron/ElephantAlpha class/method smoke 保留為 source guard，也不能單獨代表產品整合完成。
|
||||
- V10.815 production 提供 `/api/ai-automation/nemotron-decision-canary`、`scripts/ops/run_nemotron_decision_canary.py` 與每日 05:00 scheduler lane。V10.816 source 將 production dispatcher 與 canary 共用 `services/nemotron_runtime_candidate_service.py`：GCP-A/GCP-B 必須使用 exact-digest `qwen3:14b`，每台 chat 最多 60 秒；最終 111 fallback 必須使用 exact-digest `qwen3:8b`，最多 45 秒並固定 `num_ctx=4096`、`num_predict=512`，所有 payload 固定 `think=false`。Production Ollama + NIM 共用單一 180 秒 monotonic deadline，canary 的 `timeout_sec` 也是整次執行上限，不再對每個候選重新計時。每次呼叫前後都驗證 digest；畸形 `/api/tags` payload/model/details schema 必須 fail closed 並轉下一候選，不能中斷 fallback。實際 model/provider/host/fallback/attempts 必須進 logger、通知 footprint 與 receipt。
|
||||
- 模型輸出必須在選定候選前通過 deterministic tool contract：工具在 allowlist、必要欄位與型別正確、SKU 必須來自本次輸入、每個 SKU 恰好一個 call 且全數覆蓋。Production 去重使用 `services/nemotron_dispatch_reservation_service.py` 在三容器共用 `/app/data` 上執行 `flock`、file fsync、atomic replace 與 parent-directory fsync 的 process/container-shared ownership-token store；state 只保存 SHA-256 SKU key、owner token、phase 與 expiry，不保存商品名稱或原始資料。In-flight lease 必須涵蓋最大模型 deadline；每個 handler 前須由原 owner 原子寫入 `side_effect_started` 並切換為四小時 crash quarantine，寫入失敗就不得執行副作用。舊 token 不可 release/commit 新 owner，state/lock 無法讀寫、schema 損壞或平台缺少 process-shared lock 時 fail closed。本地非 production 測試才使用記憶體 backend。告警工具必須有 EventRouter delivered/durable-queue acknowledgement，推薦工具必須有 DB write 或 durable notification，KM 工具必須有持久化 insight，才可提交四小時 TTL；若 durable side effect 已完成但 dedupe commit 未驗證，結果仍按已派發計數、輸出 `dedupe_commit_unverified`，並保留 quarantine，絕不可 release 後重送。模型契約錯誤、reservation ownership lost、boundary 未建立或 handler 未回 durable outcome 時才 release 自己持有的 lease。程序若在 boundary 後、side effect 前崩潰，最多抑制該 SKU 四小時；這是避免重複通知/寫入的 at-most-once 取捨，必須由 receipt/告警揭露而非靜默重試。
|
||||
- Decision-only canary 只產生 synthetic decision，並共用 production `_validate_tool_call_contract()` 驗證完整 call set，不可只看第一筆；多餘、重複、未知、缺欄位或未綁定 SKU 的 call 都必須 fail closed。Canary 同時驗證 post-call digest，固定 `tool_execution_count=0`、`database_call_performed=false`、`writes_price_tables=false`、`writes_ai_insights=false`、`telegram_sent=false`。使用 111 成功時狀態必須是 `canary_passed_degraded_fallback`，不能偽裝成 GCP healthy；scheduler 只在 model canary 後發生命週期 acknowledgement 並原子回寫 receipt。111 的 `num_ctx=4096`、`num_predict=512` 是不可被 env 弱化的固定 NemoTron 契約；候選失敗鏈只輸出 label/tier/model/status/digest-match/error-class 的 privacy-safe footprint。
|
||||
- `AI Agent product integration truth` 會讀取 fresh NemoTron decision-only receipt；這份 shadow canary能證明模型決策路徑真的執行，但不能冒充正式商品 action、agent outcome 或完整 Controlled Apply。V10.816 的 source/test 綠燈仍需 production deploy、fresh degraded-fallback receipt 與獨立 readback 才能提升 runtime 狀態。
|
||||
- V10.812 起四 Agent activity 依角色證據判定：Hermes/OpenClaw 仍以實際 `ai_calls` 為主，NemoTron 可加入其派發/insight durable evidence，ElephantAlpha 可由其 `ai_insights`/action plan 證明 active；沒有成功 call 或 verified executed action 時只能是 `runtime_active_unverified`，不得把 deterministic artifact 數量包裝成 healthy integration。
|
||||
|
||||
---
|
||||
@@ -70,6 +72,7 @@
|
||||
## 零、LLM 路由紅線（2026-05-12）
|
||||
|
||||
- 所有 AI Agent、LLM 推理與 embedding 預設必須走 Ollama 三主機級聯：GCP-A `34.87.90.216:11434` → GCP-B `34.21.145.224:11434` → 111 `192.168.0.111:11434`。
|
||||
- NemoTron decision runtime 是 model-aware cascade，不得把同一 floating model tag盲送三台：GCP-A/GCP-B 鎖 `qwen3:14b` exact digest，111 最終備援鎖 `qwen3:8b` exact digest與較小輸出上限；GCP capacity degraded 時可維持自架決策可用性，但必須在 receipt/UI 明示 degraded fallback，並持續保留恢復 GCP capacity 的 P0。
|
||||
- `services/ollama_service.resolve_ollama_host()` 是主機解析契約；`OLLAMA_HOST`、`HERMES_URL`、`EMBEDDING_HOST`、`OLLAMA_API_BASE` 只接受 GCP-A / GCP-B / 111 或 110 的核准轉發端口。
|
||||
- 188 直連 GCP-A / GCP-B timeout 時，resolver 可先使用同順位 110 proxy rescue：GCP-A direct → `192.168.0.110:11435` → GCP-B direct → `192.168.0.110:11436` → 111。proxy rescue 只是同一順位的可用入口，不代表 GCP direct host 已恢復。
|
||||
- `OLLAMA_RESOLVE_HOST_HEALTH_SKIP_ENABLED=true` 時，resolver 會讀最近 `host_health_probes`；若 direct GCP-A/GCP-B 在視窗內已被判定不健康，會直接略過該 direct endpoint，先試同順位 proxy rescue，避免每 120 秒 cache refresh 都等待 direct timeout。此 skip 只套用 direct GCP，不套用 110 proxy。
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# AI Automation Mainline Work Items
|
||||
|
||||
> Updated: 2026-07-17 10:28 Asia/Taipei
|
||||
> Updated: 2026-07-22 Asia/Taipei
|
||||
> Governance: `global_product_governance_v2` + ADR-038
|
||||
> Current P0: `GROWTH-P0-001 comparison coverage truth + autonomous refresh`
|
||||
|
||||
@@ -17,16 +17,16 @@
|
||||
| Order | ID | Status | Work item | Exit evidence / next machine action |
|
||||
|---:|---|---|---|---|
|
||||
| 1 | `SEC-P0-001` | Completed | Deny-by-default route access control | `governance/evidence/SEC-P0-001-20260711T122758Z.json` plus the current runtime receipt prove anonymous matrix 8/8 denied, public `/metrics` 404, exact internal target up, EwoooC product markers present, Prometheus identity preserved and `momo-db` unchanged. |
|
||||
| 2 | `GROWTH-P0-001` | In progress (`runtime_partial`) | Comparison coverage truth + autonomous refresh | V10.813 is live and retains the growth runtime introduced at exact Gitea object `2647632660673e9f1533e61922c96e4fb4adcb40`. Controlled run `34999f5314054f09909663b46fc42ee2` scanned 20, verified/wrote/read back one exact offer and raised the fixed cohort to `25 ready + 1 candidate validation + 24 unmatched`, `50%` count and `NT$211,667 / NT$354,062 = 59.782%` revenue. Follow-up run `e1508ec3a49241d1965bc99d428b0325` found 19 candidates but safely wrote zero because none passed strict identity/variant/unit verification; its independent terminal is `degraded_no_safe_candidate`. Yahoo remains durably `active + enabled + write_enabled`, and the durable readback proves `formal_source_activated=true` even when the latest run is verified no-write. Formal runtime remains `2/15`. Next: retry the unresolved revenue-weighted batch only with fresh source evidence, then add the next approved structured marketplace adapter without relaxing promotion gates. |
|
||||
| 2 | `GROWTH-P0-001` | In progress (`runtime_partial`) | Comparison coverage truth + autonomous refresh | V10.815 is live and retains the growth runtime introduced at exact Gitea object `2647632660673e9f1533e61922c96e4fb4adcb40`. Controlled run `34999f5314054f09909663b46fc42ee2` scanned 20, verified/wrote/read back one exact offer and raised the fixed cohort to `25 ready + 1 candidate validation + 24 unmatched`, `50%` count and `NT$211,667 / NT$354,062 = 59.782%` revenue. Follow-up run `e1508ec3a49241d1965bc99d428b0325` found 19 candidates but safely wrote zero because none passed strict identity/variant/unit verification; its independent terminal is `degraded_no_safe_candidate`. Yahoo remains durably `active + enabled + write_enabled`, and the durable readback proves `formal_source_activated=true` even when the latest run is verified no-write. Formal runtime remains `2/15`. Next: retry the unresolved revenue-weighted batch only with fresh source evidence, then add the next approved structured marketplace adapter without relaxing promotion gates. |
|
||||
| 3 | `SEC-P0-002` | In progress (`canary_ready`) | Database identity + least-privilege RBAC | V10.789 is live and `governance/auth_identity_runtime_receipt.json` verifies required tables, two active admins, durable lockout, session revocation, trusted proxy policy and no-secret mutation audit readiness. Runtime is intentionally hybrid because zero database-admin success receipts have been captured; `auto` retires shared authority after two durable successes without a manual review gate. Next: capture database-admin login receipts and verify automatic database-only cutover. |
|
||||
| 4 | `SEC-P0-003` | In progress | Webhook trust and replay protection | Telegram secret-token verification code exists; production secret activation remains unproven. Exit: secret provisioned outside source, required mode enabled, invalid-secret 401 and authorized callback canary pass. |
|
||||
| 5 | `SUPPLY-P0-001` | In progress | Gitea-only secure software supply chain | Gitea-native checkout, secret-safe `.dockerignore`, commit-bound source receipt and governance gate are active. Exit: exact dependency lock, internal SAST/SCA/secret scan, SBOM, image digest/provenance, vulnerability SLA and production digest readback. |
|
||||
| 6 | `GOV-P0-001` | In progress | Canonical full asset graph + runtime reconciliation | `governance/ewoooc_asset_inventory.json` seeds hosts, services, data, AI, routes, supply chain, observability and recovery. Exit: same-run probe receipt for every asset; drift auto-creates work items. |
|
||||
| 7 | `GOV-P0-002` | Not started | Unified controlled-apply envelope | Introduce one `trace_id/run_id/work_item_id` across sensor, identity, SOT diff, decision, risk, dry-run, execution, verifier, rollback/retry and learning acknowledgement. Start with EventRouter + AutoHeal. |
|
||||
| 8 | `RAG-P0-001` | In progress (`source_fix_ready_runtime_canary_pending`) | Internal RAG candidate canary + NemoTron decision-only proof | V10.813 production still preserves the prior fail-closed receipt. Windows 99 audit run `534c6208-2088-428c-ba6f-f7a63a67f422` now proves the old GCP project is `DELETE_REQUESTED`, GCP-A is unreachable, and GCP-B Ollama `0.22.1` has exact `bge-m3` / `qwen3:14b` digests. Its bounded BGE-M3 probe returned 1024 dimensions with zero DB/app writes but required `116.475s`, proving the 10-second consistency timeout was a false-negative source defect. V10.814 source raises only the canary cold-start cap to 150 seconds, verifies `/api/tags` digest before embedding, permits one digest-verified GCP host as degraded shadow quorum while excluding 111, and adds a daily 05:00 NemoTron model decision with zero tool/DB/price/insight writes. Exit: deploy V10.814, pass both production canaries with Telegram/durable receipts, retain redundancy blocker until a canonical replacement GCP-A asset exists, then controlled `RAG_ENABLED` shadow activation plus query/hit/feedback telemetry. |
|
||||
| 8 | `RAG-P0-001` | In progress (`rag_canary_passed_nemotron_fallback_source_ready`) | Internal RAG candidate canary + NemoTron decision-only proof | V10.815 production run `736e6de1-fbc3-4c9b-bcfe-9b6ef488b932` passed PixelRAG receipt -> exact-digest BGE-M3 -> read-only pgvector canary with zero business writes; activation remains correctly blocked by `RAG_ENABLED=false` and degraded embedding redundancy. GCP-A is unreachable; GCP-B exact-digest `qwen3:14b` chat timed out at 300s and `qwen2.5:7b-instruct` at 240s, while 111 exact-digest `qwen3:8b` produced the required tool call in 19.053s with no tool/DB/Telegram execution. V10.816 source now implements exact-digest GCP-A -> GCP-B -> 111 model-aware fallback with one 180-second production deadline, bounded 111 context/output, deterministic one-call-per-SKU tool validation, truthful failure footprints and a process-shared `side_effect_started` boundary. Durable side effects remain quarantined for four hours even when dedupe commit verification fails, including mixed forced-review/model batches, so app/scheduler/bot cannot reopen the same SKU and duplicate the action；mixed-path dispatched/skipped metrics preserve input-count truth. Focused model/dedupe regression is `93 passed`. Next: deploy V10.816, obtain shared-store and fresh production `canary_passed_degraded_fallback` receipts plus independent zero-write readback; then restore GCP capacity/redundancy before controlled `RAG_ENABLED` shadow activation and query/hit/feedback telemetry. |
|
||||
| 9 | `MCP-P0-001` | In progress (`federation_source_ready`) | MCP/RAG production runtime closure | V10.796 source adds a strict public aggregate receipt for canonical `ewoooc` and `momo-pro-system` identities without opening authenticated internal APIs or exposing endpoint/tool payload data. Exit still requires V10.796 production `/health`, two fresh AWOOOI durable receipts with fingerprint recompute, live MCP servers/router/RAG, approved caller/tool boundary and production query canary. Current source readiness must not be reported as runtime closure. |
|
||||
| 10 | `SEC-P0-004` | Not started | Security operations lifecycle and metrics | Add durable security incident state and publish MTTA, MTTR, recurrence, false positive, human intervention, verifier pass, rollback and freshness. Exit: detect-to-learn production receipt. |
|
||||
| 11 | `REL-P0-001` | In progress (`runtime_verified_cd_degraded`) | Formal deploy and visible proof discipline | Production V10.813 runs exact Gitea object `a193acc407ef61b9cc6340e8e11aa9d0fb9a39c5`; dev merge `715f90133101e7fdcac243bf999aef05f80e7401` carries the same source. Host 110 still has no matching EwoooC runner, so formal CD remains unavailable and is not replaced by fallback evidence. Windows 99 controlled run `2ef0a084-7aa8-4748-a0db-127fb623d75d` deployed exact archive SHA-256 `65369d5f78c56376bffe8606adc790c046274dc070dfef9b933a6140cd83cc66`; rollback/evidence is retained at `/home/ollama/momo-deploy-backups/ewoooc-20260716T190911Z-a193acc-2ef0a084`. Internal/external `/health` is healthy at V10.813, all 8 source hashes match, app/scheduler/bot identities and immutable `momo-db` ID `cd092451cb5fd555d0ffff70642e109f3b742882c418beeab631793d1e9dc55d` remained unchanged. Public login visibly reports V10.813; authenticated cockpit visual proof remains unavailable in the current browser session. |
|
||||
| 11 | `REL-P0-001` | In progress (`v10815_runtime_verified_v10816_pending`) | Formal deploy and visible proof discipline | Production V10.815 runs exact Gitea object `283c8c80c631f5d97315708885413b62ee5a34ea`; dev merge `df83c646cdb3fde5e33f045bff19c5a392f84b86` has an identical tree. Host 110 still has no matching EwoooC runner, so formal CD remains unavailable and is not replaced by fallback evidence. Windows 99 controlled run `736e6de1-fbc3-4c9b-bcfe-9b6ef488b932` corrected and deployed the complete 36-file source set, verified `36/36` hashes and `12/12` runtime env checks, recreated only app/scheduler/bot, and left `momo-db` identity unchanged; rollback is retained at `/home/ollama/momo-deploy-backups/ewoooc-20260722T115155Z-283c8c8-full-736e6de1`. Internal/external `/health` is healthy at V10.815 and public JS/CSS match the tested hashes. V10.816 remains source-only until Gitea integration plus a new Windows 99 controlled deployment and independent runtime readback complete. |
|
||||
|
||||
### GROWTH-P0-001 Fixed Execution Lanes
|
||||
|
||||
@@ -36,7 +36,7 @@ These lanes are one ordered current P0, not optional side work. They must advanc
|
||||
|---|---|---|---|
|
||||
| A. Sales freshness | In progress (`sla_runtime_closed_source_redundancy_partial`) | Latest sales date `2026-07-13`; before the `2026-07-15 20:00` cutoff, raw lag is `2` but SLA lag is `0`, state is `grace` and decisions remain released. Scheduler receipt `0f24219e0bbb4740b7ad6645e7952296` and explicit canary receipt `6e2df008f65544fe8e557c11ff0dbb93` both persisted `completed_no_write`; durable decision is `no_candidate_fresh_no_write`. Live and persisted readiness now agree at Google Drive `1/4`; HTTPS, IMAP and local remain disabled. | Continue automatic report-arrival reconciliation. At/after 20:00 require `2026-07-14` or automatically block decision use, emit the bounded upstream action and verify the next arrival receipt; keep source redundancy partial until another approved source is live. |
|
||||
| B. Verified same-item evidence | In progress | TOP50 fixed cohort: `25` verified, `1` candidate/source validation, `24` unmatched. Count coverage is `50%`; revenue-weighted coverage is `NT$211,667 / NT$354,062 = 59.782%`. V10.810 preserves fingerprint `7b74504fcc1e1801c2ca2b42`; run `34999f5314054f09909663b46fc42ee2` added one independently verified offer (`+2.0pp` count, `+1.89pp` revenue), while run `e1508ec3a49241d1965bc99d428b0325` correctly ended no-write. Their durable artifact SHA-256 values are `994beba6dfa70ef8c50031f130916ee155657dd77ac8a2f59b6530b491d45d1e` and `e58be7bb07599871a8f318ad63885bd868c786b0144b81ba0980ae80cf556541`. | Retry unresolved candidates only after fresh evidence arrives, preserve deterministic identity/unit/variant gates, and publish count/revenue deltas against the same fingerprint per run. |
|
||||
| C. Platform runtime coverage | In progress (`runtime_canary_activated`) | V10.813 is live. Yahoo is durably active and exact-offer canary readback remains valid; an already-active canary now returns `already_active_verified`, `state_changed=false` and `writes_database_count=0`, while latest no-write receipts no longer erase durable activation truth. Formal runtime is `2/15`; PixelRAG remains evidence-only. | Continue bounded refresh on schedule, monitor expiry/recurrence/rollback signals, then implement the next approved structured source contract for Shopee, Coupang, ETMall, Friday or Rakuten without treating blocked pages as product data. |
|
||||
| C. Platform runtime coverage | In progress (`runtime_canary_activated`) | V10.815 is live. Yahoo is durably active and exact-offer canary readback remains valid; an already-active canary returns `already_active_verified`, `state_changed=false` and `writes_database_count=0`, while latest no-write receipts do not erase durable activation truth. Formal runtime is `2/15`; PixelRAG remains evidence-only. | Continue bounded refresh on schedule, monitor expiry/recurrence/rollback signals, then implement the next approved structured source contract for Shopee, Coupang, ETMall, Friday or Rakuten without treating blocked pages as product data. |
|
||||
|
||||
### AI Agent Product Integration Acceptance
|
||||
|
||||
@@ -45,7 +45,7 @@ This is an acceptance surface inside the current growth P0; it does not reorder
|
||||
| Layer | Current status | Exit evidence |
|
||||
|---|---|---|
|
||||
| Source and scheduler wiring | Source ready (`4/4`) | Hermes, NemoTron, OpenClaw and ElephantAlpha source markers plus scheduler ownership are machine-read and reported separately from runtime. |
|
||||
| Agent runtime activity | Production partial (`3/4` role-active; `0/4` healthy); V10.814 source canary ready | V10.813 live readback preserves Hermes/OpenClaw call evidence, zero NemoTron runtime activity and ElephantAlpha durable role-owned insights. V10.814 adds a qwen3 decision-only canary that uses the production payload and exact digest but stops before every tool/action/data write; a fresh pass proves shadow model execution only, not formal product action. Exit requires all four role-active and healthy in the bounded window, without treating class presence, configured fallback or shadow canary as an executed business outcome. |
|
||||
| Agent runtime activity | Production partial; V10.816 bounded fallback source ready | V10.815 runtime proved the RAG no-write lane and separately proved 111 `qwen3:8b` can execute the NemoTron tool contract in 19.053s, but production dispatcher does not use that model-aware fallback until V10.816 is deployed. A fresh decision-only pass proves shadow model execution only, not formal product action. Exit requires all four Agents role-active and healthy in the bounded window, without treating class presence, configured fallback or shadow canary as an executed business outcome. |
|
||||
| MCP/RAG dependency | Runtime disabled / telemetry empty | Production currently reports `MCP_ROUTER_ENABLED=false`, `RAG_ENABLED=false`, zero `mcp_calls` and zero `rag_query_log` activity. Exit requires enabled approved routes, live health, non-zero agent/product telemetry and the internal RAG candidate canary. |
|
||||
| Controlled automation closure | Runtime partial | `/api/ai-automation/agent-product-integration`, CLI and smoke must report Detect -> Normalize -> Correlate -> Decide -> Check -> Controlled Apply -> Verify -> Retry/Rollback -> Learn/Writeback. Completion requires bounded execution, linked outcome/incident verification and durable learning evidence; aggregate source presence is insufficient. |
|
||||
|
||||
@@ -57,8 +57,8 @@ This bounded interruption is closed and control returns to `GROWTH-P0-001` witho
|
||||
|---|---|---|
|
||||
| Program | Completed for the four primary analysis tabs | Daily sales, sales analysis, growth analysis and monthly summary now share one canonical day/month/range contract; cross-tab links preserve the selected period. |
|
||||
| Asset coverage | `4/4` pages and `2/2` sales async APIs verified | Daily KPI/calendar/charts/Top 10, growth KPI/series, monthly KPI/tables/charts and sales KPI/charts/YoY/detail table all use the active period. Historical current-snapshot mixing and blank monthly charts are replaced by explicit honest states. |
|
||||
| Runtime closure | Completed in V10.810 at `911393190ded015e384e438c26b68faf50ec260c` | `2026-04` daily stays inside April and bounds extreme DoD/WoW values with traceable markers; growth single-point series remain visible; sales consumes the real `chart_values` payload and aggregates repeated SKU rows before ranking; monthly requests are abortable/deduplicated and publish explicit loading/error/ready states. External HTTPS serves the exact tested JS/CSS checksums and internal/external `/health` report V10.810. |
|
||||
| Verification | Passed; formal CD still degraded | Final regression: `2,123 passed / 9 skipped / 0 failed`; focused analytics contracts: `49 passed`. The automated visual guard passed `12/12` combinations (`4` pages x desktop/tablet/mobile), covering `126` chart-target observations, period links, actual pixels/elements, honest empty states, date-tick density, console errors and overflow. Formal CD remains degraded because no matching runner executed the release. |
|
||||
| Runtime closure | Completed in V10.815 at `283c8c80c631f5d97315708885413b62ee5a34ea` | The exact `/sales_analysis` URL with blank date fields now resolves one canonical period and applies the same metric/range/filter contract to KPI, charts, YoY and detail rows. Query, chart and export logic are split into dedicated services; repeated SKU rows are aggregated before ranking, date/range links preserve state, and invalid filters fail safely. Windows 99 independent readback verified all 36 target files, runtime env, public `/health` and exact JS/CSS hashes. |
|
||||
| Verification | Passed; authenticated visual proof remains pending | V10.815 broad regression: `2,174 passed / 9 skipped / 0 failed`; focused analytics batches: `173 passed` and `70 passed`, plus Python compile, Node syntax, Jinja render and diff checks. Public target route returns the expected login redirect and public assets match SHA-256 `6bb4a985744917ef2243d6d055c85b6803b2baf5c571580f49ef69c536ccdf04` / `d98654ddaac10a49af2351750f350bbbcfc3491d62b14c8a25d02eb4d7803186`. The current browser session cannot provide authenticated chart screenshots, so that evidence is not claimed. |
|
||||
|
||||
## P1
|
||||
|
||||
@@ -69,10 +69,10 @@ This bounded interruption is closed and control returns to `GROWTH-P0-001` witho
|
||||
| 14 | `APPSEC-P1-001` | In progress | CSP and DOM/XSS hardening | Security headers are present and CSP is report-only. Collect violations, remove high-risk `innerHTML`/inline sinks, then enforce CSP by canary. |
|
||||
| 15 | `APPSEC-P1-002` | Not started | Unsafe shared-cache serialization removal | Replace writable pickle caches in dashboard/daily-sales/EDM/sales with constrained JSON or signed schema. |
|
||||
| 16 | `ARCH-P1-001` | In progress | Split oversized policy/executor/verifier modules | Current top debts include 44k-line PChome mapping and 14k-line smoke service. Split by bounded family and independent tests. |
|
||||
| 17 | `UX-P1-001` | In progress | Professional full-site UI/UX | V10.810 closes chart rendering defects across all four primary analysis tabs: real payload values render, single points stay visible, extreme percentages no longer flatten normal data, dense date labels auto-skip, Top 50 vendor content uses bounded zoom, and loading/error/empty states are explicit. The `12/12` desktop/tablet/mobile visual matrix has zero horizontal overflow. The broader site-wide first-viewport, progressive-disclosure, accessibility and loading/error/degraded-state audit remains in progress. |
|
||||
| 17 | `UX-P1-001` | In progress | Professional full-site UI/UX | V10.815 closes the exact sales-analysis period/filter linkage and keeps the prior four-tab rendering guards: real payload values, visible single points, bounded extreme percentages, date-label auto-skip, zoom bounds and explicit loading/error/empty states. Public assets are hash-verified; authenticated V10.815 visual proof and the broader site-wide first-viewport, progressive-disclosure, accessibility and loading/error/degraded-state audit remain in progress. |
|
||||
| 18 | `PIXELRAG-P1-001` | Not started | Ollama-first multimodal embedding benchmark | Verify approved visual embedding on GCP-A -> GCP-B -> 111 and design pgvector-compatible visual metadata; FAISS remains disallowed without ADR. |
|
||||
| 19 | `MARKET-P1-001` | In progress | Marketplace source contracts | Yahoo Shopping remains active in V10.810 production with public-boundary allowlists, bounded streaming/rate, provenance, current product-detail readback, stock/spec/variant guards, source-specific promotion partition, idempotent exact canary activation and durable activation readback across no-write runs. Four fresh verified Yahoo offers now contribute formal evidence across completed batches; non-exact and unit-price candidates do not. Shopee, Coupang, ETMall, Friday and Rakuten still require equivalent structured contracts, and blocked pages remain non-product data. |
|
||||
| 20 | `QA-P1-001` | In progress | Deterministic test and CI governance | V10.810 final broad regression is `2,123 passed / 9 skipped / 0 failed`; analytics contracts are `49 passed`, and the new runtime chart guard passes `12/12` page/viewport cases with actual pixel/element and console assertions. Production host parity is `18/18` files and public HTTPS parity is `9/9` key JS/CSS assets. No matching EwoooC runner executed this release, so CI/CD parity is not claimed and exact-object fallback receipts remain separate evidence. |
|
||||
| 20 | `QA-P1-001` | In progress | Deterministic test and CI governance | V10.816 source-candidate broad regression is `2,223 passed / 15 skipped / 0 failed`; focused model/dedupe regression is `93 passed`, and independent ninth-round review found no material issue. V10.815 production parity remains `36/36` source hashes plus `12/12` runtime env checks, and public HTTPS serves its exact tested sales JS/CSS. No matching EwoooC runner has executed V10.816, so formal CI/CD or production parity is not claimed and Windows 99 exact-object deployment receipts remain a separate evidence layer. |
|
||||
|
||||
## P2
|
||||
|
||||
@@ -97,7 +97,7 @@ These are reusable foundations, not proof that the full program is complete.
|
||||
| Completed | PromotionGate replay | No production write. |
|
||||
| Completed | Embedding-signature guard replay | Signature readiness only. |
|
||||
| Completed | Candidate knowledge replay | Internal RAG preview only; no DB/model call. |
|
||||
| Source ready; production pending | Internal RAG + NemoTron decision canaries | V10.814 source separates cold-start timeout from normal embedding latency, enforces exact BGE-M3/qwen3 digest readback, permits one approved GCP host only as degraded shadow quorum, and adds zero-tool/zero-data-write NemoTron runtime proof. Production canary receipts, replacement GCP-A control-plane identity, RAG shadow activation and query/outcome telemetry remain P0. |
|
||||
| Source ready; production pending | Model-aware NemoTron dispatcher fallback | V10.816 source uses one modular exact-digest candidate registry for production and canary: GCP-A/GCP-B `qwen3:14b` with 60-second attempts, then 111 `qwen3:8b` with 45 seconds, `think=false`, fixed `num_ctx=4096` / `num_predict=512` and one total deadline. Production and canary share the full one-call-per-SKU tool contract; malformed model identity schema fails closed and continues the approved candidate chain. Two-phase dedupe uses `/app/data` shared `flock` + fsync + atomic JSON ownership-token leases across Gunicorn workers and app/scheduler/bot containers. Before any handler, the owner persists `side_effect_started` with a four-hour quarantine; only an explicit DB write, durable insight, delivered notification or durable EventRouter queue acknowledgement counts as dispatched. A later commit-verification failure is surfaced but never releases the quarantine, preventing duplicate action after a durable effect. Privacy-safe state hashes SKU values; privacy-safe attempt summaries preserve label/tier/model/status/digest/error-class in structured and recipient-visible footprints. Production deployment, shared-store canary, fresh degraded-fallback receipt and independent readback remain P0; GCP-A replacement, GCP-B capacity and RAG shadow activation remain unresolved rather than hidden by the fallback. |
|
||||
| Completed | PixelRAG application portfolio | Commerce/RAG/UX/ops/marketing/governance inventory. |
|
||||
| Completed | Ollama-first VLM route readiness and replay worker | Evidence-bound artifact output; no direct price write. |
|
||||
| Completed | Platform probe worker | Shopee/Coupang barriers become structured fallback/backoff receipts. |
|
||||
|
||||
@@ -75,6 +75,7 @@
|
||||
- 2026-07-15 追記：`services/pchome_growth_same_item_reconciliation.py` 已達 880 行；同商品 identity verifier、exact DB readback、coverage post-verifier 與 durable receipt persistence 應在 `ARCH-P1-001` 拆成獨立 policy/verifier/repository，主模組只保留 bounded orchestration。
|
||||
- 2026-07-17 追記：Nemotron decision-only canary 的排程執行、Telegram acknowledgement 與 durable receipt 終局已移至 `services/nemotron_decision_canary_scheduler_task.py`（120 行）；`run_scheduler.py` 僅保留薄委派與排程註冊，清冊同步為 1,684 行，下一步仍依序拆 task registration 與 runtime startup。
|
||||
- 2026-07-22 追記：業績分析的 canonical query、metric aggregate、Other-category contract 與 period-linked Excel export 已抽到 `services/sales_analysis_query_service.py`（391 行）及 `services/sales_analysis_export_service.py`（207 行）；`routes/sales_routes.py` 降為 2,954 行，後續繼續拆 page context 與 legacy pandas API。
|
||||
- 2026-07-22 追記：V10.816 將 exact-digest model candidate/identity policy 抽成 `services/nemotron_runtime_candidate_service.py`（234 行）、process/container-shared ownership-token lease 與 side-effect crash quarantine 抽成 `services/nemotron_dispatch_reservation_service.py`（376 行），decision-only runtime proof 維持在 `services/nemotron_decision_canary_service.py`（604 行）；`services/nemoton_dispatcher_service.py` 因補齊共用 tool contract、durable outcome、commit-failure quarantine 與 privacy-safe fallback footprint增至 2,864 行。此輪先完成 P0 自動化正確性；後續 `ARCH-P1-001` 應依序抽出 delivery outcome adapter、tool execution coordinator 與 footprint renderer，dispatcher 只保留 orchestration。
|
||||
|
||||
## 達到或超過 800 行檔案清單
|
||||
|
||||
@@ -137,7 +138,7 @@
|
||||
| 2961 | `services/openclaw_strategist_service.py` | P1 strategist | prompt、query、report、notification 分離 |
|
||||
| 2383 | `services/competitor_intel_repository.py` | P1 repository | query、decision envelope、UI projection、cache 分離 |
|
||||
| 2332 | `services/elephant_alpha_autonomous_engine.py` | P1 engine | trigger、planner、executor、notification 分離 |
|
||||
| 2166 | `services/nemoton_dispatcher_service.py` | P1 dispatcher | model client、tool parser、decision、delivery 分離 |
|
||||
| 2864 | `services/nemoton_dispatcher_service.py` | P1 dispatcher | 已抽 exact-digest candidate registry 與 process-shared lease/crash-quarantine store；下一步拆 delivery outcome adapter、tool execution coordinator、footprint renderer |
|
||||
| 2011 | `services/external_market_offer_service.py` | P1 market offers | source registry、normalizer、sync、review、readiness 分離 |
|
||||
| 1917 | `services/market_intel/deployment_readiness.py` | P1 readiness | check registry、runtime probe、projection 分離 |
|
||||
| 1658 | `routes/market_intel_review_report_routes.py` | P1 review route | report query、export、route glue 分離 |
|
||||
|
||||
32
scripts/ops/run_nemotron_dispatch_reservation_canary.py
Normal file
32
scripts/ops/run_nemotron_dispatch_reservation_canary.py
Normal file
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the no-business-side-effect NemoTron shared reservation canary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from services.nemotron_dispatch_reservation_service import ( # noqa: E402
|
||||
PRODUCTION_STATE_PATH,
|
||||
run_shared_reservation_canary,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--state-path", default=str(PRODUCTION_STATE_PATH))
|
||||
args = parser.parse_args()
|
||||
payload = run_shared_reservation_canary(state_path=args.state_path)
|
||||
print(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
||||
return 0 if payload["success"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,9 @@
|
||||
"""Decision-only NemoTron runtime canary.
|
||||
|
||||
The canary exercises the production qwen3 tool-calling payload against an
|
||||
approved GCP Ollama host, validates the returned decision, and stops before any
|
||||
tool, Telegram, database, price, or insight write is allowed.
|
||||
exact-digest GCP-first candidate chain with a bounded 111 fallback, validates
|
||||
the returned decision, and stops before any tool, Telegram, database, price, or
|
||||
insight write is allowed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -18,24 +19,30 @@ from typing import Any, Mapping
|
||||
|
||||
import requests
|
||||
|
||||
from services.nemotron_runtime_candidate_service import (
|
||||
FALLBACK_EXPECTED_DIGEST,
|
||||
FALLBACK_MODEL,
|
||||
IDENTITY_TIMEOUT_SEC,
|
||||
PRIMARY_EXPECTED_DIGEST,
|
||||
PRIMARY_MODEL,
|
||||
NemotronRuntimeCandidate,
|
||||
build_nemotron_runtime_candidates,
|
||||
inspect_nemotron_model_identity,
|
||||
)
|
||||
from services.ollama_service import OLLAMA_HOST_PRIMARY, OLLAMA_HOST_SECONDARY
|
||||
|
||||
|
||||
POLICY = "controlled_nemotron_decision_only_canary_v1"
|
||||
CANARY_VERSION = "nemotron_decision_only_canary_v1"
|
||||
NEMOTRON_OLLAMA_MODEL = os.getenv("NEMOTRON_OLLAMA_MODEL", "qwen3:14b")
|
||||
NEMOTRON_OLLAMA_EXPECTED_DIGEST = os.getenv(
|
||||
"NEMOTRON_OLLAMA_EXPECTED_DIGEST",
|
||||
"bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8",
|
||||
).strip().lower()
|
||||
NEMOTRON_OLLAMA_MODEL = PRIMARY_MODEL
|
||||
NEMOTRON_OLLAMA_EXPECTED_DIGEST = PRIMARY_EXPECTED_DIGEST
|
||||
NEMOTRON_OLLAMA_FALLBACK_MODEL = FALLBACK_MODEL
|
||||
NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST = FALLBACK_EXPECTED_DIGEST
|
||||
DEFAULT_TIMEOUT_SEC = max(
|
||||
30,
|
||||
min(int(os.getenv("NEMOTRON_DECISION_CANARY_TIMEOUT_SEC", "300")), 600),
|
||||
)
|
||||
DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC = max(
|
||||
2,
|
||||
min(int(os.getenv("NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC", "10")), 30),
|
||||
)
|
||||
DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC = IDENTITY_TIMEOUT_SEC
|
||||
DEFAULT_MAX_AGE_HOURS = max(
|
||||
1,
|
||||
min(int(os.getenv("NEMOTRON_DECISION_CANARY_MAX_AGE_HOURS", "26")), 168),
|
||||
@@ -86,52 +93,21 @@ def _parse_datetime(value: Any) -> datetime | None:
|
||||
|
||||
|
||||
def _model_identity(host: str) -> dict[str, Any]:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{host.rstrip('/')}/api/tags",
|
||||
timeout=DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC,
|
||||
)
|
||||
response.raise_for_status()
|
||||
models = response.json().get("models") or []
|
||||
except Exception as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
"host": host,
|
||||
"model": NEMOTRON_OLLAMA_MODEL,
|
||||
"digest": None,
|
||||
"digest_matches": False,
|
||||
"elapsed_ms": round((time.monotonic() - started) * 1000),
|
||||
"error": f"{type(exc).__name__}: {str(exc)[:180]}",
|
||||
}
|
||||
|
||||
target = NEMOTRON_OLLAMA_MODEL
|
||||
target_without_latest = target.removesuffix(":latest")
|
||||
matched: Mapping[str, Any] = {}
|
||||
for item in models:
|
||||
names = {
|
||||
str(item.get("name") or "").strip(),
|
||||
str(item.get("model") or "").strip(),
|
||||
}
|
||||
if target in names or target_without_latest in names:
|
||||
matched = item
|
||||
break
|
||||
digest = str(matched.get("digest") or "").strip().lower()
|
||||
digest_matches = bool(digest) and digest == NEMOTRON_OLLAMA_EXPECTED_DIGEST
|
||||
return {
|
||||
"ok": bool(matched) and digest_matches,
|
||||
"host": host,
|
||||
"model": target,
|
||||
"digest": digest or None,
|
||||
"expected_digest": NEMOTRON_OLLAMA_EXPECTED_DIGEST,
|
||||
"digest_matches": digest_matches,
|
||||
"parameter_size": str((matched.get("details") or {}).get("parameter_size") or ""),
|
||||
"quantization_level": str(
|
||||
(matched.get("details") or {}).get("quantization_level") or ""
|
||||
),
|
||||
"elapsed_ms": round((time.monotonic() - started) * 1000),
|
||||
"error": None if matched else f"model_not_found:{target}",
|
||||
}
|
||||
candidate = NemotronRuntimeCandidate(
|
||||
label="compatibility_identity",
|
||||
tier="primary",
|
||||
host=host.rstrip("/"),
|
||||
model=NEMOTRON_OLLAMA_MODEL,
|
||||
expected_digest=NEMOTRON_OLLAMA_EXPECTED_DIGEST,
|
||||
request_timeout_sec=DEFAULT_TIMEOUT_SEC,
|
||||
num_predict=160,
|
||||
num_ctx=None,
|
||||
)
|
||||
return inspect_nemotron_model_identity(
|
||||
candidate,
|
||||
request_get=requests.get,
|
||||
timeout_sec=DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC,
|
||||
)
|
||||
|
||||
|
||||
def _latest_execution(root: Path, *, now: datetime | None = None) -> dict[str, Any]:
|
||||
@@ -165,7 +141,12 @@ def _latest_execution(root: Path, *, now: datetime | None = None) -> dict[str, A
|
||||
"canary_passed": payload.get("canary_passed") is True,
|
||||
"selected_host": payload.get("selected_host"),
|
||||
"model": payload.get("model"),
|
||||
"expected_digest": payload.get("expected_digest"),
|
||||
"observed_digest": payload.get("observed_digest"),
|
||||
"selected_candidate_label": payload.get("selected_candidate_label"),
|
||||
"fallback_used": payload.get("fallback_used") is True,
|
||||
"degraded_runtime": payload.get("degraded_runtime") is True,
|
||||
"attempt_count": int(payload.get("attempt_count") or 0),
|
||||
"decision_tool": payload.get("decision_tool"),
|
||||
"decision_sku": payload.get("decision_sku"),
|
||||
"model_elapsed_ms": payload.get("model_elapsed_ms"),
|
||||
@@ -283,67 +264,204 @@ def run_nemotron_decision_canary(
|
||||
),
|
||||
}
|
||||
|
||||
host_preflights = [
|
||||
_model_identity(OLLAMA_HOST_PRIMARY),
|
||||
_model_identity(OLLAMA_HOST_SECONDARY),
|
||||
]
|
||||
selected = next((item for item in host_preflights if item.get("ok")), None)
|
||||
from services.nemoton_dispatcher_service import (
|
||||
ToolCallContractError,
|
||||
_parse_content_fallback,
|
||||
_parse_tool_calls_struct,
|
||||
_remaining_timeout,
|
||||
_validate_tool_call_contract,
|
||||
build_qwen3_dispatch_payload,
|
||||
)
|
||||
|
||||
candidates = build_nemotron_runtime_candidates(
|
||||
primary_model=NEMOTRON_OLLAMA_MODEL,
|
||||
primary_expected_digest=NEMOTRON_OLLAMA_EXPECTED_DIGEST,
|
||||
fallback_model=NEMOTRON_OLLAMA_FALLBACK_MODEL,
|
||||
fallback_expected_digest=NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST,
|
||||
)
|
||||
timeout_cap = max(30, min(int(timeout_sec or DEFAULT_TIMEOUT_SEC), 600))
|
||||
canary_started = time.monotonic()
|
||||
deadline_monotonic = canary_started + timeout_cap
|
||||
host_preflights: list[dict[str, Any]] = []
|
||||
runtime_attempts: list[dict[str, Any]] = []
|
||||
selected: dict[str, Any] | None = None
|
||||
selected_candidate: NemotronRuntimeCandidate | None = None
|
||||
post_identity: dict[str, Any] = {}
|
||||
model_call_performed = False
|
||||
model_elapsed_ms = 0
|
||||
decisions: list[dict[str, Any]] = []
|
||||
decision_format = "none"
|
||||
error: str | None = None
|
||||
if selected is None:
|
||||
error = "no_approved_gcp_host_with_expected_nemotron_digest"
|
||||
else:
|
||||
from services.nemoton_dispatcher_service import (
|
||||
_parse_content_fallback,
|
||||
_parse_tool_calls_struct,
|
||||
build_qwen3_dispatch_payload,
|
||||
expected_tool = "trigger_price_alert"
|
||||
threat = _SyntheticThreat()
|
||||
for candidate in candidates:
|
||||
attempt: dict[str, Any] = {
|
||||
"candidate": candidate.as_public_dict(),
|
||||
"identity": {},
|
||||
"model_call_performed": False,
|
||||
"model_elapsed_ms": 0,
|
||||
"decision_format": "none",
|
||||
"decision_count": 0,
|
||||
"status": "identity_failed",
|
||||
"error": None,
|
||||
}
|
||||
try:
|
||||
identity_timeout = _remaining_timeout(
|
||||
deadline_monotonic,
|
||||
DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC,
|
||||
)
|
||||
except requests.Timeout as exc:
|
||||
attempt["status"] = "deadline_exhausted"
|
||||
attempt["error"] = str(exc)
|
||||
runtime_attempts.append(attempt)
|
||||
error = str(exc)
|
||||
break
|
||||
identity = inspect_nemotron_model_identity(
|
||||
candidate,
|
||||
request_get=requests.get,
|
||||
timeout_sec=identity_timeout,
|
||||
)
|
||||
host_preflights.append(identity)
|
||||
attempt["identity"] = identity
|
||||
attempt["error"] = identity.get("error")
|
||||
if identity.get("ok") is not True:
|
||||
runtime_attempts.append(attempt)
|
||||
error = str(identity.get("error") or "model identity failed")
|
||||
continue
|
||||
|
||||
threat = _SyntheticThreat()
|
||||
payload = build_qwen3_dispatch_payload(
|
||||
[threat],
|
||||
mcp_context="controlled decision-only canary; no live market data",
|
||||
num_predict=160,
|
||||
model=candidate.model,
|
||||
num_predict=candidate.num_predict,
|
||||
num_ctx=candidate.num_ctx,
|
||||
keep_alive=0,
|
||||
)
|
||||
payload["think"] = False
|
||||
started = time.monotonic()
|
||||
model_call_performed = True
|
||||
attempt["model_call_performed"] = True
|
||||
candidate_decisions: list[dict[str, Any]] = []
|
||||
candidate_decision_format = "none"
|
||||
candidate_error: str | None = None
|
||||
candidate_contract_error: str | None = None
|
||||
candidate_contract_valid = False
|
||||
try:
|
||||
request_timeout = _remaining_timeout(
|
||||
deadline_monotonic,
|
||||
candidate.request_timeout_sec,
|
||||
)
|
||||
attempt["request_timeout_sec"] = round(request_timeout, 3)
|
||||
response = requests.post(
|
||||
f"{str(selected['host']).rstrip('/')}/api/chat",
|
||||
f"{candidate.host}/api/chat",
|
||||
json=payload,
|
||||
timeout=max(30, min(int(timeout_sec or DEFAULT_TIMEOUT_SEC), 600)),
|
||||
timeout=request_timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
message = body.get("message") or {}
|
||||
decisions = _parse_tool_calls_struct(message.get("tool_calls") or [])
|
||||
if decisions:
|
||||
decision_format = "tool_calls"
|
||||
candidate_decisions = _parse_tool_calls_struct(
|
||||
message.get("tool_calls") or []
|
||||
)
|
||||
if candidate_decisions:
|
||||
candidate_decision_format = "tool_calls"
|
||||
else:
|
||||
decisions = _parse_content_fallback(str(message.get("content") or ""))
|
||||
if decisions:
|
||||
decision_format = "content_fallback"
|
||||
candidate_decisions = _parse_content_fallback(
|
||||
str(message.get("content") or "")
|
||||
)
|
||||
if candidate_decisions:
|
||||
candidate_decision_format = "content_fallback"
|
||||
try:
|
||||
candidate_decisions = _validate_tool_call_contract(
|
||||
candidate_decisions,
|
||||
[threat],
|
||||
)
|
||||
candidate_contract_valid = True
|
||||
except ToolCallContractError as exc:
|
||||
candidate_contract_error = str(exc)[:240]
|
||||
except Exception as exc:
|
||||
error = f"{type(exc).__name__}: {str(exc)[:240]}"
|
||||
candidate_error = f"{type(exc).__name__}: {str(exc)[:240]}"
|
||||
finally:
|
||||
model_elapsed_ms = round((time.monotonic() - started) * 1000)
|
||||
elapsed_ms = round((time.monotonic() - started) * 1000)
|
||||
model_elapsed_ms += elapsed_ms
|
||||
attempt["model_elapsed_ms"] = elapsed_ms
|
||||
|
||||
candidate_decision = candidate_decisions[0] if candidate_decisions else {}
|
||||
candidate_args = (
|
||||
candidate_decision.get("args")
|
||||
if isinstance(candidate_decision.get("args"), dict)
|
||||
else {}
|
||||
)
|
||||
candidate_tool = str(candidate_decision.get("tool") or "")
|
||||
candidate_sku = str((candidate_args or {}).get("sku") or "")
|
||||
candidate_post_identity: dict[str, Any] = {}
|
||||
if candidate_error is None:
|
||||
try:
|
||||
post_identity_timeout = _remaining_timeout(
|
||||
deadline_monotonic,
|
||||
DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC,
|
||||
)
|
||||
candidate_post_identity = inspect_nemotron_model_identity(
|
||||
candidate,
|
||||
request_get=requests.get,
|
||||
timeout_sec=post_identity_timeout,
|
||||
)
|
||||
except requests.Timeout as exc:
|
||||
candidate_error = f"{type(exc).__name__}: {str(exc)[:240]}"
|
||||
candidate_checks = {
|
||||
"approved_host_selected": True,
|
||||
"expected_digest_verified_before": identity.get("digest_matches") is True,
|
||||
"model_call_completed": candidate_error is None,
|
||||
"decision_present": bool(candidate_decisions),
|
||||
"decision_contract_valid": candidate_contract_valid,
|
||||
"decision_tool_allowed": candidate_tool in APPROVED_TOOLS,
|
||||
"expected_decision_tool_selected": candidate_tool == expected_tool,
|
||||
"synthetic_sku_preserved": candidate_sku == _SyntheticThreat.sku,
|
||||
"expected_digest_stable_after": (
|
||||
candidate_post_identity.get("digest_matches") is True
|
||||
),
|
||||
"tool_execution_absent": True,
|
||||
"database_call_absent": True,
|
||||
"telegram_send_absent": True,
|
||||
}
|
||||
attempt.update({
|
||||
"decision_format": candidate_decision_format,
|
||||
"decision_count": len(candidate_decisions),
|
||||
"decision_tool": candidate_tool or None,
|
||||
"decision_sku": candidate_sku or None,
|
||||
"post_model_identity": candidate_post_identity,
|
||||
"checks": candidate_checks,
|
||||
"status": "passed" if all(candidate_checks.values()) else "failed",
|
||||
"error": candidate_error or candidate_contract_error,
|
||||
"contract_error": candidate_contract_error,
|
||||
})
|
||||
runtime_attempts.append(attempt)
|
||||
if all(candidate_checks.values()):
|
||||
selected = identity
|
||||
selected_candidate = candidate
|
||||
post_identity = candidate_post_identity
|
||||
decisions = candidate_decisions
|
||||
decision_format = candidate_decision_format
|
||||
error = None
|
||||
break
|
||||
error = candidate_error or "candidate decision contract failed"
|
||||
|
||||
if selected_candidate is None:
|
||||
error = (
|
||||
"decision_total_deadline_exhausted"
|
||||
if time.monotonic() >= deadline_monotonic
|
||||
else "all_approved_model_candidates_failed"
|
||||
)
|
||||
|
||||
decision = decisions[0] if decisions else {}
|
||||
decision_tool = str(decision.get("tool") or "")
|
||||
decision_args = decision.get("args") if isinstance(decision.get("args"), dict) else {}
|
||||
decision_sku = str((decision_args or {}).get("sku") or "")
|
||||
expected_tool = "trigger_price_alert"
|
||||
post_identity = _model_identity(str(selected["host"])) if selected else {}
|
||||
checks = {
|
||||
"approved_host_selected": selected is not None,
|
||||
"approved_host_selected": selected_candidate is not None,
|
||||
"expected_digest_verified_before": bool(selected and selected.get("digest_matches")),
|
||||
"model_call_completed": model_call_performed and error is None,
|
||||
"model_call_completed": selected_candidate is not None and error is None,
|
||||
"decision_present": bool(decisions),
|
||||
"decision_contract_valid": selected_candidate is not None,
|
||||
"decision_tool_allowed": decision_tool in APPROVED_TOOLS,
|
||||
"expected_decision_tool_selected": decision_tool == expected_tool,
|
||||
"synthetic_sku_preserved": decision_sku == _SyntheticThreat.sku,
|
||||
@@ -353,7 +471,22 @@ def run_nemotron_decision_canary(
|
||||
"telegram_send_absent": True,
|
||||
}
|
||||
canary_passed = all(checks.values())
|
||||
status = "canary_passed" if canary_passed else "canary_failed"
|
||||
fallback_used = bool(selected_candidate and selected_candidate.is_fallback)
|
||||
status = (
|
||||
"canary_passed_degraded_fallback"
|
||||
if canary_passed and fallback_used
|
||||
else "canary_passed"
|
||||
if canary_passed
|
||||
else "canary_failed"
|
||||
)
|
||||
selected_model = (
|
||||
selected_candidate.model if selected_candidate else NEMOTRON_OLLAMA_MODEL
|
||||
)
|
||||
selected_expected_digest = (
|
||||
selected_candidate.expected_digest
|
||||
if selected_candidate
|
||||
else NEMOTRON_OLLAMA_EXPECTED_DIGEST
|
||||
)
|
||||
execution_receipt: dict[str, Any] = {
|
||||
"policy": POLICY,
|
||||
"canary_version": CANARY_VERSION,
|
||||
@@ -361,14 +494,24 @@ def run_nemotron_decision_canary(
|
||||
"run_identity": run_identity,
|
||||
"status": status,
|
||||
"canary_passed": canary_passed,
|
||||
"model": NEMOTRON_OLLAMA_MODEL,
|
||||
"expected_digest": NEMOTRON_OLLAMA_EXPECTED_DIGEST,
|
||||
"model": selected_model,
|
||||
"expected_digest": selected_expected_digest,
|
||||
"observed_digest": selected.get("digest") if selected else None,
|
||||
"selected_host": selected.get("host") if selected else None,
|
||||
"selected_candidate_label": (
|
||||
selected_candidate.label if selected_candidate else None
|
||||
),
|
||||
"fallback_used": fallback_used,
|
||||
"degraded_runtime": fallback_used,
|
||||
"host_preflights": host_preflights,
|
||||
"runtime_attempts": runtime_attempts,
|
||||
"attempt_count": len(runtime_attempts),
|
||||
"post_model_identity": post_identity,
|
||||
"model_call_performed": model_call_performed,
|
||||
"model_elapsed_ms": model_elapsed_ms,
|
||||
"timeout_budget_sec": timeout_cap,
|
||||
"total_elapsed_ms": round((time.monotonic() - canary_started) * 1000),
|
||||
"deadline_exhausted": time.monotonic() >= deadline_monotonic,
|
||||
"decision_format": decision_format,
|
||||
"decision_count": len(decisions),
|
||||
"decision_tool": decision_tool or None,
|
||||
@@ -388,8 +531,8 @@ def run_nemotron_decision_canary(
|
||||
"error": error,
|
||||
"rollback_terminal": "decision_only_no_tool_or_data_write",
|
||||
"source_of_truth_diff": {
|
||||
"expected_model": NEMOTRON_OLLAMA_MODEL,
|
||||
"expected_digest": NEMOTRON_OLLAMA_EXPECTED_DIGEST,
|
||||
"expected_model": selected_model,
|
||||
"expected_digest": selected_expected_digest,
|
||||
"observed_digest": selected.get("digest") if selected else None,
|
||||
"expected_decision_tool": expected_tool,
|
||||
"observed_decision_tool": decision_tool or None,
|
||||
@@ -400,7 +543,7 @@ def run_nemotron_decision_canary(
|
||||
"source_of_truth_diff_recorded": True,
|
||||
"ai_candidate_decision_recorded": bool(decisions),
|
||||
"risk_policy_decision": "medium_decision_only_no_tool_execution",
|
||||
"check_mode_passed": bool(selected),
|
||||
"check_mode_passed": canary_passed,
|
||||
"bounded_execution_performed": model_call_performed,
|
||||
"independent_verifier_passed": canary_passed,
|
||||
"rollback_or_no_write_terminal": "decision_only_no_tool_or_data_write",
|
||||
@@ -444,7 +587,9 @@ def run_nemotron_decision_canary(
|
||||
"rollback_terminal": "decision_only_no_tool_or_data_write",
|
||||
},
|
||||
"next_machine_action": (
|
||||
"continue_scheduled_nemotron_decision_canary"
|
||||
"restore_gcp_decision_runtime_capacity"
|
||||
if canary_passed and fallback_used
|
||||
else "continue_scheduled_nemotron_decision_canary"
|
||||
if canary_passed
|
||||
else "repair_nemotron_model_runtime_then_retry_decision_only_canary"
|
||||
),
|
||||
|
||||
376
services/nemotron_dispatch_reservation_service.py
Normal file
376
services/nemotron_dispatch_reservation_service.py
Normal file
@@ -0,0 +1,376 @@
|
||||
"""Process-shared reservation store for NemoTron dispatch side effects."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
try:
|
||||
import fcntl
|
||||
except ImportError: # pragma: no cover - exercised by compatibility monkeypatch
|
||||
fcntl = None
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
STATE_VERSION = 1
|
||||
PRODUCTION_STATE_PATH = Path(
|
||||
"/app/data/ai_automation/nemotron_dispatch_dedupe_state.json"
|
||||
)
|
||||
|
||||
|
||||
def production_shared_reservation_required() -> bool:
|
||||
return (
|
||||
os.getenv("FLASK_ENV", "").strip().lower() in {"prod", "production"}
|
||||
or os.getenv("USE_POSTGRESQL", "").strip().lower() == "true"
|
||||
)
|
||||
|
||||
|
||||
def _sku_key(sku: str) -> str:
|
||||
value = str(sku or "").strip()
|
||||
if not value:
|
||||
return ""
|
||||
return hashlib.sha256(f"nemotron-dispatch:{value}".encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class ReservationStateError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class SharedFileReservationStore:
|
||||
"""Atomic lease/commit state shared by processes using the same bind mount."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
state_path: str | Path,
|
||||
*,
|
||||
committed_ttl_sec: int,
|
||||
lease_ttl_sec: int,
|
||||
clock: Callable[[], float] | None = None,
|
||||
):
|
||||
self.state_path = Path(state_path)
|
||||
self.lock_path = self.state_path.with_suffix(self.state_path.suffix + ".lock")
|
||||
self.committed_ttl_sec = max(1, int(committed_ttl_sec))
|
||||
self.lease_ttl_sec = max(1, int(lease_ttl_sec))
|
||||
self.clock = clock or time.time
|
||||
|
||||
@staticmethod
|
||||
def _empty_state() -> dict:
|
||||
return {"version": STATE_VERSION, "leases": {}, "committed": {}}
|
||||
|
||||
def _read_state(self) -> dict:
|
||||
if not self.state_path.exists():
|
||||
return self._empty_state()
|
||||
try:
|
||||
payload = json.loads(self.state_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise ReservationStateError("reservation_state_unreadable") from exc
|
||||
if not isinstance(payload, dict) or payload.get("version") != STATE_VERSION:
|
||||
raise ReservationStateError("reservation_state_schema_invalid")
|
||||
leases = payload.get("leases")
|
||||
committed = payload.get("committed")
|
||||
if not isinstance(leases, dict) or not isinstance(committed, dict):
|
||||
raise ReservationStateError("reservation_state_schema_invalid")
|
||||
for lease in leases.values():
|
||||
if (
|
||||
not isinstance(lease, dict)
|
||||
or not isinstance(lease.get("token"), str)
|
||||
or not isinstance(lease.get("expires_at"), (int, float))
|
||||
or lease.get("phase", "reserved") not in {
|
||||
"reserved",
|
||||
"side_effect_started",
|
||||
}
|
||||
):
|
||||
raise ReservationStateError("reservation_lease_schema_invalid")
|
||||
for marker in committed.values():
|
||||
if (
|
||||
not isinstance(marker, dict)
|
||||
or not isinstance(marker.get("until"), (int, float))
|
||||
):
|
||||
raise ReservationStateError("reservation_commit_schema_invalid")
|
||||
return payload
|
||||
|
||||
def _write_state(self, payload: dict) -> None:
|
||||
temporary = self.state_path.with_name(
|
||||
f".{self.state_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
|
||||
)
|
||||
data = json.dumps(payload, sort_keys=True, separators=(",", ":"))
|
||||
try:
|
||||
with temporary.open("w", encoding="utf-8") as handle:
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, 0o640)
|
||||
os.replace(temporary, self.state_path)
|
||||
directory_fd = os.open(str(self.state_path.parent), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
finally:
|
||||
try:
|
||||
temporary.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _cleanup(payload: dict, now: float) -> bool:
|
||||
changed = False
|
||||
for key in [
|
||||
key
|
||||
for key, lease in payload["leases"].items()
|
||||
if float(lease["expires_at"]) <= now
|
||||
]:
|
||||
del payload["leases"][key]
|
||||
changed = True
|
||||
for key in [
|
||||
key
|
||||
for key, marker in payload["committed"].items()
|
||||
if float(marker["until"]) <= now
|
||||
]:
|
||||
del payload["committed"][key]
|
||||
changed = True
|
||||
return changed
|
||||
|
||||
def _locked(self, operation):
|
||||
if fcntl is None:
|
||||
raise ReservationStateError("process_shared_lock_unavailable")
|
||||
self.state_path.parent.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||
with self.lock_path.open("a+", encoding="utf-8") as lock_handle:
|
||||
os.chmod(self.lock_path, 0o640)
|
||||
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX)
|
||||
try:
|
||||
payload = self._read_state()
|
||||
now = float(self.clock())
|
||||
cleanup_changed = self._cleanup(payload, now)
|
||||
result, operation_changed = operation(payload, now)
|
||||
if cleanup_changed or operation_changed:
|
||||
self._write_state(payload)
|
||||
return result
|
||||
finally:
|
||||
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN)
|
||||
|
||||
def reserve(self, sku: str) -> str | None:
|
||||
key = _sku_key(sku)
|
||||
if not key:
|
||||
return None
|
||||
|
||||
def operation(payload, now):
|
||||
if key in payload["leases"] or key in payload["committed"]:
|
||||
return None, False
|
||||
token = uuid.uuid4().hex
|
||||
payload["leases"][key] = {
|
||||
"token": token,
|
||||
"expires_at": now + self.lease_ttl_sec,
|
||||
"phase": "reserved",
|
||||
}
|
||||
return token, True
|
||||
|
||||
try:
|
||||
return self._locked(operation)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"[NemotronReservation] shared reserve failed closed: %s",
|
||||
type(exc).__name__,
|
||||
)
|
||||
return None
|
||||
|
||||
def refresh(self, sku: str, token: str) -> bool:
|
||||
key = _sku_key(sku)
|
||||
|
||||
def operation(payload, now):
|
||||
lease = payload["leases"].get(key)
|
||||
if not lease or lease.get("token") != str(token):
|
||||
return False, False
|
||||
ttl = (
|
||||
self.committed_ttl_sec
|
||||
if lease.get("phase") == "side_effect_started"
|
||||
else self.lease_ttl_sec
|
||||
)
|
||||
lease["expires_at"] = now + ttl
|
||||
return True, True
|
||||
|
||||
try:
|
||||
return bool(key and self._locked(operation))
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"[NemotronReservation] shared refresh failed closed: %s",
|
||||
type(exc).__name__,
|
||||
)
|
||||
return False
|
||||
|
||||
def begin_side_effect(self, sku: str, token: str) -> bool:
|
||||
"""Persist the at-most-once boundary before invoking an external effect."""
|
||||
key = _sku_key(sku)
|
||||
|
||||
def operation(payload, now):
|
||||
lease = payload["leases"].get(key)
|
||||
if not lease or lease.get("token") != str(token):
|
||||
return False, False
|
||||
lease["phase"] = "side_effect_started"
|
||||
lease["expires_at"] = now + self.committed_ttl_sec
|
||||
return True, True
|
||||
|
||||
try:
|
||||
return bool(key and self._locked(operation))
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"[NemotronReservation] shared side-effect boundary failed closed: %s",
|
||||
type(exc).__name__,
|
||||
)
|
||||
return False
|
||||
|
||||
def commit(self, sku: str, token: str) -> bool:
|
||||
key = _sku_key(sku)
|
||||
|
||||
def operation(payload, now):
|
||||
lease = payload["leases"].get(key)
|
||||
if not lease or lease.get("token") != str(token):
|
||||
return False, False
|
||||
del payload["leases"][key]
|
||||
payload["committed"][key] = {
|
||||
"until": now + self.committed_ttl_sec,
|
||||
}
|
||||
return True, True
|
||||
|
||||
try:
|
||||
return bool(key and self._locked(operation))
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"[NemotronReservation] shared commit failed closed: %s",
|
||||
type(exc).__name__,
|
||||
)
|
||||
return False
|
||||
|
||||
def release(self, sku: str, token: str | None) -> bool:
|
||||
key = _sku_key(sku)
|
||||
|
||||
def operation(payload, _now):
|
||||
lease = payload["leases"].get(key)
|
||||
if not lease or lease.get("token") != str(token or ""):
|
||||
return False, False
|
||||
del payload["leases"][key]
|
||||
return True, True
|
||||
|
||||
try:
|
||||
return bool(key and self._locked(operation))
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"[NemotronReservation] shared release failed closed: %s",
|
||||
type(exc).__name__,
|
||||
)
|
||||
return False
|
||||
|
||||
def is_duplicate(self, sku: str) -> bool:
|
||||
key = _sku_key(sku)
|
||||
if not key:
|
||||
return True
|
||||
|
||||
def operation(payload, _now):
|
||||
return (
|
||||
key in payload["leases"] or key in payload["committed"],
|
||||
False,
|
||||
)
|
||||
|
||||
try:
|
||||
return bool(self._locked(operation))
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"[NemotronReservation] shared read failed closed: %s",
|
||||
type(exc).__name__,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def build_production_shared_reservation_store(
|
||||
*,
|
||||
committed_ttl_sec: int,
|
||||
lease_ttl_sec: int,
|
||||
) -> SharedFileReservationStore | None:
|
||||
if not production_shared_reservation_required():
|
||||
return None
|
||||
return SharedFileReservationStore(
|
||||
PRODUCTION_STATE_PATH,
|
||||
committed_ttl_sec=committed_ttl_sec,
|
||||
lease_ttl_sec=lease_ttl_sec,
|
||||
)
|
||||
|
||||
|
||||
def run_shared_reservation_canary(
|
||||
*,
|
||||
state_path: str | Path = PRODUCTION_STATE_PATH,
|
||||
committed_ttl_sec: int = 4 * 3600,
|
||||
lease_ttl_sec: int = 15 * 60,
|
||||
) -> dict:
|
||||
"""Exercise shared ownership semantics without business side effects."""
|
||||
first = SharedFileReservationStore(
|
||||
state_path,
|
||||
committed_ttl_sec=committed_ttl_sec,
|
||||
lease_ttl_sec=lease_ttl_sec,
|
||||
)
|
||||
second = SharedFileReservationStore(
|
||||
state_path,
|
||||
committed_ttl_sec=committed_ttl_sec,
|
||||
lease_ttl_sec=lease_ttl_sec,
|
||||
)
|
||||
synthetic_sku = f"CANARY-NEMOTRON-RESERVATION-{uuid.uuid4().hex}"
|
||||
token = first.reserve(synthetic_sku)
|
||||
competing_token = second.reserve(synthetic_sku) if token else None
|
||||
stale_release_rejected = (
|
||||
second.release(synthetic_sku, "stale-canary-token") is False
|
||||
if token
|
||||
else False
|
||||
)
|
||||
side_effect_boundary_persisted = (
|
||||
first.begin_side_effect(synthetic_sku, token) if token else False
|
||||
)
|
||||
owner_release_succeeded = (
|
||||
first.release(synthetic_sku, token) if token else False
|
||||
)
|
||||
terminal_clear = (
|
||||
first.is_duplicate(synthetic_sku) is False
|
||||
if owner_release_succeeded
|
||||
else False
|
||||
)
|
||||
checks = {
|
||||
"first_owner_reserved": bool(token),
|
||||
"competing_owner_rejected": competing_token is None,
|
||||
"stale_token_release_rejected": stale_release_rejected,
|
||||
"side_effect_boundary_persisted": side_effect_boundary_persisted,
|
||||
"owner_release_succeeded": owner_release_succeeded,
|
||||
"terminal_state_clear": terminal_clear,
|
||||
}
|
||||
success = all(checks.values())
|
||||
return {
|
||||
"success": success,
|
||||
"status": "shared_reservation_canary_passed" if success else "shared_reservation_canary_failed",
|
||||
"policy": "nemotron_process_shared_reservation_v1",
|
||||
"state_path": str(Path(state_path)),
|
||||
"checks": checks,
|
||||
"check_count": len(checks),
|
||||
"check_pass_count": sum(checks.values()),
|
||||
"controlled_apply": {
|
||||
"writes_shared_reservation_state": True,
|
||||
"terminal_state_clear": terminal_clear,
|
||||
"writes_database": False,
|
||||
"sends_telegram": False,
|
||||
"executes_business_tool": False,
|
||||
"reads_secret": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PRODUCTION_STATE_PATH",
|
||||
"ReservationStateError",
|
||||
"SharedFileReservationStore",
|
||||
"build_production_shared_reservation_store",
|
||||
"production_shared_reservation_required",
|
||||
"run_shared_reservation_canary",
|
||||
]
|
||||
234
services/nemotron_runtime_candidate_service.py
Normal file
234
services/nemotron_runtime_candidate_service.py
Normal file
@@ -0,0 +1,234 @@
|
||||
"""Digest-locked Ollama candidates for the decision agent runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any, Callable, Mapping
|
||||
|
||||
import requests
|
||||
|
||||
from services.ollama_service import (
|
||||
OLLAMA_HOST_FALLBACK,
|
||||
OLLAMA_HOST_PRIMARY,
|
||||
OLLAMA_HOST_SECONDARY,
|
||||
)
|
||||
|
||||
|
||||
PRIMARY_MODEL = os.getenv("NEMOTRON_OLLAMA_MODEL", "qwen3:14b").strip()
|
||||
PRIMARY_EXPECTED_DIGEST = os.getenv(
|
||||
"NEMOTRON_OLLAMA_EXPECTED_DIGEST",
|
||||
"bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8",
|
||||
).strip().lower()
|
||||
FALLBACK_MODEL = os.getenv(
|
||||
"NEMOTRON_OLLAMA_FALLBACK_MODEL",
|
||||
"qwen3:8b",
|
||||
).strip()
|
||||
FALLBACK_EXPECTED_DIGEST = os.getenv(
|
||||
"NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST",
|
||||
"500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41",
|
||||
).strip().lower()
|
||||
NEMOTRON_FALLBACK_NUM_CTX = 4096
|
||||
NEMOTRON_FALLBACK_NUM_PREDICT = 512
|
||||
|
||||
|
||||
def _bounded_int_env(name: str, default: int, minimum: int, maximum: int) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, str(default)))
|
||||
except (TypeError, ValueError):
|
||||
value = default
|
||||
return max(minimum, min(value, maximum))
|
||||
|
||||
|
||||
IDENTITY_TIMEOUT_SEC = _bounded_int_env(
|
||||
"NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC", 10, 2, 30
|
||||
)
|
||||
GCP_ATTEMPT_TIMEOUT_SEC = _bounded_int_env(
|
||||
"NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC", 60, 30, 180
|
||||
)
|
||||
FALLBACK_ATTEMPT_TIMEOUT_SEC = _bounded_int_env(
|
||||
"NEMOTRON_111_ATTEMPT_TIMEOUT_SEC", 45, 20, 120
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NemotronRuntimeCandidate:
|
||||
label: str
|
||||
tier: str
|
||||
host: str
|
||||
model: str
|
||||
expected_digest: str
|
||||
request_timeout_sec: int
|
||||
num_predict: int
|
||||
num_ctx: int | None
|
||||
|
||||
@property
|
||||
def is_fallback(self) -> bool:
|
||||
return self.tier == "fallback"
|
||||
|
||||
def as_public_dict(self) -> dict[str, Any]:
|
||||
payload = asdict(self)
|
||||
payload["is_fallback"] = self.is_fallback
|
||||
return payload
|
||||
|
||||
|
||||
def build_nemotron_runtime_candidates(
|
||||
*,
|
||||
primary_model: str | None = None,
|
||||
primary_expected_digest: str | None = None,
|
||||
fallback_model: str | None = None,
|
||||
fallback_expected_digest: str | None = None,
|
||||
) -> tuple[NemotronRuntimeCandidate, ...]:
|
||||
"""Return the approved GCP-first, 111-final candidate chain."""
|
||||
resolved_primary_model = str(primary_model or PRIMARY_MODEL).strip()
|
||||
resolved_primary_digest = str(
|
||||
primary_expected_digest or PRIMARY_EXPECTED_DIGEST
|
||||
).strip().lower()
|
||||
resolved_fallback_model = str(fallback_model or FALLBACK_MODEL).strip()
|
||||
resolved_fallback_digest = str(
|
||||
fallback_expected_digest or FALLBACK_EXPECTED_DIGEST
|
||||
).strip().lower()
|
||||
return (
|
||||
NemotronRuntimeCandidate(
|
||||
label="gcp_primary",
|
||||
tier="primary",
|
||||
host=OLLAMA_HOST_PRIMARY.rstrip("/"),
|
||||
model=resolved_primary_model,
|
||||
expected_digest=resolved_primary_digest,
|
||||
request_timeout_sec=GCP_ATTEMPT_TIMEOUT_SEC,
|
||||
num_predict=2048,
|
||||
num_ctx=None,
|
||||
),
|
||||
NemotronRuntimeCandidate(
|
||||
label="gcp_secondary",
|
||||
tier="secondary",
|
||||
host=OLLAMA_HOST_SECONDARY.rstrip("/"),
|
||||
model=resolved_primary_model,
|
||||
expected_digest=resolved_primary_digest,
|
||||
request_timeout_sec=GCP_ATTEMPT_TIMEOUT_SEC,
|
||||
num_predict=2048,
|
||||
num_ctx=None,
|
||||
),
|
||||
NemotronRuntimeCandidate(
|
||||
label="ollama_111_fallback",
|
||||
tier="fallback",
|
||||
host=OLLAMA_HOST_FALLBACK.rstrip("/"),
|
||||
model=resolved_fallback_model,
|
||||
expected_digest=resolved_fallback_digest,
|
||||
request_timeout_sec=FALLBACK_ATTEMPT_TIMEOUT_SEC,
|
||||
num_predict=NEMOTRON_FALLBACK_NUM_PREDICT,
|
||||
num_ctx=NEMOTRON_FALLBACK_NUM_CTX,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def inspect_nemotron_model_identity(
|
||||
candidate: NemotronRuntimeCandidate,
|
||||
*,
|
||||
request_get: Callable[..., Any] | None = None,
|
||||
timeout_sec: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Verify an exact model digest without loading the model."""
|
||||
started = time.monotonic()
|
||||
getter = request_get or requests.get
|
||||
timeout = max(0.1, min(float(timeout_sec or IDENTITY_TIMEOUT_SEC), 30.0))
|
||||
try:
|
||||
response = getter(f"{candidate.host}/api/tags", timeout=timeout)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
if not isinstance(payload, Mapping):
|
||||
raise ValueError("malformed_tags_payload")
|
||||
models = payload.get("models", [])
|
||||
if models is None:
|
||||
models = []
|
||||
if not isinstance(models, list):
|
||||
raise ValueError("malformed_models_schema")
|
||||
except Exception as exc:
|
||||
return {
|
||||
**candidate.as_public_dict(),
|
||||
"ok": False,
|
||||
"digest": None,
|
||||
"digest_matches": False,
|
||||
"elapsed_ms": round((time.monotonic() - started) * 1000),
|
||||
"error": f"{type(exc).__name__}: {str(exc)[:180]}",
|
||||
}
|
||||
|
||||
target = candidate.model
|
||||
target_without_latest = target.removesuffix(":latest")
|
||||
matched: Mapping[str, Any] = {}
|
||||
if any(not isinstance(item, Mapping) for item in models):
|
||||
return {
|
||||
**candidate.as_public_dict(),
|
||||
"ok": False,
|
||||
"digest": None,
|
||||
"digest_matches": False,
|
||||
"parameter_size": "",
|
||||
"quantization_level": "",
|
||||
"elapsed_ms": round((time.monotonic() - started) * 1000),
|
||||
"error": "malformed_models_schema",
|
||||
}
|
||||
if any(
|
||||
"details" in item and not isinstance(item.get("details"), Mapping)
|
||||
for item in models
|
||||
):
|
||||
return {
|
||||
**candidate.as_public_dict(),
|
||||
"ok": False,
|
||||
"digest": None,
|
||||
"digest_matches": False,
|
||||
"parameter_size": "",
|
||||
"quantization_level": "",
|
||||
"elapsed_ms": round((time.monotonic() - started) * 1000),
|
||||
"error": "malformed_model_details",
|
||||
}
|
||||
for item in models:
|
||||
names = {
|
||||
str(item.get("name") or "").strip(),
|
||||
str(item.get("model") or "").strip(),
|
||||
}
|
||||
if target in names or target_without_latest in names:
|
||||
matched = item
|
||||
break
|
||||
digest = str(matched.get("digest") or "").strip().lower()
|
||||
raw_details = matched.get("details", {}) if matched else {}
|
||||
malformed_details = bool(matched) and not isinstance(raw_details, Mapping)
|
||||
details = raw_details if isinstance(raw_details, Mapping) else {}
|
||||
digest_matches = (
|
||||
bool(digest)
|
||||
and digest == candidate.expected_digest
|
||||
and not malformed_details
|
||||
)
|
||||
error = None
|
||||
if not matched:
|
||||
error = f"model_not_found:{target}"
|
||||
elif malformed_details:
|
||||
error = "malformed_model_details"
|
||||
elif not digest_matches:
|
||||
error = "model_digest_mismatch"
|
||||
return {
|
||||
**candidate.as_public_dict(),
|
||||
"ok": bool(matched) and digest_matches,
|
||||
"digest": digest or None,
|
||||
"digest_matches": digest_matches,
|
||||
"parameter_size": str(details.get("parameter_size") or ""),
|
||||
"quantization_level": str(details.get("quantization_level") or ""),
|
||||
"elapsed_ms": round((time.monotonic() - started) * 1000),
|
||||
"error": error,
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FALLBACK_ATTEMPT_TIMEOUT_SEC",
|
||||
"FALLBACK_EXPECTED_DIGEST",
|
||||
"FALLBACK_MODEL",
|
||||
"GCP_ATTEMPT_TIMEOUT_SEC",
|
||||
"IDENTITY_TIMEOUT_SEC",
|
||||
"NemotronRuntimeCandidate",
|
||||
"NEMOTRON_FALLBACK_NUM_CTX",
|
||||
"NEMOTRON_FALLBACK_NUM_PREDICT",
|
||||
"PRIMARY_EXPECTED_DIGEST",
|
||||
"PRIMARY_MODEL",
|
||||
"build_nemotron_runtime_candidates",
|
||||
"inspect_nemotron_model_identity",
|
||||
]
|
||||
@@ -47,6 +47,14 @@ def approved_ollama_env(name: str, default: str = '') -> str:
|
||||
return default
|
||||
|
||||
|
||||
def _bounded_int_env(name: str, default: int, minimum: int, maximum: int) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, str(default)))
|
||||
except (TypeError, ValueError):
|
||||
value = default
|
||||
return max(minimum, min(value, maximum))
|
||||
|
||||
|
||||
# Ollama 設定 - 僅允許 GCP-A → GCP-B → 111 三主機
|
||||
OLLAMA_HOST_PRIMARY = approved_ollama_env('OLLAMA_HOST_PRIMARY', 'http://34.87.90.216:11434')
|
||||
OLLAMA_HOST_SECONDARY = approved_ollama_env('OLLAMA_HOST_SECONDARY', 'http://34.21.145.224:11434')
|
||||
@@ -66,8 +74,8 @@ EMBED_GCP_FAILURE_COOLDOWN_SEC = int(os.getenv('OLLAMA_EMBED_GCP_FAILURE_COOLDOW
|
||||
EMBED_GCP_FAILURE_NOTICE_SEC = int(os.getenv('OLLAMA_EMBED_GCP_FAILURE_NOTICE_SEC', '30'))
|
||||
FALLBACK_111_KEEP_ALIVE = os.getenv('OLLAMA_111_KEEP_ALIVE', '5m')
|
||||
FALLBACK_111_MAX_TIMEOUT = int(os.getenv('OLLAMA_111_MAX_TIMEOUT', '20'))
|
||||
FALLBACK_111_NUM_CTX = int(os.getenv('OLLAMA_111_NUM_CTX', '4096'))
|
||||
FALLBACK_111_NUM_PREDICT = int(os.getenv('OLLAMA_111_NUM_PREDICT', '512'))
|
||||
FALLBACK_111_NUM_CTX = _bounded_int_env('OLLAMA_111_NUM_CTX', 4096, 512, 4096)
|
||||
FALLBACK_111_NUM_PREDICT = _bounded_int_env('OLLAMA_111_NUM_PREDICT', 512, 32, 512)
|
||||
FALLBACK_111_MODEL = os.getenv('OLLAMA_111_MODEL_FALLBACK', 'llama3.2:latest')
|
||||
FALLBACK_111_CIRCUIT_CACHE_SEC = int(os.getenv('OLLAMA_111_CIRCUIT_CACHE_SEC', '60'))
|
||||
FALLBACK_111_MODEL_PATTERNS = tuple(
|
||||
|
||||
@@ -3,6 +3,8 @@ import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, payload, status_code=200):
|
||||
@@ -19,12 +21,18 @@ class _Response:
|
||||
return self._payload
|
||||
|
||||
|
||||
def _tags(service, *, digest=None):
|
||||
def _tags(service, *, model=None, digest=None):
|
||||
resolved_model = model or service.NEMOTRON_OLLAMA_MODEL
|
||||
resolved_digest = digest or (
|
||||
service.NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST
|
||||
if resolved_model == service.NEMOTRON_OLLAMA_FALLBACK_MODEL
|
||||
else service.NEMOTRON_OLLAMA_EXPECTED_DIGEST
|
||||
)
|
||||
return {
|
||||
"models": [{
|
||||
"name": service.NEMOTRON_OLLAMA_MODEL,
|
||||
"model": service.NEMOTRON_OLLAMA_MODEL,
|
||||
"digest": digest or service.NEMOTRON_OLLAMA_EXPECTED_DIGEST,
|
||||
"name": resolved_model,
|
||||
"model": resolved_model,
|
||||
"digest": resolved_digest,
|
||||
"details": {
|
||||
"parameter_size": "14.8B",
|
||||
"quantization_level": "Q4_K_M",
|
||||
@@ -33,6 +41,22 @@ def _tags(service, *, digest=None):
|
||||
}
|
||||
|
||||
|
||||
def _valid_canary_tool_call():
|
||||
return {
|
||||
"function": {
|
||||
"name": "trigger_price_alert",
|
||||
"arguments": {
|
||||
"sku": "CANARY-NEMO-001",
|
||||
"name": "NemoTron 受控決策驗證品",
|
||||
"gap_pct": 22.4,
|
||||
"sales_delta": -35.0,
|
||||
"action": "建立價格告警候選",
|
||||
"confidence": 0.91,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_nemotron_canary_readback_never_calls_network(tmp_path, monkeypatch):
|
||||
from services import nemotron_decision_canary_service as service
|
||||
|
||||
@@ -113,7 +137,8 @@ def test_nemotron_canary_executes_decision_only_on_verified_secondary(
|
||||
assert posted[0][0].endswith("/api/chat")
|
||||
assert posted[0][1]["keep_alive"] == 0
|
||||
assert posted[0][1]["think"] is False
|
||||
assert posted[0][2] == 45
|
||||
assert posted[0][1]["options"]["num_predict"] == 2048
|
||||
assert posted[0][2] == pytest.approx(45, abs=0.01)
|
||||
assert payload["latest_execution"]["canary_passed"] is True
|
||||
|
||||
acknowledged = service.acknowledge_nemotron_decision_canary(
|
||||
@@ -157,12 +182,263 @@ def test_nemotron_canary_blocks_model_digest_drift_before_model_call(
|
||||
|
||||
assert payload["status"] == "canary_failed"
|
||||
assert payload["execution"]["model_call_performed"] is False
|
||||
assert payload["execution"]["error"] == (
|
||||
"no_approved_gcp_host_with_expected_nemotron_digest"
|
||||
assert payload["execution"]["error"] == "all_approved_model_candidates_failed"
|
||||
assert payload["execution"]["attempt_count"] == 3
|
||||
assert all(
|
||||
attempt["status"] == "identity_failed"
|
||||
for attempt in payload["execution"]["runtime_attempts"]
|
||||
)
|
||||
assert model_calls == []
|
||||
|
||||
|
||||
def test_nemotron_canary_malformed_primary_tags_falls_through_secondary(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
from services import nemotron_decision_canary_service as service
|
||||
|
||||
def fake_get(url, timeout):
|
||||
if service.OLLAMA_HOST_PRIMARY in url:
|
||||
return _Response({
|
||||
"models": [{
|
||||
"name": service.NEMOTRON_OLLAMA_MODEL,
|
||||
"model": service.NEMOTRON_OLLAMA_MODEL,
|
||||
"digest": service.NEMOTRON_OLLAMA_EXPECTED_DIGEST,
|
||||
"details": [],
|
||||
}]
|
||||
})
|
||||
return _Response(_tags(service))
|
||||
|
||||
monkeypatch.setattr(service.requests, "get", fake_get)
|
||||
monkeypatch.setattr(
|
||||
service.requests,
|
||||
"post",
|
||||
lambda *_args, **_kwargs: _Response({
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [_valid_canary_tool_call()],
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
payload = service.run_nemotron_decision_canary(
|
||||
output_root=tmp_path,
|
||||
execute=True,
|
||||
write_receipt=False,
|
||||
timeout_sec=45,
|
||||
)
|
||||
|
||||
execution = payload["execution"]
|
||||
assert payload["status"] == "canary_passed"
|
||||
assert execution["selected_host"] == service.OLLAMA_HOST_SECONDARY
|
||||
assert execution["runtime_attempts"][0]["status"] == "identity_failed"
|
||||
assert execution["runtime_attempts"][0]["error"] == "malformed_model_details"
|
||||
|
||||
|
||||
def test_nemotron_canary_uses_exact_digest_111_fallback_after_gcp_timeout(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
import requests
|
||||
from services import nemotron_decision_canary_service as service
|
||||
from services.ollama_service import OLLAMA_HOST_FALLBACK
|
||||
|
||||
def fake_get(url, timeout):
|
||||
assert timeout == service.DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC
|
||||
if service.OLLAMA_HOST_PRIMARY in url:
|
||||
raise requests.ConnectionError("primary unavailable")
|
||||
if service.OLLAMA_HOST_SECONDARY in url:
|
||||
return _Response(_tags(service))
|
||||
if OLLAMA_HOST_FALLBACK in url:
|
||||
return _Response(
|
||||
_tags(service, model=service.NEMOTRON_OLLAMA_FALLBACK_MODEL)
|
||||
)
|
||||
raise AssertionError(f"unexpected identity URL: {url}")
|
||||
|
||||
posted = []
|
||||
|
||||
def fake_post(url, json, timeout):
|
||||
posted.append((url, json, timeout))
|
||||
if service.OLLAMA_HOST_SECONDARY in url:
|
||||
raise requests.Timeout("secondary saturated")
|
||||
return _Response({
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [{
|
||||
"function": {
|
||||
"name": "trigger_price_alert",
|
||||
"arguments": {
|
||||
"sku": "CANARY-NEMO-001",
|
||||
"name": "NemoTron 受控決策驗證品",
|
||||
"gap_pct": 22.4,
|
||||
"sales_delta": -35.0,
|
||||
"action": "建立價格告警候選",
|
||||
"confidence": 0.91,
|
||||
},
|
||||
}
|
||||
}],
|
||||
},
|
||||
"prompt_eval_count": 100,
|
||||
"eval_count": 20,
|
||||
})
|
||||
|
||||
monkeypatch.setattr(service.requests, "get", fake_get)
|
||||
monkeypatch.setattr(service.requests, "post", fake_post)
|
||||
|
||||
payload = service.run_nemotron_decision_canary(
|
||||
output_root=tmp_path,
|
||||
execute=True,
|
||||
write_receipt=True,
|
||||
timeout_sec=90,
|
||||
run_id="nemotron-canary-fallback-test-001",
|
||||
)
|
||||
|
||||
assert payload["status"] == "canary_passed_degraded_fallback"
|
||||
execution = payload["execution"]
|
||||
assert execution["selected_host"] == OLLAMA_HOST_FALLBACK
|
||||
assert execution["model"] == service.NEMOTRON_OLLAMA_FALLBACK_MODEL
|
||||
assert execution["fallback_used"] is True
|
||||
assert execution["degraded_runtime"] is True
|
||||
assert execution["selected_candidate_label"] == "ollama_111_fallback"
|
||||
assert execution["attempt_count"] == 3
|
||||
assert posted[0][2] == 60
|
||||
assert posted[1][2] == 45
|
||||
assert posted[1][1]["model"] == service.NEMOTRON_OLLAMA_FALLBACK_MODEL
|
||||
assert posted[1][1]["options"]["num_ctx"] == 4096
|
||||
assert posted[1][1]["options"]["num_predict"] == 512
|
||||
assert payload["next_machine_action"] == "restore_gcp_decision_runtime_capacity"
|
||||
assert payload["latest_execution"]["fallback_used"] is True
|
||||
|
||||
|
||||
def test_nemotron_canary_timeout_is_one_total_deadline(tmp_path, monkeypatch):
|
||||
import requests
|
||||
from services import nemotron_decision_canary_service as service
|
||||
|
||||
clock = {"now": 50.0}
|
||||
monkeypatch.setattr(service.time, "monotonic", lambda: clock["now"])
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"inspect_nemotron_model_identity",
|
||||
lambda candidate, **_kwargs: {
|
||||
**candidate.as_public_dict(),
|
||||
"ok": True,
|
||||
"digest": candidate.expected_digest,
|
||||
"digest_matches": True,
|
||||
"elapsed_ms": 0,
|
||||
"error": None,
|
||||
},
|
||||
)
|
||||
posted = []
|
||||
|
||||
def timeout_post(url, json, timeout):
|
||||
posted.append((url, timeout))
|
||||
clock["now"] += float(timeout)
|
||||
raise requests.Timeout("simulated timeout")
|
||||
|
||||
monkeypatch.setattr(service.requests, "post", timeout_post)
|
||||
|
||||
payload = service.run_nemotron_decision_canary(
|
||||
output_root=tmp_path,
|
||||
execute=True,
|
||||
write_receipt=False,
|
||||
timeout_sec=30,
|
||||
)
|
||||
|
||||
execution = payload["execution"]
|
||||
assert payload["status"] == "canary_failed"
|
||||
assert execution["error"] == "decision_total_deadline_exhausted"
|
||||
assert execution["deadline_exhausted"] is True
|
||||
assert execution["total_elapsed_ms"] == 30000
|
||||
assert len(posted) == 1
|
||||
assert posted[0][1] == 30
|
||||
assert execution["attempt_count"] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_calls", "error_fragment"),
|
||||
[
|
||||
(
|
||||
[
|
||||
_valid_canary_tool_call(),
|
||||
{
|
||||
"function": {
|
||||
"name": "delete_product",
|
||||
"arguments": {"sku": "CANARY-NEMO-001"},
|
||||
}
|
||||
},
|
||||
],
|
||||
"tool_call_count_mismatch",
|
||||
),
|
||||
(
|
||||
[_valid_canary_tool_call(), _valid_canary_tool_call()],
|
||||
"tool_call_count_mismatch",
|
||||
),
|
||||
(
|
||||
[{
|
||||
"function": {
|
||||
"name": "delete_product",
|
||||
"arguments": {"sku": "CANARY-NEMO-001"},
|
||||
}
|
||||
}],
|
||||
"unknown_tool",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_nemotron_canary_rejects_extra_duplicate_or_unknown_tool_calls(
|
||||
tmp_path, monkeypatch, tool_calls, error_fragment
|
||||
):
|
||||
from services import nemotron_decision_canary_service as service
|
||||
from services.ollama_service import OLLAMA_HOST_FALLBACK
|
||||
|
||||
def fake_get(url, timeout):
|
||||
model = (
|
||||
service.NEMOTRON_OLLAMA_FALLBACK_MODEL
|
||||
if OLLAMA_HOST_FALLBACK in url
|
||||
else service.NEMOTRON_OLLAMA_MODEL
|
||||
)
|
||||
return _Response(_tags(service, model=model))
|
||||
|
||||
monkeypatch.setattr(service.requests, "get", fake_get)
|
||||
monkeypatch.setattr(
|
||||
service.requests,
|
||||
"post",
|
||||
lambda *_args, **_kwargs: _Response({
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": tool_calls,
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
payload = service.run_nemotron_decision_canary(
|
||||
output_root=tmp_path,
|
||||
execute=True,
|
||||
write_receipt=False,
|
||||
timeout_sec=90,
|
||||
)
|
||||
|
||||
execution = payload["execution"]
|
||||
assert payload["status"] == "canary_failed"
|
||||
assert execution["canary_passed"] is False
|
||||
assert execution["tool_execution_count"] == 0
|
||||
assert execution["database_call_performed"] is False
|
||||
model_attempts = [
|
||||
attempt
|
||||
for attempt in execution["runtime_attempts"]
|
||||
if attempt["model_call_performed"]
|
||||
]
|
||||
assert len(model_attempts) == 3
|
||||
assert all(
|
||||
attempt["checks"]["decision_contract_valid"] is False
|
||||
for attempt in model_attempts
|
||||
)
|
||||
assert all(
|
||||
error_fragment in str(attempt.get("contract_error") or "")
|
||||
for attempt in model_attempts
|
||||
)
|
||||
|
||||
|
||||
def test_nemotron_canary_route_is_read_only():
|
||||
from routes import system_public_routes as routes
|
||||
|
||||
|
||||
@@ -62,7 +62,10 @@ def test_send_telegram_attaches_decision_envelope_to_event_router(monkeypatch):
|
||||
"severity": "P2",
|
||||
"guardrails": {"can_auto_execute": False},
|
||||
}
|
||||
module.NemotronDispatcher()._send_telegram("hello price alert", decision_envelope=envelope)
|
||||
outcome = module.NemotronDispatcher()._send_telegram(
|
||||
"hello price alert",
|
||||
decision_envelope=envelope,
|
||||
)
|
||||
|
||||
assert len(captured) == 1
|
||||
event = captured[0]
|
||||
@@ -71,6 +74,25 @@ def test_send_telegram_attaches_decision_envelope_to_event_router(monkeypatch):
|
||||
assert event["decision_envelope"] == envelope
|
||||
assert event["payload"]["decision_envelope"] == envelope
|
||||
assert event["payload"]["raw_message"] == "hello price alert"
|
||||
assert outcome["durable"] is False
|
||||
|
||||
|
||||
def test_send_telegram_requires_delivery_or_durable_queue(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
fake_event_router = types.ModuleType("services.event_router")
|
||||
fake_event_router.dispatch_sync = lambda event: {
|
||||
"delivered": False,
|
||||
"queued": False,
|
||||
"errors": ["telegram unavailable"],
|
||||
}
|
||||
monkeypatch.setitem(sys.modules, "services.event_router", fake_event_router)
|
||||
|
||||
outcome = module.NemotronDispatcher()._send_telegram("test")
|
||||
|
||||
assert outcome["durable"] is False
|
||||
assert outcome["delivered"] is False
|
||||
assert outcome["queued"] is False
|
||||
|
||||
|
||||
def test_exec_trigger_price_alert_persists_decision_envelope(monkeypatch):
|
||||
@@ -82,15 +104,18 @@ def test_exec_trigger_price_alert_persists_decision_envelope(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_send_telegram",
|
||||
lambda message, decision_envelope=None: sent.append((message, decision_envelope)),
|
||||
lambda message, decision_envelope=None: (
|
||||
sent.append((message, decision_envelope))
|
||||
or {"durable": True, "delivered": True, "queued": False}
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_sink_insight_to_km",
|
||||
lambda *args, **kwargs: stored.append((args, kwargs)),
|
||||
lambda *args, **kwargs: (stored.append((args, kwargs)) or True),
|
||||
)
|
||||
|
||||
dispatcher._exec_trigger_price_alert(
|
||||
outcome = dispatcher._exec_trigger_price_alert(
|
||||
sku="SKU-1",
|
||||
name="測試商品",
|
||||
gap_pct=12.5,
|
||||
@@ -115,3 +140,4 @@ def test_exec_trigger_price_alert_persists_decision_envelope(monkeypatch):
|
||||
assert envelope["subject"]["competitor_product_id"] == "PC-1"
|
||||
assert stored
|
||||
assert stored[0][1]["metadata"]["decision_envelope"] == envelope
|
||||
assert outcome["durable"] is True
|
||||
|
||||
281
tests/test_nemotron_dispatch_reservation_service.py
Normal file
281
tests/test_nemotron_dispatch_reservation_service.py
Normal file
@@ -0,0 +1,281 @@
|
||||
import json
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def _process_reserve(state_path, start_event, result_queue):
|
||||
from services.nemotron_dispatch_reservation_service import (
|
||||
SharedFileReservationStore,
|
||||
)
|
||||
|
||||
store = SharedFileReservationStore(
|
||||
state_path,
|
||||
committed_ttl_sec=3600,
|
||||
lease_ttl_sec=60,
|
||||
)
|
||||
start_event.wait(timeout=5)
|
||||
result_queue.put(store.reserve("SKU-PROCESS-SHARED"))
|
||||
|
||||
|
||||
def test_shared_store_coordinates_independent_instances_and_ttl(tmp_path):
|
||||
from services.nemotron_dispatch_reservation_service import (
|
||||
SharedFileReservationStore,
|
||||
)
|
||||
|
||||
clock = {"now": 1000.0}
|
||||
path = tmp_path / "dedupe" / "state.json"
|
||||
first = SharedFileReservationStore(
|
||||
path,
|
||||
committed_ttl_sec=100,
|
||||
lease_ttl_sec=20,
|
||||
clock=lambda: clock["now"],
|
||||
)
|
||||
second = SharedFileReservationStore(
|
||||
path,
|
||||
committed_ttl_sec=100,
|
||||
lease_ttl_sec=20,
|
||||
clock=lambda: clock["now"],
|
||||
)
|
||||
|
||||
token = first.reserve("SKU-SHARED")
|
||||
assert token
|
||||
assert second.reserve("SKU-SHARED") is None
|
||||
assert second.is_duplicate("SKU-SHARED") is True
|
||||
assert "SKU-SHARED" not in path.read_text(encoding="utf-8")
|
||||
assert second.release("SKU-SHARED", "stale-token") is False
|
||||
assert first.refresh("SKU-SHARED", token) is True
|
||||
assert first.commit("SKU-SHARED", token) is True
|
||||
assert second.reserve("SKU-SHARED") is None
|
||||
|
||||
clock["now"] += 101
|
||||
next_token = second.reserve("SKU-SHARED")
|
||||
assert next_token and next_token != token
|
||||
assert second.release("SKU-SHARED", next_token) is True
|
||||
|
||||
|
||||
def test_begin_side_effect_persists_extended_quarantine(tmp_path):
|
||||
from services.nemotron_dispatch_reservation_service import (
|
||||
SharedFileReservationStore,
|
||||
)
|
||||
|
||||
clock = {"now": 1000.0}
|
||||
path = tmp_path / "state.json"
|
||||
first = SharedFileReservationStore(
|
||||
path,
|
||||
committed_ttl_sec=100,
|
||||
lease_ttl_sec=20,
|
||||
clock=lambda: clock["now"],
|
||||
)
|
||||
second = SharedFileReservationStore(
|
||||
path,
|
||||
committed_ttl_sec=100,
|
||||
lease_ttl_sec=20,
|
||||
clock=lambda: clock["now"],
|
||||
)
|
||||
|
||||
token = first.reserve("SKU-SIDE-EFFECT")
|
||||
assert token
|
||||
assert first.begin_side_effect("SKU-SIDE-EFFECT", token) is True
|
||||
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
lease = next(iter(payload["leases"].values()))
|
||||
assert lease == {
|
||||
"token": token,
|
||||
"expires_at": 1100.0,
|
||||
"phase": "side_effect_started",
|
||||
}
|
||||
|
||||
clock["now"] += 21
|
||||
assert second.reserve("SKU-SIDE-EFFECT") is None
|
||||
clock["now"] += 80
|
||||
assert second.reserve("SKU-SIDE-EFFECT")
|
||||
|
||||
|
||||
def test_shared_store_is_atomic_across_processes(tmp_path):
|
||||
state_path = str(tmp_path / "process" / "state.json")
|
||||
context = multiprocessing.get_context("fork")
|
||||
start_event = context.Event()
|
||||
result_queue = context.Queue()
|
||||
processes = [
|
||||
context.Process(
|
||||
target=_process_reserve,
|
||||
args=(state_path, start_event, result_queue),
|
||||
)
|
||||
for _index in range(6)
|
||||
]
|
||||
for process in processes:
|
||||
process.start()
|
||||
start_event.set()
|
||||
tokens = [result_queue.get(timeout=10) for _process in processes]
|
||||
for process in processes:
|
||||
process.join(timeout=10)
|
||||
assert process.exitcode == 0
|
||||
|
||||
owners = [token for token in tokens if token]
|
||||
assert len(owners) == 1
|
||||
|
||||
|
||||
def test_shared_store_corruption_fails_closed(tmp_path):
|
||||
from services.nemotron_dispatch_reservation_service import (
|
||||
SharedFileReservationStore,
|
||||
)
|
||||
|
||||
path = tmp_path / "state.json"
|
||||
path.write_text("{not-json", encoding="utf-8")
|
||||
store = SharedFileReservationStore(
|
||||
path,
|
||||
committed_ttl_sec=3600,
|
||||
lease_ttl_sec=900,
|
||||
)
|
||||
|
||||
assert store.reserve("SKU-CORRUPT") is None
|
||||
assert store.refresh("SKU-CORRUPT", "token") is False
|
||||
assert store.commit("SKU-CORRUPT", "token") is False
|
||||
assert store.release("SKU-CORRUPT", "token") is False
|
||||
assert store.is_duplicate("SKU-CORRUPT") is True
|
||||
assert path.read_text(encoding="utf-8") == "{not-json"
|
||||
|
||||
|
||||
def test_atomic_write_fsyncs_file_and_parent_directory(tmp_path, monkeypatch):
|
||||
from services import nemotron_dispatch_reservation_service as service
|
||||
|
||||
fsynced = []
|
||||
real_fsync = service.os.fsync
|
||||
|
||||
def record_fsync(fd):
|
||||
fsynced.append(fd)
|
||||
return real_fsync(fd)
|
||||
|
||||
monkeypatch.setattr(service.os, "fsync", record_fsync)
|
||||
store = service.SharedFileReservationStore(
|
||||
tmp_path / "nested" / "state.json",
|
||||
committed_ttl_sec=3600,
|
||||
lease_ttl_sec=900,
|
||||
)
|
||||
|
||||
assert store.reserve("SKU-FSYNC")
|
||||
assert len(fsynced) >= 2
|
||||
|
||||
|
||||
def test_missing_process_shared_lock_fails_closed(monkeypatch, tmp_path):
|
||||
from services import nemotron_dispatch_reservation_service as service
|
||||
|
||||
monkeypatch.setattr(service, "fcntl", None)
|
||||
monkeypatch.delenv("FLASK_ENV", raising=False)
|
||||
monkeypatch.delenv("USE_POSTGRESQL", raising=False)
|
||||
|
||||
assert service.build_production_shared_reservation_store(
|
||||
committed_ttl_sec=3600,
|
||||
lease_ttl_sec=900,
|
||||
) is None
|
||||
|
||||
store = service.SharedFileReservationStore(
|
||||
tmp_path / "state.json",
|
||||
committed_ttl_sec=3600,
|
||||
lease_ttl_sec=900,
|
||||
)
|
||||
assert store.reserve("SKU-NO-FCNTL") is None
|
||||
assert store.is_duplicate("SKU-NO-FCNTL") is True
|
||||
|
||||
|
||||
def test_production_builder_forces_canonical_shared_data_path(monkeypatch):
|
||||
from services import nemotron_dispatch_reservation_service as service
|
||||
|
||||
monkeypatch.setenv("FLASK_ENV", "production")
|
||||
store = service.build_production_shared_reservation_store(
|
||||
committed_ttl_sec=3600,
|
||||
lease_ttl_sec=900,
|
||||
)
|
||||
|
||||
assert store is not None
|
||||
assert store.state_path == Path(
|
||||
"/app/data/ai_automation/nemotron_dispatch_dedupe_state.json"
|
||||
)
|
||||
assert store.lock_path == Path(
|
||||
"/app/data/ai_automation/nemotron_dispatch_dedupe_state.json.lock"
|
||||
)
|
||||
|
||||
|
||||
def test_postgres_runtime_cannot_fall_back_to_process_local_memory(monkeypatch):
|
||||
from services import nemotron_dispatch_reservation_service as service
|
||||
|
||||
monkeypatch.delenv("FLASK_ENV", raising=False)
|
||||
monkeypatch.setenv("USE_POSTGRESQL", "true")
|
||||
|
||||
assert service.production_shared_reservation_required() is True
|
||||
assert service.build_production_shared_reservation_store(
|
||||
committed_ttl_sec=3600,
|
||||
lease_ttl_sec=900,
|
||||
) is not None
|
||||
|
||||
|
||||
def test_shared_state_contains_only_hashed_sku_and_timing_metadata(tmp_path):
|
||||
from services.nemotron_dispatch_reservation_service import (
|
||||
SharedFileReservationStore,
|
||||
)
|
||||
|
||||
path = tmp_path / "state.json"
|
||||
store = SharedFileReservationStore(
|
||||
path,
|
||||
committed_ttl_sec=3600,
|
||||
lease_ttl_sec=900,
|
||||
)
|
||||
token = store.reserve("RAW-SKU-MUST-NOT-APPEAR")
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
assert token
|
||||
assert "RAW-SKU-MUST-NOT-APPEAR" not in path.read_text(encoding="utf-8")
|
||||
assert set(payload) == {"version", "leases", "committed"}
|
||||
assert len(next(iter(payload["leases"]))) == 64
|
||||
|
||||
|
||||
def test_production_containers_share_data_mount_and_force_shared_backend():
|
||||
compose = (
|
||||
Path(__file__).resolve().parents[1] / "docker-compose.yml"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert compose.count("- ./data:/app/data") >= 3
|
||||
assert compose.count("- FLASK_ENV=production") >= 3
|
||||
|
||||
|
||||
def test_shared_reservation_canary_has_no_business_side_effects(tmp_path):
|
||||
from services.nemotron_dispatch_reservation_service import (
|
||||
run_shared_reservation_canary,
|
||||
)
|
||||
|
||||
payload = run_shared_reservation_canary(
|
||||
state_path=tmp_path / "canary" / "state.json",
|
||||
)
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["check_pass_count"] == payload["check_count"] == 6
|
||||
assert payload["controlled_apply"] == {
|
||||
"writes_shared_reservation_state": True,
|
||||
"terminal_state_clear": True,
|
||||
"writes_database": False,
|
||||
"sends_telegram": False,
|
||||
"executes_business_tool": False,
|
||||
"reads_secret": False,
|
||||
}
|
||||
|
||||
|
||||
def test_shared_reservation_canary_cli_is_machine_readable(tmp_path):
|
||||
completed = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/ops/run_nemotron_dispatch_reservation_canary.py",
|
||||
"--state-path",
|
||||
str(tmp_path / "cli" / "state.json"),
|
||||
],
|
||||
cwd=Path(__file__).resolve().parents[1],
|
||||
capture_output=True,
|
||||
check=False,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
payload = json.loads(completed.stdout)
|
||||
assert payload["status"] == "shared_reservation_canary_passed"
|
||||
assert payload["controlled_apply"]["writes_database"] is False
|
||||
@@ -9,9 +9,11 @@ def _force_legacy_nim_first(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
module._ALERT_CACHE.clear()
|
||||
module._ALERT_INFLIGHT.clear()
|
||||
monkeypatch.setattr(module, "NEMOTRON_OLLAMA_FIRST", False)
|
||||
yield
|
||||
module._ALERT_CACHE.clear()
|
||||
module._ALERT_INFLIGHT.clear()
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -33,6 +35,7 @@ def _patch_execution_methods(monkeypatch, dispatcher):
|
||||
def record(kind):
|
||||
def _inner(*args, **kwargs):
|
||||
calls.append({"kind": kind, "args": args, "kwargs": kwargs})
|
||||
return {"durable": True}
|
||||
return _inner
|
||||
|
||||
monkeypatch.setattr(dispatcher, "_exec_trigger_price_alert", record("price_alert"))
|
||||
@@ -56,6 +59,207 @@ def test_dispatch_falls_back_to_hermes_rules_without_nim_api_key(monkeypatch):
|
||||
assert calls[0]["kind"] == "price_alert"
|
||||
|
||||
|
||||
def test_forced_review_failure_is_counted_as_skipped(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
monkeypatch.setattr(module, "NIM_API_KEY", "")
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_exec_flag_for_human_review",
|
||||
lambda *_args, **_kwargs: {"durable": False},
|
||||
)
|
||||
threat = FakeThreat("SKU-FORCED", "斷崖品")
|
||||
threat.sales_7d_delta_pct = -100.0
|
||||
|
||||
result = dispatcher.dispatch([threat], hermes_stats={"duration_sec": 1})
|
||||
|
||||
assert result["dispatched"] == 0
|
||||
assert result["skipped"] == 1
|
||||
assert len(result["errors"]) == 1
|
||||
assert "durable_side_effect_not_confirmed" in result["errors"][0]
|
||||
|
||||
|
||||
def test_hermes_failure_is_counted_as_skipped(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
monkeypatch.setattr(module, "NIM_API_KEY", "")
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_exec_trigger_price_alert",
|
||||
lambda *_args, **_kwargs: {"durable": False},
|
||||
)
|
||||
|
||||
result = dispatcher.dispatch(
|
||||
[FakeThreat("SKU-HERMES-FAIL", "告警品")],
|
||||
hermes_stats={"duration_sec": 1},
|
||||
)
|
||||
|
||||
assert result["dispatched"] == 0
|
||||
assert result["skipped"] == 1
|
||||
assert len(result["errors"]) == 1
|
||||
assert "durable_side_effect_not_confirmed" in result["errors"][0]
|
||||
|
||||
|
||||
def test_forced_review_commit_failure_keeps_durable_result_quarantined(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
monkeypatch.setattr(module, "NIM_API_KEY", "")
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_exec_flag_for_human_review",
|
||||
lambda *_args, **_kwargs: {"durable": True},
|
||||
)
|
||||
monkeypatch.setattr(module, "_mark_alert_dispatched", lambda *_args: False)
|
||||
released = []
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"_release_alert_reservation",
|
||||
lambda *args: (released.append(args) or True),
|
||||
)
|
||||
threat = FakeThreat("SKU-FORCED-COMMIT", "斷崖品")
|
||||
threat.sales_7d_delta_pct = -100.0
|
||||
|
||||
result = dispatcher.dispatch([threat], hermes_stats={"duration_sec": 1})
|
||||
|
||||
assert result["dispatched"] == 1
|
||||
assert result["skipped"] == 0
|
||||
assert result["errors"] == [
|
||||
"dedupe_commit_unverified:flag_for_human_review(SKU-FORCED-COMMIT)"
|
||||
]
|
||||
assert released == []
|
||||
|
||||
|
||||
def test_hermes_commit_failure_keeps_durable_result_quarantined(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
monkeypatch.setattr(module, "NIM_API_KEY", "")
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
_patch_execution_methods(monkeypatch, dispatcher)
|
||||
monkeypatch.setattr(module, "_mark_alert_dispatched", lambda *_args: False)
|
||||
released = []
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"_release_alert_reservation",
|
||||
lambda *args: (released.append(args) or True),
|
||||
)
|
||||
|
||||
result = dispatcher.dispatch(
|
||||
[FakeThreat("SKU-HERMES-COMMIT", "告警品")],
|
||||
hermes_stats={"duration_sec": 1},
|
||||
)
|
||||
|
||||
assert result["dispatched"] == 1
|
||||
assert result["skipped"] == 0
|
||||
assert result["errors"] == [
|
||||
"dedupe_commit_unverified:hermes_rule_fallback(SKU-HERMES-COMMIT)"
|
||||
]
|
||||
assert released == []
|
||||
|
||||
|
||||
def test_mixed_batch_cleanup_never_releases_forced_durable_quarantine(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
monkeypatch.setattr(module, "NIM_API_KEY", "test-key")
|
||||
monkeypatch.setattr(module, "_check_nim_quota", lambda: True)
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
_patch_execution_methods(monkeypatch, dispatcher)
|
||||
forced = FakeThreat("SKU-FORCED-MIXED", "斷崖品")
|
||||
forced.sales_7d_delta_pct = -100.0
|
||||
candidate = FakeThreat("SKU-NIM-MIXED", "模型候選")
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_call_nim",
|
||||
lambda _threats, **_kwargs: (
|
||||
[{
|
||||
"tool": "trigger_price_alert",
|
||||
"args": {
|
||||
"sku": candidate.sku,
|
||||
"name": candidate.name,
|
||||
"gap_pct": candidate.gap_pct,
|
||||
"sales_delta": candidate.sales_7d_delta_pct,
|
||||
"action": candidate.recommended_action,
|
||||
"confidence": candidate.confidence,
|
||||
},
|
||||
}],
|
||||
{"provider": "nim", "total_tokens": 1},
|
||||
),
|
||||
)
|
||||
real_mark = module._mark_alert_dispatched
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"_mark_alert_dispatched",
|
||||
lambda sku, token: (
|
||||
False if str(sku) == forced.sku else real_mark(sku, token)
|
||||
),
|
||||
)
|
||||
|
||||
result = dispatcher.dispatch(
|
||||
[forced, candidate],
|
||||
hermes_stats={"duration_sec": 1},
|
||||
)
|
||||
|
||||
assert result["dispatched"] == 2
|
||||
assert result["skipped"] == 0
|
||||
assert result["errors"] == [
|
||||
"dedupe_commit_unverified:flag_for_human_review(SKU-FORCED-MIXED)"
|
||||
]
|
||||
assert module._is_duplicate_alert(forced.sku) is True
|
||||
assert module._is_duplicate_alert(candidate.sku) is True
|
||||
|
||||
|
||||
def test_mixed_batch_counts_failed_model_candidate_as_skipped(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
monkeypatch.setattr(module, "NIM_API_KEY", "test-key")
|
||||
monkeypatch.setattr(module, "_check_nim_quota", lambda: True)
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
forced = FakeThreat("SKU-FORCED-COUNT", "斷崖品")
|
||||
forced.sales_7d_delta_pct = -100.0
|
||||
candidate = FakeThreat("SKU-NIM-COUNT", "模型候選")
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_exec_flag_for_human_review",
|
||||
lambda *_args, **_kwargs: {"durable": True},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_exec_trigger_price_alert",
|
||||
lambda *_args, **_kwargs: {"durable": False},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_call_nim",
|
||||
lambda _threats, **_kwargs: (
|
||||
[{
|
||||
"tool": "trigger_price_alert",
|
||||
"args": {
|
||||
"sku": candidate.sku,
|
||||
"name": candidate.name,
|
||||
"gap_pct": candidate.gap_pct,
|
||||
"sales_delta": candidate.sales_7d_delta_pct,
|
||||
"action": candidate.recommended_action,
|
||||
"confidence": candidate.confidence,
|
||||
},
|
||||
}],
|
||||
{"provider": "nim", "total_tokens": 1},
|
||||
),
|
||||
)
|
||||
|
||||
result = dispatcher.dispatch(
|
||||
[forced, candidate],
|
||||
hermes_stats={"duration_sec": 1},
|
||||
)
|
||||
|
||||
assert result["dispatched"] == 1
|
||||
assert result["skipped"] == 1
|
||||
assert len(result["errors"]) == 1
|
||||
assert "durable_side_effect_not_confirmed" in result["errors"][0]
|
||||
assert result["dispatched"] + result["skipped"] == 2
|
||||
|
||||
|
||||
def test_dispatch_routes_non_exact_match_to_human_review(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
@@ -83,7 +287,7 @@ def test_dispatch_falls_back_to_hermes_rules_on_nim_timeout(monkeypatch):
|
||||
monkeypatch.setattr(module, "_check_nim_quota", lambda: True)
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
calls = _patch_execution_methods(monkeypatch, dispatcher)
|
||||
monkeypatch.setattr(dispatcher, "_call_nim", lambda threats: (_ for _ in ()).throw(requests.Timeout("timeout")))
|
||||
monkeypatch.setattr(dispatcher, "_call_nim", lambda threats, **_kwargs: (_ for _ in ()).throw(requests.Timeout("timeout")))
|
||||
|
||||
result = dispatcher.dispatch([FakeThreat("SKU-2", "測試品")], hermes_stats={"duration_sec": 1})
|
||||
|
||||
@@ -101,7 +305,7 @@ def test_dispatch_falls_back_to_hermes_rules_on_zero_tool_calls(monkeypatch):
|
||||
monkeypatch.setattr(module, "_check_nim_quota", lambda: True)
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
calls = _patch_execution_methods(monkeypatch, dispatcher)
|
||||
monkeypatch.setattr(dispatcher, "_call_nim", lambda threats: ([], {"total_tokens": 10}))
|
||||
monkeypatch.setattr(dispatcher, "_call_nim", lambda threats, **_kwargs: ([], {"total_tokens": 10}))
|
||||
|
||||
result = dispatcher.dispatch([FakeThreat("SKU-3", "測試品")], hermes_stats={"duration_sec": 1})
|
||||
|
||||
|
||||
@@ -39,6 +39,20 @@ class FakeThreat:
|
||||
sales_7d_prev_amount: float = 120000.0
|
||||
|
||||
|
||||
def _valid_price_alert_call(sku="SKU-Q1"):
|
||||
return {
|
||||
"tool": "trigger_price_alert",
|
||||
"args": {
|
||||
"sku": sku,
|
||||
"name": "qwen3 測試品",
|
||||
"gap_pct": 22.4,
|
||||
"sales_delta": -35.0,
|
||||
"action": "跟進降價",
|
||||
"confidence": 0.85,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class _FakeResp:
|
||||
def __init__(self, payload: dict, status: int = 200):
|
||||
self._payload = payload
|
||||
@@ -59,6 +73,7 @@ def _noop_log_ai_call(*args, **kwargs):
|
||||
class _Ctx:
|
||||
def set_tokens(self, **_kw): pass
|
||||
def set_provider(self, *_a, **_kw): pass
|
||||
def set_model(self, *_a, **_kw): pass
|
||||
def set_error(self, *_a, **_kw): pass
|
||||
def fallback_to_caller(self, *_a, **_kw): pass
|
||||
def set_cache_hit(self, *_a, **_kw): pass
|
||||
@@ -76,11 +91,13 @@ def _reset_global_state():
|
||||
import services.nemoton_dispatcher_service as _nem
|
||||
import services.ollama_service as _oss
|
||||
_nem._ALERT_CACHE.clear()
|
||||
_nem._ALERT_INFLIGHT.clear()
|
||||
_oss._unhealthy_marks.clear()
|
||||
_oss._resolved_host_cache['host'] = None
|
||||
_oss._resolved_host_cache['ts'] = 0
|
||||
yield
|
||||
_nem._ALERT_CACHE.clear()
|
||||
_nem._ALERT_INFLIGHT.clear()
|
||||
_oss._unhealthy_marks.clear()
|
||||
_oss._resolved_host_cache['host'] = None
|
||||
_oss._resolved_host_cache['ts'] = 0
|
||||
@@ -93,6 +110,7 @@ def _patch_execution_methods(monkeypatch, dispatcher):
|
||||
def record(kind):
|
||||
def _inner(*args, **kwargs):
|
||||
calls.append({"kind": kind, "args": args, "kwargs": kwargs})
|
||||
return {"durable": True}
|
||||
return _inner
|
||||
|
||||
monkeypatch.setattr(dispatcher, "_exec_trigger_price_alert", record("price_alert"))
|
||||
@@ -102,14 +120,27 @@ def _patch_execution_methods(monkeypatch, dispatcher):
|
||||
|
||||
|
||||
def _enable_qwen3_path(monkeypatch, module):
|
||||
"""打開 NEMOTRON_OLLAMA_FIRST + 旁路 mcp/log_ai_call/resolve_host 等副作用"""
|
||||
"""打開 Ollama-first，並隔離 MCP、logger 與 model identity 網路副作用。"""
|
||||
monkeypatch.setattr(module, "NEMOTRON_OLLAMA_FIRST", True)
|
||||
monkeypatch.setattr(module, "log_ai_call", _noop_log_ai_call)
|
||||
monkeypatch.setattr(module, "build_mcp_context", lambda: "MCP-MOCK")
|
||||
# 確保即使未被呼叫，import 路徑可解析
|
||||
import services.ollama_service as ollama_module
|
||||
import services.nemotron_runtime_candidate_service as runtime_module
|
||||
|
||||
monkeypatch.setattr(ollama_module, "resolve_ollama_host", lambda: "http://34.87.90.216:11434")
|
||||
monkeypatch.setattr(ollama_module, "mark_unhealthy", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(
|
||||
runtime_module,
|
||||
"inspect_nemotron_model_identity",
|
||||
lambda candidate, **_kwargs: {
|
||||
**candidate.as_public_dict(),
|
||||
"ok": True,
|
||||
"digest": candidate.expected_digest,
|
||||
"digest_matches": True,
|
||||
"error": None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
@@ -178,12 +209,7 @@ def test_qwen3_retries_secondary_when_primary_chat_fails(monkeypatch):
|
||||
import services.ollama_service as ollama_module
|
||||
|
||||
_enable_qwen3_path(monkeypatch, module)
|
||||
hosts = iter([
|
||||
"http://34.87.90.216:11434",
|
||||
"http://34.21.145.224:11434",
|
||||
])
|
||||
marked = []
|
||||
monkeypatch.setattr(ollama_module, "resolve_ollama_host", lambda: next(hosts))
|
||||
monkeypatch.setattr(ollama_module, "mark_unhealthy", lambda host: marked.append(host))
|
||||
|
||||
fake_body = {
|
||||
@@ -227,6 +253,519 @@ def test_qwen3_retries_secondary_when_primary_chat_fails(monkeypatch):
|
||||
assert calls[0]["kind"] == "price_alert"
|
||||
|
||||
|
||||
def test_qwen3_uses_bounded_111_model_after_both_gcp_candidates_fail(monkeypatch):
|
||||
"""兩台 GCP 都逾時時，正式 dispatcher 應改用 exact-digest 111 qwen3:8b。"""
|
||||
import requests
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
_enable_qwen3_path(monkeypatch, module)
|
||||
posted = []
|
||||
fake_body = {
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [{
|
||||
"function": {
|
||||
"name": "trigger_price_alert",
|
||||
"arguments": {
|
||||
"sku": "SKU-Q1",
|
||||
"name": "qwen3 測試品",
|
||||
"gap_pct": 22.4,
|
||||
"sales_delta": -35.0,
|
||||
"action": "跟進降價",
|
||||
"confidence": 0.85,
|
||||
},
|
||||
}
|
||||
}],
|
||||
},
|
||||
"prompt_eval_count": 180,
|
||||
"eval_count": 36,
|
||||
}
|
||||
|
||||
def fake_post(url, json, timeout):
|
||||
posted.append((url, json, timeout))
|
||||
if len(posted) <= 2:
|
||||
raise requests.Timeout("GCP saturated")
|
||||
return _FakeResp(fake_body)
|
||||
|
||||
monkeypatch.setattr(module.requests, "post", fake_post)
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
calls = _patch_execution_methods(monkeypatch, dispatcher)
|
||||
|
||||
result = dispatcher.dispatch([FakeThreat()], hermes_stats={"duration_sec": 1.0})
|
||||
|
||||
stats = result["nim_stats"]
|
||||
assert result["dispatched"] == 1
|
||||
assert stats["provider"] == "ollama_111"
|
||||
assert stats["model"] == "qwen3:8b"
|
||||
assert stats["candidate_label"] == "ollama_111_fallback"
|
||||
assert stats["fallback_used"] is True
|
||||
assert [attempt["status"] for attempt in stats["attempted_candidates"]] == [
|
||||
"failed",
|
||||
"failed",
|
||||
"selected",
|
||||
]
|
||||
assert posted[2][1]["think"] is False
|
||||
assert posted[2][1]["options"]["num_predict"] == 512
|
||||
assert posted[2][2] == 45
|
||||
assert calls[0]["kind"] == "price_alert"
|
||||
assert "qwen3:8b (111 備援)" in calls[0]["kwargs"]["footprint"]
|
||||
assert "降級備援" in calls[0]["kwargs"]["footprint"]
|
||||
|
||||
structured = module._build_footprint_json(None, stats)
|
||||
assert structured["dispatcher"]["platform"] == "Ollama"
|
||||
assert structured["dispatcher"]["model"] == "qwen3:8b"
|
||||
assert structured["dispatcher"]["fallback_used"] is True
|
||||
assert [
|
||||
attempt["label"]
|
||||
for attempt in structured["dispatcher"]["candidate_attempts"]
|
||||
] == ["gcp_primary", "gcp_secondary", "ollama_111_fallback"]
|
||||
assert all(
|
||||
"host" not in attempt and "error" not in attempt
|
||||
for attempt in structured["dispatcher"]["candidate_attempts"]
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_primary_tool_contract_retries_verified_secondary(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
_enable_qwen3_path(monkeypatch, module)
|
||||
valid_body = {
|
||||
"message": {
|
||||
"tool_calls": [{"function": {
|
||||
"name": "trigger_price_alert",
|
||||
"arguments": _valid_price_alert_call()["args"],
|
||||
}}],
|
||||
"content": "",
|
||||
},
|
||||
"prompt_eval_count": 100,
|
||||
"eval_count": 20,
|
||||
}
|
||||
invalid_body = {
|
||||
"message": {
|
||||
"tool_calls": [{"function": {
|
||||
"name": "delete_product",
|
||||
"arguments": {"sku": "SKU-Q1"},
|
||||
}}],
|
||||
"content": "",
|
||||
},
|
||||
}
|
||||
responses = [_FakeResp(invalid_body), _FakeResp(valid_body)]
|
||||
monkeypatch.setattr(
|
||||
module.requests,
|
||||
"post",
|
||||
lambda *_args, **_kwargs: responses.pop(0),
|
||||
)
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
calls = _patch_execution_methods(monkeypatch, dispatcher)
|
||||
|
||||
result = dispatcher.dispatch([FakeThreat()], hermes_stats={"duration_sec": 1.0})
|
||||
|
||||
attempts = result["nim_stats"]["attempted_candidates"]
|
||||
assert [attempt["status"] for attempt in attempts] == ["failed", "selected"]
|
||||
assert "unknown_tool:delete_product" in attempts[0]["error"]
|
||||
assert result["nim_stats"]["provider"] == "ollama_secondary"
|
||||
assert result["dispatched"] == 1
|
||||
assert calls[0]["kind"] == "price_alert"
|
||||
|
||||
|
||||
def test_malformed_primary_tags_schema_falls_through_to_verified_secondary(
|
||||
monkeypatch,
|
||||
):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
import services.nemotron_runtime_candidate_service as runtime_module
|
||||
import services.ollama_service as ollama_module
|
||||
|
||||
monkeypatch.setattr(module, "NEMOTRON_OLLAMA_FIRST", True)
|
||||
monkeypatch.setattr(module, "log_ai_call", _noop_log_ai_call)
|
||||
monkeypatch.setattr(module, "build_mcp_context", lambda: "MCP-MOCK")
|
||||
monkeypatch.setattr(ollama_module, "mark_unhealthy", lambda *_args: None)
|
||||
|
||||
def fake_get(url, timeout):
|
||||
if runtime_module.OLLAMA_HOST_PRIMARY in url:
|
||||
return _FakeResp({
|
||||
"models": [
|
||||
None,
|
||||
{
|
||||
"name": runtime_module.PRIMARY_MODEL,
|
||||
"model": runtime_module.PRIMARY_MODEL,
|
||||
"digest": runtime_module.PRIMARY_EXPECTED_DIGEST,
|
||||
"details": {},
|
||||
},
|
||||
]
|
||||
})
|
||||
return _FakeResp({
|
||||
"models": [{
|
||||
"name": runtime_module.PRIMARY_MODEL,
|
||||
"model": runtime_module.PRIMARY_MODEL,
|
||||
"digest": runtime_module.PRIMARY_EXPECTED_DIGEST,
|
||||
"details": {
|
||||
"parameter_size": "14.8B",
|
||||
"quantization_level": "Q4_K_M",
|
||||
},
|
||||
}]
|
||||
})
|
||||
|
||||
valid_body = {
|
||||
"message": {
|
||||
"tool_calls": [{"function": {
|
||||
"name": "trigger_price_alert",
|
||||
"arguments": _valid_price_alert_call()["args"],
|
||||
}}],
|
||||
"content": "",
|
||||
},
|
||||
"prompt_eval_count": 100,
|
||||
"eval_count": 20,
|
||||
}
|
||||
monkeypatch.setattr(module.requests, "get", fake_get)
|
||||
monkeypatch.setattr(
|
||||
module.requests,
|
||||
"post",
|
||||
lambda *_args, **_kwargs: _FakeResp(valid_body),
|
||||
)
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
calls = _patch_execution_methods(monkeypatch, dispatcher)
|
||||
|
||||
result = dispatcher.dispatch([FakeThreat()], hermes_stats={"duration_sec": 1})
|
||||
|
||||
attempts = result["nim_stats"]["attempted_candidates"]
|
||||
assert [attempt["status"] for attempt in attempts] == [
|
||||
"identity_failed",
|
||||
"selected",
|
||||
]
|
||||
assert attempts[0]["error"] == "malformed_models_schema"
|
||||
assert result["nim_stats"]["provider"] == "ollama_secondary"
|
||||
assert result["dispatched"] == 1
|
||||
assert calls[0]["kind"] == "price_alert"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_calls", "threats", "error_fragment"),
|
||||
[
|
||||
([{"tool": "delete_product", "args": {"sku": "SKU-Q1"}}], [FakeThreat()], "unknown_tool"),
|
||||
([_valid_price_alert_call("OTHER")], [FakeThreat()], "sku_not_in_request"),
|
||||
([{"tool": "trigger_price_alert", "args": {"sku": "SKU-Q1"}}], [FakeThreat()], "missing_required_args"),
|
||||
([_valid_price_alert_call()], [FakeThreat(), FakeThreat(sku="SKU-Q2")], "tool_call_count_mismatch"),
|
||||
(
|
||||
[_valid_price_alert_call(), _valid_price_alert_call()],
|
||||
[FakeThreat(), FakeThreat(sku="SKU-Q2")],
|
||||
"duplicate_sku_tool_call",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_tool_contract_rejects_unbound_incomplete_or_duplicate_calls(
|
||||
tool_calls, threats, error_fragment
|
||||
):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
with pytest.raises(module.ToolCallContractError, match=error_fragment):
|
||||
module._validate_tool_call_contract(tool_calls, threats)
|
||||
|
||||
|
||||
def test_failed_handler_does_not_poison_alert_dedupe(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_exec_trigger_price_alert",
|
||||
lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("send failed")),
|
||||
)
|
||||
first_token = module._reserve_alert("SKU-Q1")
|
||||
assert first_token
|
||||
first = dispatcher._execute_tool_calls(
|
||||
tool_calls=[_valid_price_alert_call()],
|
||||
threats=[FakeThreat()],
|
||||
hermes_stats=None,
|
||||
nim_stats={"provider": "nim", "total_tokens": 1},
|
||||
reservation_tokens={"SKU-Q1": first_token},
|
||||
)
|
||||
|
||||
assert first["dispatched"] == 0
|
||||
assert module._is_duplicate_alert("SKU-Q1") is False
|
||||
|
||||
sent = []
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_exec_trigger_price_alert",
|
||||
lambda **kwargs: (sent.append(kwargs) or {"durable": True}),
|
||||
)
|
||||
second_token = module._reserve_alert("SKU-Q1")
|
||||
assert second_token
|
||||
second = dispatcher._execute_tool_calls(
|
||||
tool_calls=[_valid_price_alert_call()],
|
||||
threats=[FakeThreat()],
|
||||
hermes_stats=None,
|
||||
nim_stats={"provider": "nim", "total_tokens": 1},
|
||||
reservation_tokens={"SKU-Q1": second_token},
|
||||
)
|
||||
|
||||
assert second["dispatched"] == 1
|
||||
assert sent
|
||||
assert module._is_duplicate_alert("SKU-Q1") is True
|
||||
|
||||
|
||||
def test_non_durable_alert_outcome_is_failed_and_releases_reservation(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_send_telegram",
|
||||
lambda *_args, **_kwargs: {
|
||||
"durable": False,
|
||||
"delivered": False,
|
||||
"queued": False,
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(dispatcher, "_sink_insight_to_km", lambda **_kwargs: True)
|
||||
token = module._reserve_alert("SKU-Q1")
|
||||
assert token
|
||||
|
||||
result = dispatcher._execute_tool_calls(
|
||||
tool_calls=[_valid_price_alert_call()],
|
||||
threats=[FakeThreat()],
|
||||
hermes_stats=None,
|
||||
nim_stats={"provider": "nim", "total_tokens": 1},
|
||||
reservation_tokens={"SKU-Q1": token},
|
||||
)
|
||||
|
||||
assert result["dispatched"] == 0
|
||||
assert result["skipped"] == 1
|
||||
assert "durable_side_effect_not_confirmed" in result["errors"][0]
|
||||
assert module._is_duplicate_alert("SKU-Q1") is False
|
||||
|
||||
|
||||
def test_recommendation_requires_db_write_or_durable_notification(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
dispatcher = module.NemotronDispatcher(engine=None)
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_send_telegram",
|
||||
lambda *_args, **_kwargs: {"durable": False},
|
||||
)
|
||||
monkeypatch.setattr(dispatcher, "_sink_insight_to_km", lambda **_kwargs: True)
|
||||
token = module._reserve_alert("SKU-Q1")
|
||||
assert token
|
||||
|
||||
result = dispatcher._execute_tool_calls(
|
||||
tool_calls=[{
|
||||
"tool": "add_to_recommendation",
|
||||
"args": {
|
||||
"sku": "SKU-Q1",
|
||||
"name": "qwen3 測試品",
|
||||
"reason": "具價格競爭力",
|
||||
"confidence": 0.85,
|
||||
},
|
||||
}],
|
||||
threats=[FakeThreat()],
|
||||
hermes_stats=None,
|
||||
nim_stats={"provider": "nim", "total_tokens": 1},
|
||||
reservation_tokens={"SKU-Q1": token},
|
||||
)
|
||||
|
||||
assert result["dispatched"] == 0
|
||||
assert "durable_side_effect_not_confirmed" in result["errors"][0]
|
||||
assert module._is_duplicate_alert("SKU-Q1") is False
|
||||
|
||||
|
||||
def test_km_route_requires_durable_insight_write(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
monkeypatch.setattr(dispatcher, "_sink_insight_to_km", lambda **_kwargs: False)
|
||||
token = module._reserve_alert("SKU-Q1")
|
||||
assert token
|
||||
|
||||
result = dispatcher._execute_tool_calls(
|
||||
tool_calls=[{
|
||||
"tool": "route_to_km",
|
||||
"args": {
|
||||
"sku": "SKU-Q1",
|
||||
"name": "qwen3 測試品",
|
||||
"km_domain": "price_competition",
|
||||
"summary": "競價洞察",
|
||||
"confidence": 0.85,
|
||||
},
|
||||
}],
|
||||
threats=[FakeThreat()],
|
||||
hermes_stats=None,
|
||||
nim_stats={"provider": "nim", "total_tokens": 1},
|
||||
reservation_tokens={"SKU-Q1": token},
|
||||
)
|
||||
|
||||
assert result["dispatched"] == 0
|
||||
assert "durable_side_effect_not_confirmed" in result["errors"][0]
|
||||
assert module._is_duplicate_alert("SKU-Q1") is False
|
||||
|
||||
|
||||
def test_alert_reservation_is_atomic_and_releasable():
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
first_token = module._reserve_alert("SKU-Q1")
|
||||
assert first_token
|
||||
assert module._reserve_alert("SKU-Q1") is None
|
||||
assert module._release_alert_reservation("SKU-Q1", first_token) is True
|
||||
second_token = module._reserve_alert("SKU-Q1")
|
||||
assert second_token and second_token != first_token
|
||||
assert module._mark_alert_dispatched("SKU-Q1", second_token) is True
|
||||
assert module._reserve_alert("SKU-Q1") is None
|
||||
|
||||
|
||||
def test_expired_reservation_token_cannot_release_or_commit_new_owner(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
clock = {"now": 1000.0}
|
||||
monkeypatch.setattr(module.time, "time", lambda: clock["now"])
|
||||
first_token = module._reserve_alert("SKU-Q1")
|
||||
assert first_token
|
||||
|
||||
clock["now"] += module._ALERT_INFLIGHT_TTL_SEC + 1
|
||||
assert module._mark_alert_dispatched("SKU-Q1", first_token) is False
|
||||
second_token = module._reserve_alert("SKU-Q1")
|
||||
assert second_token and second_token != first_token
|
||||
|
||||
assert module._release_alert_reservation("SKU-Q1", first_token) is False
|
||||
assert module._mark_alert_dispatched("SKU-Q1", first_token) is False
|
||||
assert module._refresh_alert_reservation("SKU-Q1", first_token) is False
|
||||
assert module._refresh_alert_reservation("SKU-Q1", second_token) is True
|
||||
assert module._mark_alert_dispatched("SKU-Q1", second_token) is True
|
||||
assert module._is_duplicate_alert("SKU-Q1") is True
|
||||
|
||||
|
||||
def test_concurrent_reservation_has_exactly_one_owner():
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Barrier
|
||||
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
barrier = Barrier(8)
|
||||
|
||||
def reserve_once():
|
||||
barrier.wait()
|
||||
return module._reserve_alert("SKU-CONCURRENT")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
tokens = list(pool.map(lambda _index: reserve_once(), range(8)))
|
||||
|
||||
owners = [token for token in tokens if token is not None]
|
||||
assert len(owners) == 1
|
||||
assert module._release_alert_reservation("SKU-CONCURRENT", owners[0]) is True
|
||||
|
||||
|
||||
def test_side_effect_boundary_extends_lease_through_slow_handler(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
clock = {"now": 5000.0}
|
||||
monkeypatch.setattr(module.time, "time", lambda: clock["now"])
|
||||
old_token = module._reserve_alert("SKU-Q1")
|
||||
assert old_token
|
||||
new_owner = {}
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
|
||||
def delayed_handler(**_kwargs):
|
||||
clock["now"] += module._ALERT_INFLIGHT_TTL_SEC + 1
|
||||
new_owner["token"] = module._reserve_alert("SKU-Q1")
|
||||
return {"durable": True}
|
||||
|
||||
monkeypatch.setattr(dispatcher, "_exec_trigger_price_alert", delayed_handler)
|
||||
result = dispatcher._execute_tool_calls(
|
||||
tool_calls=[_valid_price_alert_call()],
|
||||
threats=[FakeThreat()],
|
||||
hermes_stats=None,
|
||||
nim_stats={"provider": "nim", "total_tokens": 1},
|
||||
reservation_tokens={"SKU-Q1": old_token},
|
||||
)
|
||||
|
||||
assert new_owner["token"] is None
|
||||
assert result["dispatched"] == 1
|
||||
assert result["errors"] == []
|
||||
assert module._is_duplicate_alert("SKU-Q1") is True
|
||||
|
||||
|
||||
def test_durable_tool_commit_failure_never_releases_quarantine(monkeypatch):
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
monkeypatch.setattr(
|
||||
dispatcher,
|
||||
"_exec_trigger_price_alert",
|
||||
lambda **_kwargs: {"durable": True},
|
||||
)
|
||||
token = module._reserve_alert("SKU-Q1")
|
||||
released = []
|
||||
monkeypatch.setattr(module, "_mark_alert_dispatched", lambda *_args: False)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"_release_alert_reservation",
|
||||
lambda *args: (released.append(args) or True),
|
||||
)
|
||||
|
||||
result = dispatcher._execute_tool_calls(
|
||||
tool_calls=[_valid_price_alert_call()],
|
||||
threats=[FakeThreat()],
|
||||
hermes_stats=None,
|
||||
nim_stats={"provider": "nim", "total_tokens": 1},
|
||||
reservation_tokens={"SKU-Q1": token},
|
||||
)
|
||||
|
||||
assert result["dispatched"] == 1
|
||||
assert result["skipped"] == 0
|
||||
assert result["errors"] == [
|
||||
"dedupe_commit_unverified:trigger_price_alert(SKU-Q1)"
|
||||
]
|
||||
assert released == []
|
||||
assert module._is_duplicate_alert("SKU-Q1") is True
|
||||
|
||||
|
||||
def test_dispatch_total_deadline_bounds_ollama_and_nim_chain(monkeypatch):
|
||||
import requests
|
||||
import services.nemoton_dispatcher_service as module
|
||||
|
||||
_enable_qwen3_path(monkeypatch, module)
|
||||
monkeypatch.setattr(module, "NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC", 30)
|
||||
monkeypatch.setattr(module, "NIM_API_KEY", "fake-key")
|
||||
monkeypatch.setattr(module, "_check_nim_quota", lambda: True)
|
||||
clock = {"now": 100.0}
|
||||
monkeypatch.setattr(module.time, "monotonic", lambda: clock["now"])
|
||||
posted = []
|
||||
|
||||
def timeout_post(url, json=None, timeout=None, **_kwargs):
|
||||
posted.append((url, timeout))
|
||||
clock["now"] += float(timeout or 0)
|
||||
raise requests.Timeout("simulated timeout")
|
||||
|
||||
monkeypatch.setattr(module.requests, "post", timeout_post)
|
||||
dispatcher = module.NemotronDispatcher()
|
||||
calls = _patch_execution_methods(monkeypatch, dispatcher)
|
||||
|
||||
result = dispatcher.dispatch([FakeThreat()], hermes_stats={"duration_sec": 1.0})
|
||||
|
||||
upstream = result["nim_stats"]["upstream_ollama_failure"]
|
||||
assert len(posted) == 1
|
||||
assert posted[0][1] == 30
|
||||
assert clock["now"] == 130.0
|
||||
assert upstream["deadline_exhausted"] is True
|
||||
assert upstream["attempt_count"] == 2
|
||||
assert result["nim_stats"]["provider"] == "hermes_rule_engine"
|
||||
assert "Ollama 候選鏈" in calls[0]["kwargs"]["footprint"]
|
||||
assert "gcp_primary | primary | qwen3:14b | failed | digest=ok | Timeout" in (
|
||||
calls[0]["kwargs"]["footprint"]
|
||||
)
|
||||
structured = module._build_footprint_json(None, result["nim_stats"])
|
||||
attempts = structured["dispatcher"]["candidate_attempts"]
|
||||
assert attempts[0] == {
|
||||
"label": "gcp_primary",
|
||||
"tier": "primary",
|
||||
"model": "qwen3:14b",
|
||||
"status": "failed",
|
||||
"digest_matches": True,
|
||||
"error_class": "Timeout",
|
||||
}
|
||||
assert result["dispatched"] == 1
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# T2. qwen3 沒回 tool_calls 但 content 含 JSON list → fallback 解析
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
@@ -252,7 +791,7 @@ def test_qwen3_content_only_fallback_parsing(monkeypatch):
|
||||
calls = _patch_execution_methods(monkeypatch, dispatcher)
|
||||
monkeypatch.setattr(
|
||||
dispatcher, "_call_nim",
|
||||
lambda threats: (_ for _ in ()).throw(AssertionError("NIM 不應被呼叫")),
|
||||
lambda threats, **_kwargs: (_ for _ in ()).throw(AssertionError("NIM 不應被呼叫")),
|
||||
)
|
||||
|
||||
result = dispatcher.dispatch([FakeThreat(confidence=0.45)], hermes_stats={"duration_sec": 1.0})
|
||||
@@ -325,7 +864,7 @@ def test_qwen3_connection_error_falls_back_to_nim(monkeypatch):
|
||||
calls = _patch_execution_methods(monkeypatch, dispatcher)
|
||||
nim_invoked = {"v": False}
|
||||
|
||||
def _fake_nim(threats):
|
||||
def _fake_nim(threats, **_kwargs):
|
||||
nim_invoked["v"] = True
|
||||
return (
|
||||
[{
|
||||
@@ -369,7 +908,7 @@ def test_qwen3_and_nim_both_fail_falls_back_to_hermes_rules(monkeypatch):
|
||||
# 攔 _call_nim 也擲 timeout
|
||||
monkeypatch.setattr(
|
||||
dispatcher, "_call_nim",
|
||||
lambda threats: (_ for _ in ()).throw(requests.Timeout("NIM timeout")),
|
||||
lambda threats, **_kwargs: (_ for _ in ()).throw(requests.Timeout("NIM timeout")),
|
||||
)
|
||||
|
||||
# 攔住規則引擎內部呼叫的 _exec_*，記錄 concern / reason 文字驗證 🟡 標記
|
||||
@@ -392,6 +931,7 @@ def test_qwen3_and_nim_both_fail_falls_back_to_hermes_rules(monkeypatch):
|
||||
'revenue_loss_7d', 'recommended_price'],
|
||||
args, kwargs)
|
||||
captured.append(("human_review", merged))
|
||||
return {"durable": True}
|
||||
|
||||
def record_alert(*args, **kwargs):
|
||||
merged = _merge_positional(
|
||||
@@ -400,9 +940,11 @@ def test_qwen3_and_nim_both_fail_falls_back_to_hermes_rules(monkeypatch):
|
||||
'revenue_loss_7d', 'recommended_price'],
|
||||
args, kwargs)
|
||||
captured.append(("price_alert", merged))
|
||||
return {"durable": True}
|
||||
|
||||
def record_reco(*args, **kwargs):
|
||||
captured.append(("recommendation", kwargs))
|
||||
return {"durable": True}
|
||||
|
||||
monkeypatch.setattr(dispatcher, "_exec_flag_for_human_review", record_review)
|
||||
monkeypatch.setattr(dispatcher, "_exec_trigger_price_alert", record_alert)
|
||||
@@ -448,7 +990,7 @@ def test_flag_false_preserves_nim_first_emergency_path(monkeypatch):
|
||||
calls = _patch_execution_methods(monkeypatch, dispatcher)
|
||||
monkeypatch.setattr(
|
||||
dispatcher, "_call_nim",
|
||||
lambda threats: (
|
||||
lambda threats, **_kwargs: (
|
||||
[{
|
||||
"tool": "trigger_price_alert",
|
||||
"args": {
|
||||
|
||||
205
tests/test_nemotron_runtime_candidate_service.py
Normal file
205
tests/test_nemotron_runtime_candidate_service.py
Normal file
@@ -0,0 +1,205 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
def test_candidate_chain_is_gcp_first_with_exact_digest_111_fallback():
|
||||
from services import nemotron_runtime_candidate_service as service
|
||||
|
||||
candidates = service.build_nemotron_runtime_candidates()
|
||||
|
||||
assert [candidate.label for candidate in candidates] == [
|
||||
"gcp_primary",
|
||||
"gcp_secondary",
|
||||
"ollama_111_fallback",
|
||||
]
|
||||
assert [candidate.model for candidate in candidates] == [
|
||||
"qwen3:14b",
|
||||
"qwen3:14b",
|
||||
"qwen3:8b",
|
||||
]
|
||||
assert candidates[0].is_fallback is False
|
||||
assert candidates[2].is_fallback is True
|
||||
assert candidates[0].request_timeout_sec == 60
|
||||
assert candidates[2].request_timeout_sec == 45
|
||||
assert candidates[0].num_predict == 2048
|
||||
assert candidates[2].num_predict == 512
|
||||
assert candidates[0].num_ctx is None
|
||||
assert candidates[2].num_ctx == 4096
|
||||
|
||||
|
||||
def test_model_identity_fails_closed_on_digest_drift():
|
||||
from services import nemotron_runtime_candidate_service as service
|
||||
|
||||
candidate = service.build_nemotron_runtime_candidates()[0]
|
||||
identity = service.inspect_nemotron_model_identity(
|
||||
candidate,
|
||||
request_get=lambda *_args, **_kwargs: _Response({
|
||||
"models": [{
|
||||
"name": candidate.model,
|
||||
"model": candidate.model,
|
||||
"digest": "deadbeef",
|
||||
"details": {},
|
||||
}]
|
||||
}),
|
||||
)
|
||||
|
||||
assert identity["ok"] is False
|
||||
assert identity["digest_matches"] is False
|
||||
assert identity["error"] == "model_digest_mismatch"
|
||||
|
||||
|
||||
def test_model_identity_reports_network_failure_without_loading_model():
|
||||
from services import nemotron_runtime_candidate_service as service
|
||||
|
||||
candidate = service.build_nemotron_runtime_candidates()[2]
|
||||
|
||||
def fail(*_args, **_kwargs):
|
||||
raise requests.Timeout("offline")
|
||||
|
||||
identity = service.inspect_nemotron_model_identity(
|
||||
candidate,
|
||||
request_get=fail,
|
||||
)
|
||||
|
||||
assert identity["ok"] is False
|
||||
assert identity["digest"] is None
|
||||
assert identity["error"].startswith("Timeout: offline")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("payload", "error_fragment"),
|
||||
[
|
||||
(None, "malformed_tags_payload"),
|
||||
({"models": {}}, "malformed_models_schema"),
|
||||
({"models": [None]}, "malformed_models_schema"),
|
||||
({"models": [{"name": "other", "details": None}]}, "malformed_model_details"),
|
||||
],
|
||||
)
|
||||
def test_model_identity_fails_closed_on_malformed_tags_schema(
|
||||
payload, error_fragment
|
||||
):
|
||||
from services import nemotron_runtime_candidate_service as service
|
||||
|
||||
candidate = service.build_nemotron_runtime_candidates()[0]
|
||||
identity = service.inspect_nemotron_model_identity(
|
||||
candidate,
|
||||
request_get=lambda *_args, **_kwargs: _Response(payload),
|
||||
)
|
||||
|
||||
assert identity["ok"] is False
|
||||
assert identity["digest_matches"] is False
|
||||
assert error_fragment in str(identity["error"])
|
||||
|
||||
|
||||
def test_model_identity_rejects_malformed_entry_mixed_with_valid_target():
|
||||
from services import nemotron_runtime_candidate_service as service
|
||||
|
||||
candidate = service.build_nemotron_runtime_candidates()[0]
|
||||
identity = service.inspect_nemotron_model_identity(
|
||||
candidate,
|
||||
request_get=lambda *_args, **_kwargs: _Response({
|
||||
"models": [
|
||||
None,
|
||||
{
|
||||
"name": candidate.model,
|
||||
"model": candidate.model,
|
||||
"digest": candidate.expected_digest,
|
||||
"details": {},
|
||||
},
|
||||
]
|
||||
}),
|
||||
)
|
||||
|
||||
assert identity["ok"] is False
|
||||
assert identity["digest_matches"] is False
|
||||
assert identity["error"] == "malformed_models_schema"
|
||||
|
||||
|
||||
def test_candidate_attempt_timeout_honors_env_but_111_resources_stay_fixed():
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"NEMOTRON_111_ATTEMPT_TIMEOUT_SEC": "33",
|
||||
"OLLAMA_111_NUM_CTX": "2048",
|
||||
"OLLAMA_111_NUM_PREDICT": "256",
|
||||
})
|
||||
completed = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
(
|
||||
"import json; "
|
||||
"from services.nemotron_runtime_candidate_service import "
|
||||
"build_nemotron_runtime_candidates; "
|
||||
"c=build_nemotron_runtime_candidates()[-1]; "
|
||||
"print(json.dumps(c.as_public_dict(), sort_keys=True))"
|
||||
),
|
||||
],
|
||||
cwd=os.path.dirname(os.path.dirname(__file__)),
|
||||
env=env,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
fallback = json.loads(completed.stdout.strip())
|
||||
assert fallback["request_timeout_sec"] == 33
|
||||
assert fallback["num_ctx"] == 4096
|
||||
assert fallback["num_predict"] == 512
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_ctx", "num_predict"),
|
||||
[
|
||||
("99999", "99999"),
|
||||
("-7", "-9"),
|
||||
("not-a-number", "invalid"),
|
||||
],
|
||||
)
|
||||
def test_candidate_resource_contract_cannot_be_weakened_by_environment(
|
||||
num_ctx, num_predict
|
||||
):
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"OLLAMA_111_NUM_CTX": num_ctx,
|
||||
"OLLAMA_111_NUM_PREDICT": num_predict,
|
||||
})
|
||||
completed = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
(
|
||||
"import json; "
|
||||
"from services.nemotron_runtime_candidate_service import "
|
||||
"build_nemotron_runtime_candidates; "
|
||||
"c=build_nemotron_runtime_candidates()[-1]; "
|
||||
"print(json.dumps(c.as_public_dict(), sort_keys=True))"
|
||||
),
|
||||
],
|
||||
cwd=os.path.dirname(os.path.dirname(__file__)),
|
||||
env=env,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
fallback = json.loads(completed.stdout.strip())
|
||||
assert fallback["num_ctx"] == 4096
|
||||
assert fallback["num_predict"] == 512
|
||||
@@ -132,8 +132,15 @@ def test_env_example_documents_runtime_and_ai_automation_variables():
|
||||
"N8N_USER",
|
||||
"N8N_WEBHOOK_BASE_URL",
|
||||
"NEMOTRON_OLLAMA_FIRST",
|
||||
"NEMOTRON_OLLAMA_EXPECTED_DIGEST",
|
||||
"NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST",
|
||||
"NEMOTRON_OLLAMA_FALLBACK_MODEL",
|
||||
"NEMOTRON_OLLAMA_MODEL",
|
||||
"NEMOTRON_OLLAMA_TIMEOUT",
|
||||
"NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC",
|
||||
"NEMOTRON_111_ATTEMPT_TIMEOUT_SEC",
|
||||
"NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC",
|
||||
"NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC",
|
||||
"OLLAMA_EMBED_TIMEOUT",
|
||||
"OPENCLAW_ADMIN_USER_IDS",
|
||||
"OPENCLAW_AGENT_DISPATCH",
|
||||
|
||||
@@ -7,6 +7,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
def test_qwen3_is_active_runtime_model_not_unused_ollama_weight():
|
||||
openclaw_source = (ROOT / "services" / "openclaw_strategist_service.py").read_text(encoding="utf-8")
|
||||
nemotron_source = (ROOT / "services" / "nemoton_dispatcher_service.py").read_text(encoding="utf-8")
|
||||
candidate_source = (ROOT / "services" / "nemotron_runtime_candidate_service.py").read_text(encoding="utf-8")
|
||||
router_source = (ROOT / "services" / "llm_model_router.py").read_text(encoding="utf-8")
|
||||
|
||||
assert "OPENCLAW_QA_OLLAMA_MODEL = os.getenv('OPENCLAW_QA_OLLAMA_MODEL', 'qwen3:14b')" in openclaw_source
|
||||
@@ -16,7 +17,11 @@ def test_qwen3_is_active_runtime_model_not_unused_ollama_weight():
|
||||
assert "OPENCLAW_STRATEGY_OLLAMA_KEEP_ALIVE = os.getenv('OPENCLAW_STRATEGY_OLLAMA_KEEP_ALIVE', '5m')" in openclaw_source
|
||||
assert "keep_alive=OPENCLAW_STRATEGY_OLLAMA_KEEP_ALIVE" in openclaw_source
|
||||
assert 'keep_alive="24h"' not in openclaw_source
|
||||
assert 'NEMOTRON_OLLAMA_MODEL = os.getenv("NEMOTRON_OLLAMA_MODEL", "qwen3:14b")' in nemotron_source
|
||||
assert "NEMOTRON_OLLAMA_MODEL = PRIMARY_MODEL" in nemotron_source
|
||||
assert "def _call_qwen3_dispatch(" in nemotron_source
|
||||
assert "for _attempt in range(3):" in nemotron_source
|
||||
assert "for candidate in build_nemotron_runtime_candidates(" in nemotron_source
|
||||
assert "inspect_nemotron_model_identity(" in nemotron_source
|
||||
assert '"qwen3:14b"' in candidate_source
|
||||
assert '"qwen3:8b"' in candidate_source
|
||||
assert 'label="ollama_111_fallback"' in candidate_source
|
||||
assert "'qwen3:14b'" in router_source
|
||||
|
||||
Reference in New Issue
Block a user