告警 BGE embedding 一致性異常

run_scheduler.py

@@ -855,10 +855,25 @@ def run_embed_consistency_check():
             result['max_diff'], result['signature'],
         )
         if not result['ok']:
+            max_diff = float(result.get('max_diff') or 0.0)
+            detail = (
+                "BGE-M3 embedding consistency mismatch "
+                f"reachable={result.get('reachable')} "
+                f"max_diff={max_diff:.2e} "
+                f"signature={result.get('signature')}"
+            )
             logger.error(
                 "[EmbedConsistency] ⚠️ INCONSISTENT — RAG 召回率將下降；"
                 "檢查三主機 bge-m3 模型版本是否同步（ollama list）"
             )
+            _notify_scheduler_failure(
+                "run_embed_consistency_check",
+                RuntimeError(detail),
+                source="Scheduler.RAG",
+                event_type="embed_consistency_mismatch",
+                title="BGE-M3 一致性異常",
+                dedup_ttl_sec=86400,
+            )
     except Exception as e:
         logger.error(f"[EmbedConsistency] task failed: {e}", exc_info=True)
         _notify_scheduler_failure(

tests/test_run_scheduler_embed_consistency.py

@@ -0,0 +1,74 @@
+import importlib
+
+
+def _load_run_scheduler(monkeypatch):
+    monkeypatch.setenv("MOMO_ALLOW_INSECURE_CONFIG_FOR_TESTS", "true")
+    monkeypatch.setenv("USE_POSTGRESQL", "false")
+    return importlib.import_module("run_scheduler")
+
+
+def test_embed_consistency_mismatch_notifies_event_router(monkeypatch):
+    run_scheduler = _load_run_scheduler(monkeypatch)
+    import services.rag_service as rag_service
+
+    monkeypatch.setattr(
+        rag_service,
+        "verify_embedding_consistency",
+        lambda: {
+            "ok": False,
+            "reachable": ["gcp_a", "gcp_b", "fallback_111"],
+            "max_diff": 0.125,
+            "signature": "abc123",
+        },
+    )
+
+    notifications = []
+
+    def fake_notify(task_name, error, **kwargs):
+        notifications.append((task_name, str(error), kwargs))
+
+    monkeypatch.setattr(run_scheduler, "_notify_scheduler_failure", fake_notify)
+
+    run_scheduler.run_embed_consistency_check()
+
+    assert notifications == [
+        (
+            "run_embed_consistency_check",
+            "BGE-M3 embedding consistency mismatch "
+            "reachable=['gcp_a', 'gcp_b', 'fallback_111'] "
+            "max_diff=1.25e-01 signature=abc123",
+            {
+                "source": "Scheduler.RAG",
+                "event_type": "embed_consistency_mismatch",
+                "title": "BGE-M3 一致性異常",
+                "dedup_ttl_sec": 86400,
+            },
+        )
+    ]
+
+
+def test_embed_consistency_ok_does_not_notify(monkeypatch):
+    run_scheduler = _load_run_scheduler(monkeypatch)
+    import services.rag_service as rag_service
+
+    monkeypatch.setattr(
+        rag_service,
+        "verify_embedding_consistency",
+        lambda: {
+            "ok": True,
+            "reachable": ["gcp_a", "gcp_b", "fallback_111"],
+            "max_diff": 0.0,
+            "signature": "abc123",
+        },
+    )
+
+    notifications = []
+    monkeypatch.setattr(
+        run_scheduler,
+        "_notify_scheduler_failure",
+        lambda *args, **kwargs: notifications.append((args, kwargs)),
+    )
+
+    run_scheduler.run_embed_consistency_check()
+
+    assert notifications == []