diff --git a/run_scheduler.py b/run_scheduler.py index 683acf5..c9c947a 100644 --- a/run_scheduler.py +++ b/run_scheduler.py @@ -855,10 +855,25 @@ def run_embed_consistency_check(): result['max_diff'], result['signature'], ) if not result['ok']: + max_diff = float(result.get('max_diff') or 0.0) + detail = ( + "BGE-M3 embedding consistency mismatch " + f"reachable={result.get('reachable')} " + f"max_diff={max_diff:.2e} " + f"signature={result.get('signature')}" + ) logger.error( "[EmbedConsistency] ⚠️ INCONSISTENT — RAG 召回率將下降;" "檢查三主機 bge-m3 模型版本是否同步(ollama list)" ) + _notify_scheduler_failure( + "run_embed_consistency_check", + RuntimeError(detail), + source="Scheduler.RAG", + event_type="embed_consistency_mismatch", + title="BGE-M3 一致性異常", + dedup_ttl_sec=86400, + ) except Exception as e: logger.error(f"[EmbedConsistency] task failed: {e}", exc_info=True) _notify_scheduler_failure( diff --git a/tests/test_run_scheduler_embed_consistency.py b/tests/test_run_scheduler_embed_consistency.py new file mode 100644 index 0000000..985f581 --- /dev/null +++ b/tests/test_run_scheduler_embed_consistency.py @@ -0,0 +1,74 @@ +import importlib + + +def _load_run_scheduler(monkeypatch): + monkeypatch.setenv("MOMO_ALLOW_INSECURE_CONFIG_FOR_TESTS", "true") + monkeypatch.setenv("USE_POSTGRESQL", "false") + return importlib.import_module("run_scheduler") + + +def test_embed_consistency_mismatch_notifies_event_router(monkeypatch): + run_scheduler = _load_run_scheduler(monkeypatch) + import services.rag_service as rag_service + + monkeypatch.setattr( + rag_service, + "verify_embedding_consistency", + lambda: { + "ok": False, + "reachable": ["gcp_a", "gcp_b", "fallback_111"], + "max_diff": 0.125, + "signature": "abc123", + }, + ) + + notifications = [] + + def fake_notify(task_name, error, **kwargs): + notifications.append((task_name, str(error), kwargs)) + + monkeypatch.setattr(run_scheduler, "_notify_scheduler_failure", fake_notify) + + run_scheduler.run_embed_consistency_check() + + assert notifications == [ + ( + "run_embed_consistency_check", + "BGE-M3 embedding consistency mismatch " + "reachable=['gcp_a', 'gcp_b', 'fallback_111'] " + "max_diff=1.25e-01 signature=abc123", + { + "source": "Scheduler.RAG", + "event_type": "embed_consistency_mismatch", + "title": "BGE-M3 一致性異常", + "dedup_ttl_sec": 86400, + }, + ) + ] + + +def test_embed_consistency_ok_does_not_notify(monkeypatch): + run_scheduler = _load_run_scheduler(monkeypatch) + import services.rag_service as rag_service + + monkeypatch.setattr( + rag_service, + "verify_embedding_consistency", + lambda: { + "ok": True, + "reachable": ["gcp_a", "gcp_b", "fallback_111"], + "max_diff": 0.0, + "signature": "abc123", + }, + ) + + notifications = [] + monkeypatch.setattr( + run_scheduler, + "_notify_scheduler_failure", + lambda *args, **kwargs: notifications.append((args, kwargs)), + ) + + run_scheduler.run_embed_consistency_check() + + assert notifications == []