111 lines
3.1 KiB
Python
111 lines
3.1 KiB
Python
import importlib
|
|
import inspect
|
|
|
|
|
|
def _load_run_scheduler(monkeypatch):
|
|
monkeypatch.setenv("MOMO_ALLOW_INSECURE_CONFIG_FOR_TESTS", "true")
|
|
monkeypatch.setenv("USE_POSTGRESQL", "false")
|
|
return importlib.import_module("run_scheduler")
|
|
|
|
|
|
def test_embed_consistency_mismatch_notifies_event_router(monkeypatch):
|
|
run_scheduler = _load_run_scheduler(monkeypatch)
|
|
import services.rag_service as rag_service
|
|
|
|
monkeypatch.setattr(
|
|
rag_service,
|
|
"verify_embedding_consistency",
|
|
lambda: {
|
|
"ok": False,
|
|
"reachable": ["gcp_a", "gcp_b", "fallback_111"],
|
|
"max_diff": 0.125,
|
|
"signature": "abc123",
|
|
},
|
|
)
|
|
|
|
notifications = []
|
|
|
|
def fake_notify(task_name, error, **kwargs):
|
|
notifications.append((task_name, str(error), kwargs))
|
|
|
|
monkeypatch.setattr(run_scheduler, "_notify_scheduler_failure", fake_notify)
|
|
|
|
run_scheduler.run_embed_consistency_check()
|
|
|
|
assert notifications == [
|
|
(
|
|
"run_embed_consistency_check",
|
|
"BGE-M3 embedding consistency mismatch "
|
|
"reachable=['gcp_a', 'gcp_b', 'fallback_111'] "
|
|
"max_diff=1.25e-01 signature=abc123",
|
|
{
|
|
"source": "Scheduler.RAG",
|
|
"event_type": "embed_consistency_mismatch",
|
|
"title": "BGE-M3 一致性異常",
|
|
"dedup_ttl_sec": 86400,
|
|
},
|
|
)
|
|
]
|
|
|
|
|
|
def test_embed_consistency_ok_does_not_notify(monkeypatch):
|
|
run_scheduler = _load_run_scheduler(monkeypatch)
|
|
import services.rag_service as rag_service
|
|
|
|
monkeypatch.setattr(
|
|
rag_service,
|
|
"verify_embedding_consistency",
|
|
lambda: {
|
|
"ok": True,
|
|
"reachable": ["gcp_a", "gcp_b", "fallback_111"],
|
|
"max_diff": 0.0,
|
|
"signature": "abc123",
|
|
},
|
|
)
|
|
|
|
notifications = []
|
|
monkeypatch.setattr(
|
|
run_scheduler,
|
|
"_notify_scheduler_failure",
|
|
lambda *args, **kwargs: notifications.append((args, kwargs)),
|
|
)
|
|
|
|
run_scheduler.run_embed_consistency_check()
|
|
|
|
assert notifications == []
|
|
|
|
|
|
def test_notify_scheduler_failure_without_active_exception_uses_error_trace(monkeypatch):
|
|
run_scheduler = _load_run_scheduler(monkeypatch)
|
|
import services.event_router as event_router
|
|
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
event_router,
|
|
"notify_failure",
|
|
lambda **kwargs: calls.append(kwargs),
|
|
)
|
|
|
|
run_scheduler._notify_scheduler_failure(
|
|
"synthetic_check",
|
|
RuntimeError("mismatch"),
|
|
source="Scheduler.Test",
|
|
event_type="synthetic_check_failed",
|
|
title="Synthetic check failed",
|
|
)
|
|
|
|
assert calls[0]["trace"] == "RuntimeError: mismatch"
|
|
|
|
|
|
def test_scheduler_observability_wrappers_notify_on_exception(monkeypatch):
|
|
run_scheduler = _load_run_scheduler(monkeypatch)
|
|
|
|
for fn_name in [
|
|
"run_host_health_probe",
|
|
"run_ai_calls_error_spike_check",
|
|
"run_observability_daily_summary",
|
|
"run_host_health_probe_cleanup",
|
|
]:
|
|
source = inspect.getsource(getattr(run_scheduler, fn_name))
|
|
assert "_notify_scheduler_failure(" in source
|