1435 lines
50 KiB
Python
1435 lines
50 KiB
Python
import asyncio
|
||
from datetime import datetime
|
||
from decimal import Decimal
|
||
from types import SimpleNamespace
|
||
from uuid import UUID
|
||
|
||
import pytest
|
||
from fastapi import HTTPException
|
||
|
||
import src.services.platform_operator_service as platform_operator_service
|
||
from src.api.v1.platform.operator_runs import (
|
||
AiRouteStatusResponse,
|
||
ListApprovalsResponse,
|
||
ListCallbackRepliesResponse,
|
||
ListCicdEventsResponse,
|
||
ListRunsResponse,
|
||
)
|
||
from src.services.ollama_failover_manager import OllamaEndpoint, OllamaRoutingResult
|
||
from src.services.ollama_health_monitor import HealthReport, HealthStatus
|
||
from src.services.platform_operator_service import (
|
||
_ai_route_health_map,
|
||
_ai_route_policy_order,
|
||
_build_awooop_status_chain,
|
||
_callback_reply_event_item,
|
||
_callback_reply_summary_matches_status,
|
||
_cicd_duration_seconds,
|
||
_cicd_event_item_from_row,
|
||
_collect_run_incident_ids,
|
||
_is_source_correlation_applied_link,
|
||
_legacy_mcp_timeline_status,
|
||
_legacy_mcp_timeline_summary,
|
||
_list_filter_context_limit,
|
||
_outbound_timeline_status,
|
||
_outbound_timeline_summary,
|
||
_outbound_timeline_title,
|
||
_remediation_summary_matches_incident_id,
|
||
_remediation_summary_matches_status,
|
||
_remediation_timeline_summary,
|
||
_run_callback_reply_summary,
|
||
_run_remediation_list_summary,
|
||
_score_source_correlation_event,
|
||
_source_event_correlation_context,
|
||
_timeline_sort_key,
|
||
_validate_ai_route_workload,
|
||
_validate_callback_reply_action_filter,
|
||
_validate_callback_reply_status_filter,
|
||
_validate_cicd_stage_filter,
|
||
_validate_cicd_status_filter,
|
||
)
|
||
|
||
|
||
def test_outbound_timeline_title_labels_runbook_review() -> None:
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"approval_request",
|
||
"📄 <b>RUNBOOK REVIEW｜待審核</b>\nIncident：INC-1",
|
||
)
|
||
|
||
assert title == "TELEGRAM：Runbook 待人工審核"
|
||
|
||
|
||
def test_outbound_timeline_title_labels_governance_alert() -> None:
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"final",
|
||
"⚠️ *AI 治理警報｜知識庫劣化*",
|
||
)
|
||
|
||
assert title == "TELEGRAM：AI 治理警報"
|
||
|
||
|
||
def test_outbound_timeline_title_labels_cicd_status() -> None:
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"final",
|
||
"✅ <b>[AWOOOI CI/CD]</b> | code-review\n📦 Code Review 完成・LOW",
|
||
)
|
||
|
||
assert title == "TELEGRAM：CI/CD 狀態通知"
|
||
|
||
|
||
def test_cicd_event_item_preserves_rollout_risk_summary() -> None:
|
||
item = _cicd_event_item_from_row(
|
||
{
|
||
"id": "1da1af11-fd3e-4073-ac85-fd304dbd2dc3",
|
||
"action_detail": "收到告警: CI_rollout_risk_pending",
|
||
"created_at": datetime(2026, 5, 21, 11, 46, 33),
|
||
"context": {
|
||
"source": "alertmanager",
|
||
"alert_id": "alert-20260521194633",
|
||
"labels": {
|
||
"alertname": "CI_rollout_risk_pending",
|
||
"stage": "rollout-risk",
|
||
"status": "pending",
|
||
"severity": "warning",
|
||
"commit": "8e68dc1e3595a2667831143f76794512bcb302be",
|
||
"triggered_by": "wooo",
|
||
"duration_seconds": "0",
|
||
},
|
||
"annotations": {
|
||
"summary": "AWOOOI 部署風險已恢復",
|
||
"description": "public_health_argocd_wait_http=curl_error_28",
|
||
"workflow_url": "http://192.168.0.110:3001/wooo/awoooi/actions/runs/2827",
|
||
},
|
||
},
|
||
},
|
||
project_id="awoooi",
|
||
)
|
||
|
||
assert item["stage"] == "rollout-risk"
|
||
assert item["status"] == "pending"
|
||
assert item["needs_attention"] is True
|
||
assert item["summary"] == "AWOOOI 部署風險已恢復"
|
||
assert "curl_error_28" in item["description"]
|
||
assert item["commit_sha"].startswith("8e68dc1e")
|
||
assert ListCicdEventsResponse(items=[item], total=1, limit=1).items[0].stage == (
|
||
"rollout-risk"
|
||
)
|
||
|
||
|
||
def test_cicd_event_filter_validation_and_duration_safety() -> None:
|
||
assert _validate_cicd_stage_filter("Rollout-Risk") == "rollout-risk"
|
||
assert _validate_cicd_status_filter("PENDING") == "pending"
|
||
assert _cicd_duration_seconds("-3") == 0
|
||
assert _cicd_duration_seconds("bad") == 0
|
||
with pytest.raises(HTTPException):
|
||
_validate_cicd_stage_filter("rollout risk;drop")
|
||
with pytest.raises(HTTPException):
|
||
_validate_cicd_status_filter("ignored")
|
||
|
||
|
||
def test_outbound_timeline_title_labels_auto_repair_handoff() -> None:
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"error",
|
||
"🤖❌ HANDOFF REQUIRED｜AI 自動修復失敗，已轉人工",
|
||
)
|
||
|
||
assert title == "TELEGRAM：AI 自動修復失敗，已轉人工"
|
||
|
||
|
||
def test_outbound_timeline_title_falls_back_to_human_label() -> None:
|
||
title = _outbound_timeline_title("telegram", "interim", "正在調用 MCP 工具")
|
||
|
||
assert title == "TELEGRAM：漸進式狀態回饋"
|
||
|
||
|
||
def test_outbound_timeline_title_labels_callback_reply_fallback() -> None:
|
||
callback_reply = {
|
||
"status": "callback_reply_fallback_sent",
|
||
"action": "history",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
"parse_mode": "plain_text",
|
||
}
|
||
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"final",
|
||
"事件歷史統計 INC-20260513-79ED5E",
|
||
callback_reply,
|
||
)
|
||
summary = _outbound_timeline_summary(
|
||
content_preview="事件歷史統計 INC-20260513-79ED5E",
|
||
send_error=None,
|
||
callback_reply=callback_reply,
|
||
)
|
||
|
||
assert title == "TELEGRAM：歷史回覆 fallback 已送出"
|
||
assert _outbound_timeline_status("sent", callback_reply) == (
|
||
"callback_reply_fallback_sent"
|
||
)
|
||
assert "callback=history" in summary
|
||
assert "incident=INC-20260513-79ED5E" in summary
|
||
assert "parse_mode=plain_text" in summary
|
||
|
||
|
||
def test_outbound_timeline_title_labels_callback_reply_failure() -> None:
|
||
callback_reply = {
|
||
"status": "callback_reply_failed",
|
||
"action": "detail",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
"error": "HTTP error: 400",
|
||
}
|
||
|
||
assert _outbound_timeline_title("telegram", "error", None, callback_reply) == (
|
||
"TELEGRAM：詳情回覆送出失敗"
|
||
)
|
||
assert _outbound_timeline_status("failed", callback_reply) == "callback_reply_failed"
|
||
assert "error=HTTP error: 400" in _outbound_timeline_summary(
|
||
content_preview=None,
|
||
send_error="HTTP error: 400",
|
||
callback_reply=callback_reply,
|
||
)
|
||
|
||
|
||
def test_collect_run_incident_ids_reads_source_refs_and_legacy_text() -> None:
|
||
run = SimpleNamespace(
|
||
trigger_ref="not-an-incident",
|
||
error_detail=None,
|
||
)
|
||
inbound_events = [
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"source_refs": {
|
||
"incident_ids": ["INC-20260514-F85F21", "INC-20260514-F85F21"],
|
||
}
|
||
},
|
||
content_preview="Alertmanager inbound converged",
|
||
content_redacted=None,
|
||
)
|
||
]
|
||
outbound_messages = [
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"source_refs": {
|
||
"incident_ids": ["INC-20260518-CB0001"],
|
||
},
|
||
},
|
||
content_preview="詳情：INC-20260513-79ED5E",
|
||
send_error=None,
|
||
)
|
||
]
|
||
|
||
incident_ids = _collect_run_incident_ids(
|
||
run=run,
|
||
inbound_events=inbound_events,
|
||
outbound_messages=outbound_messages,
|
||
)
|
||
|
||
assert incident_ids == [
|
||
"INC-20260514-F85F21",
|
||
"INC-20260518-CB0001",
|
||
"INC-20260513-79ED5E",
|
||
]
|
||
|
||
|
||
def test_run_callback_reply_summary_marks_latest_fallback() -> None:
|
||
summary = _run_callback_reply_summary([
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"callback_reply": {
|
||
"status": "callback_reply_sent",
|
||
"action": "detail",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
}
|
||
},
|
||
sent_at=datetime(2026, 5, 18, 6, 1, 0),
|
||
queued_at=datetime(2026, 5, 18, 6, 1, 0),
|
||
provider_message_id="100",
|
||
),
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"callback_reply": {
|
||
"status": "callback_reply_fallback_sent",
|
||
"action": "history",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
}
|
||
},
|
||
sent_at=datetime(2026, 5, 18, 6, 2, 0),
|
||
queued_at=datetime(2026, 5, 18, 6, 2, 0),
|
||
provider_message_id="101",
|
||
),
|
||
])
|
||
|
||
assert summary["status"] == "fallback_sent"
|
||
assert summary["total"] == 2
|
||
assert summary["sent"] == 1
|
||
assert summary["fallback_sent"] == 1
|
||
assert summary["latest_action"] == "history"
|
||
assert summary["latest_incident_id"] == "INC-20260513-79ED5E"
|
||
assert summary["latest_provider_message_id"] == "101"
|
||
assert summary["needs_human"] is False
|
||
|
||
|
||
def test_run_callback_reply_summary_marks_failed_as_human_attention() -> None:
|
||
summary = _run_callback_reply_summary([
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"callback_reply": {
|
||
"status": "callback_reply_failed",
|
||
"action": "detail",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
}
|
||
},
|
||
sent_at=None,
|
||
queued_at=datetime(2026, 5, 18, 6, 3, 0),
|
||
provider_message_id="telegram_callback_reply:failed",
|
||
)
|
||
])
|
||
|
||
assert summary["status"] == "failed"
|
||
assert summary["failed"] == 1
|
||
assert summary["needs_human"] is True
|
||
|
||
|
||
def test_run_callback_reply_summary_marks_no_callback() -> None:
|
||
summary = _run_callback_reply_summary([
|
||
SimpleNamespace(
|
||
source_envelope={},
|
||
sent_at=datetime(2026, 5, 18, 6, 1, 0),
|
||
queued_at=datetime(2026, 5, 18, 6, 1, 0),
|
||
provider_message_id="100",
|
||
)
|
||
])
|
||
|
||
assert summary["status"] == "no_callback"
|
||
assert summary["total"] == 0
|
||
|
||
|
||
def test_list_runs_response_preserves_callback_reply_summary() -> None:
|
||
run_id = UUID("5c0306e0-591a-5445-9a33-80f499426b38")
|
||
response = ListRunsResponse.model_validate({
|
||
"runs": [
|
||
{
|
||
"run_id": run_id,
|
||
"project_id": "awoooi",
|
||
"agent_id": "legacy-telegram-gateway",
|
||
"state": "completed",
|
||
"is_shadow": True,
|
||
"cost_usd": Decimal("0.0000"),
|
||
"step_count": 0,
|
||
"created_at": datetime(2026, 5, 18, 7, 31, 37),
|
||
"timeout_at": None,
|
||
"remediation_summary": None,
|
||
"callback_reply_summary": {
|
||
"schema_version": "awooop_run_callback_reply_summary_v1",
|
||
"status": "failed",
|
||
"total": 1,
|
||
"sent": 0,
|
||
"fallback_sent": 0,
|
||
"rescue_sent": 0,
|
||
"failed": 1,
|
||
"needs_human": True,
|
||
"latest_status": "callback_reply_failed",
|
||
"latest_action": "detail",
|
||
"latest_incident_id": "INC-20260513-79ED5E",
|
||
"latest_at": "2026-05-18T07:31:37",
|
||
"latest_provider_message_id": "telegram_callback_reply:failed",
|
||
},
|
||
}
|
||
],
|
||
"total": 1,
|
||
"page": 1,
|
||
"per_page": 1,
|
||
})
|
||
|
||
dumped = response.model_dump(mode="json")
|
||
assert dumped["runs"][0]["callback_reply_summary"]["status"] == "failed"
|
||
assert dumped["runs"][0]["callback_reply_summary"]["needs_human"] is True
|
||
|
||
|
||
def test_callback_reply_event_item_surfaces_run_link_and_human_flag() -> None:
|
||
run_id = UUID("5c0306e0-591a-5445-9a33-80f499426b38")
|
||
message_id = UUID("56cdb6ad-46a4-48f5-9d3b-b1ac9c0b2e92")
|
||
|
||
item = _callback_reply_event_item({
|
||
"message_id": message_id,
|
||
"run_id": run_id,
|
||
"project_id": "awoooi",
|
||
"channel_type": "telegram",
|
||
"message_type": "error",
|
||
"send_status": "failed",
|
||
"send_error": "HTTP error: 400",
|
||
"provider_message_id": "telegram_callback_reply:failed",
|
||
"queued_at": datetime(2026, 5, 18, 7, 31, 37),
|
||
"sent_at": None,
|
||
"triggered_by_state": "callback_reply",
|
||
"content_preview": "無法取得歷史統計",
|
||
"run_state": "completed",
|
||
"agent_id": "legacy-telegram-gateway",
|
||
"run_created_at": datetime(2026, 5, 18, 7, 30, 0),
|
||
"callback_reply": {
|
||
"status": "callback_reply_failed",
|
||
"action": "history",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
"error": "HTTP error: 400",
|
||
},
|
||
})
|
||
|
||
assert item["status"] == "failed"
|
||
assert item["needs_human"] is True
|
||
assert item["action"] == "history"
|
||
assert item["incident_id"] == "INC-20260513-79ED5E"
|
||
assert item["event_at"] == datetime(2026, 5, 18, 7, 31, 37)
|
||
assert item["run_detail_href"] == (
|
||
"/awooop/runs/5c0306e0-591a-5445-9a33-80f499426b38?project_id=awoooi"
|
||
)
|
||
|
||
|
||
def test_list_callback_replies_response_preserves_callback_evidence() -> None:
|
||
run_id = UUID("5c0306e0-591a-5445-9a33-80f499426b38")
|
||
message_id = UUID("56cdb6ad-46a4-48f5-9d3b-b1ac9c0b2e92")
|
||
response = ListCallbackRepliesResponse.model_validate({
|
||
"items": [
|
||
{
|
||
"message_id": message_id,
|
||
"run_id": run_id,
|
||
"project_id": "awoooi",
|
||
"status": "fallback_sent",
|
||
"needs_human": False,
|
||
"action": "detail",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
"event_at": datetime(2026, 5, 18, 7, 31, 37),
|
||
"channel_type": "telegram",
|
||
"message_type": "final",
|
||
"send_status": "sent",
|
||
"send_error": None,
|
||
"provider_message_id": "123",
|
||
"triggered_by_state": "callback_reply",
|
||
"content_preview": "事件詳情",
|
||
"run_state": "completed",
|
||
"agent_id": "legacy-telegram-gateway",
|
||
"run_created_at": datetime(2026, 5, 18, 7, 30, 0),
|
||
"callback_reply": {
|
||
"status": "callback_reply_fallback_sent",
|
||
"action": "detail",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
},
|
||
"awooop_status_chain": {
|
||
"schema_version": "awooop_status_chain_v1",
|
||
"repair_state": "read_only_dry_run",
|
||
"needs_human": True,
|
||
},
|
||
"km_stale_completion_summary": {
|
||
"schema_version": (
|
||
"km_stale_owner_review_completion_callback_summary_v1"
|
||
),
|
||
"project_id": "awoooi",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
"status": "matched_owner_review",
|
||
"ready_count": 3,
|
||
"blocked_count": 1,
|
||
"completed_count": 2,
|
||
"failed_count": 0,
|
||
"batch_writes_allowed": False,
|
||
"manual_review_required": True,
|
||
"related_total": 1,
|
||
"work_item": None,
|
||
"related_items": [
|
||
{
|
||
"entry_id": "km-1",
|
||
"readiness": "ready",
|
||
"next_action": "preview_stale_km_review_completion",
|
||
}
|
||
],
|
||
},
|
||
"run_detail_href": (
|
||
"/awooop/runs/5c0306e0-591a-5445-9a33-80f499426b38"
|
||
"?project_id=awoooi"
|
||
),
|
||
}
|
||
],
|
||
"total": 1,
|
||
"page": 1,
|
||
"per_page": 20,
|
||
})
|
||
|
||
dumped = response.model_dump(mode="json")
|
||
assert dumped["items"][0]["status"] == "fallback_sent"
|
||
assert dumped["items"][0]["callback_reply"]["action"] == "detail"
|
||
assert dumped["items"][0]["awooop_status_chain"]["repair_state"] == (
|
||
"read_only_dry_run"
|
||
)
|
||
assert dumped["items"][0]["km_stale_completion_summary"]["ready_count"] == 3
|
||
assert dumped["items"][0]["km_stale_completion_summary"]["related_total"] == 1
|
||
assert dumped["items"][0]["run_detail_href"].endswith("project_id=awoooi")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_km_stale_completion_summary_matches_callback_incident(
|
||
monkeypatch,
|
||
) -> None:
|
||
async def fake_query_km_stale_completion_queue(**kwargs):
|
||
assert kwargs["project_id"] == "awoooi"
|
||
assert kwargs["status_bucket"] == "all"
|
||
assert kwargs["limit"] == 100
|
||
return SimpleNamespace(
|
||
project_id="awoooi",
|
||
total=2,
|
||
returned=2,
|
||
pending_count=2,
|
||
ready_count=1,
|
||
blocked_count=1,
|
||
completed_count=0,
|
||
failed_count=0,
|
||
writes_on_read=False,
|
||
manual_review_required=True,
|
||
batch_writes_allowed=False,
|
||
items=[
|
||
SimpleNamespace(
|
||
entry_id="km-1",
|
||
title="Bitan pharmacy status drift",
|
||
dispatch_id="dispatch-1",
|
||
governance_event_id="event-1",
|
||
readiness="ready",
|
||
workflow_stage="waiting_owner_review",
|
||
next_action="preview_stale_km_review_completion",
|
||
priority_tier="P0",
|
||
recommended_completion_outcome="refresh_with_evidence",
|
||
can_preview=True,
|
||
related_incident_id="INC-20260513-79ED5E",
|
||
),
|
||
SimpleNamespace(
|
||
entry_id="km-2",
|
||
title="Other stale KM",
|
||
related_incident_id="INC-20260513-OTHER",
|
||
readiness="blocked",
|
||
),
|
||
],
|
||
)
|
||
|
||
monkeypatch.setattr(
|
||
platform_operator_service,
|
||
"query_km_stale_owner_review_completion_queue",
|
||
fake_query_km_stale_completion_queue,
|
||
)
|
||
|
||
summary = await platform_operator_service._fetch_km_stale_completion_summary_for_incident(
|
||
project_id="awoooi",
|
||
incident_id="INC-20260513-79ED5E",
|
||
queue_cache={},
|
||
)
|
||
|
||
assert summary["schema_version"] == (
|
||
"km_stale_owner_review_completion_callback_summary_v1"
|
||
)
|
||
assert summary["status"] == "matched_owner_review"
|
||
assert summary["ready_count"] == 1
|
||
assert summary["blocked_count"] == 1
|
||
assert summary["batch_writes_allowed"] is False
|
||
assert summary["related_items"][0]["entry_id"] == "km-1"
|
||
assert summary["related_items"][0]["can_preview"] is True
|
||
assert summary["work_item"] is None
|
||
|
||
|
||
def test_km_stale_completion_summary_generates_owner_review_work_item() -> None:
|
||
summary = platform_operator_service._build_km_stale_completion_summary(
|
||
queue=SimpleNamespace(
|
||
total=10,
|
||
returned=10,
|
||
pending_count=10,
|
||
ready_count=10,
|
||
blocked_count=0,
|
||
completed_count=1,
|
||
failed_count=0,
|
||
writes_on_read=False,
|
||
manual_review_required=True,
|
||
batch_writes_allowed=False,
|
||
items=[],
|
||
),
|
||
project_id="awoooi",
|
||
incident_id="INC-20260524-16109D",
|
||
)
|
||
|
||
assert summary["status"] == "no_related_owner_review"
|
||
assert summary["work_item"] == {
|
||
"schema_version": "km_stale_callback_owner_review_work_item_v1",
|
||
"work_item_id": (
|
||
"km-callback-owner-review:awoooi:INC-20260524-16109D"
|
||
),
|
||
"kind": "km_stale_callback_owner_review",
|
||
"status": "open",
|
||
"project_id": "awoooi",
|
||
"incident_id": "INC-20260524-16109D",
|
||
"reason": "no_matching_completion_item",
|
||
"title": (
|
||
"Telegram callback incident has no matching KM owner-review item"
|
||
),
|
||
"next_step": "review_or_queue_km_owner_review",
|
||
"target_surface": "awooop_runs_callback_evidence",
|
||
"target_href": (
|
||
"/awooop/runs?project_id=awoooi"
|
||
"&incident_id=INC-20260524-16109D"
|
||
"&callback_reply_status=sent"
|
||
),
|
||
"work_item_href": (
|
||
"/awooop/work-items?project_id=awoooi"
|
||
"&work_item_id=km-callback-owner-review%3Aawoooi%3AINC-20260524-16109D"
|
||
"&incident_id=INC-20260524-16109D"
|
||
),
|
||
"writes_on_read": False,
|
||
"manual_review_required": True,
|
||
"batch_writes_allowed": False,
|
||
}
|
||
|
||
|
||
def test_list_approvals_response_preserves_status_chain() -> None:
|
||
run_id = UUID("5c0306e0-591a-5445-9a33-80f499426b38")
|
||
response = ListApprovalsResponse.model_validate({
|
||
"items": [
|
||
{
|
||
"run_id": run_id,
|
||
"project_id": "awoooi",
|
||
"agent_id": "hermes-approval-router",
|
||
"created_at": datetime(2026, 5, 18, 7, 30, 0),
|
||
"timeout_at": datetime(2026, 5, 18, 7, 45, 0),
|
||
"remediation_summary": {
|
||
"status": "read_only_dry_run",
|
||
"incident_ids": ["INC-20260513-79ED5E"],
|
||
},
|
||
"awooop_status_chain": {
|
||
"schema_version": "awooop_status_chain_v1",
|
||
"source_id": "INC-20260513-79ED5E",
|
||
"repair_state": "read_only_dry_run",
|
||
"needs_human": True,
|
||
"next_step": "approve_or_escalate_from_awooop",
|
||
},
|
||
}
|
||
],
|
||
"total": 1,
|
||
})
|
||
|
||
dumped = response.model_dump(mode="json")
|
||
assert dumped["items"][0]["remediation_summary"]["status"] == (
|
||
"read_only_dry_run"
|
||
)
|
||
assert dumped["items"][0]["awooop_status_chain"]["source_id"] == (
|
||
"INC-20260513-79ED5E"
|
||
)
|
||
assert dumped["items"][0]["awooop_status_chain"]["needs_human"] is True
|
||
|
||
|
||
def test_callback_reply_action_filter_normalizes_safe_actions() -> None:
|
||
assert _validate_callback_reply_action_filter(" History ") == "history"
|
||
assert _validate_callback_reply_action_filter("incident:detail-2") == (
|
||
"incident:detail-2"
|
||
)
|
||
assert _validate_callback_reply_action_filter("") is None
|
||
|
||
|
||
def test_callback_reply_action_filter_rejects_unsafe_values() -> None:
|
||
with pytest.raises(HTTPException):
|
||
_validate_callback_reply_action_filter("detail;drop")
|
||
|
||
|
||
def test_remediation_timeline_summary_surfaces_route_and_write_flags() -> None:
|
||
summary = _remediation_timeline_summary({
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"mode": "replay",
|
||
"verification_result_preview": "degraded",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
})
|
||
|
||
assert "incident=INC-20260514-F85F21" in summary
|
||
assert "route=auto_repair_executor/ssh_diagnose/read" in summary
|
||
assert "writes_incident=False" in summary
|
||
assert "writes_auto_repair=False" in summary
|
||
|
||
|
||
def test_awooop_status_chain_marks_verified_repair() -> None:
|
||
chain = _build_awooop_status_chain(
|
||
incident_ids=["INC-20260513-79ED5E"],
|
||
source_id="INC-20260513-79ED5E",
|
||
truth_chain={
|
||
"truth_status": {
|
||
"current_stage": "execution_succeeded",
|
||
"stage_status": "success",
|
||
"needs_human": False,
|
||
"blockers": [],
|
||
},
|
||
"automation_quality": {
|
||
"verdict": "auto_repaired_verified",
|
||
"facts": {
|
||
"auto_repair_execution_records": 1,
|
||
"automation_operation_records": 1,
|
||
"verification_result": "healthy",
|
||
"mcp_gateway_total": 2,
|
||
"knowledge_entries": 1,
|
||
},
|
||
"blockers": [],
|
||
},
|
||
"mcp": {
|
||
"awooop_gateway": {
|
||
"total": 2,
|
||
"success": 1,
|
||
"failed": 1,
|
||
"blocked": 0,
|
||
"first_class_total": 2,
|
||
"legacy_bridge_total": 0,
|
||
"policy_enforced_total": 2,
|
||
"stage": "provider_failed_after_gateway",
|
||
"stage_status": "failed",
|
||
"by_tool": [
|
||
{
|
||
"tool_name": "prometheus.query",
|
||
"total": 2,
|
||
"success": 1,
|
||
"failed": 1,
|
||
"blocked": 0,
|
||
}
|
||
],
|
||
},
|
||
"legacy": {
|
||
"total": 1,
|
||
"success": 1,
|
||
"failed": 0,
|
||
"by_tool": [
|
||
{
|
||
"tool_name": "ssh_host",
|
||
"success": 1,
|
||
"failed": 0,
|
||
}
|
||
],
|
||
},
|
||
},
|
||
"execution": {
|
||
"automation_operation_log": [
|
||
{
|
||
"operation_type": "playbook_executed",
|
||
"status": "success",
|
||
"actor": "auto_repair_executor",
|
||
"input_action": "restart_service",
|
||
"input_executor": "ansible",
|
||
"input_playbook_id": "pb-host-restart",
|
||
"input_playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
|
||
}
|
||
],
|
||
"ansible": {
|
||
"considered": True,
|
||
"records": [
|
||
{
|
||
"operation_type": "ansible_check_mode_executed",
|
||
"status": "success",
|
||
"playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
|
||
"check_mode": "true",
|
||
}
|
||
],
|
||
"candidate_catalog": {
|
||
"candidates": [
|
||
{
|
||
"catalog_id": "ansible:188-ai-web",
|
||
"playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
|
||
"risk_level": "medium",
|
||
"match_score": 3,
|
||
}
|
||
]
|
||
},
|
||
"not_used_reason": None,
|
||
},
|
||
},
|
||
"channel": {
|
||
"inbound_events": [
|
||
{
|
||
"channel_type": "alertmanager",
|
||
"provider_event_id": "alert-1",
|
||
"content_type": "application/json",
|
||
"is_duplicate": False,
|
||
"received_at": "2026-05-20T00:00:00Z",
|
||
"source_envelope": {
|
||
"source_refs": {
|
||
"alert_ids": ["alert-1"],
|
||
"sentry_issue_ids": ["SENTRY-1"],
|
||
"signoz_alerts": ["signoz:abc"],
|
||
"fingerprints": ["fp-1"],
|
||
}
|
||
},
|
||
}
|
||
],
|
||
"outbound_messages": [
|
||
{
|
||
"channel_type": "telegram",
|
||
"message_type": "incident_detail",
|
||
"send_status": "sent",
|
||
"sent_at": "2026-05-20T00:01:00Z",
|
||
}
|
||
],
|
||
},
|
||
},
|
||
remediation_history={
|
||
"total": 1,
|
||
"items": [
|
||
{
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "rollout_restart",
|
||
"required_scope": "write",
|
||
"verification_result_preview": "healthy",
|
||
"writes_incident_state": True,
|
||
"writes_auto_repair_result": True,
|
||
}
|
||
],
|
||
},
|
||
)
|
||
|
||
assert chain["repair_state"] == "auto_repaired_verified"
|
||
assert chain["verification"] == "healthy"
|
||
assert chain["needs_human"] is False
|
||
assert chain["next_step"] == "monitor_for_regression"
|
||
assert chain["evidence"]["latest_route"] == "auto_repair_executor/rollout_restart/write"
|
||
assert chain["mcp"]["gateway"]["success"] == 1
|
||
assert chain["mcp"]["gateway"]["failed"] == 1
|
||
assert chain["mcp"]["gateway"]["policy_enforced_total"] == 2
|
||
assert chain["mcp"]["legacy"]["total"] == 1
|
||
assert chain["mcp"]["top_tools"][0]["tool_name"] == "prometheus.query"
|
||
assert chain["execution"]["operation_total"] == 1
|
||
assert chain["execution"]["latest_executor"] == "ansible"
|
||
assert chain["execution"]["playbook_ids"] == ["pb-host-restart"]
|
||
assert chain["execution"]["ansible"]["considered"] is True
|
||
assert chain["execution"]["ansible"]["candidate_count"] == 1
|
||
assert chain["source_refs"]["inbound_total"] == 1
|
||
assert chain["source_refs"]["outbound_total"] == 1
|
||
assert chain["source_refs"]["refs"]["sentry_issue_ids"] == ["SENTRY-1"]
|
||
assert chain["source_refs"]["refs"]["signoz_alerts"] == ["signoz:abc"]
|
||
|
||
|
||
def test_awooop_status_chain_includes_source_provider_correlation() -> None:
|
||
chain = _build_awooop_status_chain(
|
||
incident_ids=["INC-20260520-4D1124"],
|
||
source_id="INC-20260520-4D1124",
|
||
source_correlation={
|
||
"schema_version": "source_provider_correlation_v1",
|
||
"status": "candidate_found",
|
||
"verification_status": "candidate_only",
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 2,
|
||
"applied_link_total": 0,
|
||
"provider_event_total": 2,
|
||
"providers": {
|
||
"sentry": {
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 1,
|
||
"applied_link_total": 0,
|
||
},
|
||
"signoz": {
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 1,
|
||
"applied_link_total": 0,
|
||
},
|
||
},
|
||
"top_candidates": [
|
||
{
|
||
"provider": "sentry",
|
||
"provider_event_id": "sentry:issue:1",
|
||
"score": 65,
|
||
"match_type": "candidate",
|
||
"reasons": ["alertname_overlap", "target_overlap"],
|
||
}
|
||
],
|
||
},
|
||
)
|
||
|
||
correlation = chain["source_refs"]["correlation"]
|
||
assert correlation["status"] == "candidate_found"
|
||
assert correlation["candidate_total"] == 2
|
||
assert correlation["verification_status"] == "candidate_only"
|
||
assert correlation["providers"]["sentry"]["candidate_total"] == 1
|
||
assert correlation["top_candidates"][0]["provider_event_id"] == "sentry:issue:1"
|
||
|
||
|
||
def test_awooop_status_chain_preserves_applied_source_link_verification() -> None:
|
||
chain = _build_awooop_status_chain(
|
||
incident_ids=["INC-20260520-4D1124"],
|
||
source_id="INC-20260520-4D1124",
|
||
source_correlation={
|
||
"schema_version": "source_provider_correlation_v1",
|
||
"status": "linked",
|
||
"verification_status": "applied_link_verified",
|
||
"direct_ref_total": 1,
|
||
"candidate_total": 0,
|
||
"applied_link_total": 1,
|
||
"latest_applied_link_at": "2026-05-21T02:03:04",
|
||
"provider_event_total": 1,
|
||
"providers": {
|
||
"sentry": {
|
||
"direct_ref_total": 1,
|
||
"candidate_total": 0,
|
||
"applied_link_total": 1,
|
||
"latest_applied_link_at": "2026-05-21T02:03:04",
|
||
},
|
||
},
|
||
"top_candidates": [
|
||
{
|
||
"provider": "sentry",
|
||
"provider_event_id": "sentry:source_correlation_linked:issue-1",
|
||
"stage": "source_correlation_linked",
|
||
"score": 100,
|
||
"match_type": "direct",
|
||
"link_state": "applied",
|
||
"verification_status": "applied_link_verified",
|
||
"reasons": ["direct_incident_ref"],
|
||
}
|
||
],
|
||
},
|
||
)
|
||
|
||
correlation = chain["source_refs"]["correlation"]
|
||
assert correlation["status"] == "linked"
|
||
assert correlation["verification_status"] == "applied_link_verified"
|
||
assert correlation["applied_link_total"] == 1
|
||
assert correlation["latest_applied_link_at"] == "2026-05-21T02:03:04"
|
||
assert correlation["providers"]["sentry"]["applied_link_total"] == 1
|
||
assert correlation["top_candidates"][0]["link_state"] == "applied"
|
||
|
||
|
||
def test_source_correlation_scoring_distinguishes_direct_and_candidate() -> None:
|
||
incident_context = {
|
||
"incident_ids": ["INC-20260520-4D1124"],
|
||
"alertnames": ["highcpuusage"],
|
||
"severities": ["p3"],
|
||
"fingerprints": ["fp-abc"],
|
||
"namespaces": ["awoooi-prod"],
|
||
"targets": ["api"],
|
||
}
|
||
direct_event = {
|
||
"incident_ids": ["INC-20260520-4D1124"],
|
||
"alertnames": ["other"],
|
||
"severities": [],
|
||
"fingerprints": [],
|
||
"namespaces": [],
|
||
"targets": [],
|
||
}
|
||
candidate_event = {
|
||
"incident_ids": [],
|
||
"alertnames": ["highcpuusage"],
|
||
"severities": ["p3"],
|
||
"fingerprints": [],
|
||
"namespaces": ["awoooi-prod"],
|
||
"targets": ["api"],
|
||
}
|
||
unrelated_event = {
|
||
"incident_ids": [],
|
||
"alertnames": ["configdrift"],
|
||
"severities": ["p3"],
|
||
"fingerprints": [],
|
||
"namespaces": ["awoooi-prod"],
|
||
"targets": [],
|
||
}
|
||
|
||
direct = _score_source_correlation_event(incident_context, direct_event)
|
||
candidate = _score_source_correlation_event(incident_context, candidate_event)
|
||
unrelated = _score_source_correlation_event(incident_context, unrelated_event)
|
||
|
||
assert direct["is_direct"] is True
|
||
assert direct["is_candidate"] is True
|
||
assert candidate["is_direct"] is False
|
||
assert candidate["is_candidate"] is True
|
||
assert "alertname_overlap" in candidate["reasons"]
|
||
assert unrelated["is_candidate"] is False
|
||
|
||
|
||
def test_source_correlation_applied_link_requires_stage_and_direct_match() -> None:
|
||
incident_context = {
|
||
"incident_ids": ["INC-20260520-4D1124"],
|
||
"alertnames": ["sentry issue"],
|
||
"severities": ["error"],
|
||
"fingerprints": ["fp-sentry-1"],
|
||
"namespaces": ["awoooi-prod"],
|
||
"targets": ["web"],
|
||
}
|
||
event_context = _source_event_correlation_context({
|
||
"provider": "sentry",
|
||
"provider_event_id": "sentry:source_correlation_linked:issue-1",
|
||
"received_at": datetime(2026, 5, 21, 2, 3, 4),
|
||
"source_envelope": {
|
||
"provider": "sentry",
|
||
"stage": "source_correlation_linked",
|
||
"source_refs": {
|
||
"incident_ids": ["INC-20260520-4D1124"],
|
||
"fingerprints": ["fp-sentry-1"],
|
||
},
|
||
"log_correlation": {
|
||
"alertname": "Sentry Issue",
|
||
"severity": "error",
|
||
"namespace": "awoooi-prod",
|
||
"target_resource": "web",
|
||
},
|
||
},
|
||
})
|
||
scored = _score_source_correlation_event(incident_context, event_context)
|
||
|
||
assert scored["is_direct"] is True
|
||
assert _is_source_correlation_applied_link(event_context, scored) is True
|
||
|
||
candidate_only_context = {
|
||
**event_context,
|
||
"incident_ids": [],
|
||
"fingerprints": [],
|
||
}
|
||
candidate_only = _score_source_correlation_event(
|
||
incident_context,
|
||
candidate_only_context,
|
||
)
|
||
non_link_stage = {
|
||
**event_context,
|
||
"stage": "upstream_canary",
|
||
}
|
||
|
||
assert candidate_only["is_direct"] is False
|
||
assert candidate_only["is_candidate"] is True
|
||
assert _is_source_correlation_applied_link(
|
||
candidate_only_context,
|
||
candidate_only,
|
||
) is False
|
||
assert _is_source_correlation_applied_link(non_link_stage, scored) is False
|
||
|
||
|
||
def test_awooop_status_chain_marks_read_only_manual_gate() -> None:
|
||
chain = _build_awooop_status_chain(
|
||
incident_ids=["INC-20260513-79ED5E"],
|
||
source_id="INC-20260513-79ED5E",
|
||
truth_chain={
|
||
"truth_status": {
|
||
"current_stage": "approval_required",
|
||
"stage_status": "waiting",
|
||
"needs_human": True,
|
||
"blockers": ["pending_human_approval"],
|
||
},
|
||
"automation_quality": {
|
||
"verdict": "approval_required",
|
||
"facts": {
|
||
"auto_repair_execution_records": 0,
|
||
"automation_operation_records": 0,
|
||
"verification_result": "missing",
|
||
"mcp_gateway_total": 1,
|
||
"knowledge_entries": 0,
|
||
},
|
||
"blockers": [],
|
||
},
|
||
},
|
||
remediation_history={
|
||
"total": 2,
|
||
"items": [
|
||
{
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
"agent_id": "investigator",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"verification_result_preview": "degraded",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
},
|
||
)
|
||
|
||
assert chain["repair_state"] == "read_only_dry_run"
|
||
assert chain["needs_human"] is True
|
||
assert chain["next_step"] == "approve_or_escalate_from_awooop"
|
||
assert chain["blockers"] == ["pending_human_approval"]
|
||
|
||
|
||
def test_legacy_mcp_timeline_summary_surfaces_tool_context() -> None:
|
||
record = {
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"agent_role": "pre_decision_investigator",
|
||
"flywheel_node": "investigator",
|
||
"duration_ms": 127,
|
||
"success": True,
|
||
"error_message": None,
|
||
}
|
||
|
||
assert _legacy_mcp_timeline_status(record) == "success"
|
||
summary = _legacy_mcp_timeline_summary(record)
|
||
|
||
assert "incident=INC-20260514-F85F21" in summary
|
||
assert "agent=pre_decision_investigator" in summary
|
||
assert "node=investigator" in summary
|
||
assert "duration_ms=127" in summary
|
||
|
||
|
||
def test_legacy_mcp_timeline_status_marks_failed_and_unknown() -> None:
|
||
assert _legacy_mcp_timeline_status({"success": False}) == "failed"
|
||
assert _legacy_mcp_timeline_status({"success": None}) == "warning"
|
||
|
||
|
||
def test_run_remediation_list_summary_marks_read_only_dry_run() -> None:
|
||
run = SimpleNamespace(state="waiting_approval")
|
||
|
||
summary = _run_remediation_list_summary(
|
||
run=run,
|
||
incident_ids=["INC-20260514-F85F21"],
|
||
items=[
|
||
{
|
||
"created_at": "2026-05-14T23:04:00+00:00",
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"mode": "replay",
|
||
"verification_result_preview": "degraded",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
)
|
||
|
||
assert summary["status"] == "read_only_dry_run"
|
||
assert summary["has_dry_run"] is True
|
||
assert summary["is_read_only"] is True
|
||
assert summary["human_gate_open"] is True
|
||
assert summary["latest_route"] == "auto_repair_executor/ssh_diagnose/read"
|
||
|
||
|
||
def test_run_remediation_list_summary_marks_mcp_observed_without_dry_run() -> None:
|
||
run = SimpleNamespace(state="completed")
|
||
|
||
summary = _run_remediation_list_summary(
|
||
run=run,
|
||
incident_ids=["INC-20260518-792684"],
|
||
items=[],
|
||
legacy_mcp_records=[
|
||
{
|
||
"created_at": "2026-05-18T04:31:30+00:00",
|
||
"incident_id": "INC-20260518-792684",
|
||
"agent_role": "pre_decision_investigator",
|
||
"mcp_server": "ssh_host",
|
||
"tool_name": "ssh_diagnose",
|
||
"success": True,
|
||
},
|
||
{
|
||
"created_at": "2026-05-18T04:31:29+00:00",
|
||
"incident_id": "INC-20260518-792684",
|
||
"agent_role": "pre_decision_investigator",
|
||
"mcp_server": "signoz",
|
||
"tool_name": "query_logs",
|
||
"success": False,
|
||
},
|
||
],
|
||
)
|
||
|
||
assert summary["status"] == "mcp_observed"
|
||
assert summary["source"] == "mcp_audit_log"
|
||
assert summary["total"] == 0
|
||
assert summary["evidence_total"] == 2
|
||
assert summary["has_dry_run"] is False
|
||
assert summary["has_mcp_investigation"] is True
|
||
assert summary["mcp_observation_total"] == 2
|
||
assert summary["mcp_observation_success"] == 1
|
||
assert summary["mcp_observation_failed"] == 1
|
||
assert summary["latest_route"] == "pre_decision_investigator/ssh_host.ssh_diagnose/read"
|
||
|
||
|
||
def test_run_remediation_list_summary_flags_write_observed() -> None:
|
||
run = SimpleNamespace(state="completed")
|
||
|
||
summary = _run_remediation_list_summary(
|
||
run=run,
|
||
incident_ids=["INC-20260514-F85F21"],
|
||
items=[
|
||
{
|
||
"created_at": "2026-05-14T23:05:00+00:00",
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "state_update",
|
||
"required_scope": "write",
|
||
"writes_incident_state": True,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
)
|
||
|
||
assert summary["status"] == "write_observed"
|
||
assert summary["is_read_only"] is False
|
||
assert summary["writes_incident_state"] is True
|
||
|
||
|
||
def test_remediation_summary_matches_status_filter() -> None:
|
||
assert _remediation_summary_matches_status(
|
||
{"status": "mcp_observed"},
|
||
"mcp_observed",
|
||
)
|
||
assert _remediation_summary_matches_status(
|
||
{"status": "read_only_dry_run"},
|
||
"read_only_dry_run",
|
||
)
|
||
assert not _remediation_summary_matches_status(
|
||
{"status": "write_observed"},
|
||
"read_only_dry_run",
|
||
)
|
||
assert _remediation_summary_matches_status(None, "no_evidence")
|
||
|
||
|
||
def test_callback_reply_summary_matches_status_filter() -> None:
|
||
assert _callback_reply_summary_matches_status(
|
||
{"status": "failed"},
|
||
"failed",
|
||
)
|
||
assert _callback_reply_summary_matches_status(
|
||
{"status": "fallback_sent"},
|
||
"fallback_sent",
|
||
)
|
||
assert not _callback_reply_summary_matches_status(
|
||
{"status": "sent"},
|
||
"failed",
|
||
)
|
||
assert _callback_reply_summary_matches_status(None, "no_callback")
|
||
|
||
|
||
def test_callback_reply_status_filter_rejects_unknown_value() -> None:
|
||
_validate_callback_reply_status_filter("failed")
|
||
with pytest.raises(HTTPException) as exc_info:
|
||
_validate_callback_reply_status_filter("telegram_error")
|
||
|
||
assert exc_info.value.status_code == 422
|
||
assert "callback_reply_status" in str(exc_info.value.detail)
|
||
|
||
|
||
def test_remediation_summary_matches_incident_id_filter() -> None:
|
||
assert _remediation_summary_matches_incident_id(
|
||
{"incident_ids": ["INC-20260514-F85F21"]},
|
||
"INC-20260514-F85F21",
|
||
)
|
||
assert not _remediation_summary_matches_incident_id(
|
||
{"incident_ids": ["INC-20260514-F85F21"]},
|
||
"INC-20260513-79ED5E",
|
||
)
|
||
assert _remediation_summary_matches_incident_id(None, None)
|
||
|
||
|
||
def test_list_filter_context_limit_scales_with_candidate_rows() -> None:
|
||
assert _list_filter_context_limit(2) == 500
|
||
assert _list_filter_context_limit(4176) == 16704
|
||
assert _list_filter_context_limit(10000) == 20000
|
||
|
||
|
||
def test_timeline_sort_key_normalizes_datetime_and_iso_string() -> None:
|
||
fallback = datetime(2026, 5, 14, 10, 0, 0)
|
||
keys = [
|
||
_timeline_sort_key({"ts": datetime(2026, 5, 14, 10, 0, 1)}, fallback),
|
||
_timeline_sort_key({"ts": "2026-05-14T10:00:02+00:00"}, fallback),
|
||
_timeline_sort_key({"ts": None}, fallback),
|
||
]
|
||
|
||
assert keys == [
|
||
"2026-05-14T10:00:01",
|
||
"2026-05-14T10:00:02+00:00",
|
||
"2026-05-14T10:00:00",
|
||
]
|
||
assert sorted(keys) == [
|
||
"2026-05-14T10:00:00",
|
||
"2026-05-14T10:00:01",
|
||
"2026-05-14T10:00:02+00:00",
|
||
]
|
||
|
||
|
||
def test_ai_route_policy_order_exposes_global_ollama_then_gemini() -> None:
|
||
policy = _ai_route_policy_order("deep_rca")
|
||
|
||
assert [item["provider_name"] for item in policy] == [
|
||
"ollama_gcp_a",
|
||
"ollama_gcp_b",
|
||
"ollama_local",
|
||
"gemini",
|
||
]
|
||
assert policy[-1]["role"] == "final_fallback"
|
||
assert policy[-1]["runtime"] == "cloud"
|
||
|
||
|
||
def test_ai_route_health_map_marks_standby_as_not_checked() -> None:
|
||
route = OllamaRoutingResult(
|
||
primary=OllamaEndpoint(
|
||
url="http://gcp-a:11434",
|
||
provider_name="ollama_gcp_a",
|
||
model="qwen3:14b",
|
||
),
|
||
fallback_chain=[
|
||
OllamaEndpoint(
|
||
url="http://gcp-b:11434",
|
||
provider_name="ollama_gcp_b",
|
||
model="qwen3:14b",
|
||
),
|
||
OllamaEndpoint(
|
||
url="http://local-111:11434",
|
||
provider_name="ollama_local",
|
||
model="qwen3:14b",
|
||
),
|
||
],
|
||
routing_reason="primary healthy",
|
||
health_gcp_a=HealthReport(
|
||
status=HealthStatus.HEALTHY,
|
||
host="http://gcp-a:11434",
|
||
latency_ms=123.4,
|
||
reason="ok",
|
||
),
|
||
health_gcp_b=None,
|
||
health_local=None,
|
||
)
|
||
|
||
health = _ai_route_health_map(route)
|
||
|
||
assert health["ollama_gcp_a"]["status"] == "healthy"
|
||
assert health["ollama_gcp_a"]["checked"] is True
|
||
assert health["ollama_gcp_b"]["status"] == "not_checked"
|
||
assert health["ollama_local"]["reason"] == "standby_not_checked_primary_healthy"
|
||
|
||
|
||
def test_ai_route_status_response_preserves_route_fields() -> None:
|
||
response = AiRouteStatusResponse.model_validate({
|
||
"schema_version": "awooop_ai_route_status_v1",
|
||
"workload_type": "deep_rca",
|
||
"policy_order": _ai_route_policy_order("deep_rca"),
|
||
"selected_provider": "ollama_gcp_a",
|
||
"selected_url": "http://gcp-a:11434",
|
||
"selected_model": "qwen3:14b",
|
||
"fallback_chain": [
|
||
{
|
||
"priority": 2,
|
||
"provider_name": "ollama_gcp_b",
|
||
"url": "http://gcp-b:11434",
|
||
"model": "qwen3:14b",
|
||
"runtime": "ollama",
|
||
}
|
||
],
|
||
"route_reason": "primary healthy",
|
||
"route_source": "ollama_failover_manager",
|
||
"route_error": None,
|
||
"health": {
|
||
"ollama_gcp_a": {
|
||
"status": "healthy",
|
||
"host": "http://gcp-a:11434",
|
||
"latency_ms": 123.4,
|
||
"reason": "ok",
|
||
"checked_at": 0,
|
||
"from_cache": False,
|
||
"checked": True,
|
||
},
|
||
},
|
||
"checked_at": datetime(2026, 5, 19, 12, 0, 0),
|
||
})
|
||
|
||
dumped = response.model_dump(mode="json")
|
||
assert dumped["policy_order"][-1]["provider_name"] == "gemini"
|
||
assert dumped["selected_provider"] == "ollama_gcp_a"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_ai_route_status_times_out_before_slow_provider_checks(monkeypatch) -> None:
|
||
class SlowFailoverManager:
|
||
async def select_provider(self, task_type: str = "general") -> None:
|
||
await asyncio.sleep(0.05)
|
||
|
||
async def fake_connectivity(endpoint):
|
||
if endpoint.provider_name == "ollama_gcp_a":
|
||
return HealthReport(
|
||
status=HealthStatus.OFFLINE,
|
||
host=endpoint.url,
|
||
reason="timeout",
|
||
)
|
||
return HealthReport(
|
||
status=HealthStatus.HEALTHY,
|
||
host=endpoint.url,
|
||
latency_ms=12.3,
|
||
reason="status_only_connectivity_ok",
|
||
)
|
||
|
||
monkeypatch.setattr(
|
||
platform_operator_service,
|
||
"_AI_ROUTE_STATUS_SELECT_TIMEOUT_SECONDS",
|
||
0.001,
|
||
)
|
||
monkeypatch.setattr(
|
||
platform_operator_service,
|
||
"get_ollama_failover_manager",
|
||
lambda: SlowFailoverManager(),
|
||
)
|
||
monkeypatch.setattr(
|
||
platform_operator_service,
|
||
"_ai_route_probe_connectivity",
|
||
fake_connectivity,
|
||
)
|
||
|
||
response = await platform_operator_service.get_ai_route_status("deep_rca")
|
||
|
||
assert response["route_reason"] == (
|
||
"route_check_timeout; lightweight connectivity selected ollama_gcp_b"
|
||
)
|
||
assert response["route_error"] is None
|
||
assert response["route_source"] == "lightweight_connectivity_fallback"
|
||
assert response["selected_provider"] == "ollama_gcp_b"
|
||
assert response["health"]["ollama_gcp_a"]["status"] == "offline"
|
||
assert response["health"]["ollama_gcp_b"]["status"] == "healthy"
|
||
assert [item["provider_name"] for item in response["fallback_chain"]] == [
|
||
"ollama_local",
|
||
"gemini",
|
||
]
|
||
assert [item["provider_name"] for item in response["policy_order"]] == [
|
||
"ollama_gcp_a",
|
||
"ollama_gcp_b",
|
||
"ollama_local",
|
||
"gemini",
|
||
]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_ai_route_status_lightweight_fallback_keeps_gemini_policy_only(
|
||
monkeypatch,
|
||
) -> None:
|
||
class SlowFailoverManager:
|
||
async def select_provider(self, task_type: str = "general") -> None:
|
||
await asyncio.sleep(0.05)
|
||
|
||
async def fake_offline_connectivity(endpoint):
|
||
return HealthReport(
|
||
status=HealthStatus.OFFLINE,
|
||
host=endpoint.url,
|
||
reason="offline",
|
||
)
|
||
|
||
monkeypatch.setattr(
|
||
platform_operator_service,
|
||
"_AI_ROUTE_STATUS_SELECT_TIMEOUT_SECONDS",
|
||
0.001,
|
||
)
|
||
monkeypatch.setattr(
|
||
platform_operator_service,
|
||
"get_ollama_failover_manager",
|
||
lambda: SlowFailoverManager(),
|
||
)
|
||
monkeypatch.setattr(
|
||
platform_operator_service,
|
||
"_ai_route_probe_connectivity",
|
||
fake_offline_connectivity,
|
||
)
|
||
|
||
response = await platform_operator_service.get_ai_route_status("deep_rca")
|
||
|
||
assert response["selected_provider"] == "gemini"
|
||
assert response["selected_model"] is None
|
||
assert response["route_source"] == "lightweight_connectivity_fallback"
|
||
assert response["route_error"] is None
|
||
assert "final fallback policy is Gemini" in response["route_reason"]
|
||
assert all(item["status"] == "offline" for item in response["health"].values())
|
||
|
||
|
||
def test_ai_route_workload_validation_rejects_unknown_value() -> None:
|
||
assert _validate_ai_route_workload(" hermes ") == "hermes"
|
||
with pytest.raises(HTTPException) as exc_info:
|
||
_validate_ai_route_workload("charge_money")
|
||
|
||
assert "Unsupported workload_type" in str(exc_info.value.detail)
|