591 lines
20 KiB
Python
591 lines
20 KiB
Python
from datetime import datetime
|
||
from decimal import Decimal
|
||
from types import SimpleNamespace
|
||
from uuid import UUID
|
||
|
||
import pytest
|
||
from fastapi import HTTPException
|
||
|
||
from src.api.v1.platform.operator_runs import (
|
||
ListCallbackRepliesResponse,
|
||
ListRunsResponse,
|
||
)
|
||
from src.services.platform_operator_service import (
|
||
_callback_reply_summary_matches_status,
|
||
_collect_run_incident_ids,
|
||
_callback_reply_event_item,
|
||
_legacy_mcp_timeline_status,
|
||
_legacy_mcp_timeline_summary,
|
||
_list_filter_context_limit,
|
||
_outbound_timeline_title,
|
||
_outbound_timeline_status,
|
||
_outbound_timeline_summary,
|
||
_remediation_summary_matches_incident_id,
|
||
_remediation_summary_matches_status,
|
||
_remediation_timeline_summary,
|
||
_run_callback_reply_summary,
|
||
_run_remediation_list_summary,
|
||
_timeline_sort_key,
|
||
_validate_callback_reply_action_filter,
|
||
_validate_callback_reply_status_filter,
|
||
)
|
||
|
||
|
||
def test_outbound_timeline_title_labels_runbook_review() -> None:
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"approval_request",
|
||
"📄 <b>RUNBOOK REVIEW｜待審核</b>\nIncident：INC-1",
|
||
)
|
||
|
||
assert title == "TELEGRAM：Runbook 待人工審核"
|
||
|
||
|
||
def test_outbound_timeline_title_labels_governance_alert() -> None:
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"final",
|
||
"⚠️ *AI 治理警報｜知識庫劣化*",
|
||
)
|
||
|
||
assert title == "TELEGRAM：AI 治理警報"
|
||
|
||
|
||
def test_outbound_timeline_title_labels_cicd_status() -> None:
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"final",
|
||
"✅ <b>[AWOOOI CI/CD]</b> | code-review\n📦 Code Review 完成・LOW",
|
||
)
|
||
|
||
assert title == "TELEGRAM：CI/CD 狀態通知"
|
||
|
||
|
||
def test_outbound_timeline_title_labels_auto_repair_handoff() -> None:
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"error",
|
||
"🤖❌ HANDOFF REQUIRED｜AI 自動修復失敗，已轉人工",
|
||
)
|
||
|
||
assert title == "TELEGRAM：AI 自動修復失敗，已轉人工"
|
||
|
||
|
||
def test_outbound_timeline_title_falls_back_to_human_label() -> None:
|
||
title = _outbound_timeline_title("telegram", "interim", "正在調用 MCP 工具")
|
||
|
||
assert title == "TELEGRAM：漸進式狀態回饋"
|
||
|
||
|
||
def test_outbound_timeline_title_labels_callback_reply_fallback() -> None:
|
||
callback_reply = {
|
||
"status": "callback_reply_fallback_sent",
|
||
"action": "history",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
"parse_mode": "plain_text",
|
||
}
|
||
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"final",
|
||
"事件歷史統計 INC-20260513-79ED5E",
|
||
callback_reply,
|
||
)
|
||
summary = _outbound_timeline_summary(
|
||
content_preview="事件歷史統計 INC-20260513-79ED5E",
|
||
send_error=None,
|
||
callback_reply=callback_reply,
|
||
)
|
||
|
||
assert title == "TELEGRAM：歷史回覆 fallback 已送出"
|
||
assert _outbound_timeline_status("sent", callback_reply) == (
|
||
"callback_reply_fallback_sent"
|
||
)
|
||
assert "callback=history" in summary
|
||
assert "incident=INC-20260513-79ED5E" in summary
|
||
assert "parse_mode=plain_text" in summary
|
||
|
||
|
||
def test_outbound_timeline_title_labels_callback_reply_failure() -> None:
|
||
callback_reply = {
|
||
"status": "callback_reply_failed",
|
||
"action": "detail",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
"error": "HTTP error: 400",
|
||
}
|
||
|
||
assert _outbound_timeline_title("telegram", "error", None, callback_reply) == (
|
||
"TELEGRAM：詳情回覆送出失敗"
|
||
)
|
||
assert _outbound_timeline_status("failed", callback_reply) == "callback_reply_failed"
|
||
assert "error=HTTP error: 400" in _outbound_timeline_summary(
|
||
content_preview=None,
|
||
send_error="HTTP error: 400",
|
||
callback_reply=callback_reply,
|
||
)
|
||
|
||
|
||
def test_collect_run_incident_ids_reads_source_refs_and_legacy_text() -> None:
|
||
run = SimpleNamespace(
|
||
trigger_ref="not-an-incident",
|
||
error_detail=None,
|
||
)
|
||
inbound_events = [
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"source_refs": {
|
||
"incident_ids": ["INC-20260514-F85F21", "INC-20260514-F85F21"],
|
||
}
|
||
},
|
||
content_preview="Alertmanager inbound converged",
|
||
content_redacted=None,
|
||
)
|
||
]
|
||
outbound_messages = [
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"source_refs": {
|
||
"incident_ids": ["INC-20260518-CB0001"],
|
||
},
|
||
},
|
||
content_preview="詳情：INC-20260513-79ED5E",
|
||
send_error=None,
|
||
)
|
||
]
|
||
|
||
incident_ids = _collect_run_incident_ids(
|
||
run=run,
|
||
inbound_events=inbound_events,
|
||
outbound_messages=outbound_messages,
|
||
)
|
||
|
||
assert incident_ids == [
|
||
"INC-20260514-F85F21",
|
||
"INC-20260518-CB0001",
|
||
"INC-20260513-79ED5E",
|
||
]
|
||
|
||
|
||
def test_run_callback_reply_summary_marks_latest_fallback() -> None:
|
||
summary = _run_callback_reply_summary([
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"callback_reply": {
|
||
"status": "callback_reply_sent",
|
||
"action": "detail",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
}
|
||
},
|
||
sent_at=datetime(2026, 5, 18, 6, 1, 0),
|
||
queued_at=datetime(2026, 5, 18, 6, 1, 0),
|
||
provider_message_id="100",
|
||
),
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"callback_reply": {
|
||
"status": "callback_reply_fallback_sent",
|
||
"action": "history",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
}
|
||
},
|
||
sent_at=datetime(2026, 5, 18, 6, 2, 0),
|
||
queued_at=datetime(2026, 5, 18, 6, 2, 0),
|
||
provider_message_id="101",
|
||
),
|
||
])
|
||
|
||
assert summary["status"] == "fallback_sent"
|
||
assert summary["total"] == 2
|
||
assert summary["sent"] == 1
|
||
assert summary["fallback_sent"] == 1
|
||
assert summary["latest_action"] == "history"
|
||
assert summary["latest_incident_id"] == "INC-20260513-79ED5E"
|
||
assert summary["latest_provider_message_id"] == "101"
|
||
assert summary["needs_human"] is False
|
||
|
||
|
||
def test_run_callback_reply_summary_marks_failed_as_human_attention() -> None:
|
||
summary = _run_callback_reply_summary([
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"callback_reply": {
|
||
"status": "callback_reply_failed",
|
||
"action": "detail",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
}
|
||
},
|
||
sent_at=None,
|
||
queued_at=datetime(2026, 5, 18, 6, 3, 0),
|
||
provider_message_id="telegram_callback_reply:failed",
|
||
)
|
||
])
|
||
|
||
assert summary["status"] == "failed"
|
||
assert summary["failed"] == 1
|
||
assert summary["needs_human"] is True
|
||
|
||
|
||
def test_run_callback_reply_summary_marks_no_callback() -> None:
|
||
summary = _run_callback_reply_summary([
|
||
SimpleNamespace(
|
||
source_envelope={},
|
||
sent_at=datetime(2026, 5, 18, 6, 1, 0),
|
||
queued_at=datetime(2026, 5, 18, 6, 1, 0),
|
||
provider_message_id="100",
|
||
)
|
||
])
|
||
|
||
assert summary["status"] == "no_callback"
|
||
assert summary["total"] == 0
|
||
|
||
|
||
def test_list_runs_response_preserves_callback_reply_summary() -> None:
|
||
run_id = UUID("5c0306e0-591a-5445-9a33-80f499426b38")
|
||
response = ListRunsResponse.model_validate({
|
||
"runs": [
|
||
{
|
||
"run_id": run_id,
|
||
"project_id": "awoooi",
|
||
"agent_id": "legacy-telegram-gateway",
|
||
"state": "completed",
|
||
"is_shadow": True,
|
||
"cost_usd": Decimal("0.0000"),
|
||
"step_count": 0,
|
||
"created_at": datetime(2026, 5, 18, 7, 31, 37),
|
||
"timeout_at": None,
|
||
"remediation_summary": None,
|
||
"callback_reply_summary": {
|
||
"schema_version": "awooop_run_callback_reply_summary_v1",
|
||
"status": "failed",
|
||
"total": 1,
|
||
"sent": 0,
|
||
"fallback_sent": 0,
|
||
"rescue_sent": 0,
|
||
"failed": 1,
|
||
"needs_human": True,
|
||
"latest_status": "callback_reply_failed",
|
||
"latest_action": "detail",
|
||
"latest_incident_id": "INC-20260513-79ED5E",
|
||
"latest_at": "2026-05-18T07:31:37",
|
||
"latest_provider_message_id": "telegram_callback_reply:failed",
|
||
},
|
||
}
|
||
],
|
||
"total": 1,
|
||
"page": 1,
|
||
"per_page": 1,
|
||
})
|
||
|
||
dumped = response.model_dump(mode="json")
|
||
assert dumped["runs"][0]["callback_reply_summary"]["status"] == "failed"
|
||
assert dumped["runs"][0]["callback_reply_summary"]["needs_human"] is True
|
||
|
||
|
||
def test_callback_reply_event_item_surfaces_run_link_and_human_flag() -> None:
|
||
run_id = UUID("5c0306e0-591a-5445-9a33-80f499426b38")
|
||
message_id = UUID("56cdb6ad-46a4-48f5-9d3b-b1ac9c0b2e92")
|
||
|
||
item = _callback_reply_event_item({
|
||
"message_id": message_id,
|
||
"run_id": run_id,
|
||
"project_id": "awoooi",
|
||
"channel_type": "telegram",
|
||
"message_type": "error",
|
||
"send_status": "failed",
|
||
"send_error": "HTTP error: 400",
|
||
"provider_message_id": "telegram_callback_reply:failed",
|
||
"queued_at": datetime(2026, 5, 18, 7, 31, 37),
|
||
"sent_at": None,
|
||
"triggered_by_state": "callback_reply",
|
||
"content_preview": "無法取得歷史統計",
|
||
"run_state": "completed",
|
||
"agent_id": "legacy-telegram-gateway",
|
||
"run_created_at": datetime(2026, 5, 18, 7, 30, 0),
|
||
"callback_reply": {
|
||
"status": "callback_reply_failed",
|
||
"action": "history",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
"error": "HTTP error: 400",
|
||
},
|
||
})
|
||
|
||
assert item["status"] == "failed"
|
||
assert item["needs_human"] is True
|
||
assert item["action"] == "history"
|
||
assert item["incident_id"] == "INC-20260513-79ED5E"
|
||
assert item["event_at"] == datetime(2026, 5, 18, 7, 31, 37)
|
||
assert item["run_detail_href"] == (
|
||
"/awooop/runs/5c0306e0-591a-5445-9a33-80f499426b38?project_id=awoooi"
|
||
)
|
||
|
||
|
||
def test_list_callback_replies_response_preserves_callback_evidence() -> None:
|
||
run_id = UUID("5c0306e0-591a-5445-9a33-80f499426b38")
|
||
message_id = UUID("56cdb6ad-46a4-48f5-9d3b-b1ac9c0b2e92")
|
||
response = ListCallbackRepliesResponse.model_validate({
|
||
"items": [
|
||
{
|
||
"message_id": message_id,
|
||
"run_id": run_id,
|
||
"project_id": "awoooi",
|
||
"status": "fallback_sent",
|
||
"needs_human": False,
|
||
"action": "detail",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
"event_at": datetime(2026, 5, 18, 7, 31, 37),
|
||
"channel_type": "telegram",
|
||
"message_type": "final",
|
||
"send_status": "sent",
|
||
"send_error": None,
|
||
"provider_message_id": "123",
|
||
"triggered_by_state": "callback_reply",
|
||
"content_preview": "事件詳情",
|
||
"run_state": "completed",
|
||
"agent_id": "legacy-telegram-gateway",
|
||
"run_created_at": datetime(2026, 5, 18, 7, 30, 0),
|
||
"callback_reply": {
|
||
"status": "callback_reply_fallback_sent",
|
||
"action": "detail",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
},
|
||
"run_detail_href": (
|
||
"/awooop/runs/5c0306e0-591a-5445-9a33-80f499426b38"
|
||
"?project_id=awoooi"
|
||
),
|
||
}
|
||
],
|
||
"total": 1,
|
||
"page": 1,
|
||
"per_page": 20,
|
||
})
|
||
|
||
dumped = response.model_dump(mode="json")
|
||
assert dumped["items"][0]["status"] == "fallback_sent"
|
||
assert dumped["items"][0]["callback_reply"]["action"] == "detail"
|
||
assert dumped["items"][0]["run_detail_href"].endswith("project_id=awoooi")
|
||
|
||
|
||
def test_callback_reply_action_filter_normalizes_safe_actions() -> None:
|
||
assert _validate_callback_reply_action_filter(" History ") == "history"
|
||
assert _validate_callback_reply_action_filter("incident:detail-2") == (
|
||
"incident:detail-2"
|
||
)
|
||
assert _validate_callback_reply_action_filter("") is None
|
||
|
||
|
||
def test_callback_reply_action_filter_rejects_unsafe_values() -> None:
|
||
with pytest.raises(HTTPException):
|
||
_validate_callback_reply_action_filter("detail;drop")
|
||
|
||
|
||
def test_remediation_timeline_summary_surfaces_route_and_write_flags() -> None:
|
||
summary = _remediation_timeline_summary({
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"mode": "replay",
|
||
"verification_result_preview": "degraded",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
})
|
||
|
||
assert "incident=INC-20260514-F85F21" in summary
|
||
assert "route=auto_repair_executor/ssh_diagnose/read" in summary
|
||
assert "writes_incident=False" in summary
|
||
assert "writes_auto_repair=False" in summary
|
||
|
||
|
||
def test_legacy_mcp_timeline_summary_surfaces_tool_context() -> None:
|
||
record = {
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"agent_role": "pre_decision_investigator",
|
||
"flywheel_node": "investigator",
|
||
"duration_ms": 127,
|
||
"success": True,
|
||
"error_message": None,
|
||
}
|
||
|
||
assert _legacy_mcp_timeline_status(record) == "success"
|
||
summary = _legacy_mcp_timeline_summary(record)
|
||
|
||
assert "incident=INC-20260514-F85F21" in summary
|
||
assert "agent=pre_decision_investigator" in summary
|
||
assert "node=investigator" in summary
|
||
assert "duration_ms=127" in summary
|
||
|
||
|
||
def test_legacy_mcp_timeline_status_marks_failed_and_unknown() -> None:
|
||
assert _legacy_mcp_timeline_status({"success": False}) == "failed"
|
||
assert _legacy_mcp_timeline_status({"success": None}) == "warning"
|
||
|
||
|
||
def test_run_remediation_list_summary_marks_read_only_dry_run() -> None:
|
||
run = SimpleNamespace(state="waiting_approval")
|
||
|
||
summary = _run_remediation_list_summary(
|
||
run=run,
|
||
incident_ids=["INC-20260514-F85F21"],
|
||
items=[
|
||
{
|
||
"created_at": "2026-05-14T23:04:00+00:00",
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"mode": "replay",
|
||
"verification_result_preview": "degraded",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
)
|
||
|
||
assert summary["status"] == "read_only_dry_run"
|
||
assert summary["has_dry_run"] is True
|
||
assert summary["is_read_only"] is True
|
||
assert summary["human_gate_open"] is True
|
||
assert summary["latest_route"] == "auto_repair_executor/ssh_diagnose/read"
|
||
|
||
|
||
def test_run_remediation_list_summary_marks_mcp_observed_without_dry_run() -> None:
|
||
run = SimpleNamespace(state="completed")
|
||
|
||
summary = _run_remediation_list_summary(
|
||
run=run,
|
||
incident_ids=["INC-20260518-792684"],
|
||
items=[],
|
||
legacy_mcp_records=[
|
||
{
|
||
"created_at": "2026-05-18T04:31:30+00:00",
|
||
"incident_id": "INC-20260518-792684",
|
||
"agent_role": "pre_decision_investigator",
|
||
"mcp_server": "ssh_host",
|
||
"tool_name": "ssh_diagnose",
|
||
"success": True,
|
||
},
|
||
{
|
||
"created_at": "2026-05-18T04:31:29+00:00",
|
||
"incident_id": "INC-20260518-792684",
|
||
"agent_role": "pre_decision_investigator",
|
||
"mcp_server": "signoz",
|
||
"tool_name": "query_logs",
|
||
"success": False,
|
||
},
|
||
],
|
||
)
|
||
|
||
assert summary["status"] == "mcp_observed"
|
||
assert summary["source"] == "mcp_audit_log"
|
||
assert summary["total"] == 0
|
||
assert summary["evidence_total"] == 2
|
||
assert summary["has_dry_run"] is False
|
||
assert summary["has_mcp_investigation"] is True
|
||
assert summary["mcp_observation_total"] == 2
|
||
assert summary["mcp_observation_success"] == 1
|
||
assert summary["mcp_observation_failed"] == 1
|
||
assert summary["latest_route"] == "pre_decision_investigator/ssh_host.ssh_diagnose/read"
|
||
|
||
|
||
def test_run_remediation_list_summary_flags_write_observed() -> None:
|
||
run = SimpleNamespace(state="completed")
|
||
|
||
summary = _run_remediation_list_summary(
|
||
run=run,
|
||
incident_ids=["INC-20260514-F85F21"],
|
||
items=[
|
||
{
|
||
"created_at": "2026-05-14T23:05:00+00:00",
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "state_update",
|
||
"required_scope": "write",
|
||
"writes_incident_state": True,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
)
|
||
|
||
assert summary["status"] == "write_observed"
|
||
assert summary["is_read_only"] is False
|
||
assert summary["writes_incident_state"] is True
|
||
|
||
|
||
def test_remediation_summary_matches_status_filter() -> None:
|
||
assert _remediation_summary_matches_status(
|
||
{"status": "mcp_observed"},
|
||
"mcp_observed",
|
||
)
|
||
assert _remediation_summary_matches_status(
|
||
{"status": "read_only_dry_run"},
|
||
"read_only_dry_run",
|
||
)
|
||
assert not _remediation_summary_matches_status(
|
||
{"status": "write_observed"},
|
||
"read_only_dry_run",
|
||
)
|
||
assert _remediation_summary_matches_status(None, "no_evidence")
|
||
|
||
|
||
def test_callback_reply_summary_matches_status_filter() -> None:
|
||
assert _callback_reply_summary_matches_status(
|
||
{"status": "failed"},
|
||
"failed",
|
||
)
|
||
assert _callback_reply_summary_matches_status(
|
||
{"status": "fallback_sent"},
|
||
"fallback_sent",
|
||
)
|
||
assert not _callback_reply_summary_matches_status(
|
||
{"status": "sent"},
|
||
"failed",
|
||
)
|
||
assert _callback_reply_summary_matches_status(None, "no_callback")
|
||
|
||
|
||
def test_callback_reply_status_filter_rejects_unknown_value() -> None:
|
||
_validate_callback_reply_status_filter("failed")
|
||
with pytest.raises(HTTPException) as exc_info:
|
||
_validate_callback_reply_status_filter("telegram_error")
|
||
|
||
assert exc_info.value.status_code == 422
|
||
assert "callback_reply_status" in str(exc_info.value.detail)
|
||
|
||
|
||
def test_remediation_summary_matches_incident_id_filter() -> None:
|
||
assert _remediation_summary_matches_incident_id(
|
||
{"incident_ids": ["INC-20260514-F85F21"]},
|
||
"INC-20260514-F85F21",
|
||
)
|
||
assert not _remediation_summary_matches_incident_id(
|
||
{"incident_ids": ["INC-20260514-F85F21"]},
|
||
"INC-20260513-79ED5E",
|
||
)
|
||
assert _remediation_summary_matches_incident_id(None, None)
|
||
|
||
|
||
def test_list_filter_context_limit_scales_with_candidate_rows() -> None:
|
||
assert _list_filter_context_limit(2) == 500
|
||
assert _list_filter_context_limit(4176) == 16704
|
||
assert _list_filter_context_limit(10000) == 20000
|
||
|
||
|
||
def test_timeline_sort_key_normalizes_datetime_and_iso_string() -> None:
|
||
fallback = datetime(2026, 5, 14, 10, 0, 0)
|
||
keys = [
|
||
_timeline_sort_key({"ts": datetime(2026, 5, 14, 10, 0, 1)}, fallback),
|
||
_timeline_sort_key({"ts": "2026-05-14T10:00:02+00:00"}, fallback),
|
||
_timeline_sort_key({"ts": None}, fallback),
|
||
]
|
||
|
||
assert keys == [
|
||
"2026-05-14T10:00:01",
|
||
"2026-05-14T10:00:02+00:00",
|
||
"2026-05-14T10:00:00",
|
||
]
|
||
assert sorted(keys) == [
|
||
"2026-05-14T10:00:00",
|
||
"2026-05-14T10:00:01",
|
||
"2026-05-14T10:00:02+00:00",
|
||
]
|