1820 lines
65 KiB
Python
1820 lines
65 KiB
Python
"""
|
||
test_telegram_message_templates.py - Telegram 訊息模板測試
|
||
|
||
2026-03-29 ogt: 新增，驗證 6 種新訊息模板格式正確
|
||
符合 feedback_no_mock_testing.md 規範 - 測試實際格式化輸出
|
||
"""
|
||
|
||
import pytest
|
||
|
||
import src.services.telegram_gateway as telegram_gateway_module
|
||
from src.services.decision_manager import _format_auto_repair_status_line
|
||
from src.services.telegram_gateway import (
|
||
DailySummaryMessage,
|
||
DeploySuccessMessage,
|
||
RateLimitMessage,
|
||
RepairReportMessage,
|
||
ResourceWarnMessage,
|
||
SentryErrorMessage,
|
||
TelegramGateway,
|
||
TelegramMessage,
|
||
)
|
||
|
||
|
||
def test_auto_repair_status_line_distinguishes_handoff_required() -> None:
|
||
"""自動化失敗 reply 必須明確標示轉人工，且不把 raw error 當純文字噴出。"""
|
||
result = _format_auto_repair_status_line(
|
||
incident_id="INC-20260507-AAAAAA",
|
||
target="node-exporter-110",
|
||
action='ssh 192.168.0.110 "ps aux --sort=-%cpu | head -15"',
|
||
success=False,
|
||
error="Unsupported <scheme> & %d format: a real number is required, not str",
|
||
)
|
||
|
||
assert "HANDOFF REQUIRED｜AI 自動修復失敗，已轉人工" in result
|
||
assert "自動化已停止，不再重試" in result
|
||
assert "請 SRE 依 AwoooP Run / 原告警卡處理" in result
|
||
assert "<scheme> & %d format" in result
|
||
assert "<scheme>" not in result
|
||
|
||
|
||
def test_auto_repair_status_line_distinguishes_auto_resolved() -> None:
|
||
"""自動化成功 reply 必須明確標示已自動解決。"""
|
||
result = _format_auto_repair_status_line(
|
||
incident_id="INC-20260507-BBBBBB",
|
||
target="awoooi-api",
|
||
action="kubectl rollout restart deployment/awoooi-api",
|
||
success=True,
|
||
metrics_delta_text="CPU 92%->30%",
|
||
)
|
||
|
||
assert "AUTO RESOLVED｜AI 自動修復完成" in result
|
||
assert "自動化已完成，等待後驗證觀察" in result
|
||
assert "CPU 92%->30%" in result
|
||
|
||
|
||
def test_telegram_html_chunks_preserve_complete_lines() -> None:
|
||
"""歷史/詳情長訊息不得用 text[:500] 切壞 HTML tag。"""
|
||
lines = [
|
||
"📊 <b>事件歷史統計</b>",
|
||
"",
|
||
"<code>INC-20260513-79ED5E</code>",
|
||
"🧭 <b>DB Truth-chain</b>",
|
||
"階段: <code>blocked</code> / <code>manual_required</code>",
|
||
] * 90
|
||
|
||
chunks = telegram_gateway_module._telegram_html_chunks(lines, limit=900)
|
||
|
||
assert len(chunks) > 1
|
||
assert all(len(chunk) <= 900 for chunk in chunks)
|
||
assert all(chunk.count("<code>") == chunk.count("</code>") for chunk in chunks)
|
||
assert all(chunk.count("<b>") == chunk.count("</b>") for chunk in chunks)
|
||
|
||
|
||
def test_telegram_html_chunks_render_single_overlong_html_line_as_safe_text() -> None:
|
||
"""單行過長時不得切出未閉合 <code>，否則 Telegram 會 400。"""
|
||
line = "告警鍵: <code>" + ("node<188>&" * 80) + "</code>"
|
||
|
||
chunks = telegram_gateway_module._telegram_html_chunks([line], limit=120)
|
||
|
||
assert len(chunks) == 1
|
||
assert len(chunks[0]) <= 120
|
||
assert "<code>" not in chunks[0]
|
||
assert "</code>" not in chunks[0]
|
||
assert "告警鍵" in chunks[0]
|
||
assert "<" in chunks[0]
|
||
|
||
|
||
def test_awooop_status_chain_lines_show_verified_auto_repair_stage() -> None:
|
||
"""詳情/歷史要直接說清楚是否已 AI 自動修復、驗證到哪裡。"""
|
||
lines = telegram_gateway_module._format_awooop_status_chain_lines(
|
||
truth_chain={
|
||
"truth_status": {
|
||
"current_stage": "execution_succeeded",
|
||
"stage_status": "success",
|
||
"needs_human": False,
|
||
"blockers": [],
|
||
},
|
||
"automation_quality": {
|
||
"verdict": "auto_repaired_verified",
|
||
"facts": {
|
||
"auto_repair_execution_records": 1,
|
||
"automation_operation_records": 1,
|
||
"verification_result": "healthy",
|
||
"mcp_gateway_total": 3,
|
||
"knowledge_entries": 2,
|
||
},
|
||
"blockers": [],
|
||
},
|
||
},
|
||
remediation_history={
|
||
"total": 1,
|
||
"items": [
|
||
{
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "rollout_restart",
|
||
"required_scope": "write",
|
||
"verification_result_preview": "healthy",
|
||
"writes_incident_state": True,
|
||
"writes_auto_repair_result": True,
|
||
}
|
||
],
|
||
},
|
||
)
|
||
|
||
joined = "\n".join(lines)
|
||
assert "AwoooP 狀態鏈" in joined
|
||
assert "auto_repaired_verified" in joined
|
||
assert "驗證: <code>healthy</code>" in joined
|
||
assert "auto-repair <code>1</code>" in joined
|
||
assert "auto_repair_executor/rollout_restart/write" in joined
|
||
assert "人工: <code>no</code>" in joined
|
||
assert "monitor_for_regression" in joined
|
||
assert "處置結論" in joined
|
||
assert "已驗證自動修復完成" in joined
|
||
|
||
|
||
def test_awooop_status_chain_lines_show_read_only_manual_gate() -> None:
|
||
"""只讀試跑不能被說成已自動修復，必須顯示等待審批/人工。"""
|
||
lines = telegram_gateway_module._format_awooop_status_chain_lines(
|
||
truth_chain={
|
||
"truth_status": {
|
||
"current_stage": "approval_required",
|
||
"stage_status": "waiting",
|
||
"needs_human": True,
|
||
"blockers": ["pending_human_approval"],
|
||
},
|
||
"automation_quality": {
|
||
"verdict": "approval_required",
|
||
"facts": {
|
||
"auto_repair_execution_records": 0,
|
||
"automation_operation_records": 0,
|
||
"verification_result": "missing",
|
||
"mcp_gateway_total": 1,
|
||
"knowledge_entries": 0,
|
||
},
|
||
"blockers": [],
|
||
},
|
||
},
|
||
remediation_history={
|
||
"total": 2,
|
||
"items": [
|
||
{
|
||
"agent_id": "investigator",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"verification_result_preview": "degraded",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
},
|
||
)
|
||
|
||
joined = "\n".join(lines)
|
||
assert "read_only_dry_run" in joined
|
||
assert "investigator/ssh_diagnose/read" in joined
|
||
assert "人工: <code>yes</code>" in joined
|
||
assert "approve_or_escalate_from_awooop" in joined
|
||
assert "pending_human_approval" in joined
|
||
|
||
|
||
def test_awooop_status_chain_lines_do_not_treat_audit_ops_as_repair() -> None:
|
||
lines = telegram_gateway_module._format_awooop_status_chain_lines(
|
||
truth_chain={
|
||
"truth_status": {
|
||
"current_stage": "manual_required",
|
||
"stage_status": "blocked",
|
||
"needs_human": True,
|
||
"blockers": ["approval_resolved_no_action_without_execution"],
|
||
},
|
||
"automation_quality": {
|
||
"verdict": "auto_repaired_verification_degraded",
|
||
"facts": {
|
||
"auto_repair_execution_records": 0,
|
||
"automation_operation_records": 1,
|
||
"effective_execution_records": 0,
|
||
"verification_result": "degraded",
|
||
"mcp_gateway_total": 2,
|
||
"knowledge_entries": 1,
|
||
},
|
||
"blockers": ["verification_recorded"],
|
||
},
|
||
},
|
||
remediation_history={"total": 0},
|
||
)
|
||
|
||
joined = "\n".join(lines)
|
||
assert "diagnostic_or_audit_recorded" in joined
|
||
assert "manual_review_or_collect_repair_evidence" in joined
|
||
assert "executed_pending_verification" not in joined
|
||
assert "處置結論" in joined
|
||
assert "只完成診斷/觀察，尚未證明修復" in joined
|
||
assert "通知：<code>telegram_sre_war_room,awooop_operator_console</code>" in joined
|
||
|
||
|
||
def test_awooop_agent_evidence_lines_show_mcp_source_execution_playbook_km() -> None:
|
||
"""Telegram 詳情/歷史要像前端一樣顯示五段 AI Agent 證據鏈。"""
|
||
lines = telegram_gateway_module._format_awooop_agent_evidence_lines(
|
||
truth_chain={
|
||
"automation_quality": {
|
||
"verdict": "approval_required",
|
||
"facts": {
|
||
"auto_repair_execution_records": 0,
|
||
"automation_operation_records": 1,
|
||
"verification_result": "degraded",
|
||
"mcp_gateway_total": 2,
|
||
"knowledge_entries": 1,
|
||
},
|
||
},
|
||
"mcp": {
|
||
"awooop_gateway": {
|
||
"total": 2,
|
||
"success": 1,
|
||
"failed": 1,
|
||
"blocked": 0,
|
||
"first_class_total": 2,
|
||
"legacy_bridge_total": 0,
|
||
"policy_enforced_total": 2,
|
||
"by_tool": [
|
||
{
|
||
"tool_name": "prometheus.query",
|
||
"total": 2,
|
||
"success": 1,
|
||
"failed": 1,
|
||
"blocked": 0,
|
||
}
|
||
],
|
||
},
|
||
"legacy": {
|
||
"total": 1,
|
||
"success": 1,
|
||
"failed": 0,
|
||
},
|
||
},
|
||
"execution": {
|
||
"automation_operation_log": [
|
||
{
|
||
"operation_type": "ansible_candidate_matched",
|
||
"status": "dry_run",
|
||
"actor": "Hermes",
|
||
"input_executor": "ansible",
|
||
"input_action": "check_mode",
|
||
}
|
||
],
|
||
"ansible": {
|
||
"considered": True,
|
||
"records": [
|
||
{
|
||
"operation_type": "ansible_candidate_matched",
|
||
"status": "dry_run",
|
||
"playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
|
||
"check_mode": True,
|
||
}
|
||
],
|
||
"candidate_catalog": {
|
||
"candidates": [
|
||
{
|
||
"catalog_id": "ansible:188-ai-web",
|
||
"playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
|
||
"risk_level": "medium",
|
||
"match_score": 3,
|
||
}
|
||
]
|
||
},
|
||
},
|
||
},
|
||
},
|
||
remediation_history={
|
||
"total": 1,
|
||
"items": [
|
||
{
|
||
"verification_result_preview": "degraded",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
},
|
||
source_correlation={
|
||
"status": "candidate_found",
|
||
"verification_status": "candidate_only",
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 1,
|
||
"applied_link_total": 0,
|
||
"providers": {
|
||
"sentry": {
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 1,
|
||
"applied_link_total": 0,
|
||
},
|
||
"signoz": {
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 0,
|
||
"applied_link_total": 0,
|
||
},
|
||
},
|
||
},
|
||
)
|
||
|
||
joined = "\n".join(lines)
|
||
assert "AI Agent 證據鏈" in joined
|
||
assert "MCP / 自建 MCP" in joined
|
||
assert "Gateway <code>1/2</code>" in joined
|
||
assert "prometheus.query" in joined
|
||
assert "Sentry/SigNoz" in joined
|
||
assert "candidate <code>1</code>" in joined
|
||
assert "sentry 0/1/0" in joined
|
||
assert "Executor: <code>ansible/dry_run</code>" in joined
|
||
assert "op <code>ansible_candidate_matched</code>" in joined
|
||
assert "PlayBook / Ansible" in joined
|
||
assert "infra/ansible/playbooks/188-ai-web.yml" in joined
|
||
assert "ansible <code>yes</code>" in joined
|
||
assert "check <code>0</code>" in joined
|
||
assert "apply <code>0</code>" in joined
|
||
assert "rc <code>--</code>" in joined
|
||
assert "KM / Learning" in joined
|
||
assert "KM <code>1</code>" in joined
|
||
assert "verify <code>degraded</code>" in joined
|
||
|
||
|
||
def test_awooop_status_chain_lines_show_ansible_apply_proof() -> None:
|
||
"""詳情/歷史必須能看出 Ansible 是否真的 apply、由誰批准、returncode 幾。"""
|
||
truth_chain = {
|
||
"truth_status": {
|
||
"current_stage": "execution_succeeded",
|
||
"stage_status": "success",
|
||
"needs_human": False,
|
||
"blockers": [],
|
||
},
|
||
"automation_quality": {
|
||
"verdict": "execution_succeeded",
|
||
"facts": {
|
||
"auto_repair_execution_records": 0,
|
||
"automation_operation_records": 2,
|
||
"effective_execution_records": 1,
|
||
"verification_result": "healthy",
|
||
"mcp_gateway_total": 2,
|
||
"knowledge_entries": 1,
|
||
},
|
||
"blockers": [],
|
||
},
|
||
"execution": {
|
||
"automation_operation_log": [
|
||
{
|
||
"operation_type": "ansible_apply_executed",
|
||
"status": "success",
|
||
"actor": "platform_operator",
|
||
"input_executor": "ansible",
|
||
"input_action": "apply_playbook",
|
||
"input_catalog_id": "ansible:188-momo-backup-user",
|
||
"input_playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml",
|
||
}
|
||
],
|
||
"ansible": {
|
||
"considered": True,
|
||
"records": [
|
||
{
|
||
"operation_type": "ansible_apply_executed",
|
||
"status": "success",
|
||
"actor": "platform_operator",
|
||
"catalog_id": "ansible:188-momo-backup-user",
|
||
"playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml",
|
||
"execution_mode": "apply",
|
||
"check_mode": False,
|
||
"apply_executed": True,
|
||
"approval_source": "user_chat_approved_continue",
|
||
"returncode": 0,
|
||
},
|
||
{
|
||
"operation_type": "ansible_check_mode_executed",
|
||
"status": "success",
|
||
"actor": "ansible_check_mode_worker",
|
||
"catalog_id": "ansible:188-momo-backup-user",
|
||
"playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml",
|
||
"execution_mode": "check_mode",
|
||
"check_mode": True,
|
||
"apply_executed": False,
|
||
"returncode": 0,
|
||
},
|
||
],
|
||
"candidate_catalog": {"candidates": []},
|
||
},
|
||
},
|
||
}
|
||
|
||
lines = telegram_gateway_module._format_awooop_status_chain_lines(
|
||
truth_chain=truth_chain,
|
||
remediation_history={"total": 0},
|
||
)
|
||
joined = "\n".join(lines)
|
||
assert "Ansible: check <code>1</code> / apply <code>1</code>" in joined
|
||
assert "ansible_apply_executed</code>/<code>success</code>" in joined
|
||
assert "rc <code>0</code>" in joined
|
||
assert "PlayBook: <code>ansible:188-momo-backup-user</code>" in joined
|
||
assert "approval <code>user_chat_approved_continue</code>" in joined
|
||
|
||
snapshot = telegram_gateway_module._callback_reply_awooop_execution_snapshot(truth_chain)
|
||
ansible = snapshot["ansible"]
|
||
assert ansible["check_mode_total"] == 1
|
||
assert ansible["apply_total"] == 1
|
||
assert ansible["applied"] is True
|
||
assert ansible["controlled_apply"] is True
|
||
assert ansible["latest_catalog_id"] == "ansible:188-momo-backup-user"
|
||
assert ansible["latest_execution_mode"] == "apply"
|
||
assert ansible["latest_returncode"] == 0
|
||
assert ansible["approval_source"] == "user_chat_approved_continue"
|
||
|
||
|
||
def test_callback_reply_awooop_status_chain_snapshot_marks_manual_gate() -> None:
|
||
"""Callback evidence 要保存當下 AwoooP 狀態鏈，不只保存 live query 結果。"""
|
||
snapshot = telegram_gateway_module._callback_reply_awooop_status_chain_snapshot(
|
||
incident_id="INC-20260514-F85F21",
|
||
truth_chain={
|
||
"truth_status": {
|
||
"current_stage": "approval_required",
|
||
"stage_status": "waiting",
|
||
"needs_human": True,
|
||
"blockers": ["pending_human_approval"],
|
||
},
|
||
"automation_quality": {
|
||
"verdict": "approval_required",
|
||
"facts": {
|
||
"auto_repair_execution_records": 0,
|
||
"automation_operation_records": 0,
|
||
"verification_result": "missing",
|
||
"mcp_gateway_total": 1,
|
||
"knowledge_entries": 0,
|
||
},
|
||
"blockers": [],
|
||
},
|
||
"mcp": {
|
||
"awooop_gateway": {
|
||
"total": 2,
|
||
"success": 1,
|
||
"failed": 1,
|
||
"blocked": 0,
|
||
"first_class_total": 2,
|
||
"legacy_bridge_total": 0,
|
||
"policy_enforced_total": 2,
|
||
"by_tool": [
|
||
{
|
||
"tool_name": "prometheus.query",
|
||
"total": 2,
|
||
"success": 1,
|
||
"failed": 1,
|
||
"blocked": 0,
|
||
}
|
||
],
|
||
},
|
||
"legacy": {
|
||
"total": 1,
|
||
"success": 1,
|
||
"failed": 0,
|
||
},
|
||
},
|
||
"execution": {
|
||
"automation_operation_log": [
|
||
{
|
||
"operation_type": "playbook_previewed",
|
||
"status": "waiting_approval",
|
||
"actor": "Hermes",
|
||
"input_executor": "ansible",
|
||
"input_playbook_id": "pb-host-restart",
|
||
"input_playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
|
||
}
|
||
],
|
||
"ansible": {
|
||
"considered": True,
|
||
"records": [
|
||
{
|
||
"operation_type": "ansible_check_mode_previewed",
|
||
"status": "waiting_approval",
|
||
"playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
|
||
"check_mode": True,
|
||
}
|
||
],
|
||
"candidate_catalog": {
|
||
"candidates": [
|
||
{
|
||
"catalog_id": "ansible:188-ai-web",
|
||
"playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
|
||
"risk_level": "medium",
|
||
"match_score": 3,
|
||
}
|
||
]
|
||
},
|
||
},
|
||
},
|
||
"channel": {
|
||
"inbound_events": [
|
||
{
|
||
"channel_type": "sentry",
|
||
"provider_event_id": "sentry:ISSUE-1",
|
||
"source_envelope": {
|
||
"source_refs": {
|
||
"sentry_issue_ids": ["ISSUE-1"],
|
||
"signoz_alerts": ["signoz:alert-1"],
|
||
"fingerprints": ["fp-1"],
|
||
}
|
||
},
|
||
}
|
||
],
|
||
"outbound_messages": [],
|
||
},
|
||
},
|
||
remediation_history={
|
||
"total": 1,
|
||
"items": [
|
||
{
|
||
"agent_id": "investigator",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"safety_level": "read_only",
|
||
"verification_result_preview": "degraded",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
},
|
||
source_correlation={
|
||
"schema_version": "source_provider_correlation_v1",
|
||
"status": "candidate_found",
|
||
"verification_status": "candidate_only",
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 1,
|
||
"applied_link_total": 0,
|
||
"provider_event_total": 1,
|
||
"latest_applied_link_at": None,
|
||
"providers": {
|
||
"sentry": {
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 1,
|
||
"applied_link_total": 0,
|
||
"latest_event_at": "2026-05-20T00:00:00Z",
|
||
},
|
||
"signoz": {
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 0,
|
||
"applied_link_total": 0,
|
||
"latest_event_at": None,
|
||
},
|
||
},
|
||
"top_candidates": [
|
||
{
|
||
"provider": "sentry",
|
||
"provider_event_id": "sentry:ISSUE-1",
|
||
"score": 2,
|
||
"link_state": "candidate",
|
||
}
|
||
],
|
||
},
|
||
)
|
||
|
||
assert snapshot is not None
|
||
assert snapshot["schema_version"] == (
|
||
"awooop_status_chain_callback_reply_snapshot_v1"
|
||
)
|
||
assert snapshot["repair_state"] == "read_only_dry_run"
|
||
assert snapshot["needs_human"] is True
|
||
assert snapshot["next_step"] == "approve_or_escalate_from_awooop"
|
||
assert snapshot["evidence"]["mcp_gateway_total"] == 1
|
||
assert snapshot["evidence"]["latest_route"] == "investigator/ssh_diagnose/read"
|
||
assert snapshot["writes"]["incident"] is False
|
||
assert snapshot["blockers"] == ["pending_human_approval"]
|
||
assert snapshot["mcp"]["gateway"]["policy_enforced_total"] == 2
|
||
assert snapshot["mcp"]["top_tools"][0]["tool_name"] == "prometheus.query"
|
||
assert snapshot["execution"]["latest_executor"] == "ansible"
|
||
assert snapshot["execution"]["playbook_paths"] == [
|
||
"infra/ansible/playbooks/188-ai-web.yml"
|
||
]
|
||
assert snapshot["execution"]["ansible"]["candidate_count"] == 1
|
||
assert snapshot["source_refs"]["refs"]["sentry_issue_ids"] == ["ISSUE-1"]
|
||
assert snapshot["source_refs"]["refs"]["signoz_alerts"] == ["signoz:alert-1"]
|
||
assert snapshot["source_refs"]["correlation"]["status"] == "candidate_found"
|
||
|
||
|
||
def test_km_stale_completion_lines_show_owner_review_queue_state() -> None:
|
||
"""詳情/歷史要顯示 KM owner-review completion queue 是否卡住或可處理。"""
|
||
lines = telegram_gateway_module._format_km_stale_completion_lines({
|
||
"schema_version": "km_stale_owner_review_completion_telegram_summary_v1",
|
||
"status": "ok",
|
||
"incident_id": "INC-20260513-205814",
|
||
"ready_count": 10,
|
||
"blocked_count": 0,
|
||
"completed_count": 1,
|
||
"failed_count": 0,
|
||
"writes_on_read": False,
|
||
"batch_writes_allowed": False,
|
||
"items": [
|
||
{
|
||
"entry_id": "bf81a30d-6abe-4c0c-b4ba-9c0ba0d761bf",
|
||
"readiness": "ready",
|
||
"next_action": "preview_stale_km_review_completion",
|
||
}
|
||
],
|
||
})
|
||
|
||
joined = "\n".join(lines)
|
||
assert "KM Owner Review" in joined
|
||
assert "ready <code>10</code>" in joined
|
||
assert "batch-write <code>no</code>" in joined
|
||
assert "bf81a30d-6abe-4c0c-b4ba-9c0ba0d761bf" in joined
|
||
assert "preview_stale_km_review_completion" in joined
|
||
|
||
|
||
def test_km_stale_completion_lines_show_callback_triage_state() -> None:
|
||
"""詳情/歷史要顯示 callback owner-review 缺口的流程、主責與卡點。"""
|
||
lines = telegram_gateway_module._format_km_stale_completion_lines({
|
||
"schema_version": "km_stale_owner_review_completion_telegram_summary_v1",
|
||
"status": "no_related_owner_review",
|
||
"incident_id": "INC-20260524-16109D",
|
||
"ready_count": 10,
|
||
"blocked_count": 0,
|
||
"completed_count": 1,
|
||
"failed_count": 0,
|
||
"writes_on_read": False,
|
||
"batch_writes_allowed": False,
|
||
"items": [],
|
||
"work_item": {
|
||
"schema_version": "km_stale_callback_owner_review_work_item_v1",
|
||
"status": "open",
|
||
"triage": {
|
||
"schema_version": "km_stale_callback_owner_review_triage_v1",
|
||
"flow_stage": "callback_observed_owner_review_link_missing",
|
||
"ai_lead_agent": "Hermes",
|
||
"supporting_agents": ["OpenClaw", "ElephantAlpha"],
|
||
"automation_state": "manual_owner_review_required",
|
||
"safe_to_auto_repair": False,
|
||
"blocking_reason": "no_matching_completion_item",
|
||
"matching_strategy": "related_incident_id_exact_match",
|
||
},
|
||
},
|
||
})
|
||
|
||
joined = "\n".join(lines)
|
||
assert "no_related_owner_review" in joined
|
||
assert "callback_observed_owner_review_link_missing" in joined
|
||
assert "related_incident_id_exact_match" in joined
|
||
assert "Hermes" in joined
|
||
assert "OpenClaw / ElephantAlpha" in joined
|
||
assert "manual_owner_review_required" in joined
|
||
assert "safe-auto <code>no</code>" in joined
|
||
assert "no_matching_completion_item" in joined
|
||
|
||
|
||
def test_awooop_runs_url_for_incident_uses_public_incident_filter() -> None:
|
||
"""Telegram URL button 必須導到公開 AwoooP Run list，並帶 incident filter。"""
|
||
url = telegram_gateway_module._awooop_runs_url_for_incident("INC-20260514-F85F21")
|
||
|
||
assert url == (
|
||
"https://awoooi.wooo.work/zh-TW/awooop/runs"
|
||
"?project_id=awoooi&incident_id=INC-20260514-F85F21"
|
||
)
|
||
|
||
|
||
def test_awooop_alerts_url_for_incident_is_canonical_truth_chain() -> None:
|
||
"""Telegram 詳情/歷史要能直接導到 Alerts Incident truth-chain。"""
|
||
url = telegram_gateway_module._awooop_alerts_url_for_incident(
|
||
"INC-20260514-F85F21"
|
||
)
|
||
|
||
assert url == (
|
||
"https://awoooi.wooo.work/zh-TW/alerts"
|
||
"?project_id=awoooi&incident_id=INC-20260514-F85F21"
|
||
)
|
||
|
||
|
||
def test_awooop_reply_markup_prefers_truth_chain_then_runs() -> None:
|
||
"""詳情/歷史 callback reply 的第一顆 URL 要是告警真相鏈。"""
|
||
reply_markup = telegram_gateway_module._awooop_runs_reply_markup(
|
||
"INC-20260514-F85F21"
|
||
)
|
||
|
||
assert reply_markup == {
|
||
"inline_keyboard": [[
|
||
{
|
||
"text": "🔎 真相鏈",
|
||
"url": (
|
||
"https://awoooi.wooo.work/zh-TW/alerts"
|
||
"?project_id=awoooi&incident_id=INC-20260514-F85F21"
|
||
),
|
||
},
|
||
{
|
||
"text": "🧭 Runs",
|
||
"url": (
|
||
"https://awoooi.wooo.work/zh-TW/awooop/runs"
|
||
"?project_id=awoooi&incident_id=INC-20260514-F85F21"
|
||
),
|
||
},
|
||
]]
|
||
}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_build_inline_keyboard_includes_awooop_deep_link() -> None:
|
||
"""主告警卡的 read-only 按鈕列要能回到 Incident truth-chain。"""
|
||
gateway = TelegramGateway()
|
||
|
||
keyboard = await gateway._build_inline_keyboard(
|
||
approval_id="INC-20260514-F85F21",
|
||
include_auto_tuning=False,
|
||
incident_id="INC-20260514-F85F21",
|
||
)
|
||
buttons = [
|
||
button
|
||
for row in keyboard["inline_keyboard"]
|
||
for button in row
|
||
]
|
||
|
||
assert {
|
||
"text": "🔎 真相鏈",
|
||
"url": (
|
||
"https://awoooi.wooo.work/zh-TW/alerts"
|
||
"?project_id=awoooi&incident_id=INC-20260514-F85F21"
|
||
),
|
||
} in buttons
|
||
assert {
|
||
"text": "🧭 Runs",
|
||
"url": (
|
||
"https://awoooi.wooo.work/zh-TW/awooop/runs"
|
||
"?project_id=awoooi&incident_id=INC-20260514-F85F21"
|
||
),
|
||
} in buttons
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_build_inline_keyboard_hides_approval_for_no_action() -> None:
|
||
"""OBSERVE / NO_ACTION 卡片不能提供會誤導成修復執行的批准入口。"""
|
||
gateway = TelegramGateway()
|
||
|
||
keyboard = await gateway._build_inline_keyboard(
|
||
approval_id="approval-no-repair-1",
|
||
include_auto_tuning=False,
|
||
incident_id="INC-20260611-NOOP",
|
||
suggested_action="NO_ACTION - REPAIR_CANDIDATE_MISSING: LLM 分析失敗",
|
||
)
|
||
buttons = [
|
||
button
|
||
for row in keyboard["inline_keyboard"]
|
||
for button in row
|
||
]
|
||
button_texts = {button["text"] for button in buttons}
|
||
|
||
assert "✅ 批准" not in button_texts
|
||
assert "❌ 拒絕" not in button_texts
|
||
assert "🧰 處置包" in button_texts
|
||
assert "🔄 重診" in button_texts
|
||
assert "🔕 靜默" in button_texts
|
||
assert {
|
||
"text": "🧭 Runs",
|
||
"url": (
|
||
"https://awoooi.wooo.work/zh-TW/awooop/runs"
|
||
"?project_id=awoooi&incident_id=INC-20260611-NOOP"
|
||
),
|
||
} in buttons
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_request_strips_awooop_callback_metadata_before_telegram_api(monkeypatch):
|
||
"""AwoooP truth-chain metadata must be mirrored, not sent to Telegram Bot API."""
|
||
captured = {}
|
||
gateway = TelegramGateway()
|
||
gateway._initialized = True
|
||
|
||
class FakeResponse:
|
||
def raise_for_status(self):
|
||
return None
|
||
|
||
def json(self):
|
||
return {"ok": True, "result": {"message_id": 456}}
|
||
|
||
class FakeClient:
|
||
async def post(self, url, json):
|
||
captured["payload"] = dict(json)
|
||
return FakeResponse()
|
||
|
||
async def fake_mirror_outbound_message(
|
||
*,
|
||
method,
|
||
payload,
|
||
provider_message_id,
|
||
source_envelope_extra=None,
|
||
):
|
||
captured["mirror"] = {
|
||
"method": method,
|
||
"payload": dict(payload),
|
||
"provider_message_id": provider_message_id,
|
||
"source_envelope_extra": source_envelope_extra,
|
||
}
|
||
|
||
gateway._http_client = FakeClient()
|
||
monkeypatch.setattr(gateway, "_mirror_outbound_message", fake_mirror_outbound_message)
|
||
|
||
result = await gateway._send_request(
|
||
"sendMessage",
|
||
{
|
||
"chat_id": "chat",
|
||
"text": "事件歷史統計 INC-20260513-79ED5E",
|
||
"_skip_incident_thread_reply": True,
|
||
"_awooop_source_envelope_extra": {
|
||
"callback_reply": {
|
||
"status": "callback_reply_sent",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
},
|
||
"awooop_status_chain": {
|
||
"schema_version": "awooop_status_chain_callback_reply_snapshot_v1",
|
||
"repair_state": "read_only_dry_run",
|
||
"needs_human": True,
|
||
"evidence": {
|
||
"auto_repair_records": 0,
|
||
"operation_records": 0,
|
||
},
|
||
},
|
||
"km_stale_completion_summary": {
|
||
"schema_version": (
|
||
"km_stale_owner_review_callback_reply_snapshot_v1"
|
||
),
|
||
"status": "no_related_owner_review",
|
||
"ready_count": 2,
|
||
"triage": {
|
||
"flow_stage": "callback_observed_owner_review_link_missing",
|
||
},
|
||
},
|
||
},
|
||
},
|
||
)
|
||
|
||
assert result["ok"] is True
|
||
assert "_awooop_source_envelope_extra" not in captured["payload"]
|
||
assert "_skip_incident_thread_reply" not in captured["payload"]
|
||
assert captured["mirror"]["source_envelope_extra"]["callback_reply"]["status"] == (
|
||
"callback_reply_sent"
|
||
)
|
||
assert captured["mirror"]["source_envelope_extra"]["awooop_status_chain"][
|
||
"repair_state"
|
||
] == "read_only_dry_run"
|
||
assert captured["mirror"]["source_envelope_extra"][
|
||
"km_stale_completion_summary"
|
||
]["ready_count"] == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_html_line_message_falls_back_to_plain_text_on_parse_error(monkeypatch):
|
||
"""Telegram HTML parse 400 時要送純文字 fallback，不可回報成歷史查詢失敗。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
reply_markup = telegram_gateway_module._awooop_runs_reply_markup("INC-20260514-F85F21")
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
if payload.get("parse_mode") == "HTML":
|
||
raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400")
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
await gateway._send_html_line_message(
|
||
["📊 <b>事件歷史統計</b>", "階段: <code>blocked</code>"],
|
||
chat_id="chat",
|
||
failure_context="test_history",
|
||
reply_markup=reply_markup,
|
||
)
|
||
|
||
assert len(sent_requests) == 2
|
||
assert sent_requests[0][1]["parse_mode"] == "HTML"
|
||
assert sent_requests[0][1]["reply_markup"] == reply_markup
|
||
assert "parse_mode" not in sent_requests[1][1]
|
||
assert sent_requests[1][1]["reply_markup"] == reply_markup
|
||
assert "<code>" not in sent_requests[1][1]["text"]
|
||
assert "blocked" in sent_requests[1][1]["text"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch):
|
||
"""詳情/歷史 callback reply 要在 AwoooP envelope 標明 sent/fallback 狀態。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
reply_markup = telegram_gateway_module._awooop_runs_reply_markup("INC-20260514-F85F21")
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
if payload.get("parse_mode") == "HTML":
|
||
raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400")
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
await gateway._send_html_line_message(
|
||
["📊 <b>事件歷史統計</b>", "🔖 <code>INC-20260514-F85F21</code>"],
|
||
chat_id="chat",
|
||
failure_context="incident_history",
|
||
reply_markup=reply_markup,
|
||
incident_id="INC-20260514-F85F21",
|
||
callback_action="history",
|
||
awooop_status_chain={
|
||
"schema_version": "awooop_status_chain_callback_reply_snapshot_v1",
|
||
"source": "telegram_callback_reply_snapshot",
|
||
"source_id": "INC-20260514-F85F21",
|
||
"incident_ids": ["INC-20260514-F85F21"],
|
||
"current_stage": "approval_required",
|
||
"stage_status": "waiting",
|
||
"verdict": "approval_required",
|
||
"repair_state": "read_only_dry_run",
|
||
"verification": "missing",
|
||
"needs_human": True,
|
||
"next_step": "approve_or_escalate_from_awooop",
|
||
"evidence": {
|
||
"auto_repair_records": 0,
|
||
"operation_records": 0,
|
||
"mcp_gateway_total": 1,
|
||
"knowledge_entries": 0,
|
||
},
|
||
"writes": {
|
||
"incident": False,
|
||
"auto_repair": False,
|
||
},
|
||
},
|
||
km_stale_completion_summary={
|
||
"schema_version": "km_stale_owner_review_completion_callback_summary_v1",
|
||
"project_id": "awoooi",
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"status": "no_related_owner_review",
|
||
"ready_count": 3,
|
||
"blocked_count": 1,
|
||
"completed_count": 2,
|
||
"failed_count": 0,
|
||
"writes_on_read": False,
|
||
"batch_writes_allowed": False,
|
||
"manual_review_required": True,
|
||
"related_total": 0,
|
||
"work_item": {
|
||
"triage": {
|
||
"schema_version": "km_stale_callback_owner_review_triage_v1",
|
||
"flow_stage": "callback_observed_owner_review_link_missing",
|
||
"ai_lead_agent": "Hermes",
|
||
"supporting_agents": ["OpenClaw", "ElephantAlpha"],
|
||
"automation_state": "manual_owner_review_required",
|
||
"safe_to_auto_repair": False,
|
||
"blocking_reason": "no_matching_completion_item",
|
||
"matching_strategy": "related_incident_id_exact_match",
|
||
},
|
||
},
|
||
},
|
||
)
|
||
|
||
first_source_extra = sent_requests[0][1]["_awooop_source_envelope_extra"]
|
||
fallback_source_extra = sent_requests[1][1]["_awooop_source_envelope_extra"]
|
||
first_extra = first_source_extra["callback_reply"]
|
||
fallback_extra = fallback_source_extra["callback_reply"]
|
||
|
||
assert first_extra["status"] == "callback_reply_sent"
|
||
assert first_extra["action"] == "history"
|
||
assert first_extra["parse_mode"] == "HTML"
|
||
assert first_source_extra["awooop_status_chain"]["repair_state"] == (
|
||
"read_only_dry_run"
|
||
)
|
||
assert first_source_extra["awooop_status_chain"]["needs_human"] is True
|
||
assert first_source_extra["km_stale_completion_summary"]["ready_count"] == 3
|
||
assert first_source_extra["km_stale_completion_summary"]["triage"]["flow_stage"] == (
|
||
"callback_observed_owner_review_link_missing"
|
||
)
|
||
assert fallback_extra["status"] == "callback_reply_fallback_sent"
|
||
assert fallback_extra["incident_id"] == "INC-20260514-F85F21"
|
||
assert fallback_extra["parse_mode"] == "plain_text"
|
||
assert fallback_source_extra["awooop_status_chain"]["next_step"] == (
|
||
"approve_or_escalate_from_awooop"
|
||
)
|
||
assert fallback_source_extra["km_stale_completion_summary"]["triage"][
|
||
"ai_lead_agent"
|
||
] == "Hermes"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_html_line_message_uses_rescue_when_markup_fallback_fails(monkeypatch):
|
||
"""詳情/歷史 fallback 若仍被 Telegram 拒收，要再送無按鈕純文字救援。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
reply_markup = telegram_gateway_module._awooop_runs_reply_markup("INC-20260514-F85F21")
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
if len(sent_requests) < 3:
|
||
raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400")
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
await gateway._send_html_line_message(
|
||
["📊 <b>事件歷史統計</b>", "🔖 <code>INC-20260514-F85F21</code>"],
|
||
chat_id="chat",
|
||
failure_context="test_history",
|
||
reply_markup=reply_markup,
|
||
)
|
||
|
||
assert len(sent_requests) == 3
|
||
assert sent_requests[1][1]["reply_markup"] == reply_markup
|
||
assert "reply_markup" not in sent_requests[2][1]
|
||
assert sent_requests[2][1]["_skip_incident_thread_reply"] is True
|
||
assert "<code>" not in sent_requests[2][1]["text"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_html_line_message_attaches_awooop_markup_to_first_chunk(monkeypatch):
|
||
"""詳情/歷史這類 HTML reply 要能帶 AwoooP evidence URL，且長訊息只掛第一段。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
reply_markup = telegram_gateway_module._awooop_runs_reply_markup("INC-20260514-F85F21")
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
await gateway._send_html_line_message(
|
||
["📊 <b>事件歷史統計</b>"] + [
|
||
f"<code>INC-20260514-F85F21 trace line {index:03d}</code>"
|
||
for index in range(80)
|
||
],
|
||
chat_id="chat",
|
||
failure_context="test_history",
|
||
reply_markup=reply_markup,
|
||
)
|
||
|
||
assert len(sent_requests) > 1
|
||
assert sent_requests[0][1]["reply_markup"] == reply_markup
|
||
assert all("reply_markup" not in payload for _, payload in sent_requests[1:])
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_info_callback_sends_history_when_answer_callback_is_stale(monkeypatch):
|
||
"""Telegram answerCallbackQuery 400 不得阻斷詳情/歷史 DB truth-chain reply。"""
|
||
gateway = TelegramGateway()
|
||
sent_history = []
|
||
|
||
monkeypatch.setattr(gateway._security, "is_whitelisted", lambda _user_id: True)
|
||
|
||
async def stale_answer(*_args, **_kwargs):
|
||
raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400")
|
||
|
||
async def fake_history(incident_id: str):
|
||
sent_history.append(incident_id)
|
||
|
||
monkeypatch.setattr(gateway, "_answer_callback", stale_answer)
|
||
monkeypatch.setattr(gateway, "_send_incident_history", fake_history)
|
||
|
||
result = await gateway.handle_callback(
|
||
callback_query_id="stale-callback",
|
||
callback_data="history:INC-20260513-79ED5E",
|
||
user_id=123456,
|
||
message_id=789,
|
||
)
|
||
|
||
assert result["success"] is True
|
||
assert result["info_action"] is True
|
||
assert sent_history == ["INC-20260513-79ED5E"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_notification_retries_plain_text_on_html_parse_400(monkeypatch):
|
||
"""簡短 HTML 若被 Telegram 拒收，要用純文字重送，不回報成 HTTP 400。"""
|
||
gateway = TelegramGateway()
|
||
sent_requests = []
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
if len(sent_requests) == 1:
|
||
raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400")
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
result = await gateway.send_notification(
|
||
"⚠️ 無法取得歷史統計: <b>broken",
|
||
chat_id="chat",
|
||
)
|
||
|
||
assert result == {"ok": True}
|
||
assert sent_requests[0][1]["parse_mode"] == "HTML"
|
||
assert "parse_mode" not in sent_requests[1][1]
|
||
assert sent_requests[1][1]["text"] == "⚠️ 無法取得歷史統計: broken"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_notification_long_html_uses_plain_text_without_cutting_tags(monkeypatch):
|
||
"""send_notification 的 500 字限制不可切壞 HTML tag。"""
|
||
gateway = TelegramGateway()
|
||
sent_requests = []
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
long_text = "📊 <b>事件歷史統計</b>\n告警鍵: <code>" + ("node<188>&" * 90) + "</code>"
|
||
|
||
await gateway.send_notification(long_text, chat_id="chat")
|
||
|
||
assert len(sent_requests) == 1
|
||
payload = sent_requests[0][1]
|
||
assert "parse_mode" not in payload
|
||
assert len(payload["text"]) <= 500
|
||
assert "<code>" not in payload["text"]
|
||
assert "</code>" not in payload["text"]
|
||
assert "事件歷史統計" in payload["text"]
|
||
|
||
|
||
class TestTelegramMessageFormat:
|
||
"""測試現有 TelegramMessage 格式化"""
|
||
|
||
def test_telegram_message_format_basic(self):
|
||
"""測試基本訊息格式化"""
|
||
msg = TelegramMessage(
|
||
status_emoji="🚨",
|
||
risk_level="CRITICAL",
|
||
resource_name="test-pod-123",
|
||
root_cause="Test root cause",
|
||
suggested_action="Restart pod",
|
||
estimated_downtime="~30s",
|
||
approval_id="INC-20260329-0001",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "🚨" in result
|
||
assert "嚴重" in result
|
||
assert "test-pod-123" in result
|
||
assert "處置狀態" in result
|
||
assert "規則建議待審批" in result
|
||
assert "AI 自動化鏈路" in result
|
||
assert "OpenClaw" in result
|
||
assert "NemoTron" in result
|
||
assert "ElephantAlpha" in result
|
||
assert len(result) <= 4096 # Telegram HTML message limit
|
||
|
||
def test_telegram_message_ai_proposal_marks_approval_wait(self):
|
||
"""有 AI 信心分數的修復建議必須標示為 AI 待審批。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="⚠️",
|
||
risk_level="MEDIUM",
|
||
resource_name="awoooi-api",
|
||
root_cause="CPU sustained high",
|
||
suggested_action="kubectl rollout restart deployment/awoooi-api",
|
||
estimated_downtime="~30s",
|
||
approval_id="INC-20260506-0000",
|
||
confidence=0.82,
|
||
ai_provider="ollama_gcp_a",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "處置狀態" in result
|
||
assert "AI 已提出修復建議，等待人工批准" in result
|
||
|
||
def test_telegram_message_no_action_marks_manual_judgement(self):
|
||
"""NO_ACTION 卡片必須一眼看得出需要人工處置包。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="ℹ️",
|
||
risk_level="LOW",
|
||
resource_name="node-exporter-110",
|
||
root_cause="規則命中但沒有安全可執行動作",
|
||
suggested_action="NO_ACTION",
|
||
estimated_downtime="unknown",
|
||
approval_id="INC-20260506-0001",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "處置狀態" in result
|
||
assert "未自動修復，已產生人工處置包" in result
|
||
assert "人工處置包" in result
|
||
assert "補證據：node_exporter target up" in result
|
||
assert "AwoooP 建立修復候選" in result
|
||
assert "execution result、verifier、KM / PlayBook trust" in result
|
||
assert "等待人工批准" not in result
|
||
|
||
def test_telegram_message_diagnosis_state_is_not_auto_repair(self):
|
||
"""SSH 只讀診斷 lane 不得被顯示成自動修復。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="⚠️",
|
||
risk_level="MEDIUM",
|
||
resource_name="node-110",
|
||
root_cause="SSH diagnosis collected",
|
||
suggested_action="ssh 192.168.0.110 'uptime'",
|
||
estimated_downtime="unknown",
|
||
approval_id="INC-20260506-DIAG",
|
||
automation_state="diagnosis_collected_manual_required",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "AI 已完成只讀診斷，需人工判斷" in result
|
||
assert "AI 自動修復失敗" not in result
|
||
|
||
def test_telegram_message_diagnosis_failure_state(self):
|
||
"""SSH 診斷工具失敗必須標成診斷失敗，而不是修復失敗。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="🚨",
|
||
risk_level="CRITICAL",
|
||
resource_name="node-110",
|
||
root_cause="SSH MCP execution failed",
|
||
suggested_action="ssh 192.168.0.110 'uptime'",
|
||
estimated_downtime="unknown",
|
||
approval_id="INC-20260506-DIAGFAIL",
|
||
automation_state="diagnosis_failed_manual_required",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "AI 診斷工具失敗，需人工排查" in result
|
||
assert "AI 自動修復失敗" not in result
|
||
|
||
def test_telegram_message_surfaces_read_only_remediation_evidence(self):
|
||
"""主告警卡必須顯示 ADR-100 只讀補救試跑與寫入旗標。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="⚠️",
|
||
risk_level="MEDIUM",
|
||
resource_name="awoooi-auto-repair-canary",
|
||
root_cause="post approval verification drift",
|
||
suggested_action="kubectl rollout restart deployment/awoooi-api",
|
||
estimated_downtime="~30s",
|
||
approval_id="INC-20260513-79ED5E",
|
||
confidence=0.82,
|
||
remediation_summary={
|
||
"schema_version": "adr100_remediation_history_v1",
|
||
"total": 3,
|
||
"items": [
|
||
{
|
||
"mode": "replay",
|
||
"allowed": True,
|
||
"success": True,
|
||
"safety_level": "read_only",
|
||
"verification_result_preview": "degraded",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
},
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "AI 已完成只讀補救試跑，等待人工審批" in result
|
||
assert "AI 證據" in result
|
||
assert "只讀試跑 3 次" in result
|
||
assert "auto_repair_executor/ssh_diagnose/read" in result
|
||
assert "incident <code>false</code>" in result
|
||
assert "auto-repair <code>false</code>" in result
|
||
|
||
def test_telegram_message_surfaces_missing_remediation_evidence(self):
|
||
"""沒有補救試跑紀錄時，主卡要明確說明，不讓值班者猜。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="ℹ️",
|
||
risk_level="LOW",
|
||
resource_name="awoooi-auto-repair-canary",
|
||
root_cause="safe canary",
|
||
suggested_action="NO_ACTION",
|
||
estimated_downtime="unknown",
|
||
approval_id="INC-20260513-EMPTY",
|
||
remediation_summary={
|
||
"schema_version": "adr100_remediation_history_v1",
|
||
"total": 0,
|
||
"items": [],
|
||
},
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "AI 證據" in result
|
||
assert "尚無補救試跑紀錄" in result
|
||
|
||
def test_telegram_message_with_token_cost(self):
|
||
"""測試含 Token/Cost 的訊息"""
|
||
msg = TelegramMessage(
|
||
status_emoji="⚠️",
|
||
risk_level="MEDIUM",
|
||
resource_name="api-pod",
|
||
root_cause="High CPU",
|
||
suggested_action="Scale up",
|
||
estimated_downtime="0s",
|
||
approval_id="INC-20260329-0002",
|
||
ai_tokens=1500,
|
||
ai_cost=0.0015,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "💰 Tokens: 1,500 / $0.0015" in result
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_append_incident_update_deduplicates_same_status(monkeypatch):
|
||
"""同一 Incident 的相同狀態更新 5 分鐘內不可重複洗版。"""
|
||
|
||
class FakeRedis:
|
||
def __init__(self):
|
||
self.values = {}
|
||
|
||
async def get(self, key):
|
||
assert key == "tg_msg:INC-DEDUP"
|
||
return "12345"
|
||
|
||
async def set(self, key, value, **kwargs):
|
||
assert kwargs["nx"] is True
|
||
assert kwargs["ex"] > 0
|
||
if key in self.values:
|
||
return False
|
||
self.values[key] = value
|
||
return True
|
||
|
||
fake_redis = FakeRedis()
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: fake_redis)
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
status_line = "🤖❌ <b>[AUTO] AI 自動修復失敗，已升級人工介入</b>"
|
||
|
||
assert await gateway.append_incident_update("INC-DEDUP", status_line) is True
|
||
assert await gateway.append_incident_update("INC-DEDUP", status_line) is True
|
||
|
||
assert [method for method, _ in sent_requests] == [
|
||
"editMessageReplyMarkup",
|
||
"sendMessage",
|
||
]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_append_incident_update_suppresses_duplicate_failure_across_incidents(monkeypatch):
|
||
"""不同 Incident 卡在相同失敗摘要時，只回覆第一則，避免 Telegram 洗版。"""
|
||
|
||
class FakeRedis:
|
||
def __init__(self):
|
||
self.values = {}
|
||
|
||
async def get(self, key):
|
||
if key == "tg_msg:INC-A":
|
||
return "111"
|
||
if key == "tg_msg:INC-B":
|
||
return "222"
|
||
return None
|
||
|
||
async def set(self, key, value, **kwargs):
|
||
assert kwargs["nx"] is True
|
||
assert kwargs["ex"] > 0
|
||
if key in self.values:
|
||
return False
|
||
self.values[key] = value
|
||
return True
|
||
|
||
fake_redis = FakeRedis()
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: fake_redis)
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
status_line = (
|
||
"🤖❌ <b>[AUTO] AI 自動修復失敗，已升級人工介入</b>\n"
|
||
"├ 動作: <code>ssh 192.168.0.110 uptime</code>\n"
|
||
"└ 錯誤: unsupported action"
|
||
)
|
||
|
||
assert await gateway.append_incident_update("INC-A", status_line) is True
|
||
assert await gateway.append_incident_update("INC-B", status_line) is True
|
||
|
||
assert [method for method, _ in sent_requests] == [
|
||
"editMessageReplyMarkup",
|
||
"sendMessage",
|
||
"editMessageReplyMarkup",
|
||
]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_approval_card_includes_remediation_summary(monkeypatch):
|
||
"""send_approval_card 要把 durable 補救試跑歷史帶進 Telegram 主卡。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True, "result": {}}
|
||
|
||
async def fake_keyboard(**kwargs):
|
||
return {"inline_keyboard": []}
|
||
|
||
async def fake_remediation_summary(**kwargs):
|
||
assert kwargs["incident_id"] == "INC-20260513-79ED5E"
|
||
return {
|
||
"schema_version": "adr100_remediation_history_v1",
|
||
"total": 1,
|
||
"items": [
|
||
{
|
||
"allowed": True,
|
||
"success": True,
|
||
"safety_level": "read_only",
|
||
"verification_result_preview": "degraded",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
}
|
||
|
||
monkeypatch.setattr(TelegramGateway, "alert_chat_id", property(lambda _self: "chat"))
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
monkeypatch.setattr(gateway, "_build_inline_keyboard", fake_keyboard)
|
||
monkeypatch.setattr(
|
||
telegram_gateway_module,
|
||
"_fetch_remediation_summary_for_card",
|
||
fake_remediation_summary,
|
||
)
|
||
|
||
await gateway.send_approval_card(
|
||
approval_id="approval-1",
|
||
risk_level="medium",
|
||
resource_name="awoooi-auto-repair-canary",
|
||
root_cause="post approval verification drift",
|
||
suggested_action="kubectl rollout restart deployment/awoooi-api",
|
||
incident_id="INC-20260513-79ED5E",
|
||
confidence=0.82,
|
||
)
|
||
|
||
assert sent_requests
|
||
text = sent_requests[0][1]["text"]
|
||
assert "AI 已完成只讀補救試跑，等待人工審批" in text
|
||
assert "auto_repair_executor/ssh_diagnose/read" in text
|
||
|
||
|
||
def test_outbound_message_type_inference():
|
||
"""Legacy Telegram 訊息 mirror 到 Channel Hub 時，必須映射成有限分類。"""
|
||
|
||
assert (
|
||
telegram_gateway_module._infer_outbound_message_type(
|
||
"ℹ️ ACTION REQUIRED | 低風險",
|
||
{"reply_markup": {"inline_keyboard": []}},
|
||
)
|
||
== "approval_request"
|
||
)
|
||
assert (
|
||
telegram_gateway_module._infer_outbound_message_type(
|
||
"🤖❌ [AUTO] AI 自動修復失敗",
|
||
{"reply_to_message_id": 123},
|
||
)
|
||
== "error"
|
||
)
|
||
assert (
|
||
telegram_gateway_module._infer_outbound_message_type(
|
||
"✅ 執行成功",
|
||
{"reply_parameters": {"message_id": 123}},
|
||
)
|
||
== "final"
|
||
)
|
||
assert (
|
||
telegram_gateway_module._infer_outbound_message_type(
|
||
"📄 <b>RUNBOOK REVIEW｜待審核</b>",
|
||
{"reply_parameters": {"message_id": 123}},
|
||
)
|
||
== "approval_request"
|
||
)
|
||
|
||
|
||
def test_extract_incident_id_from_text():
|
||
"""Telegram 出站文字可擷取 Incident ID，供後續訊息接回原告警卡片。"""
|
||
|
||
assert (
|
||
telegram_gateway_module._extract_incident_id_from_text(
|
||
"Incident: INC-20260506-E54736\nEntry ID: abc"
|
||
)
|
||
== "INC-20260506-E54736"
|
||
)
|
||
assert telegram_gateway_module._extract_incident_id_from_text("沒有事件編號") is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_attach_incident_thread_reply(monkeypatch):
|
||
"""非主卡、含 Incident ID 的後續訊息，應自動 reply 原告警 message_id。"""
|
||
|
||
class FakeRedis:
|
||
async def get(self, key):
|
||
assert key == "tg_msg:INC-20260506-E54736"
|
||
return "9876"
|
||
|
||
gateway = TelegramGateway()
|
||
payload = {
|
||
"chat_id": gateway.alert_chat_id,
|
||
"text": "📄 RUNBOOK REVIEW｜待審核\nIncident: INC-20260506-E54736",
|
||
}
|
||
|
||
monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: FakeRedis())
|
||
|
||
await gateway._attach_incident_thread_reply("sendMessage", payload)
|
||
|
||
assert payload["reply_parameters"] == {
|
||
"message_id": 9876,
|
||
"allow_sending_without_reply": True,
|
||
}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_attach_incident_thread_skips_root_action_card(monkeypatch):
|
||
"""ACTION REQUIRED 主告警卡不應自動 reply 舊訊息。"""
|
||
|
||
class FakeRedis:
|
||
async def get(self, key): # pragma: no cover - 不應被呼叫
|
||
raise AssertionError(key)
|
||
|
||
gateway = TelegramGateway()
|
||
payload = {
|
||
"chat_id": gateway.alert_chat_id,
|
||
"text": (
|
||
"ℹ️ ACTION REQUIRED | 低風險\n"
|
||
"📋 INC-20260506-E54736\n"
|
||
"🤖 AI 自動化鏈路"
|
||
),
|
||
}
|
||
|
||
monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: FakeRedis())
|
||
|
||
await gateway._attach_incident_thread_reply("sendMessage", payload)
|
||
|
||
assert "reply_parameters" not in payload
|
||
|
||
|
||
def test_legacy_outbound_run_id_is_stable():
|
||
"""沒有正式 run_id 的 legacy Telegram 發送，要有穩定 soft run_id 供查詢串接。"""
|
||
|
||
first = telegram_gateway_module._legacy_outbound_run_id("-1001", "42")
|
||
second = telegram_gateway_module._legacy_outbound_run_id("-1001", "42")
|
||
other = telegram_gateway_module._legacy_outbound_run_id("-1001", "43")
|
||
|
||
assert first == second
|
||
assert first != other
|
||
|
||
|
||
class TestSentryErrorMessage:
|
||
"""測試 Sentry 錯誤訊息"""
|
||
|
||
def test_sentry_error_format_basic(self):
|
||
"""測試基本 Sentry 錯誤格式"""
|
||
msg = SentryErrorMessage(
|
||
error_id="SENTRY-abc123",
|
||
error_type="TypeError",
|
||
error_message="Cannot read property 'x' of undefined",
|
||
service_name="awoooi-api",
|
||
file_location="src/api/v1/incidents.py:123",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "🐛" in result
|
||
assert "SENTRY ERROR" in result
|
||
assert "TypeError" in result
|
||
assert "awoooi-api" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_sentry_error_with_stack_trace(self):
|
||
"""測試含 Stack Trace 的 Sentry 錯誤"""
|
||
msg = SentryErrorMessage(
|
||
error_id="SENTRY-xyz789",
|
||
error_type="ValueError",
|
||
error_message="Invalid input",
|
||
service_name="awoooi-web",
|
||
file_location="src/components/App.tsx:45",
|
||
occurrence_count=15,
|
||
affected_users=3,
|
||
first_seen="10 分鐘前",
|
||
stack_trace=[
|
||
"incidents.py:123 in get_incident",
|
||
"service.py:45 in fetch_data",
|
||
"db.py:89 in query",
|
||
],
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "發生次數: <code>15</code>" in result
|
||
assert "影響用戶: <code>3</code>" in result
|
||
assert "Stack Trace" in result
|
||
|
||
|
||
class TestResourceWarnMessage:
|
||
"""測試資源告警訊息"""
|
||
|
||
def test_resource_warn_format_basic(self):
|
||
"""測試基本資源告警格式"""
|
||
msg = ResourceWarnMessage(
|
||
resource_id="RES-20260329-0001",
|
||
pod_name="awoooi-api-7d4b8c9f5-abc12",
|
||
namespace="awoooi-prod",
|
||
cpu_percent=92.5,
|
||
memory_percent=78.0,
|
||
disk_percent=45.0,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "⚠️" in result
|
||
assert "資源告警" in result
|
||
assert "CPU: 🔴" in result # 92.5% > 90%
|
||
assert "Memory: 🟡" in result # 78% >= 70%
|
||
assert "Disk: 🟢" in result # 45% < 70%
|
||
assert len(result) <= 900
|
||
|
||
def test_resource_warn_with_limits(self):
|
||
"""測試含限制資訊的資源告警"""
|
||
msg = ResourceWarnMessage(
|
||
resource_id="RES-20260329-0002",
|
||
pod_name="test-pod",
|
||
cpu_percent=85.0,
|
||
cpu_limit="500m",
|
||
memory_percent=60.0,
|
||
memory_limit="512Mi",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "(limit: 500m)" in result
|
||
assert "(limit: 512Mi)" in result
|
||
|
||
|
||
class TestRepairReportMessage:
|
||
"""測試自動修復報告"""
|
||
|
||
def test_repair_report_format_basic(self):
|
||
"""測試基本修復報告格式"""
|
||
msg = RepairReportMessage(
|
||
report_date="2026-03-29",
|
||
total_repairs=12,
|
||
success_count=10,
|
||
failure_count=2,
|
||
saved_minutes=45,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "🔧" in result
|
||
assert "自動修復報告" in result
|
||
assert "總修復次數: <code>12</code>" in result
|
||
assert "成功: ✅ <code>10</code> (83%)" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_repair_report_with_top_issues(self):
|
||
"""測試含 Top 問題的修復報告"""
|
||
msg = RepairReportMessage(
|
||
report_date="2026-03-29",
|
||
total_repairs=12,
|
||
success_count=10,
|
||
failure_count=2,
|
||
top_issues=[
|
||
("Pod CrashLoopBackOff", 5),
|
||
("OOM Killed", 4),
|
||
("Image Pull Failed", 3),
|
||
],
|
||
ai_cost_gemini=0.0234,
|
||
ai_tokens_total=1823,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "Top 3 問題" in result
|
||
assert "Pod CrashLoopBackOff" in result
|
||
assert "Gemini: $0.0234" in result
|
||
|
||
|
||
class TestDailySummaryMessage:
|
||
"""測試每日摘要"""
|
||
|
||
def test_daily_summary_format_basic(self):
|
||
"""測試基本每日摘要格式"""
|
||
msg = DailySummaryMessage(
|
||
summary_date="2026-03-29",
|
||
alert_total=45,
|
||
alert_critical=2,
|
||
alert_medium=18,
|
||
alert_low=25,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "📊" in result
|
||
assert "每日摘要" in result
|
||
assert "總數: <code>45</code>" in result
|
||
assert "Critical: <code>2</code>" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_daily_summary_with_full_stats(self):
|
||
"""測試完整統計的每日摘要"""
|
||
msg = DailySummaryMessage(
|
||
summary_date="2026-03-29",
|
||
alert_total=45,
|
||
auto_repair_count=30,
|
||
manual_approval_count=10,
|
||
ignored_count=5,
|
||
api_availability=99.95,
|
||
ai_cost=0.15,
|
||
budget_remaining=9.85,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "自動修復: <code>30</code>" in result
|
||
assert "API: <code>99.95%</code>" in result
|
||
assert "預算剩餘: $9.85" in result
|
||
|
||
|
||
class TestDeploySuccessMessage:
|
||
"""測試部署成功訊息"""
|
||
|
||
def test_deploy_success_format_basic(self):
|
||
"""測試基本部署成功格式"""
|
||
msg = DeploySuccessMessage(
|
||
commit_sha="abc1234567",
|
||
triggered_by="ogt",
|
||
environment="Production",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "✅" in result
|
||
assert "部署成功" in result
|
||
assert "abc12345" in result # 前 8 字元
|
||
assert "@ogt" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_deploy_success_with_e2e(self):
|
||
"""測試含 E2E 結果的部署成功"""
|
||
msg = DeploySuccessMessage(
|
||
commit_sha="abc1234567",
|
||
triggered_by="ogt",
|
||
api_version="v1.2.3",
|
||
web_version="v1.2.3",
|
||
duration_seconds=225, # 3m 45s
|
||
e2e_passed=26,
|
||
e2e_total=26,
|
||
health_check_passed=True,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "v1.2.3" in result
|
||
assert "3m 45s" in result
|
||
assert "✅ 26/26 PASSED" in result
|
||
|
||
|
||
class TestRateLimitMessage:
|
||
"""測試 API 限額警告"""
|
||
|
||
def test_rate_limit_format_basic(self):
|
||
"""測試基本限額警告格式"""
|
||
msg = RateLimitMessage(
|
||
provider="gemini",
|
||
daily_usage=450,
|
||
daily_limit=500,
|
||
token_usage=85000,
|
||
token_limit=100000,
|
||
cost_usd=0.08,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "⚠️" in result
|
||
assert "API 限額警告" in result
|
||
assert "GEMINI API" in result
|
||
assert "450/500" in result
|
||
assert "(90%)" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_rate_limit_with_suggestions(self):
|
||
"""測試含建議的限額警告"""
|
||
msg = RateLimitMessage(
|
||
provider="openai",
|
||
daily_usage=90,
|
||
daily_limit=100,
|
||
suggestions=[
|
||
"考慮切換到 Ollama 優先",
|
||
"或增加每日限額",
|
||
],
|
||
reset_time="明日 00:00",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "建議" in result
|
||
assert "切換到 Ollama" in result
|
||
assert "明日 00:00" in result
|