624 lines
20 KiB
Python
624 lines
20 KiB
Python
"""
|
||
test_telegram_message_templates.py - Telegram 訊息模板測試
|
||
|
||
2026-03-29 ogt: 新增，驗證 6 種新訊息模板格式正確
|
||
符合 feedback_no_mock_testing.md 規範 - 測試實際格式化輸出
|
||
"""
|
||
|
||
import pytest
|
||
|
||
import src.services.telegram_gateway as telegram_gateway_module
|
||
from src.services.decision_manager import _format_auto_repair_status_line
|
||
from src.services.telegram_gateway import (
|
||
DailySummaryMessage,
|
||
DeploySuccessMessage,
|
||
RateLimitMessage,
|
||
RepairReportMessage,
|
||
ResourceWarnMessage,
|
||
SentryErrorMessage,
|
||
TelegramGateway,
|
||
TelegramMessage,
|
||
)
|
||
|
||
|
||
def test_auto_repair_status_line_distinguishes_handoff_required() -> None:
|
||
"""自動化失敗 reply 必須明確標示轉人工，且不把 raw error 當純文字噴出。"""
|
||
result = _format_auto_repair_status_line(
|
||
incident_id="INC-20260507-AAAAAA",
|
||
target="node-exporter-110",
|
||
action='ssh 192.168.0.110 "ps aux --sort=-%cpu | head -15"',
|
||
success=False,
|
||
error="Unsupported <scheme> & %d format: a real number is required, not str",
|
||
)
|
||
|
||
assert "HANDOFF REQUIRED｜AI 自動修復失敗，已轉人工" in result
|
||
assert "自動化已停止，不再重試" in result
|
||
assert "請 SRE 依 AwoooP Run / 原告警卡處理" in result
|
||
assert "<scheme> & %d format" in result
|
||
assert "<scheme>" not in result
|
||
|
||
|
||
def test_auto_repair_status_line_distinguishes_auto_resolved() -> None:
|
||
"""自動化成功 reply 必須明確標示已自動解決。"""
|
||
result = _format_auto_repair_status_line(
|
||
incident_id="INC-20260507-BBBBBB",
|
||
target="awoooi-api",
|
||
action="kubectl rollout restart deployment/awoooi-api",
|
||
success=True,
|
||
metrics_delta_text="CPU 92%->30%",
|
||
)
|
||
|
||
assert "AUTO RESOLVED｜AI 自動修復完成" in result
|
||
assert "自動化已完成，等待後驗證觀察" in result
|
||
assert "CPU 92%->30%" in result
|
||
|
||
|
||
class TestTelegramMessageFormat:
|
||
"""測試現有 TelegramMessage 格式化"""
|
||
|
||
def test_telegram_message_format_basic(self):
|
||
"""測試基本訊息格式化"""
|
||
msg = TelegramMessage(
|
||
status_emoji="🚨",
|
||
risk_level="CRITICAL",
|
||
resource_name="test-pod-123",
|
||
root_cause="Test root cause",
|
||
suggested_action="Restart pod",
|
||
estimated_downtime="~30s",
|
||
approval_id="INC-20260329-0001",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "🚨" in result
|
||
assert "嚴重" in result
|
||
assert "test-pod-123" in result
|
||
assert "處置狀態" in result
|
||
assert "規則建議待審批" in result
|
||
assert "AI 自動化鏈路" in result
|
||
assert "OpenClaw" in result
|
||
assert "NemoTron" in result
|
||
assert "ElephantAlpha" in result
|
||
assert len(result) <= 4096 # Telegram HTML message limit
|
||
|
||
def test_telegram_message_ai_proposal_marks_approval_wait(self):
|
||
"""有 AI 信心分數的修復建議必須標示為 AI 待審批。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="⚠️",
|
||
risk_level="MEDIUM",
|
||
resource_name="awoooi-api",
|
||
root_cause="CPU sustained high",
|
||
suggested_action="kubectl rollout restart deployment/awoooi-api",
|
||
estimated_downtime="~30s",
|
||
approval_id="INC-20260506-0000",
|
||
confidence=0.82,
|
||
ai_provider="ollama_gcp_a",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "處置狀態" in result
|
||
assert "AI 已提出修復建議，等待人工批准" in result
|
||
|
||
def test_telegram_message_no_action_marks_manual_judgement(self):
|
||
"""NO_ACTION 卡片必須一眼看得出需要人工判斷。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="ℹ️",
|
||
risk_level="LOW",
|
||
resource_name="node-exporter-110",
|
||
root_cause="規則命中但沒有安全可執行動作",
|
||
suggested_action="NO_ACTION",
|
||
estimated_downtime="unknown",
|
||
approval_id="INC-20260506-0001",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "處置狀態" in result
|
||
assert "AI 無可安全執行動作，需人工判斷" in result
|
||
|
||
def test_telegram_message_diagnosis_state_is_not_auto_repair(self):
|
||
"""SSH 只讀診斷 lane 不得被顯示成自動修復。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="⚠️",
|
||
risk_level="MEDIUM",
|
||
resource_name="node-110",
|
||
root_cause="SSH diagnosis collected",
|
||
suggested_action="ssh 192.168.0.110 'uptime'",
|
||
estimated_downtime="unknown",
|
||
approval_id="INC-20260506-DIAG",
|
||
automation_state="diagnosis_collected_manual_required",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "AI 已完成只讀診斷，需人工判斷" in result
|
||
assert "AI 自動修復失敗" not in result
|
||
|
||
def test_telegram_message_diagnosis_failure_state(self):
|
||
"""SSH 診斷工具失敗必須標成診斷失敗，而不是修復失敗。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="🚨",
|
||
risk_level="CRITICAL",
|
||
resource_name="node-110",
|
||
root_cause="SSH MCP execution failed",
|
||
suggested_action="ssh 192.168.0.110 'uptime'",
|
||
estimated_downtime="unknown",
|
||
approval_id="INC-20260506-DIAGFAIL",
|
||
automation_state="diagnosis_failed_manual_required",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "AI 診斷工具失敗，需人工排查" in result
|
||
assert "AI 自動修復失敗" not in result
|
||
|
||
def test_telegram_message_with_token_cost(self):
|
||
"""測試含 Token/Cost 的訊息"""
|
||
msg = TelegramMessage(
|
||
status_emoji="⚠️",
|
||
risk_level="MEDIUM",
|
||
resource_name="api-pod",
|
||
root_cause="High CPU",
|
||
suggested_action="Scale up",
|
||
estimated_downtime="0s",
|
||
approval_id="INC-20260329-0002",
|
||
ai_tokens=1500,
|
||
ai_cost=0.0015,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "💰 Tokens: 1,500 / $0.0015" in result
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_append_incident_update_deduplicates_same_status(monkeypatch):
|
||
"""同一 Incident 的相同狀態更新 5 分鐘內不可重複洗版。"""
|
||
|
||
class FakeRedis:
|
||
def __init__(self):
|
||
self.values = {}
|
||
|
||
async def get(self, key):
|
||
assert key == "tg_msg:INC-DEDUP"
|
||
return "12345"
|
||
|
||
async def set(self, key, value, **kwargs):
|
||
assert kwargs["nx"] is True
|
||
assert kwargs["ex"] > 0
|
||
if key in self.values:
|
||
return False
|
||
self.values[key] = value
|
||
return True
|
||
|
||
fake_redis = FakeRedis()
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: fake_redis)
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
status_line = "🤖❌ <b>[AUTO] AI 自動修復失敗，已升級人工介入</b>"
|
||
|
||
assert await gateway.append_incident_update("INC-DEDUP", status_line) is True
|
||
assert await gateway.append_incident_update("INC-DEDUP", status_line) is True
|
||
|
||
assert [method for method, _ in sent_requests] == [
|
||
"editMessageReplyMarkup",
|
||
"sendMessage",
|
||
]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_append_incident_update_suppresses_duplicate_failure_across_incidents(monkeypatch):
|
||
"""不同 Incident 卡在相同失敗摘要時，只回覆第一則，避免 Telegram 洗版。"""
|
||
|
||
class FakeRedis:
|
||
def __init__(self):
|
||
self.values = {}
|
||
|
||
async def get(self, key):
|
||
if key == "tg_msg:INC-A":
|
||
return "111"
|
||
if key == "tg_msg:INC-B":
|
||
return "222"
|
||
return None
|
||
|
||
async def set(self, key, value, **kwargs):
|
||
assert kwargs["nx"] is True
|
||
assert kwargs["ex"] > 0
|
||
if key in self.values:
|
||
return False
|
||
self.values[key] = value
|
||
return True
|
||
|
||
fake_redis = FakeRedis()
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: fake_redis)
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
status_line = (
|
||
"🤖❌ <b>[AUTO] AI 自動修復失敗，已升級人工介入</b>\n"
|
||
"├ 動作: <code>ssh 192.168.0.110 uptime</code>\n"
|
||
"└ 錯誤: unsupported action"
|
||
)
|
||
|
||
assert await gateway.append_incident_update("INC-A", status_line) is True
|
||
assert await gateway.append_incident_update("INC-B", status_line) is True
|
||
|
||
assert [method for method, _ in sent_requests] == [
|
||
"editMessageReplyMarkup",
|
||
"sendMessage",
|
||
"editMessageReplyMarkup",
|
||
]
|
||
|
||
|
||
def test_outbound_message_type_inference():
|
||
"""Legacy Telegram 訊息 mirror 到 Channel Hub 時，必須映射成有限分類。"""
|
||
|
||
assert (
|
||
telegram_gateway_module._infer_outbound_message_type(
|
||
"ℹ️ ACTION REQUIRED | 低風險",
|
||
{"reply_markup": {"inline_keyboard": []}},
|
||
)
|
||
== "approval_request"
|
||
)
|
||
assert (
|
||
telegram_gateway_module._infer_outbound_message_type(
|
||
"🤖❌ [AUTO] AI 自動修復失敗",
|
||
{"reply_to_message_id": 123},
|
||
)
|
||
== "error"
|
||
)
|
||
assert (
|
||
telegram_gateway_module._infer_outbound_message_type(
|
||
"✅ 執行成功",
|
||
{"reply_parameters": {"message_id": 123}},
|
||
)
|
||
== "final"
|
||
)
|
||
assert (
|
||
telegram_gateway_module._infer_outbound_message_type(
|
||
"📄 <b>RUNBOOK REVIEW｜待審核</b>",
|
||
{"reply_parameters": {"message_id": 123}},
|
||
)
|
||
== "approval_request"
|
||
)
|
||
|
||
|
||
def test_extract_incident_id_from_text():
|
||
"""Telegram 出站文字可擷取 Incident ID，供後續訊息接回原告警卡片。"""
|
||
|
||
assert (
|
||
telegram_gateway_module._extract_incident_id_from_text(
|
||
"Incident: INC-20260506-E54736\nEntry ID: abc"
|
||
)
|
||
== "INC-20260506-E54736"
|
||
)
|
||
assert telegram_gateway_module._extract_incident_id_from_text("沒有事件編號") is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_attach_incident_thread_reply(monkeypatch):
|
||
"""非主卡、含 Incident ID 的後續訊息，應自動 reply 原告警 message_id。"""
|
||
|
||
class FakeRedis:
|
||
async def get(self, key):
|
||
assert key == "tg_msg:INC-20260506-E54736"
|
||
return "9876"
|
||
|
||
gateway = TelegramGateway()
|
||
payload = {
|
||
"chat_id": gateway.alert_chat_id,
|
||
"text": "📄 RUNBOOK REVIEW｜待審核\nIncident: INC-20260506-E54736",
|
||
}
|
||
|
||
monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: FakeRedis())
|
||
|
||
await gateway._attach_incident_thread_reply("sendMessage", payload)
|
||
|
||
assert payload["reply_parameters"] == {
|
||
"message_id": 9876,
|
||
"allow_sending_without_reply": True,
|
||
}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_attach_incident_thread_skips_root_action_card(monkeypatch):
|
||
"""ACTION REQUIRED 主告警卡不應自動 reply 舊訊息。"""
|
||
|
||
class FakeRedis:
|
||
async def get(self, key): # pragma: no cover - 不應被呼叫
|
||
raise AssertionError(key)
|
||
|
||
gateway = TelegramGateway()
|
||
payload = {
|
||
"chat_id": gateway.alert_chat_id,
|
||
"text": (
|
||
"ℹ️ ACTION REQUIRED | 低風險\n"
|
||
"📋 INC-20260506-E54736\n"
|
||
"🤖 AI 自動化鏈路"
|
||
),
|
||
}
|
||
|
||
monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: FakeRedis())
|
||
|
||
await gateway._attach_incident_thread_reply("sendMessage", payload)
|
||
|
||
assert "reply_parameters" not in payload
|
||
|
||
|
||
def test_legacy_outbound_run_id_is_stable():
|
||
"""沒有正式 run_id 的 legacy Telegram 發送，要有穩定 soft run_id 供查詢串接。"""
|
||
|
||
first = telegram_gateway_module._legacy_outbound_run_id("-1001", "42")
|
||
second = telegram_gateway_module._legacy_outbound_run_id("-1001", "42")
|
||
other = telegram_gateway_module._legacy_outbound_run_id("-1001", "43")
|
||
|
||
assert first == second
|
||
assert first != other
|
||
|
||
|
||
class TestSentryErrorMessage:
|
||
"""測試 Sentry 錯誤訊息"""
|
||
|
||
def test_sentry_error_format_basic(self):
|
||
"""測試基本 Sentry 錯誤格式"""
|
||
msg = SentryErrorMessage(
|
||
error_id="SENTRY-abc123",
|
||
error_type="TypeError",
|
||
error_message="Cannot read property 'x' of undefined",
|
||
service_name="awoooi-api",
|
||
file_location="src/api/v1/incidents.py:123",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "🐛" in result
|
||
assert "SENTRY ERROR" in result
|
||
assert "TypeError" in result
|
||
assert "awoooi-api" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_sentry_error_with_stack_trace(self):
|
||
"""測試含 Stack Trace 的 Sentry 錯誤"""
|
||
msg = SentryErrorMessage(
|
||
error_id="SENTRY-xyz789",
|
||
error_type="ValueError",
|
||
error_message="Invalid input",
|
||
service_name="awoooi-web",
|
||
file_location="src/components/App.tsx:45",
|
||
occurrence_count=15,
|
||
affected_users=3,
|
||
first_seen="10 分鐘前",
|
||
stack_trace=[
|
||
"incidents.py:123 in get_incident",
|
||
"service.py:45 in fetch_data",
|
||
"db.py:89 in query",
|
||
],
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "發生次數: <code>15</code>" in result
|
||
assert "影響用戶: <code>3</code>" in result
|
||
assert "Stack Trace" in result
|
||
|
||
|
||
class TestResourceWarnMessage:
|
||
"""測試資源告警訊息"""
|
||
|
||
def test_resource_warn_format_basic(self):
|
||
"""測試基本資源告警格式"""
|
||
msg = ResourceWarnMessage(
|
||
resource_id="RES-20260329-0001",
|
||
pod_name="awoooi-api-7d4b8c9f5-abc12",
|
||
namespace="awoooi-prod",
|
||
cpu_percent=92.5,
|
||
memory_percent=78.0,
|
||
disk_percent=45.0,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "⚠️" in result
|
||
assert "資源告警" in result
|
||
assert "CPU: 🔴" in result # 92.5% > 90%
|
||
assert "Memory: 🟡" in result # 78% >= 70%
|
||
assert "Disk: 🟢" in result # 45% < 70%
|
||
assert len(result) <= 900
|
||
|
||
def test_resource_warn_with_limits(self):
|
||
"""測試含限制資訊的資源告警"""
|
||
msg = ResourceWarnMessage(
|
||
resource_id="RES-20260329-0002",
|
||
pod_name="test-pod",
|
||
cpu_percent=85.0,
|
||
cpu_limit="500m",
|
||
memory_percent=60.0,
|
||
memory_limit="512Mi",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "(limit: 500m)" in result
|
||
assert "(limit: 512Mi)" in result
|
||
|
||
|
||
class TestRepairReportMessage:
|
||
"""測試自動修復報告"""
|
||
|
||
def test_repair_report_format_basic(self):
|
||
"""測試基本修復報告格式"""
|
||
msg = RepairReportMessage(
|
||
report_date="2026-03-29",
|
||
total_repairs=12,
|
||
success_count=10,
|
||
failure_count=2,
|
||
saved_minutes=45,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "🔧" in result
|
||
assert "自動修復報告" in result
|
||
assert "總修復次數: <code>12</code>" in result
|
||
assert "成功: ✅ <code>10</code> (83%)" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_repair_report_with_top_issues(self):
|
||
"""測試含 Top 問題的修復報告"""
|
||
msg = RepairReportMessage(
|
||
report_date="2026-03-29",
|
||
total_repairs=12,
|
||
success_count=10,
|
||
failure_count=2,
|
||
top_issues=[
|
||
("Pod CrashLoopBackOff", 5),
|
||
("OOM Killed", 4),
|
||
("Image Pull Failed", 3),
|
||
],
|
||
ai_cost_gemini=0.0234,
|
||
ai_tokens_total=1823,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "Top 3 問題" in result
|
||
assert "Pod CrashLoopBackOff" in result
|
||
assert "Gemini: $0.0234" in result
|
||
|
||
|
||
class TestDailySummaryMessage:
|
||
"""測試每日摘要"""
|
||
|
||
def test_daily_summary_format_basic(self):
|
||
"""測試基本每日摘要格式"""
|
||
msg = DailySummaryMessage(
|
||
summary_date="2026-03-29",
|
||
alert_total=45,
|
||
alert_critical=2,
|
||
alert_medium=18,
|
||
alert_low=25,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "📊" in result
|
||
assert "每日摘要" in result
|
||
assert "總數: <code>45</code>" in result
|
||
assert "Critical: <code>2</code>" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_daily_summary_with_full_stats(self):
|
||
"""測試完整統計的每日摘要"""
|
||
msg = DailySummaryMessage(
|
||
summary_date="2026-03-29",
|
||
alert_total=45,
|
||
auto_repair_count=30,
|
||
manual_approval_count=10,
|
||
ignored_count=5,
|
||
api_availability=99.95,
|
||
ai_cost=0.15,
|
||
budget_remaining=9.85,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "自動修復: <code>30</code>" in result
|
||
assert "API: <code>99.95%</code>" in result
|
||
assert "預算剩餘: $9.85" in result
|
||
|
||
|
||
class TestDeploySuccessMessage:
|
||
"""測試部署成功訊息"""
|
||
|
||
def test_deploy_success_format_basic(self):
|
||
"""測試基本部署成功格式"""
|
||
msg = DeploySuccessMessage(
|
||
commit_sha="abc1234567",
|
||
triggered_by="ogt",
|
||
environment="Production",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "✅" in result
|
||
assert "部署成功" in result
|
||
assert "abc12345" in result # 前 8 字元
|
||
assert "@ogt" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_deploy_success_with_e2e(self):
|
||
"""測試含 E2E 結果的部署成功"""
|
||
msg = DeploySuccessMessage(
|
||
commit_sha="abc1234567",
|
||
triggered_by="ogt",
|
||
api_version="v1.2.3",
|
||
web_version="v1.2.3",
|
||
duration_seconds=225, # 3m 45s
|
||
e2e_passed=26,
|
||
e2e_total=26,
|
||
health_check_passed=True,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "v1.2.3" in result
|
||
assert "3m 45s" in result
|
||
assert "✅ 26/26 PASSED" in result
|
||
|
||
|
||
class TestRateLimitMessage:
|
||
"""測試 API 限額警告"""
|
||
|
||
def test_rate_limit_format_basic(self):
|
||
"""測試基本限額警告格式"""
|
||
msg = RateLimitMessage(
|
||
provider="gemini",
|
||
daily_usage=450,
|
||
daily_limit=500,
|
||
token_usage=85000,
|
||
token_limit=100000,
|
||
cost_usd=0.08,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "⚠️" in result
|
||
assert "API 限額警告" in result
|
||
assert "GEMINI API" in result
|
||
assert "450/500" in result
|
||
assert "(90%)" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_rate_limit_with_suggestions(self):
|
||
"""測試含建議的限額警告"""
|
||
msg = RateLimitMessage(
|
||
provider="openai",
|
||
daily_usage=90,
|
||
daily_limit=100,
|
||
suggestions=[
|
||
"考慮切換到 Ollama 優先",
|
||
"或增加每日限額",
|
||
],
|
||
reset_time="明日 00:00",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "建議" in result
|
||
assert "切換到 Ollama" in result
|
||
assert "明日 00:00" in result
|