2346 lines
86 KiB
Python
2346 lines
86 KiB
Python
"""
|
||
test_telegram_message_templates.py - Telegram 訊息模板測試
|
||
|
||
2026-03-29 ogt: 新增，驗證 6 種新訊息模板格式正確
|
||
符合 feedback_no_mock_testing.md 規範 - 測試實際格式化輸出
|
||
"""
|
||
|
||
import pytest
|
||
|
||
import src.services.telegram_gateway as telegram_gateway_module
|
||
from src.services.decision_manager import _format_auto_repair_status_line
|
||
from src.services.telegram_gateway import (
|
||
DailySummaryMessage,
|
||
DeploySuccessMessage,
|
||
RateLimitMessage,
|
||
RepairReportMessage,
|
||
ResourceWarnMessage,
|
||
SentryErrorMessage,
|
||
TelegramGateway,
|
||
TelegramMessage,
|
||
WeeklyReportMessage,
|
||
format_aiops_signal_alert_card,
|
||
format_host_resource_alert_card,
|
||
normalize_telegram_send_message_payload,
|
||
)
|
||
|
||
|
||
def test_auto_repair_status_line_distinguishes_handoff_required() -> None:
|
||
"""自動化失敗 reply 必須明確標示轉人工，且不把 raw error 當純文字噴出。"""
|
||
result = _format_auto_repair_status_line(
|
||
incident_id="INC-20260507-AAAAAA",
|
||
target="node-exporter-110",
|
||
action='ssh 192.168.0.110 "ps aux --sort=-%cpu | head -15"',
|
||
success=False,
|
||
error="Unsupported <scheme> & %d format: a real number is required, not str",
|
||
)
|
||
|
||
assert "HANDOFF REQUIRED｜AI 自動修復失敗，已轉人工" in result
|
||
assert "自動化已停止，不再重試" in result
|
||
assert "請 SRE 依 AwoooP Run / 原告警卡處理" in result
|
||
assert "<scheme> & %d format" in result
|
||
assert "<scheme>" not in result
|
||
|
||
|
||
def test_auto_repair_status_line_distinguishes_auto_resolved() -> None:
|
||
"""自動化成功 reply 必須明確標示已自動解決。"""
|
||
result = _format_auto_repair_status_line(
|
||
incident_id="INC-20260507-BBBBBB",
|
||
target="awoooi-api",
|
||
action="kubectl rollout restart deployment/awoooi-api",
|
||
success=True,
|
||
metrics_delta_text="CPU 92%->30%",
|
||
)
|
||
|
||
assert "AUTO RESOLVED｜AI 自動修復完成" in result
|
||
assert "自動化已完成，等待後驗證觀察" in result
|
||
assert "CPU 92%->30%" in result
|
||
|
||
|
||
def test_host_resource_alert_card_becomes_ai_automation_event_packet() -> None:
|
||
"""Host resource raw dump 必須轉成 AI 自動化事件包，而不是直接洗版。"""
|
||
raw_alert = """WARN h110-gitea 🔴 CPU 警告: used=42.9% load=8.04
|
||
|
||
WARN h110-gitea ⚠️ 容器內 root Node.js 進程:
|
||
root 235 1.3 0.0 1092656 52564 ? Sl 05:55 0:00 npm run build:web
|
||
root 247 1.7 0.0 1093752 54044 ? Sl 05:55 0:00 npm run build
|
||
root 259 184 0.9 28062752 604824 ? Sl 05:55 0:33 node /workspace/wooo/stockplatform-v2/nodemodules/.bin/next build
|
||
root 340 9.8 0.1 2586268 130016 ? Sl 05:56 0:01 node /workspace/wooo/stockplatform-v2/apps/web/.next/build/postcss.js 37591
|
||
root 364 181 0.7 3491396 494608 ? Rl 05:56 0:18 /opt/hostedtoolcache/node/20.20.2/x64/bin/node /workspace/wooo/stockplatform-v2/nodemodules/next/dist/compiled/jest-worker/processChild.js
|
||
"""
|
||
|
||
result = format_host_resource_alert_card(raw_alert)
|
||
|
||
assert "P1 主機資源壓力｜h110-gitea" in result
|
||
assert "ai_automation_alert_card_v1" in result
|
||
assert "一眼摘要" in result
|
||
assert "CPU 使用率" in result
|
||
assert "Load" in result
|
||
assert "容器 root 進程" in result
|
||
assert "AI 自動化判讀" in result
|
||
assert "runner_build_resource_pressure" in result
|
||
assert "candidate_only" in result
|
||
assert "runtime_write_gate=0" in result
|
||
assert "Top evidence" in result
|
||
assert "PID 259" in result
|
||
assert "Next.js build" in result
|
||
assert "PID 364" in result
|
||
assert "Next.js build worker" in result
|
||
assert "建議下一步" in result
|
||
assert "禁止事項" in result
|
||
assert "不 kill process" in result
|
||
assert "root 259" not in result
|
||
assert "/workspace/wooo/" not in result
|
||
assert "processChild.js" not in result
|
||
|
||
|
||
def test_orphan_browser_alert_becomes_runaway_process_event_packet() -> None:
|
||
"""HostOrphanBrowserSmokeHighCpu 必須變成 runaway process 專屬事件包。"""
|
||
raw_alert = (
|
||
'alertname="HostOrphanBrowserSmokeHighCpu" host="110" '
|
||
'rule="stockplatform_headless_smoke" '
|
||
"description=\"orphan Chrome smoke group detected\""
|
||
)
|
||
|
||
result = format_host_resource_alert_card(raw_alert)
|
||
|
||
assert "P3 主機資源壓力｜110" in result
|
||
assert "ai_automation_alert_card_v1" in result
|
||
assert "orphan_browser_smoke_runaway_process" in result
|
||
assert "HostOrphanBrowserSmokeHighCpu" in result
|
||
assert "stockplatform_headless_smoke" in result
|
||
assert "host-runaway-process-remediation.py" in result
|
||
assert "dry-run" in result
|
||
assert "gated SIGTERM" in result
|
||
assert "KM / PlayBook / Verifier" in result
|
||
assert "runtime_write_gate=0" in result
|
||
assert "不 kill process" in result
|
||
assert "Docker" in result
|
||
|
||
|
||
def test_ci_runner_load_alert_becomes_capacity_event_packet() -> None:
|
||
"""HostCiRunnerLoadSaturation 不可被誤導成可 kill 的 runaway process。"""
|
||
raw_alert = (
|
||
'alertname="HostCiRunnerLoadSaturation" host="110" '
|
||
"awoooi_host_gitea_actions_active_container_count 2"
|
||
)
|
||
|
||
result = format_host_resource_alert_card(raw_alert)
|
||
|
||
assert "P3 主機資源壓力｜110" in result
|
||
assert "ci_runner_load_saturation" in result
|
||
assert "CI load evidence packet" in result
|
||
assert "Gitea Actions run" in result
|
||
assert "合法 CI" in result
|
||
assert "不做 process remediation" in result
|
||
assert "runtime_write_gate=0" in result
|
||
assert "不 kill process" in result
|
||
|
||
|
||
def test_wazuh_alert_becomes_aiops_signal_event_packet() -> None:
|
||
"""Wazuh raw alert 不可把內網 IP、完整路徑或 token 直接送進 Telegram。"""
|
||
raw_alert = """
|
||
Wazuh alert fired rule.level=12 agent.name="web-110"
|
||
full_log=/var/ossec/logs/alerts/alerts.json srcip=192.168.0.110
|
||
Authorization: Bearer abcdefghijklmnopqrstuvwxyz token=super-secret
|
||
syscheck integrity changed on /etc/nginx/sites-enabled/awoooi.conf
|
||
"""
|
||
|
||
result = format_aiops_signal_alert_card(raw_alert)
|
||
|
||
assert "P1 AIOps 訊號｜Wazuh 入侵訊號" in result
|
||
assert "ai_automation_alert_card_v1" in result
|
||
assert "wazuh_intrusion_signal" in result
|
||
assert "security_intrusion_triage" in result
|
||
assert "candidate_only" in result
|
||
assert "runtime_write_gate=0" in result
|
||
assert "Top evidence" in result
|
||
assert "Timeline / KM / PlayBook" in result
|
||
assert "不封鎖 IP" in result
|
||
assert "192.168.0.110" not in result
|
||
assert "/var/ossec" not in result
|
||
assert "/etc/nginx" not in result
|
||
assert "abcdefghijkl" not in result
|
||
assert "super-secret" not in result
|
||
|
||
|
||
def test_wazuh_dashboard_api_degraded_alert_becomes_readback_gap_event_packet() -> None:
|
||
"""Wazuh Dashboard/API mismatch 必須轉成讀回退化事件卡，不可誤報成已修復。"""
|
||
raw_alert = """
|
||
wazuh_dashboard_api_readback_degraded dashboard agent list disappeared
|
||
POST /api/check-stored-api status=429 POST /api/check-api status=500
|
||
https://127.0.0.1:55000 is unreachable manager registry readback blocked
|
||
full_log=/var/ossec/logs/alerts/alerts.json Authorization: Bearer abcdefghijklmnopqrstuvwxyz
|
||
"""
|
||
|
||
result = format_aiops_signal_alert_card(raw_alert)
|
||
|
||
assert "P1 AIOps 訊號｜Wazuh Dashboard / API 讀回退化" in result
|
||
assert "ai_automation_alert_card_v1" in result
|
||
assert "wazuh_dashboard_api_readback_degraded" in result
|
||
assert "siem_observability_readback_degraded" in result
|
||
assert "candidate_only" in result
|
||
assert "runtime_write_gate=0" in result
|
||
assert "manager registry 只讀計數" in result
|
||
assert "no-false-green" in result
|
||
assert "不重啟 Wazuh" in result
|
||
assert "不重新註冊 agent" in result
|
||
assert "127.0.0.1:55000" not in result
|
||
assert "/var/ossec" not in result
|
||
assert "abcdefghijkl" not in result
|
||
|
||
|
||
def test_nginx_drift_alert_becomes_public_gateway_event_packet() -> None:
|
||
"""Nginx drift 應進 Public Gateway gate，而不是直接要求 reload。"""
|
||
raw_alert = """
|
||
alertname="nginx_config_drift" target="188-gateway"
|
||
nginx -t failed for /etc/nginx/sites-enabled/awoooi.conf
|
||
diff_url=https://internal.example.invalid/raw/live.conf
|
||
"""
|
||
|
||
result = format_aiops_signal_alert_card(raw_alert)
|
||
|
||
assert "AIOps 訊號｜Nginx 配置漂移" in result
|
||
assert "nginx_config_drift" in result
|
||
assert "public_gateway_config_drift" in result
|
||
assert "redacted live export" in result
|
||
assert "runtime_write_gate=0" in result
|
||
assert "不執行 nginx -t" in result
|
||
assert "不 reload" in result
|
||
assert "/etc/nginx" not in result
|
||
assert "internal.example.invalid" not in result
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("raw_alert", "event_type", "lane"),
|
||
[
|
||
(
|
||
'alertname="BackupAggregateRunFailed" host="110" escrow_missing=5 restic failed',
|
||
"backup_restore_escrow_signal",
|
||
"backup_restore_escrow_triage",
|
||
),
|
||
(
|
||
'alertname="SourceProviderIngestionStale" service="sentry" freshness window stale',
|
||
"provider_freshness_signal",
|
||
"source_provider_freshness_triage",
|
||
),
|
||
(
|
||
"dependency_supply_chain_drift package=next CVE-2026-0001 npm audit vulnerability",
|
||
"supply_chain_drift",
|
||
"supply_chain_drift_review",
|
||
),
|
||
(
|
||
"kali_assessment_signal target=public-gateway nmap finding high risk",
|
||
"kali_assessment_signal",
|
||
"security_assessment_review",
|
||
),
|
||
],
|
||
)
|
||
def test_aiops_signal_formatter_covers_non_host_alert_lanes(
|
||
raw_alert: str,
|
||
event_type: str,
|
||
lane: str,
|
||
) -> None:
|
||
"""非 host 類告警也必須進 AI lane / Gate / Verifier 語義。"""
|
||
result = format_aiops_signal_alert_card(raw_alert)
|
||
|
||
assert "ai_automation_alert_card_v1" in result
|
||
assert event_type in result
|
||
assert lane in result
|
||
assert "candidate_only" in result
|
||
assert "runtime_write_gate=0" in result
|
||
assert "Top evidence" in result
|
||
assert "禁止事項" in result
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_alert_notification_normalizes_host_resource_raw_dump(monkeypatch) -> None:
|
||
"""send_alert_notification 是最後出口，必須自動套用 AI 自動化事件包。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(TelegramGateway, "alert_chat_id", property(lambda _self: "chat"))
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
await gateway.send_alert_notification(
|
||
text=(
|
||
"WARN h110-gitea 🔴 CPU 警告: used=42.9% load=8.04\n"
|
||
"root 259 184 0.9 28062752 604824 ? Sl 05:55 0:33 "
|
||
"node /workspace/wooo/stockplatform-v2/nodemodules/.bin/next build"
|
||
),
|
||
parse_mode="HTML",
|
||
)
|
||
|
||
assert sent_requests
|
||
payload = sent_requests[0][1]
|
||
assert payload["chat_id"] == "chat"
|
||
assert payload["parse_mode"] == "HTML"
|
||
assert "ai_automation_alert_card_v1" in payload["text"]
|
||
assert "AI 自動化判讀" in payload["text"]
|
||
assert "root 259" not in payload["text"]
|
||
assert "/workspace/wooo/" not in payload["text"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_alert_notification_normalizes_aiops_signal_alert(monkeypatch) -> None:
|
||
"""send_alert_notification 也必須把 Nginx / Wazuh 類訊號轉成 AI 事件包。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(TelegramGateway, "alert_chat_id", property(lambda _self: "chat"))
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
await gateway.send_alert_notification(
|
||
text=(
|
||
'alertname="nginx_config_drift" target="188-gateway" '
|
||
"nginx -t failed for /etc/nginx/sites-enabled/awoooi.conf"
|
||
),
|
||
parse_mode="MarkdownV2",
|
||
)
|
||
|
||
payload = sent_requests[0][1]
|
||
assert payload["parse_mode"] == "HTML"
|
||
assert "ai_automation_alert_card_v1" in payload["text"]
|
||
assert "public_gateway_config_drift" in payload["text"]
|
||
assert "runtime_write_gate=0" in payload["text"]
|
||
assert "/etc/nginx" not in payload["text"]
|
||
|
||
|
||
def test_prisma_generate_alert_redacts_raw_process_json_and_urls() -> None:
|
||
"""Prisma generate 類 root Node.js 告警不得把路徑、URL 或 JSON 直接送出。"""
|
||
raw_alert = """WARN h110-gitea 🔴 CPU 警告: used=29.8% load=8.62
|
||
|
||
WARN h110-gitea ⚠️ 容器內 root Node.js 進程:
|
||
root 365 27.5 0.1 1283324 108564 ? Sl 06:27 0:00 node /opt/hostedtoolcache/node/20.20.2/x64/bin/pnpm prisma generate
|
||
root 376 15.5 0.3 11756860 217220 ? Rl 06:27 0:03 node ./node_modules/.bin/../prisma/build/index.js generate
|
||
root 392 0.0 0.0 1096836 53400 ? Ssl 06:27 0:00 /opt/hostedtoolcache/node/20.20.2/x64/bin/node /workspace/wooo/vibework/node_modules/.pnpm/prisma@7.8.0_types+react@18.3.30_react@18.3.30/node_modules/prisma/build/index.js {"product":"prisma","version":"7.8.0","endpoint":"https://checkpoint.prisma.io","command":"generate"}
|
||
"""
|
||
|
||
result = format_host_resource_alert_card(raw_alert)
|
||
|
||
assert "P1 主機資源壓力｜h110-gitea" in result
|
||
assert "runner_prisma_generate_resource_pressure" in result
|
||
assert "Prisma generate" in result
|
||
assert "容器 root 進程：<code>3</code>" in result
|
||
assert "套件來源" in result
|
||
assert "runtime_write_gate=0" in result
|
||
assert "root 365" not in result
|
||
assert "checkpoint.prisma.io" not in result
|
||
assert "node_modules" not in result
|
||
assert "/opt/hostedtoolcache" not in result
|
||
assert "/workspace/wooo" not in result
|
||
assert '"product":"prisma"' not in result
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_alert_notification_forces_html_card_for_markdown_host_alert(monkeypatch) -> None:
|
||
"""即使呼叫端用 Markdown，host raw dump 仍必須被最後出口改成 HTML 卡。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(TelegramGateway, "alert_chat_id", property(lambda _self: "chat"))
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
await gateway.send_alert_notification(
|
||
text=(
|
||
"WARN h110-gitea 🔴 CPU 警告: used=29.8% load=8.62\n"
|
||
"root 365 27.5 0.1 1283324 108564 ? Sl 06:27 0:00 "
|
||
"node /opt/hostedtoolcache/node/20.20.2/x64/bin/pnpm prisma generate"
|
||
),
|
||
parse_mode="MarkdownV2",
|
||
)
|
||
|
||
payload = sent_requests[0][1]
|
||
assert payload["parse_mode"] == "HTML"
|
||
assert "ai_automation_alert_card_v1" in payload["text"]
|
||
assert "runner_prisma_generate_resource_pressure" in payload["text"]
|
||
assert "/opt/hostedtoolcache" not in payload["text"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_text_normalizes_host_resource_alert(monkeypatch) -> None:
|
||
"""send_text 旁路也不能把 host resource raw dump 直接送出。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(TelegramGateway, "alert_chat_id", property(lambda _self: "chat"))
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
await gateway.send_text(
|
||
text=(
|
||
"WARN h110-gitea 🔴 CPU 警告: used=29.8% load=8.62\n"
|
||
"root 365 27.5 0.1 1283324 108564 ? Sl 06:27 0:00 "
|
||
"node /workspace/wooo/vibework/node_modules/.bin/prisma generate"
|
||
),
|
||
)
|
||
|
||
payload = sent_requests[0][1]
|
||
assert payload["parse_mode"] == "HTML"
|
||
assert "P1 主機資源壓力" in payload["text"]
|
||
assert "node_modules" not in payload["text"]
|
||
assert "/workspace/wooo" not in payload["text"]
|
||
|
||
|
||
def test_send_request_payload_normalizer_blocks_direct_host_raw_dump() -> None:
|
||
"""direct _send_request("sendMessage") 也必須在最後出口被轉成事件卡。"""
|
||
raw_alert = (
|
||
"WARN h110-gitea 🔴 CPU 警告: used=80.4% load=19.10\n"
|
||
"WARN h110-gitea ⚠️ 容器內 root Node.js 進程:\n"
|
||
"root 311 185 0.9 29242688 606596 ? Sl 07:11 0:33 "
|
||
"node /workspace/wooo/stockplatform-v2/nodemodules/.bin/next build\n"
|
||
"root 830 0.0 0.0 1126624 57588 ? Rsl 07:14 0:00 "
|
||
'/opt/hostedtoolcache/node/22.22.3/x64/bin/node /workspace/wooo/2026FIFAWorldCup/platform/web/nodemodules/prisma/build/child '
|
||
'{"product":"prisma","version":"5.20.0","endpoint":"https://checkpoint.prisma.io"}'
|
||
)
|
||
|
||
payload = {
|
||
"chat_id": "chat",
|
||
"text": raw_alert,
|
||
"parse_mode": "MarkdownV2",
|
||
"disable_web_page_preview": True,
|
||
}
|
||
|
||
result = normalize_telegram_send_message_payload("sendMessage", payload)
|
||
|
||
assert result["parse_mode"] == "HTML"
|
||
assert "P1 主機資源壓力｜h110-gitea" in result["text"]
|
||
assert "ai_automation_alert_card_v1" in result["text"]
|
||
assert "runtime_write_gate=0" in result["text"]
|
||
assert "root 311" not in result["text"]
|
||
assert "checkpoint.prisma.io" not in result["text"]
|
||
assert "/workspace/wooo" not in result["text"]
|
||
assert "/opt/hostedtoolcache" not in result["text"]
|
||
assert '"product":"prisma"' not in result["text"]
|
||
|
||
|
||
def test_weekly_report_marks_all_zero_as_low_trust_anomaly() -> None:
|
||
report = WeeklyReportMessage(
|
||
week_range="2026-W24",
|
||
report_date="2026-06-12 10:00",
|
||
stats_source_ok=False,
|
||
k3s_source_ok=True,
|
||
git_source_ok=False,
|
||
cost_source_ok=False,
|
||
all_zero_actionable_anomaly=True,
|
||
)
|
||
|
||
body = report.format()
|
||
|
||
assert "報表資料信任度" in body
|
||
assert "判定: <b>低可信</b>" in body
|
||
assert "統計=失效" in body
|
||
assert "Git=失效" in body
|
||
assert "成本=缺資料" in body
|
||
assert "全 0: <code>actionable_anomaly</code>" in body
|
||
assert "資料缺口 / 下一步" in body
|
||
assert "全 0 不是健康" in body
|
||
assert "report-source-gap:stats_api" in body
|
||
assert "report-source-gap:gitea_activity" in body
|
||
assert "告警統計" in body
|
||
|
||
|
||
def test_weekly_report_keeps_nonzero_source_status_visible() -> None:
|
||
report = WeeklyReportMessage(
|
||
week_range="2026-W24",
|
||
report_date="2026-06-12 10:00",
|
||
alert_total=3,
|
||
ai_proposal_count=2,
|
||
commits_count=5,
|
||
deploy_count=1,
|
||
ai_tokens_week=1200,
|
||
stats_source_ok=True,
|
||
k3s_source_ok=True,
|
||
git_source_ok=True,
|
||
cost_source_ok=True,
|
||
)
|
||
|
||
body = report.format()
|
||
|
||
assert "判定: <b>可參考</b>" in body
|
||
assert "全 0: <code>no</code>" in body
|
||
assert "資料源通過" in body
|
||
assert "Commits: <code>5</code>" in body
|
||
assert "Tokens: <code>1,200</code>" in body
|
||
|
||
|
||
def test_weekly_report_includes_report_source_health_assets() -> None:
|
||
report = WeeklyReportMessage(
|
||
week_range="2026-W25",
|
||
report_date="2026-06-18 19:40",
|
||
alert_total=4,
|
||
ai_proposal_count=1,
|
||
commits_count=2,
|
||
deploy_count=1,
|
||
stats_source_ok=True,
|
||
k3s_source_ok=True,
|
||
git_source_ok=True,
|
||
cost_source_ok=False,
|
||
report_source_confidence_percent=40,
|
||
report_source_ok_count=2,
|
||
report_source_total_count=5,
|
||
report_source_gap_ids=[
|
||
"report-source-gap:incident_summary",
|
||
"report-source-gap:resolution_stats",
|
||
"report-source-gap:ai_performance",
|
||
],
|
||
report_asset_state_lines=[
|
||
"KM: draft_ready 3/6",
|
||
"PlayBook: draft_required 0/3",
|
||
"腳本: readback_only 1/1",
|
||
"排程: no_send_preview 3/3",
|
||
"Verifier: source_health_ready 1/4",
|
||
],
|
||
)
|
||
|
||
body = report.format()
|
||
|
||
assert "報表資料源 / 沉澱" in body
|
||
assert "來源: <code>2/5</code>" in body
|
||
assert "信心: <code>40%</code>" in body
|
||
assert "report-source-gap:incident_summary" in body
|
||
assert "report-source-gap:resolution_stats" in body
|
||
assert "report-source-gap:ai_performance" in body
|
||
assert "KM: draft_ready 3/6" in body
|
||
assert "PlayBook: draft_required 0/3" in body
|
||
assert "Verifier: source_health_ready 1/4" in body
|
||
assert "不自動改排程" in body
|
||
|
||
|
||
def test_telegram_html_chunks_preserve_complete_lines() -> None:
|
||
"""歷史/詳情長訊息不得用 text[:500] 切壞 HTML tag。"""
|
||
lines = [
|
||
"📊 <b>事件歷史統計</b>",
|
||
"",
|
||
"<code>INC-20260513-79ED5E</code>",
|
||
"🧭 <b>DB Truth-chain</b>",
|
||
"階段: <code>blocked</code> / <code>manual_required</code>",
|
||
] * 90
|
||
|
||
chunks = telegram_gateway_module._telegram_html_chunks(lines, limit=900)
|
||
|
||
assert len(chunks) > 1
|
||
assert all(len(chunk) <= 900 for chunk in chunks)
|
||
assert all(chunk.count("<code>") == chunk.count("</code>") for chunk in chunks)
|
||
assert all(chunk.count("<b>") == chunk.count("</b>") for chunk in chunks)
|
||
|
||
|
||
def test_telegram_html_chunks_render_single_overlong_html_line_as_safe_text() -> None:
|
||
"""單行過長時不得切出未閉合 <code>，否則 Telegram 會 400。"""
|
||
line = "告警鍵: <code>" + ("node<188>&" * 80) + "</code>"
|
||
|
||
chunks = telegram_gateway_module._telegram_html_chunks([line], limit=120)
|
||
|
||
assert len(chunks) == 1
|
||
assert len(chunks[0]) <= 120
|
||
assert "<code>" not in chunks[0]
|
||
assert "</code>" not in chunks[0]
|
||
assert "告警鍵" in chunks[0]
|
||
assert "<" in chunks[0]
|
||
|
||
|
||
def test_awooop_status_chain_lines_show_verified_auto_repair_stage() -> None:
|
||
"""詳情/歷史要直接說清楚是否已 AI 自動修復、驗證到哪裡。"""
|
||
lines = telegram_gateway_module._format_awooop_status_chain_lines(
|
||
truth_chain={
|
||
"truth_status": {
|
||
"current_stage": "execution_succeeded",
|
||
"stage_status": "success",
|
||
"needs_human": False,
|
||
"blockers": [],
|
||
},
|
||
"automation_quality": {
|
||
"verdict": "auto_repaired_verified",
|
||
"facts": {
|
||
"auto_repair_execution_records": 1,
|
||
"automation_operation_records": 1,
|
||
"verification_result": "healthy",
|
||
"mcp_gateway_total": 3,
|
||
"knowledge_entries": 2,
|
||
},
|
||
"blockers": [],
|
||
},
|
||
},
|
||
remediation_history={
|
||
"total": 1,
|
||
"items": [
|
||
{
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "rollout_restart",
|
||
"required_scope": "write",
|
||
"verification_result_preview": "healthy",
|
||
"writes_incident_state": True,
|
||
"writes_auto_repair_result": True,
|
||
}
|
||
],
|
||
},
|
||
)
|
||
|
||
joined = "\n".join(lines)
|
||
assert "AwoooP 狀態鏈" in joined
|
||
assert "auto_repaired_verified" in joined
|
||
assert "驗證: <code>healthy</code>" in joined
|
||
assert "auto-repair <code>1</code>" in joined
|
||
assert "auto_repair_executor/rollout_restart/write" in joined
|
||
assert "人工: <code>no</code>" in joined
|
||
assert "monitor_for_regression" in joined
|
||
assert "處置結論" in joined
|
||
assert "已驗證自動修復完成" in joined
|
||
assert "自動化資產沉澱" in joined
|
||
assert "KM：<code>2</code>" in joined
|
||
assert "Verifier：<code>ready</code>" in joined
|
||
|
||
|
||
def test_awooop_status_chain_lines_show_read_only_manual_gate() -> None:
|
||
"""只讀試跑不能被說成已自動修復，必須顯示等待審批/人工。"""
|
||
lines = telegram_gateway_module._format_awooop_status_chain_lines(
|
||
truth_chain={
|
||
"truth_status": {
|
||
"current_stage": "approval_required",
|
||
"stage_status": "waiting",
|
||
"needs_human": True,
|
||
"blockers": ["pending_human_approval"],
|
||
},
|
||
"automation_quality": {
|
||
"verdict": "approval_required",
|
||
"facts": {
|
||
"auto_repair_execution_records": 0,
|
||
"automation_operation_records": 0,
|
||
"verification_result": "missing",
|
||
"mcp_gateway_total": 1,
|
||
"knowledge_entries": 0,
|
||
},
|
||
"blockers": [],
|
||
},
|
||
},
|
||
remediation_history={
|
||
"total": 2,
|
||
"items": [
|
||
{
|
||
"agent_id": "investigator",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"verification_result_preview": "degraded",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
},
|
||
)
|
||
|
||
joined = "\n".join(lines)
|
||
assert "read_only_dry_run" in joined
|
||
assert "investigator/ssh_diagnose/read" in joined
|
||
assert "人工: <code>yes</code>" in joined
|
||
assert "approve_or_escalate_from_awooop" in joined
|
||
assert "pending_human_approval" in joined
|
||
assert "自動化資產沉澱" in joined
|
||
assert "PlayBook：<code>--</code>" in joined
|
||
|
||
|
||
def test_awooop_status_chain_lines_do_not_treat_audit_ops_as_repair() -> None:
|
||
lines = telegram_gateway_module._format_awooop_status_chain_lines(
|
||
truth_chain={
|
||
"truth_status": {
|
||
"current_stage": "manual_required",
|
||
"stage_status": "blocked",
|
||
"needs_human": True,
|
||
"blockers": ["approval_resolved_no_action_without_execution"],
|
||
},
|
||
"automation_quality": {
|
||
"verdict": "auto_repaired_verification_degraded",
|
||
"facts": {
|
||
"auto_repair_execution_records": 0,
|
||
"automation_operation_records": 1,
|
||
"effective_execution_records": 0,
|
||
"verification_result": "degraded",
|
||
"mcp_gateway_total": 2,
|
||
"knowledge_entries": 1,
|
||
},
|
||
"blockers": ["verification_recorded"],
|
||
},
|
||
},
|
||
remediation_history={"total": 0},
|
||
)
|
||
|
||
joined = "\n".join(lines)
|
||
assert "diagnostic_or_audit_recorded" in joined
|
||
assert "manual_review_or_collect_repair_evidence" in joined
|
||
assert "executed_pending_verification" not in joined
|
||
assert "處置結論" in joined
|
||
assert "只完成診斷/觀察，尚未證明修復" in joined
|
||
assert "通知：<code>telegram_sre_war_room,awooop_operator_console</code>" in joined
|
||
|
||
|
||
def test_awooop_agent_evidence_lines_show_mcp_source_execution_playbook_km() -> None:
|
||
"""Telegram 詳情/歷史要像前端一樣顯示五段 AI Agent 證據鏈。"""
|
||
lines = telegram_gateway_module._format_awooop_agent_evidence_lines(
|
||
truth_chain={
|
||
"automation_quality": {
|
||
"verdict": "approval_required",
|
||
"facts": {
|
||
"auto_repair_execution_records": 0,
|
||
"automation_operation_records": 1,
|
||
"verification_result": "degraded",
|
||
"mcp_gateway_total": 2,
|
||
"knowledge_entries": 1,
|
||
},
|
||
},
|
||
"mcp": {
|
||
"awooop_gateway": {
|
||
"total": 2,
|
||
"success": 1,
|
||
"failed": 1,
|
||
"blocked": 0,
|
||
"first_class_total": 2,
|
||
"legacy_bridge_total": 0,
|
||
"policy_enforced_total": 2,
|
||
"by_tool": [
|
||
{
|
||
"tool_name": "prometheus.query",
|
||
"total": 2,
|
||
"success": 1,
|
||
"failed": 1,
|
||
"blocked": 0,
|
||
}
|
||
],
|
||
},
|
||
"legacy": {
|
||
"total": 1,
|
||
"success": 1,
|
||
"failed": 0,
|
||
},
|
||
},
|
||
"execution": {
|
||
"automation_operation_log": [
|
||
{
|
||
"operation_type": "ansible_candidate_matched",
|
||
"status": "dry_run",
|
||
"actor": "Hermes",
|
||
"input_executor": "ansible",
|
||
"input_action": "check_mode",
|
||
}
|
||
],
|
||
"ansible": {
|
||
"considered": True,
|
||
"records": [
|
||
{
|
||
"operation_type": "ansible_candidate_matched",
|
||
"status": "dry_run",
|
||
"playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
|
||
"check_mode": True,
|
||
}
|
||
],
|
||
"candidate_catalog": {
|
||
"candidates": [
|
||
{
|
||
"catalog_id": "ansible:188-ai-web",
|
||
"playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
|
||
"risk_level": "medium",
|
||
"match_score": 3,
|
||
}
|
||
]
|
||
},
|
||
},
|
||
},
|
||
},
|
||
remediation_history={
|
||
"total": 1,
|
||
"items": [
|
||
{
|
||
"verification_result_preview": "degraded",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
},
|
||
source_correlation={
|
||
"status": "candidate_found",
|
||
"verification_status": "candidate_only",
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 1,
|
||
"applied_link_total": 0,
|
||
"providers": {
|
||
"sentry": {
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 1,
|
||
"applied_link_total": 0,
|
||
},
|
||
"signoz": {
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 0,
|
||
"applied_link_total": 0,
|
||
},
|
||
},
|
||
},
|
||
)
|
||
|
||
joined = "\n".join(lines)
|
||
assert "AI Agent 證據鏈" in joined
|
||
assert "MCP / 自建 MCP" in joined
|
||
assert "Gateway <code>1/2</code>" in joined
|
||
assert "prometheus.query" in joined
|
||
assert "Sentry/SigNoz" in joined
|
||
assert "candidate <code>1</code>" in joined
|
||
assert "sentry 0/1/0" in joined
|
||
assert "Executor: <code>ansible/dry_run</code>" in joined
|
||
assert "op <code>ansible_candidate_matched</code>" in joined
|
||
assert "PlayBook / Ansible" in joined
|
||
assert "infra/ansible/playbooks/188-ai-web.yml" in joined
|
||
assert "ansible <code>yes</code>" in joined
|
||
assert "check <code>0</code>" in joined
|
||
assert "apply <code>0</code>" in joined
|
||
assert "rc <code>--</code>" in joined
|
||
assert "KM / Learning" in joined
|
||
assert "KM <code>1</code>" in joined
|
||
assert "verify <code>degraded</code>" in joined
|
||
|
||
|
||
def test_awooop_status_chain_lines_show_ansible_apply_proof() -> None:
|
||
"""詳情/歷史必須能看出 Ansible 是否真的 apply、由誰批准、returncode 幾。"""
|
||
truth_chain = {
|
||
"truth_status": {
|
||
"current_stage": "execution_succeeded",
|
||
"stage_status": "success",
|
||
"needs_human": False,
|
||
"blockers": [],
|
||
},
|
||
"automation_quality": {
|
||
"verdict": "execution_succeeded",
|
||
"facts": {
|
||
"auto_repair_execution_records": 0,
|
||
"automation_operation_records": 2,
|
||
"effective_execution_records": 1,
|
||
"verification_result": "healthy",
|
||
"mcp_gateway_total": 2,
|
||
"knowledge_entries": 1,
|
||
},
|
||
"blockers": [],
|
||
},
|
||
"execution": {
|
||
"automation_operation_log": [
|
||
{
|
||
"operation_type": "ansible_apply_executed",
|
||
"status": "success",
|
||
"actor": "platform_operator",
|
||
"input_executor": "ansible",
|
||
"input_action": "apply_playbook",
|
||
"input_catalog_id": "ansible:188-momo-backup-user",
|
||
"input_playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml",
|
||
}
|
||
],
|
||
"ansible": {
|
||
"considered": True,
|
||
"records": [
|
||
{
|
||
"operation_type": "ansible_apply_executed",
|
||
"status": "success",
|
||
"actor": "platform_operator",
|
||
"catalog_id": "ansible:188-momo-backup-user",
|
||
"playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml",
|
||
"execution_mode": "apply",
|
||
"check_mode": False,
|
||
"apply_executed": True,
|
||
"approval_source": "user_chat_approved_continue",
|
||
"returncode": 0,
|
||
},
|
||
{
|
||
"operation_type": "ansible_check_mode_executed",
|
||
"status": "success",
|
||
"actor": "ansible_check_mode_worker",
|
||
"catalog_id": "ansible:188-momo-backup-user",
|
||
"playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml",
|
||
"execution_mode": "check_mode",
|
||
"check_mode": True,
|
||
"apply_executed": False,
|
||
"returncode": 0,
|
||
},
|
||
],
|
||
"candidate_catalog": {"candidates": []},
|
||
},
|
||
},
|
||
}
|
||
|
||
lines = telegram_gateway_module._format_awooop_status_chain_lines(
|
||
truth_chain=truth_chain,
|
||
remediation_history={"total": 0},
|
||
)
|
||
joined = "\n".join(lines)
|
||
assert "Ansible: check <code>1</code> / apply <code>1</code>" in joined
|
||
assert "ansible_apply_executed</code>/<code>success</code>" in joined
|
||
assert "rc <code>0</code>" in joined
|
||
assert "PlayBook: <code>ansible:188-momo-backup-user</code>" in joined
|
||
assert "approval <code>user_chat_approved_continue</code>" in joined
|
||
|
||
snapshot = telegram_gateway_module._callback_reply_awooop_execution_snapshot(truth_chain)
|
||
ansible = snapshot["ansible"]
|
||
assert ansible["check_mode_total"] == 1
|
||
assert ansible["apply_total"] == 1
|
||
assert ansible["applied"] is True
|
||
assert ansible["controlled_apply"] is True
|
||
assert ansible["latest_catalog_id"] == "ansible:188-momo-backup-user"
|
||
assert ansible["latest_execution_mode"] == "apply"
|
||
assert ansible["latest_returncode"] == 0
|
||
assert ansible["approval_source"] == "user_chat_approved_continue"
|
||
|
||
|
||
def test_callback_reply_awooop_status_chain_snapshot_marks_manual_gate() -> None:
|
||
"""Callback evidence 要保存當下 AwoooP 狀態鏈，不只保存 live query 結果。"""
|
||
snapshot = telegram_gateway_module._callback_reply_awooop_status_chain_snapshot(
|
||
incident_id="INC-20260514-F85F21",
|
||
truth_chain={
|
||
"truth_status": {
|
||
"current_stage": "approval_required",
|
||
"stage_status": "waiting",
|
||
"needs_human": True,
|
||
"blockers": ["pending_human_approval"],
|
||
},
|
||
"automation_quality": {
|
||
"verdict": "approval_required",
|
||
"facts": {
|
||
"auto_repair_execution_records": 0,
|
||
"automation_operation_records": 0,
|
||
"verification_result": "missing",
|
||
"mcp_gateway_total": 1,
|
||
"knowledge_entries": 0,
|
||
},
|
||
"blockers": [],
|
||
},
|
||
"mcp": {
|
||
"awooop_gateway": {
|
||
"total": 2,
|
||
"success": 1,
|
||
"failed": 1,
|
||
"blocked": 0,
|
||
"first_class_total": 2,
|
||
"legacy_bridge_total": 0,
|
||
"policy_enforced_total": 2,
|
||
"by_tool": [
|
||
{
|
||
"tool_name": "prometheus.query",
|
||
"total": 2,
|
||
"success": 1,
|
||
"failed": 1,
|
||
"blocked": 0,
|
||
}
|
||
],
|
||
},
|
||
"legacy": {
|
||
"total": 1,
|
||
"success": 1,
|
||
"failed": 0,
|
||
},
|
||
},
|
||
"execution": {
|
||
"automation_operation_log": [
|
||
{
|
||
"operation_type": "playbook_previewed",
|
||
"status": "waiting_approval",
|
||
"actor": "Hermes",
|
||
"input_executor": "ansible",
|
||
"input_playbook_id": "pb-host-restart",
|
||
"input_playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
|
||
}
|
||
],
|
||
"ansible": {
|
||
"considered": True,
|
||
"records": [
|
||
{
|
||
"operation_type": "ansible_check_mode_previewed",
|
||
"status": "waiting_approval",
|
||
"playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
|
||
"check_mode": True,
|
||
}
|
||
],
|
||
"candidate_catalog": {
|
||
"candidates": [
|
||
{
|
||
"catalog_id": "ansible:188-ai-web",
|
||
"playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
|
||
"risk_level": "medium",
|
||
"match_score": 3,
|
||
}
|
||
]
|
||
},
|
||
},
|
||
},
|
||
"channel": {
|
||
"inbound_events": [
|
||
{
|
||
"channel_type": "sentry",
|
||
"provider_event_id": "sentry:ISSUE-1",
|
||
"source_envelope": {
|
||
"source_refs": {
|
||
"sentry_issue_ids": ["ISSUE-1"],
|
||
"signoz_alerts": ["signoz:alert-1"],
|
||
"fingerprints": ["fp-1"],
|
||
}
|
||
},
|
||
}
|
||
],
|
||
"outbound_messages": [],
|
||
},
|
||
},
|
||
remediation_history={
|
||
"total": 1,
|
||
"items": [
|
||
{
|
||
"agent_id": "investigator",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"safety_level": "read_only",
|
||
"verification_result_preview": "degraded",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
},
|
||
source_correlation={
|
||
"schema_version": "source_provider_correlation_v1",
|
||
"status": "candidate_found",
|
||
"verification_status": "candidate_only",
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 1,
|
||
"applied_link_total": 0,
|
||
"provider_event_total": 1,
|
||
"latest_applied_link_at": None,
|
||
"providers": {
|
||
"sentry": {
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 1,
|
||
"applied_link_total": 0,
|
||
"latest_event_at": "2026-05-20T00:00:00Z",
|
||
},
|
||
"signoz": {
|
||
"direct_ref_total": 0,
|
||
"candidate_total": 0,
|
||
"applied_link_total": 0,
|
||
"latest_event_at": None,
|
||
},
|
||
},
|
||
"top_candidates": [
|
||
{
|
||
"provider": "sentry",
|
||
"provider_event_id": "sentry:ISSUE-1",
|
||
"score": 2,
|
||
"link_state": "candidate",
|
||
}
|
||
],
|
||
},
|
||
)
|
||
|
||
assert snapshot is not None
|
||
assert snapshot["schema_version"] == (
|
||
"awooop_status_chain_callback_reply_snapshot_v1"
|
||
)
|
||
assert snapshot["repair_state"] == "read_only_dry_run"
|
||
assert snapshot["needs_human"] is True
|
||
assert snapshot["next_step"] == "approve_or_escalate_from_awooop"
|
||
assert snapshot["evidence"]["mcp_gateway_total"] == 1
|
||
assert snapshot["evidence"]["latest_route"] == "investigator/ssh_diagnose/read"
|
||
assert snapshot["writes"]["incident"] is False
|
||
assert snapshot["blockers"] == ["pending_human_approval"]
|
||
assert snapshot["mcp"]["gateway"]["policy_enforced_total"] == 2
|
||
assert snapshot["mcp"]["top_tools"][0]["tool_name"] == "prometheus.query"
|
||
assert snapshot["execution"]["latest_executor"] == "ansible"
|
||
assert snapshot["execution"]["playbook_paths"] == [
|
||
"infra/ansible/playbooks/188-ai-web.yml"
|
||
]
|
||
assert snapshot["execution"]["ansible"]["candidate_count"] == 1
|
||
assert snapshot["source_refs"]["refs"]["sentry_issue_ids"] == ["ISSUE-1"]
|
||
assert snapshot["source_refs"]["refs"]["signoz_alerts"] == ["signoz:alert-1"]
|
||
assert snapshot["source_refs"]["correlation"]["status"] == "candidate_found"
|
||
|
||
|
||
def test_km_stale_completion_lines_show_owner_review_queue_state() -> None:
|
||
"""詳情/歷史要顯示 KM owner-review completion queue 是否卡住或可處理。"""
|
||
lines = telegram_gateway_module._format_km_stale_completion_lines({
|
||
"schema_version": "km_stale_owner_review_completion_telegram_summary_v1",
|
||
"status": "ok",
|
||
"incident_id": "INC-20260513-205814",
|
||
"ready_count": 10,
|
||
"blocked_count": 0,
|
||
"completed_count": 1,
|
||
"failed_count": 0,
|
||
"writes_on_read": False,
|
||
"batch_writes_allowed": False,
|
||
"items": [
|
||
{
|
||
"entry_id": "bf81a30d-6abe-4c0c-b4ba-9c0ba0d761bf",
|
||
"readiness": "ready",
|
||
"next_action": "preview_stale_km_review_completion",
|
||
}
|
||
],
|
||
})
|
||
|
||
joined = "\n".join(lines)
|
||
assert "KM Owner Review" in joined
|
||
assert "ready <code>10</code>" in joined
|
||
assert "batch-write <code>no</code>" in joined
|
||
assert "bf81a30d-6abe-4c0c-b4ba-9c0ba0d761bf" in joined
|
||
assert "preview_stale_km_review_completion" in joined
|
||
|
||
|
||
def test_km_stale_completion_lines_show_callback_triage_state() -> None:
|
||
"""詳情/歷史要顯示 callback owner-review 缺口的流程、主責與卡點。"""
|
||
lines = telegram_gateway_module._format_km_stale_completion_lines({
|
||
"schema_version": "km_stale_owner_review_completion_telegram_summary_v1",
|
||
"status": "no_related_owner_review",
|
||
"incident_id": "INC-20260524-16109D",
|
||
"ready_count": 10,
|
||
"blocked_count": 0,
|
||
"completed_count": 1,
|
||
"failed_count": 0,
|
||
"writes_on_read": False,
|
||
"batch_writes_allowed": False,
|
||
"items": [],
|
||
"work_item": {
|
||
"schema_version": "km_stale_callback_owner_review_work_item_v1",
|
||
"status": "open",
|
||
"triage": {
|
||
"schema_version": "km_stale_callback_owner_review_triage_v1",
|
||
"flow_stage": "callback_observed_owner_review_link_missing",
|
||
"ai_lead_agent": "Hermes",
|
||
"supporting_agents": ["OpenClaw", "ElephantAlpha"],
|
||
"automation_state": "manual_owner_review_required",
|
||
"safe_to_auto_repair": False,
|
||
"blocking_reason": "no_matching_completion_item",
|
||
"matching_strategy": "related_incident_id_exact_match",
|
||
},
|
||
},
|
||
})
|
||
|
||
joined = "\n".join(lines)
|
||
assert "no_related_owner_review" in joined
|
||
assert "callback_observed_owner_review_link_missing" in joined
|
||
assert "related_incident_id_exact_match" in joined
|
||
assert "Hermes" in joined
|
||
assert "OpenClaw / ElephantAlpha" in joined
|
||
assert "manual_owner_review_required" in joined
|
||
assert "safe-auto <code>no</code>" in joined
|
||
assert "no_matching_completion_item" in joined
|
||
|
||
|
||
def test_awooop_runs_url_for_incident_uses_public_incident_filter() -> None:
|
||
"""Telegram URL button 必須導到公開 AwoooP Run list，並帶 incident filter。"""
|
||
url = telegram_gateway_module._awooop_runs_url_for_incident("INC-20260514-F85F21")
|
||
|
||
assert url == (
|
||
"https://awoooi.wooo.work/zh-TW/awooop/runs"
|
||
"?project_id=awoooi&incident_id=INC-20260514-F85F21"
|
||
)
|
||
|
||
|
||
def test_awooop_alerts_url_for_incident_is_canonical_truth_chain() -> None:
|
||
"""Telegram 詳情/歷史要能直接導到 Alerts Incident truth-chain。"""
|
||
url = telegram_gateway_module._awooop_alerts_url_for_incident(
|
||
"INC-20260514-F85F21"
|
||
)
|
||
|
||
assert url == (
|
||
"https://awoooi.wooo.work/zh-TW/alerts"
|
||
"?project_id=awoooi&incident_id=INC-20260514-F85F21"
|
||
)
|
||
|
||
|
||
def test_awooop_reply_markup_prefers_truth_chain_then_runs() -> None:
|
||
"""詳情/歷史 callback reply 的第一顆 URL 要是告警真相鏈。"""
|
||
reply_markup = telegram_gateway_module._awooop_runs_reply_markup(
|
||
"INC-20260514-F85F21"
|
||
)
|
||
|
||
assert reply_markup == {
|
||
"inline_keyboard": [[
|
||
{
|
||
"text": "🔎 真相鏈",
|
||
"url": (
|
||
"https://awoooi.wooo.work/zh-TW/alerts"
|
||
"?project_id=awoooi&incident_id=INC-20260514-F85F21"
|
||
),
|
||
},
|
||
{
|
||
"text": "🧭 Runs",
|
||
"url": (
|
||
"https://awoooi.wooo.work/zh-TW/awooop/runs"
|
||
"?project_id=awoooi&incident_id=INC-20260514-F85F21"
|
||
),
|
||
},
|
||
]]
|
||
}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_build_inline_keyboard_includes_awooop_deep_link() -> None:
|
||
"""主告警卡的 read-only 按鈕列要能回到 Incident truth-chain。"""
|
||
gateway = TelegramGateway()
|
||
|
||
keyboard = await gateway._build_inline_keyboard(
|
||
approval_id="INC-20260514-F85F21",
|
||
include_auto_tuning=False,
|
||
incident_id="INC-20260514-F85F21",
|
||
)
|
||
buttons = [
|
||
button
|
||
for row in keyboard["inline_keyboard"]
|
||
for button in row
|
||
]
|
||
|
||
assert {
|
||
"text": "🔎 真相鏈",
|
||
"url": (
|
||
"https://awoooi.wooo.work/zh-TW/alerts"
|
||
"?project_id=awoooi&incident_id=INC-20260514-F85F21"
|
||
),
|
||
} in buttons
|
||
assert {
|
||
"text": "🧭 Runs",
|
||
"url": (
|
||
"https://awoooi.wooo.work/zh-TW/awooop/runs"
|
||
"?project_id=awoooi&incident_id=INC-20260514-F85F21"
|
||
),
|
||
} in buttons
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_build_inline_keyboard_hides_approval_for_no_action() -> None:
|
||
"""OBSERVE / NO_ACTION 卡片不能提供會誤導成修復執行的批准入口。"""
|
||
gateway = TelegramGateway()
|
||
|
||
keyboard = await gateway._build_inline_keyboard(
|
||
approval_id="approval-no-repair-1",
|
||
include_auto_tuning=False,
|
||
incident_id="INC-20260611-NOOP",
|
||
suggested_action="NO_ACTION - REPAIR_CANDIDATE_MISSING: LLM 分析失敗",
|
||
)
|
||
buttons = [
|
||
button
|
||
for row in keyboard["inline_keyboard"]
|
||
for button in row
|
||
]
|
||
button_texts = {button["text"] for button in buttons}
|
||
|
||
assert "✅ 批准" not in button_texts
|
||
assert "❌ 拒絕" not in button_texts
|
||
assert "🧰 處置包" in button_texts
|
||
assert "🔄 重診" in button_texts
|
||
assert "🔕 靜默" in button_texts
|
||
assert {
|
||
"text": "🧭 Runs",
|
||
"url": (
|
||
"https://awoooi.wooo.work/zh-TW/awooop/runs"
|
||
"?project_id=awoooi&incident_id=INC-20260611-NOOP"
|
||
),
|
||
} in buttons
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_request_strips_awooop_callback_metadata_before_telegram_api(monkeypatch):
|
||
"""AwoooP truth-chain metadata must be mirrored, not sent to Telegram Bot API."""
|
||
captured = {}
|
||
gateway = TelegramGateway()
|
||
gateway._initialized = True
|
||
|
||
class FakeResponse:
|
||
def raise_for_status(self):
|
||
return None
|
||
|
||
def json(self):
|
||
return {"ok": True, "result": {"message_id": 456}}
|
||
|
||
class FakeClient:
|
||
async def post(self, url, json):
|
||
captured["payload"] = dict(json)
|
||
return FakeResponse()
|
||
|
||
async def fake_mirror_outbound_message(
|
||
*,
|
||
method,
|
||
payload,
|
||
provider_message_id,
|
||
source_envelope_extra=None,
|
||
):
|
||
captured["mirror"] = {
|
||
"method": method,
|
||
"payload": dict(payload),
|
||
"provider_message_id": provider_message_id,
|
||
"source_envelope_extra": source_envelope_extra,
|
||
}
|
||
|
||
gateway._http_client = FakeClient()
|
||
monkeypatch.setattr(gateway, "_mirror_outbound_message", fake_mirror_outbound_message)
|
||
|
||
result = await gateway._send_request(
|
||
"sendMessage",
|
||
{
|
||
"chat_id": "chat",
|
||
"text": "事件歷史統計 INC-20260513-79ED5E",
|
||
"_skip_incident_thread_reply": True,
|
||
"_awooop_source_envelope_extra": {
|
||
"callback_reply": {
|
||
"status": "callback_reply_sent",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
},
|
||
"awooop_status_chain": {
|
||
"schema_version": "awooop_status_chain_callback_reply_snapshot_v1",
|
||
"repair_state": "read_only_dry_run",
|
||
"needs_human": True,
|
||
"evidence": {
|
||
"auto_repair_records": 0,
|
||
"operation_records": 0,
|
||
},
|
||
},
|
||
"km_stale_completion_summary": {
|
||
"schema_version": (
|
||
"km_stale_owner_review_callback_reply_snapshot_v1"
|
||
),
|
||
"status": "no_related_owner_review",
|
||
"ready_count": 2,
|
||
"triage": {
|
||
"flow_stage": "callback_observed_owner_review_link_missing",
|
||
},
|
||
},
|
||
},
|
||
},
|
||
)
|
||
|
||
assert result["ok"] is True
|
||
assert "_awooop_source_envelope_extra" not in captured["payload"]
|
||
assert "_skip_incident_thread_reply" not in captured["payload"]
|
||
assert captured["mirror"]["source_envelope_extra"]["callback_reply"]["status"] == (
|
||
"callback_reply_sent"
|
||
)
|
||
assert captured["mirror"]["source_envelope_extra"]["awooop_status_chain"][
|
||
"repair_state"
|
||
] == "read_only_dry_run"
|
||
assert captured["mirror"]["source_envelope_extra"][
|
||
"km_stale_completion_summary"
|
||
]["ready_count"] == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_html_line_message_falls_back_to_plain_text_on_parse_error(monkeypatch):
|
||
"""Telegram HTML parse 400 時要送純文字 fallback，不可回報成歷史查詢失敗。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
reply_markup = telegram_gateway_module._awooop_runs_reply_markup("INC-20260514-F85F21")
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
if payload.get("parse_mode") == "HTML":
|
||
raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400")
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
await gateway._send_html_line_message(
|
||
["📊 <b>事件歷史統計</b>", "階段: <code>blocked</code>"],
|
||
chat_id="chat",
|
||
failure_context="test_history",
|
||
reply_markup=reply_markup,
|
||
)
|
||
|
||
assert len(sent_requests) == 2
|
||
assert sent_requests[0][1]["parse_mode"] == "HTML"
|
||
assert sent_requests[0][1]["reply_markup"] == reply_markup
|
||
assert "parse_mode" not in sent_requests[1][1]
|
||
assert sent_requests[1][1]["reply_markup"] == reply_markup
|
||
assert "<code>" not in sent_requests[1][1]["text"]
|
||
assert "blocked" in sent_requests[1][1]["text"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch):
|
||
"""詳情/歷史 callback reply 要在 AwoooP envelope 標明 sent/fallback 狀態。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
reply_markup = telegram_gateway_module._awooop_runs_reply_markup("INC-20260514-F85F21")
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
if payload.get("parse_mode") == "HTML":
|
||
raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400")
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
await gateway._send_html_line_message(
|
||
["📊 <b>事件歷史統計</b>", "🔖 <code>INC-20260514-F85F21</code>"],
|
||
chat_id="chat",
|
||
failure_context="incident_history",
|
||
reply_markup=reply_markup,
|
||
incident_id="INC-20260514-F85F21",
|
||
callback_action="history",
|
||
awooop_status_chain={
|
||
"schema_version": "awooop_status_chain_callback_reply_snapshot_v1",
|
||
"source": "telegram_callback_reply_snapshot",
|
||
"source_id": "INC-20260514-F85F21",
|
||
"incident_ids": ["INC-20260514-F85F21"],
|
||
"current_stage": "approval_required",
|
||
"stage_status": "waiting",
|
||
"verdict": "approval_required",
|
||
"repair_state": "read_only_dry_run",
|
||
"verification": "missing",
|
||
"needs_human": True,
|
||
"next_step": "approve_or_escalate_from_awooop",
|
||
"evidence": {
|
||
"auto_repair_records": 0,
|
||
"operation_records": 0,
|
||
"mcp_gateway_total": 1,
|
||
"knowledge_entries": 0,
|
||
},
|
||
"writes": {
|
||
"incident": False,
|
||
"auto_repair": False,
|
||
},
|
||
},
|
||
km_stale_completion_summary={
|
||
"schema_version": "km_stale_owner_review_completion_callback_summary_v1",
|
||
"project_id": "awoooi",
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"status": "no_related_owner_review",
|
||
"ready_count": 3,
|
||
"blocked_count": 1,
|
||
"completed_count": 2,
|
||
"failed_count": 0,
|
||
"writes_on_read": False,
|
||
"batch_writes_allowed": False,
|
||
"manual_review_required": True,
|
||
"related_total": 0,
|
||
"work_item": {
|
||
"triage": {
|
||
"schema_version": "km_stale_callback_owner_review_triage_v1",
|
||
"flow_stage": "callback_observed_owner_review_link_missing",
|
||
"ai_lead_agent": "Hermes",
|
||
"supporting_agents": ["OpenClaw", "ElephantAlpha"],
|
||
"automation_state": "manual_owner_review_required",
|
||
"safe_to_auto_repair": False,
|
||
"blocking_reason": "no_matching_completion_item",
|
||
"matching_strategy": "related_incident_id_exact_match",
|
||
},
|
||
},
|
||
},
|
||
)
|
||
|
||
first_source_extra = sent_requests[0][1]["_awooop_source_envelope_extra"]
|
||
fallback_source_extra = sent_requests[1][1]["_awooop_source_envelope_extra"]
|
||
first_extra = first_source_extra["callback_reply"]
|
||
fallback_extra = fallback_source_extra["callback_reply"]
|
||
|
||
assert first_extra["status"] == "callback_reply_sent"
|
||
assert first_extra["action"] == "history"
|
||
assert first_extra["parse_mode"] == "HTML"
|
||
assert first_source_extra["awooop_status_chain"]["repair_state"] == (
|
||
"read_only_dry_run"
|
||
)
|
||
assert first_source_extra["awooop_status_chain"]["needs_human"] is True
|
||
assert first_source_extra["km_stale_completion_summary"]["ready_count"] == 3
|
||
assert first_source_extra["km_stale_completion_summary"]["triage"]["flow_stage"] == (
|
||
"callback_observed_owner_review_link_missing"
|
||
)
|
||
assert fallback_extra["status"] == "callback_reply_fallback_sent"
|
||
assert fallback_extra["incident_id"] == "INC-20260514-F85F21"
|
||
assert fallback_extra["parse_mode"] == "plain_text"
|
||
assert fallback_source_extra["awooop_status_chain"]["next_step"] == (
|
||
"approve_or_escalate_from_awooop"
|
||
)
|
||
assert fallback_source_extra["km_stale_completion_summary"]["triage"][
|
||
"ai_lead_agent"
|
||
] == "Hermes"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_html_line_message_uses_rescue_when_markup_fallback_fails(monkeypatch):
|
||
"""詳情/歷史 fallback 若仍被 Telegram 拒收，要再送無按鈕純文字救援。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
reply_markup = telegram_gateway_module._awooop_runs_reply_markup("INC-20260514-F85F21")
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
if len(sent_requests) < 3:
|
||
raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400")
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
await gateway._send_html_line_message(
|
||
["📊 <b>事件歷史統計</b>", "🔖 <code>INC-20260514-F85F21</code>"],
|
||
chat_id="chat",
|
||
failure_context="test_history",
|
||
reply_markup=reply_markup,
|
||
)
|
||
|
||
assert len(sent_requests) == 3
|
||
assert sent_requests[1][1]["reply_markup"] == reply_markup
|
||
assert "reply_markup" not in sent_requests[2][1]
|
||
assert sent_requests[2][1]["_skip_incident_thread_reply"] is True
|
||
assert "<code>" not in sent_requests[2][1]["text"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_html_line_message_attaches_awooop_markup_to_first_chunk(monkeypatch):
|
||
"""詳情/歷史這類 HTML reply 要能帶 AwoooP evidence URL，且長訊息只掛第一段。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
reply_markup = telegram_gateway_module._awooop_runs_reply_markup("INC-20260514-F85F21")
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
await gateway._send_html_line_message(
|
||
["📊 <b>事件歷史統計</b>"] + [
|
||
f"<code>INC-20260514-F85F21 trace line {index:03d}</code>"
|
||
for index in range(80)
|
||
],
|
||
chat_id="chat",
|
||
failure_context="test_history",
|
||
reply_markup=reply_markup,
|
||
)
|
||
|
||
assert len(sent_requests) > 1
|
||
assert sent_requests[0][1]["reply_markup"] == reply_markup
|
||
assert all("reply_markup" not in payload for _, payload in sent_requests[1:])
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_info_callback_sends_history_when_answer_callback_is_stale(monkeypatch):
|
||
"""Telegram answerCallbackQuery 400 不得阻斷詳情/歷史 DB truth-chain reply。"""
|
||
gateway = TelegramGateway()
|
||
sent_history = []
|
||
|
||
monkeypatch.setattr(gateway._security, "is_whitelisted", lambda _user_id: True)
|
||
|
||
async def stale_answer(*_args, **_kwargs):
|
||
raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400")
|
||
|
||
async def fake_history(incident_id: str):
|
||
sent_history.append(incident_id)
|
||
|
||
monkeypatch.setattr(gateway, "_answer_callback", stale_answer)
|
||
monkeypatch.setattr(gateway, "_send_incident_history", fake_history)
|
||
|
||
result = await gateway.handle_callback(
|
||
callback_query_id="stale-callback",
|
||
callback_data="history:INC-20260513-79ED5E",
|
||
user_id=123456,
|
||
message_id=789,
|
||
)
|
||
|
||
assert result["success"] is True
|
||
assert result["info_action"] is True
|
||
assert sent_history == ["INC-20260513-79ED5E"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_notification_retries_plain_text_on_html_parse_400(monkeypatch):
|
||
"""簡短 HTML 若被 Telegram 拒收，要用純文字重送，不回報成 HTTP 400。"""
|
||
gateway = TelegramGateway()
|
||
sent_requests = []
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
if len(sent_requests) == 1:
|
||
raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400")
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
result = await gateway.send_notification(
|
||
"⚠️ 無法取得歷史統計: <b>broken",
|
||
chat_id="chat",
|
||
)
|
||
|
||
assert result == {"ok": True}
|
||
assert sent_requests[0][1]["parse_mode"] == "HTML"
|
||
assert "parse_mode" not in sent_requests[1][1]
|
||
assert sent_requests[1][1]["text"] == "⚠️ 無法取得歷史統計: broken"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_notification_long_html_uses_plain_text_without_cutting_tags(monkeypatch):
|
||
"""send_notification 的 500 字限制不可切壞 HTML tag。"""
|
||
gateway = TelegramGateway()
|
||
sent_requests = []
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
long_text = "📊 <b>事件歷史統計</b>\n告警鍵: <code>" + ("node<188>&" * 90) + "</code>"
|
||
|
||
await gateway.send_notification(long_text, chat_id="chat")
|
||
|
||
assert len(sent_requests) == 1
|
||
payload = sent_requests[0][1]
|
||
assert "parse_mode" not in payload
|
||
assert len(payload["text"]) <= 500
|
||
assert "<code>" not in payload["text"]
|
||
assert "</code>" not in payload["text"]
|
||
assert "事件歷史統計" in payload["text"]
|
||
|
||
|
||
class TestTelegramMessageFormat:
|
||
"""測試現有 TelegramMessage 格式化"""
|
||
|
||
def test_telegram_message_format_basic(self):
|
||
"""測試基本訊息格式化"""
|
||
msg = TelegramMessage(
|
||
status_emoji="🚨",
|
||
risk_level="CRITICAL",
|
||
resource_name="test-pod-123",
|
||
root_cause="Test root cause",
|
||
suggested_action="Restart pod",
|
||
estimated_downtime="~30s",
|
||
approval_id="INC-20260329-0001",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "🚨" in result
|
||
assert "嚴重" in result
|
||
assert "test-pod-123" in result
|
||
assert "處置狀態" in result
|
||
assert "規則建議待審批" in result
|
||
assert "AI 自動化鏈路" in result
|
||
assert "OpenClaw" in result
|
||
assert "NemoTron" in result
|
||
assert "ElephantAlpha" in result
|
||
assert "自動化資產沉澱" in result
|
||
assert "KM" in result
|
||
assert "PlayBook" in result
|
||
assert "腳本/Ansible" in result
|
||
assert "排程/來源" in result
|
||
assert "Verifier" in result
|
||
assert len(result) <= 4096 # Telegram HTML message limit
|
||
|
||
def test_telegram_message_ai_proposal_marks_approval_wait(self):
|
||
"""有 AI 信心分數的修復建議必須標示為 AI 待審批。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="⚠️",
|
||
risk_level="MEDIUM",
|
||
resource_name="awoooi-api",
|
||
root_cause="CPU sustained high",
|
||
suggested_action="kubectl rollout restart deployment/awoooi-api",
|
||
estimated_downtime="~30s",
|
||
approval_id="INC-20260506-0000",
|
||
confidence=0.82,
|
||
ai_provider="ollama_gcp_a",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "處置狀態" in result
|
||
assert "AI 已提出修復建議，等待人工批准" in result
|
||
|
||
def test_telegram_message_no_action_marks_manual_judgement(self):
|
||
"""NO_ACTION 卡片必須一眼看得出需要人工處置包。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="ℹ️",
|
||
risk_level="LOW",
|
||
resource_name="node-exporter-110",
|
||
root_cause="規則命中但沒有安全可執行動作",
|
||
suggested_action="NO_ACTION",
|
||
estimated_downtime="unknown",
|
||
approval_id="INC-20260506-0001",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "處置狀態" in result
|
||
assert "未自動修復，已產生人工處置包" in result
|
||
assert "人工處置包" in result
|
||
assert "補證據：node_exporter target up" in result
|
||
assert "AwoooP 建立修復候選" in result
|
||
assert "沉澱資產：KM、PlayBook、腳本/Ansible、排程/監控規則、Verifier 結果" in result
|
||
assert "Runs、Work Items、Knowledge Base 要顯示資產 ID、owner、狀態與下一步" in result
|
||
assert "等待人工批准" not in result
|
||
|
||
def test_telegram_message_diagnosis_state_is_not_auto_repair(self):
|
||
"""SSH 只讀診斷 lane 不得被顯示成自動修復。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="⚠️",
|
||
risk_level="MEDIUM",
|
||
resource_name="node-110",
|
||
root_cause="SSH diagnosis collected",
|
||
suggested_action="ssh 192.168.0.110 'uptime'",
|
||
estimated_downtime="unknown",
|
||
approval_id="INC-20260506-DIAG",
|
||
automation_state="diagnosis_collected_manual_required",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "AI 已完成只讀診斷，需人工判斷" in result
|
||
assert "AI 自動修復失敗" not in result
|
||
|
||
def test_telegram_message_diagnosis_failure_state(self):
|
||
"""SSH 診斷工具失敗必須標成診斷失敗，而不是修復失敗。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="🚨",
|
||
risk_level="CRITICAL",
|
||
resource_name="node-110",
|
||
root_cause="SSH MCP execution failed",
|
||
suggested_action="ssh 192.168.0.110 'uptime'",
|
||
estimated_downtime="unknown",
|
||
approval_id="INC-20260506-DIAGFAIL",
|
||
automation_state="diagnosis_failed_manual_required",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "AI 診斷工具失敗，需人工排查" in result
|
||
assert "AI 自動修復失敗" not in result
|
||
|
||
def test_telegram_message_surfaces_read_only_remediation_evidence(self):
|
||
"""主告警卡必須顯示 ADR-100 只讀補救試跑與寫入旗標。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="⚠️",
|
||
risk_level="MEDIUM",
|
||
resource_name="awoooi-auto-repair-canary",
|
||
root_cause="post approval verification drift",
|
||
suggested_action="kubectl rollout restart deployment/awoooi-api",
|
||
estimated_downtime="~30s",
|
||
approval_id="INC-20260513-79ED5E",
|
||
confidence=0.82,
|
||
remediation_summary={
|
||
"schema_version": "adr100_remediation_history_v1",
|
||
"total": 3,
|
||
"items": [
|
||
{
|
||
"mode": "replay",
|
||
"allowed": True,
|
||
"success": True,
|
||
"safety_level": "read_only",
|
||
"verification_result_preview": "degraded",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
},
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "AI 已完成只讀補救試跑，等待人工審批" in result
|
||
assert "AI 證據" in result
|
||
assert "只讀試跑 3 次" in result
|
||
assert "auto_repair_executor/ssh_diagnose/read" in result
|
||
assert "incident <code>false</code>" in result
|
||
assert "auto-repair <code>false</code>" in result
|
||
assert "自動化資產沉澱" in result
|
||
assert "Verifier：<code>blocked</code>" in result
|
||
|
||
def test_telegram_message_asset_deposition_shows_km_playbook_verifier(self):
|
||
"""主告警卡必須把 KM / PlayBook / Verifier 沉澱結果前移到可掃描區。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="⚠️",
|
||
risk_level="MEDIUM",
|
||
resource_name="awoooi-api",
|
||
root_cause="已完成診斷並產生修復候選",
|
||
suggested_action="等待人工批准",
|
||
estimated_downtime="unknown",
|
||
approval_id="INC-20260618-ASSET",
|
||
playbook_name="infra/ansible/playbooks/110-devops.yml",
|
||
automation_quality={
|
||
"verdict": "approval_required",
|
||
"facts": {
|
||
"knowledge_entries": 2,
|
||
"verification_result": "healthy",
|
||
"mcp_gateway_total": 7,
|
||
"auto_repair_execution_records": 0,
|
||
"automation_operation_records": 0,
|
||
},
|
||
"blockers": [],
|
||
},
|
||
remediation_summary={
|
||
"total": 1,
|
||
"items": [
|
||
{
|
||
"agent_id": "investigator",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"verification_result_preview": "healthy",
|
||
}
|
||
],
|
||
},
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "自動化資產沉澱" in result
|
||
assert "KM：<code>2</code>" in result
|
||
assert "PlayBook：<code>1</code>" in result
|
||
assert "Verifier：<code>ready</code>" in result
|
||
assert "完成 3 / 卡點 0" in result
|
||
|
||
def test_telegram_message_surfaces_missing_remediation_evidence(self):
|
||
"""沒有補救試跑紀錄時，主卡要明確說明，不讓值班者猜。"""
|
||
msg = TelegramMessage(
|
||
status_emoji="ℹ️",
|
||
risk_level="LOW",
|
||
resource_name="awoooi-auto-repair-canary",
|
||
root_cause="safe canary",
|
||
suggested_action="NO_ACTION",
|
||
estimated_downtime="unknown",
|
||
approval_id="INC-20260513-EMPTY",
|
||
remediation_summary={
|
||
"schema_version": "adr100_remediation_history_v1",
|
||
"total": 0,
|
||
"items": [],
|
||
},
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "AI 證據" in result
|
||
assert "尚無補救試跑紀錄" in result
|
||
|
||
def test_telegram_message_with_token_cost(self):
|
||
"""測試含 Token/Cost 的訊息"""
|
||
msg = TelegramMessage(
|
||
status_emoji="⚠️",
|
||
risk_level="MEDIUM",
|
||
resource_name="api-pod",
|
||
root_cause="High CPU",
|
||
suggested_action="Scale up",
|
||
estimated_downtime="0s",
|
||
approval_id="INC-20260329-0002",
|
||
ai_tokens=1500,
|
||
ai_cost=0.0015,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "💰 Tokens: 1,500 / $0.0015" in result
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_append_incident_update_deduplicates_same_status(monkeypatch):
|
||
"""同一 Incident 的相同狀態更新 5 分鐘內不可重複洗版。"""
|
||
|
||
class FakeRedis:
|
||
def __init__(self):
|
||
self.values = {}
|
||
|
||
async def get(self, key):
|
||
assert key == "tg_msg:INC-DEDUP"
|
||
return "12345"
|
||
|
||
async def set(self, key, value, **kwargs):
|
||
assert kwargs["nx"] is True
|
||
assert kwargs["ex"] > 0
|
||
if key in self.values:
|
||
return False
|
||
self.values[key] = value
|
||
return True
|
||
|
||
fake_redis = FakeRedis()
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: fake_redis)
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
status_line = "🤖❌ <b>[AUTO] AI 自動修復失敗，已升級人工介入</b>"
|
||
|
||
assert await gateway.append_incident_update("INC-DEDUP", status_line) is True
|
||
assert await gateway.append_incident_update("INC-DEDUP", status_line) is True
|
||
|
||
assert [method for method, _ in sent_requests] == [
|
||
"editMessageReplyMarkup",
|
||
"sendMessage",
|
||
]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_append_incident_update_suppresses_duplicate_failure_across_incidents(monkeypatch):
|
||
"""不同 Incident 卡在相同失敗摘要時，只回覆第一則，避免 Telegram 洗版。"""
|
||
|
||
class FakeRedis:
|
||
def __init__(self):
|
||
self.values = {}
|
||
|
||
async def get(self, key):
|
||
if key == "tg_msg:INC-A":
|
||
return "111"
|
||
if key == "tg_msg:INC-B":
|
||
return "222"
|
||
return None
|
||
|
||
async def set(self, key, value, **kwargs):
|
||
assert kwargs["nx"] is True
|
||
assert kwargs["ex"] > 0
|
||
if key in self.values:
|
||
return False
|
||
self.values[key] = value
|
||
return True
|
||
|
||
fake_redis = FakeRedis()
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True}
|
||
|
||
monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: fake_redis)
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
|
||
status_line = (
|
||
"🤖❌ <b>[AUTO] AI 自動修復失敗，已升級人工介入</b>\n"
|
||
"├ 動作: <code>ssh 192.168.0.110 uptime</code>\n"
|
||
"└ 錯誤: unsupported action"
|
||
)
|
||
|
||
assert await gateway.append_incident_update("INC-A", status_line) is True
|
||
assert await gateway.append_incident_update("INC-B", status_line) is True
|
||
|
||
assert [method for method, _ in sent_requests] == [
|
||
"editMessageReplyMarkup",
|
||
"sendMessage",
|
||
"editMessageReplyMarkup",
|
||
]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_send_approval_card_includes_remediation_summary(monkeypatch):
|
||
"""send_approval_card 要把 durable 補救試跑歷史帶進 Telegram 主卡。"""
|
||
sent_requests = []
|
||
gateway = TelegramGateway()
|
||
|
||
async def fake_send_request(method, payload):
|
||
sent_requests.append((method, payload))
|
||
return {"ok": True, "result": {}}
|
||
|
||
async def fake_keyboard(**kwargs):
|
||
return {"inline_keyboard": []}
|
||
|
||
async def fake_remediation_summary(**kwargs):
|
||
assert kwargs["incident_id"] == "INC-20260513-79ED5E"
|
||
return {
|
||
"schema_version": "adr100_remediation_history_v1",
|
||
"total": 1,
|
||
"items": [
|
||
{
|
||
"allowed": True,
|
||
"success": True,
|
||
"safety_level": "read_only",
|
||
"verification_result_preview": "degraded",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
}
|
||
|
||
monkeypatch.setattr(TelegramGateway, "alert_chat_id", property(lambda _self: "chat"))
|
||
monkeypatch.setattr(gateway, "_send_request", fake_send_request)
|
||
monkeypatch.setattr(gateway, "_build_inline_keyboard", fake_keyboard)
|
||
monkeypatch.setattr(
|
||
telegram_gateway_module,
|
||
"_fetch_remediation_summary_for_card",
|
||
fake_remediation_summary,
|
||
)
|
||
|
||
await gateway.send_approval_card(
|
||
approval_id="approval-1",
|
||
risk_level="medium",
|
||
resource_name="awoooi-auto-repair-canary",
|
||
root_cause="post approval verification drift",
|
||
suggested_action="kubectl rollout restart deployment/awoooi-api",
|
||
incident_id="INC-20260513-79ED5E",
|
||
confidence=0.82,
|
||
)
|
||
|
||
assert sent_requests
|
||
text = sent_requests[0][1]["text"]
|
||
assert "AI 已完成只讀補救試跑，等待人工審批" in text
|
||
assert "auto_repair_executor/ssh_diagnose/read" in text
|
||
|
||
|
||
def test_outbound_message_type_inference():
|
||
"""Legacy Telegram 訊息 mirror 到 Channel Hub 時，必須映射成有限分類。"""
|
||
|
||
assert (
|
||
telegram_gateway_module._infer_outbound_message_type(
|
||
"ℹ️ ACTION REQUIRED | 低風險",
|
||
{"reply_markup": {"inline_keyboard": []}},
|
||
)
|
||
== "approval_request"
|
||
)
|
||
assert (
|
||
telegram_gateway_module._infer_outbound_message_type(
|
||
"🤖❌ [AUTO] AI 自動修復失敗",
|
||
{"reply_to_message_id": 123},
|
||
)
|
||
== "error"
|
||
)
|
||
assert (
|
||
telegram_gateway_module._infer_outbound_message_type(
|
||
"✅ 執行成功",
|
||
{"reply_parameters": {"message_id": 123}},
|
||
)
|
||
== "final"
|
||
)
|
||
assert (
|
||
telegram_gateway_module._infer_outbound_message_type(
|
||
"📄 <b>RUNBOOK REVIEW｜待審核</b>",
|
||
{"reply_parameters": {"message_id": 123}},
|
||
)
|
||
== "approval_request"
|
||
)
|
||
|
||
|
||
def test_extract_incident_id_from_text():
|
||
"""Telegram 出站文字可擷取 Incident ID，供後續訊息接回原告警卡片。"""
|
||
|
||
assert (
|
||
telegram_gateway_module._extract_incident_id_from_text(
|
||
"Incident: INC-20260506-E54736\nEntry ID: abc"
|
||
)
|
||
== "INC-20260506-E54736"
|
||
)
|
||
assert telegram_gateway_module._extract_incident_id_from_text("沒有事件編號") is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_attach_incident_thread_reply(monkeypatch):
|
||
"""非主卡、含 Incident ID 的後續訊息，應自動 reply 原告警 message_id。"""
|
||
|
||
class FakeRedis:
|
||
async def get(self, key):
|
||
assert key == "tg_msg:INC-20260506-E54736"
|
||
return "9876"
|
||
|
||
gateway = TelegramGateway()
|
||
payload = {
|
||
"chat_id": gateway.alert_chat_id,
|
||
"text": "📄 RUNBOOK REVIEW｜待審核\nIncident: INC-20260506-E54736",
|
||
}
|
||
|
||
monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: FakeRedis())
|
||
|
||
await gateway._attach_incident_thread_reply("sendMessage", payload)
|
||
|
||
assert payload["reply_parameters"] == {
|
||
"message_id": 9876,
|
||
"allow_sending_without_reply": True,
|
||
}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_attach_incident_thread_skips_root_action_card(monkeypatch):
|
||
"""ACTION REQUIRED 主告警卡不應自動 reply 舊訊息。"""
|
||
|
||
class FakeRedis:
|
||
async def get(self, key): # pragma: no cover - 不應被呼叫
|
||
raise AssertionError(key)
|
||
|
||
gateway = TelegramGateway()
|
||
payload = {
|
||
"chat_id": gateway.alert_chat_id,
|
||
"text": (
|
||
"ℹ️ ACTION REQUIRED | 低風險\n"
|
||
"📋 INC-20260506-E54736\n"
|
||
"🤖 AI 自動化鏈路"
|
||
),
|
||
}
|
||
|
||
monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: FakeRedis())
|
||
|
||
await gateway._attach_incident_thread_reply("sendMessage", payload)
|
||
|
||
assert "reply_parameters" not in payload
|
||
|
||
|
||
def test_legacy_outbound_run_id_is_stable():
|
||
"""沒有正式 run_id 的 legacy Telegram 發送，要有穩定 soft run_id 供查詢串接。"""
|
||
|
||
first = telegram_gateway_module._legacy_outbound_run_id("-1001", "42")
|
||
second = telegram_gateway_module._legacy_outbound_run_id("-1001", "42")
|
||
other = telegram_gateway_module._legacy_outbound_run_id("-1001", "43")
|
||
|
||
assert first == second
|
||
assert first != other
|
||
|
||
|
||
class TestSentryErrorMessage:
|
||
"""測試 Sentry 錯誤訊息"""
|
||
|
||
def test_sentry_error_format_basic(self):
|
||
"""測試基本 Sentry 錯誤格式"""
|
||
msg = SentryErrorMessage(
|
||
error_id="SENTRY-abc123",
|
||
error_type="TypeError",
|
||
error_message="Cannot read property 'x' of undefined",
|
||
service_name="awoooi-api",
|
||
file_location="src/api/v1/incidents.py:123",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "🐛" in result
|
||
assert "SENTRY ERROR" in result
|
||
assert "TypeError" in result
|
||
assert "awoooi-api" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_sentry_error_with_stack_trace(self):
|
||
"""測試含 Stack Trace 的 Sentry 錯誤"""
|
||
msg = SentryErrorMessage(
|
||
error_id="SENTRY-xyz789",
|
||
error_type="ValueError",
|
||
error_message="Invalid input",
|
||
service_name="awoooi-web",
|
||
file_location="src/components/App.tsx:45",
|
||
occurrence_count=15,
|
||
affected_users=3,
|
||
first_seen="10 分鐘前",
|
||
stack_trace=[
|
||
"incidents.py:123 in get_incident",
|
||
"service.py:45 in fetch_data",
|
||
"db.py:89 in query",
|
||
],
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "發生次數: <code>15</code>" in result
|
||
assert "影響用戶: <code>3</code>" in result
|
||
assert "Stack Trace" in result
|
||
|
||
|
||
class TestResourceWarnMessage:
|
||
"""測試資源告警訊息"""
|
||
|
||
def test_resource_warn_format_basic(self):
|
||
"""測試基本資源告警格式"""
|
||
msg = ResourceWarnMessage(
|
||
resource_id="RES-20260329-0001",
|
||
pod_name="awoooi-api-7d4b8c9f5-abc12",
|
||
namespace="awoooi-prod",
|
||
cpu_percent=92.5,
|
||
memory_percent=78.0,
|
||
disk_percent=45.0,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "⚠️" in result
|
||
assert "資源告警" in result
|
||
assert "CPU: 🔴" in result # 92.5% > 90%
|
||
assert "Memory: 🟡" in result # 78% >= 70%
|
||
assert "Disk: 🟢" in result # 45% < 70%
|
||
assert len(result) <= 900
|
||
|
||
def test_resource_warn_with_limits(self):
|
||
"""測試含限制資訊的資源告警"""
|
||
msg = ResourceWarnMessage(
|
||
resource_id="RES-20260329-0002",
|
||
pod_name="test-pod",
|
||
cpu_percent=85.0,
|
||
cpu_limit="500m",
|
||
memory_percent=60.0,
|
||
memory_limit="512Mi",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "(limit: 500m)" in result
|
||
assert "(limit: 512Mi)" in result
|
||
|
||
|
||
class TestRepairReportMessage:
|
||
"""測試自動修復報告"""
|
||
|
||
def test_repair_report_format_basic(self):
|
||
"""測試基本修復報告格式"""
|
||
msg = RepairReportMessage(
|
||
report_date="2026-03-29",
|
||
total_repairs=12,
|
||
success_count=10,
|
||
failure_count=2,
|
||
saved_minutes=45,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "🔧" in result
|
||
assert "自動修復報告" in result
|
||
assert "總修復次數: <code>12</code>" in result
|
||
assert "成功: ✅ <code>10</code> (83%)" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_repair_report_with_top_issues(self):
|
||
"""測試含 Top 問題的修復報告"""
|
||
msg = RepairReportMessage(
|
||
report_date="2026-03-29",
|
||
total_repairs=12,
|
||
success_count=10,
|
||
failure_count=2,
|
||
top_issues=[
|
||
("Pod CrashLoopBackOff", 5),
|
||
("OOM Killed", 4),
|
||
("Image Pull Failed", 3),
|
||
],
|
||
ai_cost_gemini=0.0234,
|
||
ai_tokens_total=1823,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "Top 3 問題" in result
|
||
assert "Pod CrashLoopBackOff" in result
|
||
assert "Gemini: $0.0234" in result
|
||
|
||
|
||
class TestDailySummaryMessage:
|
||
"""測試每日摘要"""
|
||
|
||
def test_daily_summary_format_basic(self):
|
||
"""測試基本每日摘要格式"""
|
||
msg = DailySummaryMessage(
|
||
summary_date="2026-03-29",
|
||
alert_total=45,
|
||
alert_critical=2,
|
||
alert_medium=18,
|
||
alert_low=25,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "📊" in result
|
||
assert "每日摘要" in result
|
||
assert "總數: <code>45</code>" in result
|
||
assert "Critical: <code>2</code>" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_daily_summary_with_full_stats(self):
|
||
"""測試完整統計的每日摘要"""
|
||
msg = DailySummaryMessage(
|
||
summary_date="2026-03-29",
|
||
alert_total=45,
|
||
auto_repair_count=30,
|
||
manual_approval_count=10,
|
||
ignored_count=5,
|
||
api_availability=99.95,
|
||
ai_cost=0.15,
|
||
budget_remaining=9.85,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "自動修復: <code>30</code>" in result
|
||
assert "API: <code>99.95%</code>" in result
|
||
assert "預算剩餘: $9.85" in result
|
||
|
||
|
||
class TestDeploySuccessMessage:
|
||
"""測試部署成功訊息"""
|
||
|
||
def test_deploy_success_format_basic(self):
|
||
"""測試基本部署成功格式"""
|
||
msg = DeploySuccessMessage(
|
||
commit_sha="abc1234567",
|
||
triggered_by="ogt",
|
||
environment="Production",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "✅" in result
|
||
assert "部署成功" in result
|
||
assert "abc12345" in result # 前 8 字元
|
||
assert "@ogt" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_deploy_success_with_e2e(self):
|
||
"""測試含 E2E 結果的部署成功"""
|
||
msg = DeploySuccessMessage(
|
||
commit_sha="abc1234567",
|
||
triggered_by="ogt",
|
||
api_version="v1.2.3",
|
||
web_version="v1.2.3",
|
||
duration_seconds=225, # 3m 45s
|
||
e2e_passed=26,
|
||
e2e_total=26,
|
||
health_check_passed=True,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "v1.2.3" in result
|
||
assert "3m 45s" in result
|
||
assert "✅ 26/26 PASSED" in result
|
||
|
||
|
||
class TestRateLimitMessage:
|
||
"""測試 API 限額警告"""
|
||
|
||
def test_rate_limit_format_basic(self):
|
||
"""測試基本限額警告格式"""
|
||
msg = RateLimitMessage(
|
||
provider="gemini",
|
||
daily_usage=450,
|
||
daily_limit=500,
|
||
token_usage=85000,
|
||
token_limit=100000,
|
||
cost_usd=0.08,
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "⚠️" in result
|
||
assert "API 限額警告" in result
|
||
assert "GEMINI API" in result
|
||
assert "450/500" in result
|
||
assert "(90%)" in result
|
||
assert len(result) <= 900
|
||
|
||
def test_rate_limit_with_suggestions(self):
|
||
"""測試含建議的限額警告"""
|
||
msg = RateLimitMessage(
|
||
provider="openai",
|
||
daily_usage=90,
|
||
daily_limit=100,
|
||
suggestions=[
|
||
"考慮切換到 Ollama 優先",
|
||
"或增加每日限額",
|
||
],
|
||
reset_time="明日 00:00",
|
||
)
|
||
|
||
result = msg.format()
|
||
|
||
assert "建議" in result
|
||
assert "切換到 Ollama" in result
|
||
assert "明日 00:00" in result
|