327 lines
11 KiB
Python
327 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from src.services.channel_hub import (
|
|
_db_timestamp_now,
|
|
build_alertmanager_provider_event_id,
|
|
build_alertmanager_run_id,
|
|
build_external_alert_provider_event_id,
|
|
build_grouped_alert_provider_event_id,
|
|
build_grouped_alert_run_id,
|
|
build_inbound_source_envelope,
|
|
ensure_completed_shadow_run,
|
|
format_alertmanager_event_content,
|
|
format_grouped_alert_digest_text,
|
|
format_grouped_alert_event_content,
|
|
mirror_inbound_event,
|
|
record_outbound_message,
|
|
)
|
|
|
|
|
|
def test_db_timestamp_now_is_naive_utc_for_asyncpg() -> None:
|
|
assert _db_timestamp_now().tzinfo is None
|
|
|
|
|
|
def test_build_grouped_alert_provider_event_id_is_deterministic() -> None:
|
|
event_id = build_grouped_alert_provider_event_id(
|
|
"INC-20260507-ABCD12",
|
|
"1234567890abcdef" * 4,
|
|
)
|
|
|
|
assert event_id == "alert-group:INC-20260507-ABCD12:1234567890abcdef1234567890abcdef"
|
|
assert len(event_id) < 256
|
|
|
|
|
|
def test_build_grouped_alert_run_id_is_stable() -> None:
|
|
provider_event_id = build_grouped_alert_provider_event_id(
|
|
"INC-20260507-ABCD12",
|
|
"1234567890abcdef" * 4,
|
|
)
|
|
|
|
first = build_grouped_alert_run_id("awoooi", provider_event_id)
|
|
second = build_grouped_alert_run_id("awoooi", provider_event_id)
|
|
other_project = build_grouped_alert_run_id("ewoooc", provider_event_id)
|
|
|
|
assert first == second
|
|
assert first != other_project
|
|
|
|
|
|
def test_build_alertmanager_provider_event_id_keeps_fingerprint() -> None:
|
|
event_id = build_alertmanager_provider_event_id(
|
|
"alert-20260513213000",
|
|
"abcdef1234567890" * 4,
|
|
"incident_linked",
|
|
)
|
|
|
|
assert event_id == "alertmanager:incident_linked:alert-20260513213000:abcdef1234567890abcdef1234567890"
|
|
assert len(event_id) < 256
|
|
|
|
|
|
def test_build_alertmanager_run_id_is_stable() -> None:
|
|
provider_event_id = build_alertmanager_provider_event_id(
|
|
"alert-20260513213000",
|
|
"abcdef1234567890" * 4,
|
|
"received",
|
|
)
|
|
|
|
assert build_alertmanager_run_id("awoooi", provider_event_id) == build_alertmanager_run_id(
|
|
"awoooi", provider_event_id
|
|
)
|
|
assert build_alertmanager_run_id("awoooi", provider_event_id) != build_alertmanager_run_id(
|
|
"ewoooc", provider_event_id
|
|
)
|
|
|
|
|
|
def test_build_external_alert_provider_event_id_is_stable() -> None:
|
|
event_id = build_external_alert_provider_event_id(
|
|
"Sentry",
|
|
"12345678901234567890",
|
|
"approval_linked",
|
|
)
|
|
|
|
assert event_id == "sentry:approval_linked:12345678901234567890"
|
|
assert len(event_id) < 256
|
|
|
|
|
|
def test_build_inbound_source_envelope_redacts_and_keeps_refs() -> None:
|
|
envelope = build_inbound_source_envelope(
|
|
provider="alertmanager",
|
|
stage="received",
|
|
provider_event_id="alertmanager:received:alert-1:fp-1",
|
|
raw_event_id="alert-1",
|
|
raw_content="ACTION REQUIRED INC-20260513-ABCDEF token 1234567890:abcdefghijklmnopqrstuvwxyzABCDEFGH",
|
|
alertname="DockerContainerUnhealthy",
|
|
severity="warning",
|
|
namespace="default",
|
|
target_resource="bitan-pharmacy-bitan-1",
|
|
fingerprint="fp-1",
|
|
incident_id="INC-20260513-ABCDEF",
|
|
approval_id="approval-1",
|
|
labels={"token": "should-not-leak", "container": "bitan-pharmacy-bitan-1"},
|
|
)
|
|
|
|
rendered = json.dumps(envelope, ensure_ascii=False)
|
|
assert envelope["schema_version"] == "inbound_source_envelope_v1"
|
|
assert envelope["source_refs"]["event_ids"] == ["alert-1"]
|
|
assert envelope["source_refs"]["incident_ids"] == ["INC-20260513-ABCDEF"]
|
|
assert envelope["source_refs"]["approval_ids"] == ["approval-1"]
|
|
assert envelope["source_refs"]["alert_ids"] == ["alert-1", "alertmanager:received:alert-1:fp-1"]
|
|
assert envelope["source_refs"]["fingerprints"] == ["fp-1"]
|
|
assert len(envelope["content_sha256"]) == 64
|
|
assert "1234567890:" not in rendered
|
|
assert "should-not-leak" not in rendered
|
|
|
|
|
|
def test_format_alertmanager_event_content_keeps_incident_first() -> None:
|
|
content = format_alertmanager_event_content(
|
|
stage="incident_linked",
|
|
alert_id="alert-20260513213000",
|
|
alertname="DockerContainerExited",
|
|
severity="warning",
|
|
namespace="default",
|
|
target_resource="bitan-pharmacy-bitan-1",
|
|
fingerprint="fp-123",
|
|
notification_type="TYPE-3",
|
|
alert_category="host_resource",
|
|
incident_id="INC-20260513-ABCDEF",
|
|
approval_id="approval-1",
|
|
repeat_count=3,
|
|
)
|
|
|
|
assert content.startswith("Alertmanager inbound incident_linked\nIncident: INC-20260513-ABCDEF")
|
|
assert "Approval: approval-1" in content
|
|
assert "Fingerprint: fp-123" in content
|
|
assert "Repeat Count: 3" in content
|
|
|
|
|
|
def test_format_grouped_alert_event_content_keeps_operator_context() -> None:
|
|
content = format_grouped_alert_event_content(
|
|
alert_id="INC-20260507-ABCD12",
|
|
alertname="DockerContainerRestartSpike",
|
|
severity="critical",
|
|
namespace="default",
|
|
target_resource="sentry-self-hosted-events-consumer-1",
|
|
group_key="DockerContainerRestartSpike:default",
|
|
count=4,
|
|
parent_fingerprint="parent-fp",
|
|
fingerprint="child-fp",
|
|
)
|
|
|
|
assert "告警已收斂，不發 Telegram" in content
|
|
assert "Alert: DockerContainerRestartSpike" in content
|
|
assert "Target: sentry-self-hosted-events-consumer-1" in content
|
|
assert "Group Count: 4" in content
|
|
assert "Parent Fingerprint: parent-fp" in content
|
|
|
|
|
|
def test_format_grouped_alert_digest_text_is_html_safe() -> None:
|
|
content = format_grouped_alert_digest_text(
|
|
alertname="Docker<Restart>",
|
|
severity="critical",
|
|
namespace="default",
|
|
target_resource="sentry&snuba",
|
|
group_key="Docker<Restart>:default",
|
|
count=7,
|
|
)
|
|
|
|
assert "告警已收斂到父卡" in content
|
|
assert "Docker<Restart>" in content
|
|
assert "sentry&snuba" in content
|
|
assert "<b>7</b> 筆同組告警" in content
|
|
assert "AwoooP Run 監控" in content
|
|
|
|
|
|
class _FakeResult:
|
|
def fetchone(self) -> tuple[str] | None:
|
|
return ("created",)
|
|
|
|
|
|
class _FakeSession:
|
|
def __init__(self) -> None:
|
|
self.statement = ""
|
|
self.params = {}
|
|
self.statements = []
|
|
self.param_sets = []
|
|
|
|
async def execute(self, statement, params): # noqa: ANN001
|
|
self.statement = str(statement)
|
|
self.params = params
|
|
self.statements.append(str(statement))
|
|
self.param_sets.append(params)
|
|
return _FakeResult()
|
|
|
|
|
|
async def test_completed_shadow_run_sets_run_state_not_null_defaults() -> None:
|
|
session = _FakeSession()
|
|
run_id = build_grouped_alert_run_id("awoooi", "provider-event")
|
|
|
|
inserted = await ensure_completed_shadow_run(
|
|
session, # type: ignore[arg-type]
|
|
project_id="awoooi",
|
|
run_id=run_id,
|
|
agent_id="legacy-telegram-gateway",
|
|
trigger_type="legacy_outbound",
|
|
trigger_ref="12713",
|
|
input_payload={"message_type": "final"},
|
|
)
|
|
|
|
assert inserted is True
|
|
assert "attempt_count, max_attempts, cost_usd, step_count" in session.statement
|
|
assert "0, 3, 0.0000, 0" in session.statement
|
|
assert session.params["project_id"] == "awoooi"
|
|
assert session.params["run_id"] == run_id
|
|
|
|
|
|
async def test_mirror_inbound_event_writes_redacted_replay_fields() -> None:
|
|
session = _FakeSession()
|
|
envelope = build_inbound_source_envelope(
|
|
provider="sentry",
|
|
stage="received",
|
|
provider_event_id="sentry:received:123",
|
|
raw_event_id="123",
|
|
raw_content="Sentry token 1234567890:abcdefghijklmnopqrstuvwxyzABCDEFGH",
|
|
alertname="Sentry issue",
|
|
severity="error",
|
|
namespace="sentry",
|
|
target_resource="frontend",
|
|
)
|
|
|
|
await mirror_inbound_event(
|
|
session, # type: ignore[arg-type]
|
|
project_id="awoooi",
|
|
channel_type="internal",
|
|
provider_event_id="sentry:received:123",
|
|
platform_subject_id="sentry",
|
|
raw_content="Sentry token 1234567890:abcdefghijklmnopqrstuvwxyzABCDEFGH",
|
|
source_envelope=envelope,
|
|
)
|
|
|
|
assert "content_redacted" in session.statement
|
|
assert "source_envelope" in session.statement
|
|
assert session.params["redaction_version"] == "audit_sink_v1"
|
|
assert len(json.loads(session.params["source_envelope"])["content_sha256"]) == 64
|
|
assert "1234567890:" not in session.params["content_redacted"]
|
|
assert "1234567890:" not in session.params["source_envelope"]
|
|
|
|
|
|
def test_sentry_and_signoz_source_refs_keep_raw_event_ids() -> None:
|
|
sentry_envelope = build_inbound_source_envelope(
|
|
provider="sentry",
|
|
stage="received",
|
|
provider_event_id="sentry:received:issue-123",
|
|
raw_event_id="issue-123",
|
|
raw_content="Sentry issue",
|
|
alertname="Sentry issue",
|
|
fingerprint="sentry-issue-123",
|
|
)
|
|
signoz_envelope = build_inbound_source_envelope(
|
|
provider="signoz",
|
|
stage="received",
|
|
provider_event_id="signoz:received:fp-456",
|
|
raw_event_id="fp-456",
|
|
raw_content="SignOz alert",
|
|
alertname="HighLatency",
|
|
fingerprint="fp-456",
|
|
)
|
|
|
|
assert sentry_envelope["source_refs"]["event_ids"] == ["issue-123"]
|
|
assert sentry_envelope["source_refs"]["sentry_issue_ids"] == [
|
|
"issue-123",
|
|
"sentry:received:issue-123",
|
|
]
|
|
assert signoz_envelope["source_refs"]["event_ids"] == ["fp-456"]
|
|
assert signoz_envelope["source_refs"]["signoz_alerts"] == ["HighLatency", "fp-456"]
|
|
|
|
|
|
def test_source_provider_heartbeat_refs_do_not_claim_real_provider_alerts() -> None:
|
|
sentry_envelope = build_inbound_source_envelope(
|
|
provider="sentry",
|
|
stage="heartbeat",
|
|
provider_event_id="sentry:heartbeat:heartbeat-1",
|
|
raw_event_id="heartbeat-1",
|
|
raw_content="Sentry heartbeat",
|
|
alertname="SourceProviderHeartbeat",
|
|
fingerprint="source-provider-heartbeat:sentry",
|
|
)
|
|
signoz_envelope = build_inbound_source_envelope(
|
|
provider="signoz",
|
|
stage="heartbeat",
|
|
provider_event_id="signoz:heartbeat:heartbeat-1",
|
|
raw_event_id="heartbeat-1",
|
|
raw_content="SignOz heartbeat",
|
|
alertname="SourceProviderHeartbeat",
|
|
fingerprint="source-provider-heartbeat:signoz",
|
|
)
|
|
|
|
assert sentry_envelope["source_refs"]["event_ids"] == ["heartbeat-1"]
|
|
assert sentry_envelope["source_refs"]["sentry_issue_ids"] == []
|
|
assert signoz_envelope["source_refs"]["event_ids"] == ["heartbeat-1"]
|
|
assert signoz_envelope["source_refs"]["signoz_alerts"] == []
|
|
|
|
|
|
async def test_record_outbound_message_sets_sent_at_for_sent_messages() -> None:
|
|
session = _FakeSession()
|
|
run_id = build_grouped_alert_run_id("awoooi", "telegram-message-13152")
|
|
|
|
await record_outbound_message(
|
|
session, # type: ignore[arg-type]
|
|
project_id="awoooi",
|
|
run_id=run_id,
|
|
channel_type="telegram",
|
|
channel_chat_id="-100123",
|
|
message_type="approval_request",
|
|
content="ACTION REQUIRED INC-20260513-9B082D",
|
|
provider_message_id="13152",
|
|
send_status="sent",
|
|
triggered_by_state="legacy_gateway",
|
|
is_shadow=False,
|
|
)
|
|
|
|
insert_statement = session.statements[-1]
|
|
assert "sent_at" in insert_statement
|
|
assert ":sent_at" in insert_statement
|
|
assert session.param_sets[-1]["send_status"] == "sent"
|
|
assert session.param_sets[-1]["sent_at"] is not None
|
|
assert session.param_sets[-1]["sent_at"].tzinfo is None
|