同步 incidents 相容欄位寫入
All checks were successful
CD Pipeline / deploy (push) Successful in 56s

This commit is contained in:
OoO
2026-05-12 23:31:33 +08:00
parent 539dea12c5
commit caa6263872
4 changed files with 159 additions and 2 deletions

View File

@@ -111,16 +111,19 @@ class Incident(Base):
task_name = Column(String(100), nullable=False)
error_type = Column(String(50), nullable=False)
error_message = Column(Text, nullable=False)
error_traceback = Column(Text)
traceback_str = Column(Text)
severity = Column(String(20), default='medium')
status = Column(String(20), default='open') # open/healing/closed/escalated
retry_count = Column(Integer, default=0)
playbook_id = Column(Integer, ForeignKey('playbooks.id'), nullable=True)
matched_playbook_id = Column(Integer, ForeignKey('playbooks.id'), nullable=True)
resolved_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, default=datetime.now)
updated_at = Column(DateTime, default=datetime.now)
# Relationship
playbook = relationship("Playbook", backref="incidents")
playbook = relationship("Playbook", foreign_keys=[matched_playbook_id], backref="incidents")
class Playbook(Base):

View File

@@ -0,0 +1,85 @@
-- =============================================================================
-- Migration 036: normalize incidents legacy/current columns
-- 日期: 2026-05-12 台北
-- =============================================================================
-- 背景:
-- migration 013 建立 incidents.error_traceback / playbook_id / resolved_at
-- 後續 ORM 改用 traceback_str / matched_playbook_id。migration 031 已加新欄位,
-- 但 create_all-only 舊環境可能缺 legacy 欄位,且新寫入只更新新欄位。
--
-- 設計:
-- 1. 保留雙欄做向下相容,不做破壞性 drop/rename。
-- 2. 補齊缺欄,並用 COALESCE 雙向回填,讓分析查詢可安全 coalesce。
-- 3. 補 playbook_id FK 與 matched_playbook_id FK已存在則略過。
-- =============================================================================
ALTER TABLE IF EXISTS incidents
ADD COLUMN IF NOT EXISTS error_traceback TEXT;
ALTER TABLE IF EXISTS incidents
ADD COLUMN IF NOT EXISTS traceback_str TEXT;
ALTER TABLE IF EXISTS incidents
ADD COLUMN IF NOT EXISTS playbook_id INTEGER;
ALTER TABLE IF EXISTS incidents
ADD COLUMN IF NOT EXISTS matched_playbook_id INTEGER;
ALTER TABLE IF EXISTS incidents
ADD COLUMN IF NOT EXISTS resolved_at TIMESTAMP;
ALTER TABLE IF EXISTS incidents
ALTER COLUMN severity TYPE VARCHAR(20);
UPDATE incidents
SET traceback_str = error_traceback
WHERE traceback_str IS NULL
AND error_traceback IS NOT NULL;
UPDATE incidents
SET error_traceback = traceback_str
WHERE error_traceback IS NULL
AND traceback_str IS NOT NULL;
UPDATE incidents
SET matched_playbook_id = playbook_id
WHERE matched_playbook_id IS NULL
AND playbook_id IS NOT NULL;
UPDATE incidents
SET playbook_id = matched_playbook_id
WHERE playbook_id IS NULL
AND matched_playbook_id IS NOT NULL;
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'incidents_playbook_id_fkey'
) THEN
ALTER TABLE incidents
ADD CONSTRAINT incidents_playbook_id_fkey
FOREIGN KEY (playbook_id)
REFERENCES playbooks(id);
END IF;
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'incidents_matched_playbook_id_fkey'
) THEN
ALTER TABLE incidents
ADD CONSTRAINT incidents_matched_playbook_id_fkey
FOREIGN KEY (matched_playbook_id)
REFERENCES playbooks(id);
END IF;
END $$;
CREATE INDEX IF NOT EXISTS idx_incidents_matched_playbook_id
ON incidents(matched_playbook_id);
DO $$
BEGIN
RAISE NOTICE 'Migration 036 done: incidents legacy/current columns normalized';
END $$;

View File

@@ -232,6 +232,11 @@ class AutoHealService:
record_autoheal_action(action="NO_PLAYBOOK", error_type=error_type, success=False)
return AutoHealResult(success=False, action=None, message=msg)
playbook_id = int(playbook.get("id") or 0)
if playbook_id > 0:
context["matched_playbook_id"] = playbook_id
context["playbook_id"] = playbook_id
if playbook["action_type"] not in _ALLOWED_ACTION_TYPES:
msg = f"action_type '{playbook['action_type']}' is not allowed"
self._log.warning("[AutoHeal] %s", msg)
@@ -317,6 +322,7 @@ class AutoHealService:
error_type=error_type,
error_message=str(context.get("exception") or context.get("error_message") or error_type)[:2000],
traceback_str=str(context.get("traceback_str") or "")[:8000] or None,
error_traceback=str(context.get("traceback_str") or "")[:8000] or None,
severity=str(context.get("severity") or "medium"),
status="healing",
retry_count=0,
@@ -569,15 +575,28 @@ class AutoHealService:
session = None
try:
session = get_session()
status = "closed" if result.success else "escalated"
playbook_id = int(
context.get("matched_playbook_id")
or context.get("playbook_id")
or 0
) or None
session.execute(
text("""
UPDATE incidents
SET status = :status,
matched_playbook_id = COALESCE(:playbook_id, matched_playbook_id),
playbook_id = COALESCE(:playbook_id, playbook_id),
resolved_at = CASE
WHEN :status = 'closed' THEN COALESCE(resolved_at, NOW())
ELSE resolved_at
END,
updated_at = NOW()
WHERE id = :incident_id
"""),
{
"status": "closed" if result.success else "escalated",
"status": status,
"playbook_id": playbook_id,
"incident_id": int(incident_id),
},
)

View File

@@ -40,3 +40,53 @@ def test_auto_heal_code_fix_writes_audit(monkeypatch):
assert result.success is True
assert calls
assert calls[0][2].action == "CODE_FIX"
def test_incident_model_keeps_legacy_and_current_columns():
from database.manager import Base
columns = set(Base.metadata.tables["incidents"].columns.keys())
assert {
"error_traceback",
"traceback_str",
"playbook_id",
"matched_playbook_id",
"resolved_at",
} <= columns
def test_auto_heal_status_update_backfills_dual_playbook_columns(monkeypatch):
from services.auto_heal_service import AutoHealResult, AutoHealService
captured = {}
class Session:
def execute(self, stmt, params):
captured["sql"] = str(stmt)
captured["params"] = params
def commit(self):
captured["committed"] = True
def rollback(self):
captured["rolled_back"] = True
def close(self):
captured["closed"] = True
monkeypatch.setattr("services.auto_heal_service.get_session", lambda: Session())
svc = AutoHealService()
svc._update_incident_status(
{"incident_id": 123, "matched_playbook_id": 77},
AutoHealResult(True, "ALERT_ONLY", "ok"),
)
assert "matched_playbook_id" in captured["sql"]
assert "playbook_id" in captured["sql"]
assert "resolved_at" in captured["sql"]
assert captured["params"]["status"] == "closed"
assert captured["params"]["playbook_id"] == 77
assert captured["params"]["incident_id"] == 123
assert captured["committed"] is True