This commit is contained in:
@@ -111,16 +111,19 @@ class Incident(Base):
|
||||
task_name = Column(String(100), nullable=False)
|
||||
error_type = Column(String(50), nullable=False)
|
||||
error_message = Column(Text, nullable=False)
|
||||
error_traceback = Column(Text)
|
||||
traceback_str = Column(Text)
|
||||
severity = Column(String(20), default='medium')
|
||||
status = Column(String(20), default='open') # open/healing/closed/escalated
|
||||
retry_count = Column(Integer, default=0)
|
||||
playbook_id = Column(Integer, ForeignKey('playbooks.id'), nullable=True)
|
||||
matched_playbook_id = Column(Integer, ForeignKey('playbooks.id'), nullable=True)
|
||||
resolved_at = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.now)
|
||||
updated_at = Column(DateTime, default=datetime.now)
|
||||
|
||||
# Relationship
|
||||
playbook = relationship("Playbook", backref="incidents")
|
||||
playbook = relationship("Playbook", foreign_keys=[matched_playbook_id], backref="incidents")
|
||||
|
||||
|
||||
class Playbook(Base):
|
||||
|
||||
85
migrations/036_normalize_incidents_dual_columns.sql
Normal file
85
migrations/036_normalize_incidents_dual_columns.sql
Normal file
@@ -0,0 +1,85 @@
|
||||
-- =============================================================================
|
||||
-- Migration 036: normalize incidents legacy/current columns
|
||||
-- 日期: 2026-05-12 台北
|
||||
-- =============================================================================
|
||||
-- 背景:
|
||||
-- migration 013 建立 incidents.error_traceback / playbook_id / resolved_at;
|
||||
-- 後續 ORM 改用 traceback_str / matched_playbook_id。migration 031 已加新欄位,
|
||||
-- 但 create_all-only 舊環境可能缺 legacy 欄位,且新寫入只更新新欄位。
|
||||
--
|
||||
-- 設計:
|
||||
-- 1. 保留雙欄做向下相容,不做破壞性 drop/rename。
|
||||
-- 2. 補齊缺欄,並用 COALESCE 雙向回填,讓分析查詢可安全 coalesce。
|
||||
-- 3. 補 playbook_id FK 與 matched_playbook_id FK;已存在則略過。
|
||||
-- =============================================================================
|
||||
|
||||
ALTER TABLE IF EXISTS incidents
|
||||
ADD COLUMN IF NOT EXISTS error_traceback TEXT;
|
||||
|
||||
ALTER TABLE IF EXISTS incidents
|
||||
ADD COLUMN IF NOT EXISTS traceback_str TEXT;
|
||||
|
||||
ALTER TABLE IF EXISTS incidents
|
||||
ADD COLUMN IF NOT EXISTS playbook_id INTEGER;
|
||||
|
||||
ALTER TABLE IF EXISTS incidents
|
||||
ADD COLUMN IF NOT EXISTS matched_playbook_id INTEGER;
|
||||
|
||||
ALTER TABLE IF EXISTS incidents
|
||||
ADD COLUMN IF NOT EXISTS resolved_at TIMESTAMP;
|
||||
|
||||
ALTER TABLE IF EXISTS incidents
|
||||
ALTER COLUMN severity TYPE VARCHAR(20);
|
||||
|
||||
UPDATE incidents
|
||||
SET traceback_str = error_traceback
|
||||
WHERE traceback_str IS NULL
|
||||
AND error_traceback IS NOT NULL;
|
||||
|
||||
UPDATE incidents
|
||||
SET error_traceback = traceback_str
|
||||
WHERE error_traceback IS NULL
|
||||
AND traceback_str IS NOT NULL;
|
||||
|
||||
UPDATE incidents
|
||||
SET matched_playbook_id = playbook_id
|
||||
WHERE matched_playbook_id IS NULL
|
||||
AND playbook_id IS NOT NULL;
|
||||
|
||||
UPDATE incidents
|
||||
SET playbook_id = matched_playbook_id
|
||||
WHERE playbook_id IS NULL
|
||||
AND matched_playbook_id IS NOT NULL;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_constraint
|
||||
WHERE conname = 'incidents_playbook_id_fkey'
|
||||
) THEN
|
||||
ALTER TABLE incidents
|
||||
ADD CONSTRAINT incidents_playbook_id_fkey
|
||||
FOREIGN KEY (playbook_id)
|
||||
REFERENCES playbooks(id);
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_constraint
|
||||
WHERE conname = 'incidents_matched_playbook_id_fkey'
|
||||
) THEN
|
||||
ALTER TABLE incidents
|
||||
ADD CONSTRAINT incidents_matched_playbook_id_fkey
|
||||
FOREIGN KEY (matched_playbook_id)
|
||||
REFERENCES playbooks(id);
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_incidents_matched_playbook_id
|
||||
ON incidents(matched_playbook_id);
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
RAISE NOTICE 'Migration 036 done: incidents legacy/current columns normalized';
|
||||
END $$;
|
||||
@@ -232,6 +232,11 @@ class AutoHealService:
|
||||
record_autoheal_action(action="NO_PLAYBOOK", error_type=error_type, success=False)
|
||||
return AutoHealResult(success=False, action=None, message=msg)
|
||||
|
||||
playbook_id = int(playbook.get("id") or 0)
|
||||
if playbook_id > 0:
|
||||
context["matched_playbook_id"] = playbook_id
|
||||
context["playbook_id"] = playbook_id
|
||||
|
||||
if playbook["action_type"] not in _ALLOWED_ACTION_TYPES:
|
||||
msg = f"action_type '{playbook['action_type']}' is not allowed"
|
||||
self._log.warning("[AutoHeal] %s", msg)
|
||||
@@ -317,6 +322,7 @@ class AutoHealService:
|
||||
error_type=error_type,
|
||||
error_message=str(context.get("exception") or context.get("error_message") or error_type)[:2000],
|
||||
traceback_str=str(context.get("traceback_str") or "")[:8000] or None,
|
||||
error_traceback=str(context.get("traceback_str") or "")[:8000] or None,
|
||||
severity=str(context.get("severity") or "medium"),
|
||||
status="healing",
|
||||
retry_count=0,
|
||||
@@ -569,15 +575,28 @@ class AutoHealService:
|
||||
session = None
|
||||
try:
|
||||
session = get_session()
|
||||
status = "closed" if result.success else "escalated"
|
||||
playbook_id = int(
|
||||
context.get("matched_playbook_id")
|
||||
or context.get("playbook_id")
|
||||
or 0
|
||||
) or None
|
||||
session.execute(
|
||||
text("""
|
||||
UPDATE incidents
|
||||
SET status = :status,
|
||||
matched_playbook_id = COALESCE(:playbook_id, matched_playbook_id),
|
||||
playbook_id = COALESCE(:playbook_id, playbook_id),
|
||||
resolved_at = CASE
|
||||
WHEN :status = 'closed' THEN COALESCE(resolved_at, NOW())
|
||||
ELSE resolved_at
|
||||
END,
|
||||
updated_at = NOW()
|
||||
WHERE id = :incident_id
|
||||
"""),
|
||||
{
|
||||
"status": "closed" if result.success else "escalated",
|
||||
"status": status,
|
||||
"playbook_id": playbook_id,
|
||||
"incident_id": int(incident_id),
|
||||
},
|
||||
)
|
||||
|
||||
@@ -40,3 +40,53 @@ def test_auto_heal_code_fix_writes_audit(monkeypatch):
|
||||
assert result.success is True
|
||||
assert calls
|
||||
assert calls[0][2].action == "CODE_FIX"
|
||||
|
||||
|
||||
def test_incident_model_keeps_legacy_and_current_columns():
|
||||
from database.manager import Base
|
||||
|
||||
columns = set(Base.metadata.tables["incidents"].columns.keys())
|
||||
|
||||
assert {
|
||||
"error_traceback",
|
||||
"traceback_str",
|
||||
"playbook_id",
|
||||
"matched_playbook_id",
|
||||
"resolved_at",
|
||||
} <= columns
|
||||
|
||||
|
||||
def test_auto_heal_status_update_backfills_dual_playbook_columns(monkeypatch):
|
||||
from services.auto_heal_service import AutoHealResult, AutoHealService
|
||||
|
||||
captured = {}
|
||||
|
||||
class Session:
|
||||
def execute(self, stmt, params):
|
||||
captured["sql"] = str(stmt)
|
||||
captured["params"] = params
|
||||
|
||||
def commit(self):
|
||||
captured["committed"] = True
|
||||
|
||||
def rollback(self):
|
||||
captured["rolled_back"] = True
|
||||
|
||||
def close(self):
|
||||
captured["closed"] = True
|
||||
|
||||
monkeypatch.setattr("services.auto_heal_service.get_session", lambda: Session())
|
||||
|
||||
svc = AutoHealService()
|
||||
svc._update_incident_status(
|
||||
{"incident_id": 123, "matched_playbook_id": 77},
|
||||
AutoHealResult(True, "ALERT_ONLY", "ok"),
|
||||
)
|
||||
|
||||
assert "matched_playbook_id" in captured["sql"]
|
||||
assert "playbook_id" in captured["sql"]
|
||||
assert "resolved_at" in captured["sql"]
|
||||
assert captured["params"]["status"] == "closed"
|
||||
assert captured["params"]["playbook_id"] == 77
|
||||
assert captured["params"]["incident_id"] == 123
|
||||
assert captured["committed"] is True
|
||||
|
||||
Reference in New Issue
Block a user