feat(awooop): expose incident reconciliation state
All checks were successful
Code Review / ai-code-review (push) Successful in 11s
CD Pipeline / tests (push) Successful in 59s
CD Pipeline / build-and-deploy (push) Successful in 7m3s
CD Pipeline / post-deploy-checks (push) Successful in 1m15s

This commit is contained in:
Your Name
2026-05-13 09:02:16 +08:00
parent 54814bc65e
commit 1003fa4246
2 changed files with 184 additions and 1 deletions

View File

@@ -97,6 +97,127 @@ def _operation_ids(automation_ops: list[dict[str, Any]]) -> list[str]:
return [str(row["op_id"]) for row in automation_ops if row.get("op_id")]
def _build_reconciliation(
*,
incident: dict[str, Any] | None,
approvals: list[dict[str, Any]],
evidence_rows: list[dict[str, Any]],
automation_ops: list[dict[str, Any]],
timeline_events: list[dict[str, Any]],
) -> dict[str, Any]:
"""Build a read-only consistency report across incident lifecycle tables."""
if incident is None:
return {
"schema_version": "incident_reconciliation_v1",
"applicable": False,
"consistency_status": "not_applicable",
"operator_next_state": "not_applicable",
"facts": {},
"mismatches": [],
}
incident_status = str(incident.get("status") or "unknown").upper()
incident_closed = incident_status in {"RESOLVED", "CLOSED"}
latest_approval = approvals[0] if approvals else None
approval_status = str((latest_approval or {}).get("status") or "none").upper()
approval_action = str((latest_approval or {}).get("action") or "")
approval_resolved = bool((latest_approval or {}).get("resolved_at"))
attempted = sum(int(row.get("sensors_attempted") or 0) for row in evidence_rows)
succeeded = sum(int(row.get("sensors_succeeded") or 0) for row in evidence_rows)
executed_ops = [
row
for row in automation_ops
if str(row.get("status") or "").lower()
in {"success", "completed", "executed"}
]
mismatches: list[dict[str, Any]] = []
def add(code: str, severity: str, message: str) -> None:
mismatches.append({
"code": code,
"severity": severity,
"message": message,
})
if (
latest_approval
and not incident_closed
and (approval_resolved or approval_status in {"APPROVED", "REJECTED"})
):
add(
"incident_open_after_approval_resolved",
"high",
"Approval reached a terminal state while the incident is still open.",
)
if approval_status == "APPROVED" and not automation_ops:
add(
"approval_approved_without_execution_record",
"high",
"Approval is approved but automation_operation_log has no linked execution record.",
)
if (
approval_status == "APPROVED"
and "NO_ACTION" in approval_action.upper()
and not executed_ops
):
add(
"approval_no_action_without_execution",
"high",
"Approval resolved to NO_ACTION and no executor produced a successful operation.",
)
if attempted > 0 and succeeded == 0:
add(
"evidence_all_sensors_failed",
"medium",
"Evidence collection attempted sensors but none succeeded.",
)
if latest_approval and not timeline_events:
add(
"timeline_missing_for_approval",
"medium",
"Approval exists but timeline_events has no linked lifecycle entries.",
)
high_count = sum(1 for row in mismatches if row["severity"] == "high")
medium_count = sum(1 for row in mismatches if row["severity"] == "medium")
if high_count:
consistency_status = "blocked"
operator_next_state = "manual_required"
elif medium_count:
consistency_status = "degraded"
operator_next_state = "investigate"
else:
consistency_status = "consistent"
operator_next_state = "continue"
return {
"schema_version": "incident_reconciliation_v1",
"applicable": True,
"consistency_status": consistency_status,
"operator_next_state": operator_next_state,
"facts": {
"incident_id": incident.get("incident_id"),
"incident_status": incident_status,
"incident_closed": incident_closed,
"latest_approval_id": (latest_approval or {}).get("id"),
"latest_approval_status": approval_status,
"latest_approval_action": approval_action,
"approval_resolved": approval_resolved,
"evidence_records": len(evidence_rows),
"sensors_attempted": attempted,
"sensors_succeeded": succeeded,
"automation_operation_records": len(automation_ops),
"executed_operation_records": len(executed_ops),
"timeline_events": len(timeline_events),
},
"mismatches": mismatches,
}
def _truth_status(
*,
incident: dict[str, Any] | None,
@@ -573,6 +694,13 @@ async def fetch_truth_chain(source_id: str, project_id: str = "awoooi") -> dict[
legacy_mcp_total=legacy_mcp_summary["total"],
outbound_visible_total=len(outbound_rows),
)
reconciliation = _build_reconciliation(
incident=incident,
approvals=approvals,
evidence_rows=evidence_rows,
automation_ops=automation_ops,
timeline_events=timeline_events,
)
evidence_totals = {
"records": len(evidence_rows),
@@ -618,6 +746,7 @@ async def fetch_truth_chain(source_id: str, project_id: str = "awoooi") -> dict[
"automation_operation_log": automation_ops,
"ansible": build_ansible_truth(automation_ops, incident=incident, drift=drift),
},
"reconciliation": reconciliation,
"learning": {
"knowledge_entries": km_entries,
},

View File

@@ -7,7 +7,11 @@ from src.services.awooop_ansible_audit_service import (
build_ansible_decision_audit_payload,
build_ansible_truth,
)
from src.services.awooop_truth_chain_service import _clean_row, _truth_status
from src.services.awooop_truth_chain_service import (
_build_reconciliation,
_clean_row,
_truth_status,
)
from src.services.drift_repeat_state import (
build_drift_fingerprint,
build_drift_repeat_state,
@@ -157,6 +161,56 @@ def test_drift_repeat_state_counts_matching_fingerprint_only() -> None:
]
def test_reconciliation_blocks_open_incident_after_no_action_approval() -> None:
reconciliation = _build_reconciliation(
incident={"incident_id": "INC-1", "status": "INVESTIGATING"},
approvals=[
{
"id": "approval-1",
"status": "APPROVED",
"action": "未知操作 | NO_ACTION",
"resolved_at": "2026-05-13T01:00:00+00:00",
}
],
evidence_rows=[{"sensors_attempted": 8, "sensors_succeeded": 0}],
automation_ops=[],
timeline_events=[],
)
codes = {row["code"] for row in reconciliation["mismatches"]}
assert reconciliation["schema_version"] == "incident_reconciliation_v1"
assert reconciliation["consistency_status"] == "blocked"
assert reconciliation["operator_next_state"] == "manual_required"
assert reconciliation["facts"]["incident_closed"] is False
assert reconciliation["facts"]["automation_operation_records"] == 0
assert "incident_open_after_approval_resolved" in codes
assert "approval_approved_without_execution_record" in codes
assert "approval_no_action_without_execution" in codes
assert "evidence_all_sensors_failed" in codes
assert "timeline_missing_for_approval" in codes
def test_reconciliation_marks_consistent_resolved_execution() -> None:
reconciliation = _build_reconciliation(
incident={"incident_id": "INC-2", "status": "RESOLVED"},
approvals=[
{
"id": "approval-2",
"status": "APPROVED",
"action": "restart service",
"resolved_at": "2026-05-13T01:00:00+00:00",
}
],
evidence_rows=[{"sensors_attempted": 8, "sensors_succeeded": 7}],
automation_ops=[{"status": "success"}],
timeline_events=[{"event_type": "executor", "status": "success"}],
)
assert reconciliation["consistency_status"] == "consistent"
assert reconciliation["operator_next_state"] == "continue"
assert reconciliation["mismatches"] == []
def test_ansible_truth_surfaces_audited_check_mode_record() -> None:
truth = build_ansible_truth(
[