diff --git a/apps/api/src/services/awooop_truth_chain_service.py b/apps/api/src/services/awooop_truth_chain_service.py index a084469a..ebcedef9 100644 --- a/apps/api/src/services/awooop_truth_chain_service.py +++ b/apps/api/src/services/awooop_truth_chain_service.py @@ -620,6 +620,91 @@ def _automation_quality_score_bucket(score: int) -> str: return "red" +def _int_value(value: Any, fallback: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return fallback + + +def _execution_backend_summary(records: list[dict[str, Any]]) -> dict[str, Any]: + summary = { + "operation_records_total": 0, + "effective_execution_records_total": 0, + "noop_operation_records_total": 0, + "audit_only_operation_records_total": 0, + "auto_repair_execution_records_total": 0, + "ansible_considered_total": 0, + "ansible_audit_record_total": 0, + "ansible_candidate_total": 0, + "ansible_check_mode_total": 0, + "ansible_apply_total": 0, + "ansible_rollback_total": 0, + "ansible_pending_check_mode_total": 0, + } + + for record in records: + quality = record.get("automation_quality") if isinstance(record.get("automation_quality"), dict) else {} + facts = quality.get("facts") if isinstance(quality.get("facts"), dict) else {} + execution = record.get("execution") if isinstance(record.get("execution"), dict) else {} + ops = ( + execution.get("automation_operation_log") + if isinstance(execution.get("automation_operation_log"), list) + else [] + ) + auto_repair_executions = ( + execution.get("auto_repair_executions") + if isinstance(execution.get("auto_repair_executions"), list) + else [] + ) + ansible = execution.get("ansible") if isinstance(execution.get("ansible"), dict) else {} + ansible_records = ansible.get("records") if isinstance(ansible.get("records"), list) else [] + catalog = ( + ansible.get("candidate_catalog") + if isinstance(ansible.get("candidate_catalog"), dict) + else {} + ) + candidates = catalog.get("candidates") if isinstance(catalog.get("candidates"), list) else [] + + summary["operation_records_total"] += _int_value(facts.get("automation_operation_records"), len(ops)) + summary["effective_execution_records_total"] += _int_value( + facts.get("effective_execution_records"), + len(_effective_execution_ops([row for row in ops if isinstance(row, dict)])), + ) + summary["noop_operation_records_total"] += _int_value( + facts.get("noop_operation_records"), + sum(1 for row in ops if isinstance(row, dict) and _is_no_action_operation(row)), + ) + summary["audit_only_operation_records_total"] += _int_value( + facts.get("audit_only_operation_records"), + sum(1 for row in ops if isinstance(row, dict) and _is_audit_only_operation(row)), + ) + summary["auto_repair_execution_records_total"] += _int_value( + facts.get("auto_repair_execution_records"), + len(auto_repair_executions), + ) + + if ansible.get("considered") is True: + summary["ansible_considered_total"] += 1 + summary["ansible_audit_record_total"] += len(ansible_records) + summary["ansible_candidate_total"] += len(candidates) + + for row in ansible_records: + if not isinstance(row, dict): + continue + operation_type = str(row.get("operation_type") or "") + if operation_type == "ansible_check_mode_executed": + summary["ansible_check_mode_total"] += 1 + elif operation_type == "ansible_apply_executed": + summary["ansible_apply_total"] += 1 + elif operation_type == "ansible_rollback_executed": + summary["ansible_rollback_total"] += 1 + elif operation_type == "ansible_candidate_matched": + summary["ansible_pending_check_mode_total"] += 1 + + return summary + + def summarize_automation_quality_records( *, project_id: str, @@ -724,6 +809,7 @@ def summarize_automation_quality_records( "score_buckets": score_buckets, "by_verdict": by_verdict, "gate_failures": failing_gates, + "execution_backend_summary": _execution_backend_summary(records), "examples": examples[:25], "production_claim": { "can_claim_full_auto_repair": evaluated_total > 0 and verified_total == evaluated_total, @@ -1491,6 +1577,7 @@ async def fetch_automation_quality_summary( "incident": truth_chain.get("incident") or incident, "truth_status": truth_chain.get("truth_status") or {}, "automation_quality": truth_chain.get("automation_quality") or {}, + "execution": truth_chain.get("execution") or {}, } records = [ diff --git a/apps/api/tests/test_awooop_truth_chain_service.py b/apps/api/tests/test_awooop_truth_chain_service.py index d89b9337..4aaab2bb 100644 --- a/apps/api/tests/test_awooop_truth_chain_service.py +++ b/apps/api/tests/test_awooop_truth_chain_service.py @@ -1,7 +1,7 @@ from __future__ import annotations import inspect -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from types import SimpleNamespace from src.services.awooop_ansible_audit_service import ( @@ -258,7 +258,7 @@ def test_drift_fingerprint_is_stable_across_item_order() -> None: def test_drift_repeat_state_counts_matching_fingerprint_only() -> None: - now = datetime(2026, 5, 13, 1, 0, tzinfo=timezone.utc) + now = datetime(2026, 5, 13, 1, 0, tzinfo=UTC) report = { "report_id": "drift-now", "namespace": "awoooi-prod", @@ -303,7 +303,7 @@ def test_drift_repeat_state_counts_matching_fingerprint_only() -> None: def test_drift_repeat_state_can_group_semantic_shape_without_values() -> None: - now = datetime(2026, 5, 19, 1, 0, tzinfo=timezone.utc) + now = datetime(2026, 5, 19, 1, 0, tzinfo=UTC) report = { "report_id": "drift-now", "namespace": "awoooi-prod", @@ -565,11 +565,32 @@ def test_automation_quality_summary_denies_full_claim_when_unverified() -> None: "applicable": True, "verdict": "auto_repaired_verified", "score": 100, + "facts": { + "automation_operation_records": 1, + "effective_execution_records": 1, + "noop_operation_records": 0, + "audit_only_operation_records": 0, + "auto_repair_execution_records": 1, + }, "gates": [ {"name": "verification_recorded", "status": "passed"}, ], "blockers": [], }, + "execution": { + "automation_operation_log": [ + { + "operation_type": "playbook_executed", + "status": "success", + } + ], + "auto_repair_executions": [{"success": True}], + "ansible": { + "considered": False, + "records": [], + "candidate_catalog": {"candidates": []}, + }, + }, }, { "incident": { @@ -588,12 +609,51 @@ def test_automation_quality_summary_denies_full_claim_when_unverified() -> None: "applicable": True, "verdict": "execution_unverified", "score": 65, + "facts": { + "automation_operation_records": 2, + "effective_execution_records": 0, + "noop_operation_records": 0, + "audit_only_operation_records": 2, + "auto_repair_execution_records": 0, + }, "gates": [ {"name": "verification_recorded", "status": "missing"}, {"name": "learning_recorded", "status": "missing"}, ], "blockers": ["verification_recorded", "learning_recorded"], }, + "execution": { + "automation_operation_log": [ + { + "operation_type": "ansible_candidate_matched", + "status": "dry_run", + }, + { + "operation_type": "ansible_check_mode_executed", + "status": "dry_run", + }, + ], + "auto_repair_executions": [], + "ansible": { + "considered": True, + "records": [ + { + "operation_type": "ansible_candidate_matched", + "status": "dry_run", + }, + { + "operation_type": "ansible_check_mode_executed", + "status": "dry_run", + }, + ], + "candidate_catalog": { + "candidates": [ + {"catalog_id": "ansible:188-ai-web"}, + {"catalog_id": "ansible:nginx-sync"}, + ] + }, + }, + }, }, ], ) @@ -613,6 +673,20 @@ def test_automation_quality_summary_denies_full_claim_when_unverified() -> None: "learning_recorded": 1, "verification_recorded": 1, } + assert summary["execution_backend_summary"] == { + "operation_records_total": 3, + "effective_execution_records_total": 1, + "noop_operation_records_total": 0, + "audit_only_operation_records_total": 2, + "auto_repair_execution_records_total": 1, + "ansible_considered_total": 1, + "ansible_audit_record_total": 2, + "ansible_candidate_total": 2, + "ansible_check_mode_total": 1, + "ansible_apply_total": 0, + "ansible_rollback_total": 0, + "ansible_pending_check_mode_total": 1, + } assert summary["examples"][1]["incident_id"] == "INC-GAP" assert summary["examples"][1]["score_bucket"] == "yellow" diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index bdb83cf8..9f48f2ff 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -252,6 +252,7 @@ "autoRepair": "Auto repair", "qualityDetail": "Average {score}, red {red}", "qualityPending": "Quality summary is still calculating; other evidence is already shown", + "executionBackendDetail": "Execution evidence: operations {operations} (effective {effective} / audit {auditOnly}), auto-repair {autoRepair}; Ansible audit {ansibleRecords}, candidates {ansibleCandidates}, check-mode {checkMode}, apply {apply}, pending wiring {pending}", "humanGap": "Human gap", "humanGapDetail": "{gate} missing {count}", "humanGapClear": "Quality summary has no top gap", diff --git a/apps/web/messages/zh-TW.json b/apps/web/messages/zh-TW.json index 44a14381..ed187700 100644 --- a/apps/web/messages/zh-TW.json +++ b/apps/web/messages/zh-TW.json @@ -253,6 +253,7 @@ "autoRepair": "自動修復", "qualityDetail": "平均 {score}，紅燈 {red}", "qualityPending": "品質摘要計算中，其他證據已先顯示", + "executionBackendDetail": "執行證據：操作 {operations}（有效 {effective} / 稽核 {auditOnly}），自動修復 {autoRepair}；Ansible 稽核 {ansibleRecords}，候選 {ansibleCandidates}，check-mode {checkMode}，apply {apply}，待接線 {pending}", "humanGap": "人工缺口", "humanGapDetail": "{gate} 缺 {count} 筆", "humanGapClear": "品質摘要未列出主要缺口", diff --git a/apps/web/src/components/dashboard/automation-evidence-card.tsx b/apps/web/src/components/dashboard/automation-evidence-card.tsx index 5888dd67..21794900 100644 --- a/apps/web/src/components/dashboard/automation-evidence-card.tsx +++ b/apps/web/src/components/dashboard/automation-evidence-card.tsx @@ -39,6 +39,18 @@ interface AutomationQualitySummary { gate: string total: number }> + execution_backend_summary?: { + operation_records_total?: number + effective_execution_records_total?: number + audit_only_operation_records_total?: number + auto_repair_execution_records_total?: number + ansible_considered_total?: number + ansible_audit_record_total?: number + ansible_candidate_total?: number + ansible_check_mode_total?: number + ansible_apply_total?: number + ansible_pending_check_mode_total?: number + } } interface DossierCoverageResponse { @@ -325,6 +337,7 @@ export function AutomationEvidenceCard() { ) const topGate = quality?.gate_failures?.[0] + const executionBackend = quality?.execution_backend_summary ?? null const qualityLoaded = Boolean(quality) const claimReady = Boolean(quality?.production_claim?.can_claim_full_auto_repair) const route = snapshot?.route ?? null @@ -360,6 +373,7 @@ export function AutomationEvidenceCard() { fallback, routeDetail, routeTone: routeTone(route), + executionBackend, } }, [snapshot, t]) @@ -484,6 +498,38 @@ export function AutomationEvidenceCard() { /> + {derived.executionBackend && ( +
+
+ )} + {derived.topGate && (