feat(awooop): expose ai decision wiring readback
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 19s
CD Pipeline / build-and-deploy (push) Successful in 4m32s
CD Pipeline / post-deploy-checks (push) Successful in 58s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 19s
CD Pipeline / build-and-deploy (push) Successful in 4m32s
CD Pipeline / post-deploy-checks (push) Successful in 58s
This commit is contained in:
@@ -596,10 +596,196 @@ def _build_log_integration_taxonomy(
|
||||
}
|
||||
|
||||
|
||||
def _decision_wiring_stage(
|
||||
*,
|
||||
stage_id: str,
|
||||
display_name: str,
|
||||
evidence_sources: list[str],
|
||||
total: int,
|
||||
recent: int,
|
||||
required_for_decision_wiring: bool,
|
||||
feeds_next_stage: str,
|
||||
next_action_if_missing: str,
|
||||
) -> dict[str, Any]:
|
||||
present = total > 0
|
||||
return {
|
||||
"stage_id": stage_id,
|
||||
"display_name": display_name,
|
||||
"evidence_sources": evidence_sources,
|
||||
"present": present,
|
||||
"total": max(0, total),
|
||||
"recent": max(0, recent),
|
||||
"required_for_decision_wiring": required_for_decision_wiring,
|
||||
"feeds_next_stage": feeds_next_stage,
|
||||
"next_action_if_missing": None if present else next_action_if_missing,
|
||||
}
|
||||
|
||||
|
||||
def _build_agent_decision_wiring(
|
||||
*,
|
||||
operation_summary: Mapping[str, Any],
|
||||
verifier_summary: Mapping[str, Any],
|
||||
km_summary: Mapping[str, Any],
|
||||
mcp_gateway_summary: Mapping[str, Any],
|
||||
legacy_mcp_summary: Mapping[str, Any],
|
||||
service_log_summary: Mapping[str, Any],
|
||||
timeline_summary: Mapping[str, Any],
|
||||
playbook_trust_summary: Mapping[str, Any],
|
||||
log_integration_taxonomy: Mapping[str, Any],
|
||||
loop_ledger: Mapping[str, Any],
|
||||
latest_flow_closure: Mapping[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Summarize live evidence-to-execution wiring for the AI Agent path."""
|
||||
|
||||
taxonomy_rollups = log_integration_taxonomy.get("rollups")
|
||||
if not isinstance(taxonomy_rollups, Mapping):
|
||||
taxonomy_rollups = {}
|
||||
source_family_count = _int_value(taxonomy_rollups.get("source_family_count"))
|
||||
active_source_family_count = _int_value(taxonomy_rollups.get("active_source_family_count"))
|
||||
all_sources_active = source_family_count > 0 and active_source_family_count == source_family_count
|
||||
evidence_total = (
|
||||
_trace_total(mcp_gateway_summary)
|
||||
+ _trace_total(legacy_mcp_summary)
|
||||
+ _trace_total(service_log_summary)
|
||||
+ _trace_total(timeline_summary)
|
||||
)
|
||||
evidence_recent = (
|
||||
_trace_recent(mcp_gateway_summary)
|
||||
+ _trace_recent(legacy_mcp_summary)
|
||||
+ _trace_recent(service_log_summary)
|
||||
+ _trace_recent(timeline_summary)
|
||||
)
|
||||
rag_context_total = _trace_total(km_summary) + _trace_total(playbook_trust_summary)
|
||||
rag_context_recent = _trace_recent(km_summary) + _trace_recent(playbook_trust_summary)
|
||||
candidate_total = _trace_total(operation_summary, "ansible_candidate_matched")
|
||||
candidate_recent = _trace_recent(operation_summary, "ansible_candidate_matched")
|
||||
check_mode_total = _trace_total(operation_summary, "ansible_check_mode_executed")
|
||||
check_mode_recent = _trace_recent(operation_summary, "ansible_check_mode_executed")
|
||||
apply_total = _trace_total(operation_summary, "ansible_apply_executed")
|
||||
apply_recent = _trace_recent(operation_summary, "ansible_apply_executed")
|
||||
rollback_total = _trace_total(operation_summary, "ansible_rollback_executed")
|
||||
verifier_total = _trace_total(verifier_summary)
|
||||
verifier_recent = _trace_recent(verifier_summary)
|
||||
|
||||
stages = [
|
||||
_decision_wiring_stage(
|
||||
stage_id="labeled_evidence_sources",
|
||||
display_name="Labeled log / MCP / timeline evidence available",
|
||||
evidence_sources=["log_integration_taxonomy", "mcp", "service_logs", "timeline_events"],
|
||||
total=evidence_total if all_sources_active else 0,
|
||||
recent=evidence_recent,
|
||||
required_for_decision_wiring=True,
|
||||
feeds_next_stage="rag_context_retrieval",
|
||||
next_action_if_missing="keep_p1a_source_family_ingestion_active_until_10_of_10",
|
||||
),
|
||||
_decision_wiring_stage(
|
||||
stage_id="rag_context_retrieval",
|
||||
display_name="RAG / KM / PlayBook trust context available",
|
||||
evidence_sources=["knowledge_entries", "playbooks"],
|
||||
total=rag_context_total,
|
||||
recent=rag_context_recent,
|
||||
required_for_decision_wiring=True,
|
||||
feeds_next_stage="playbook_candidate_selection",
|
||||
next_action_if_missing="retrieve_similar_km_entries_and_playbook_trust_before_candidate",
|
||||
),
|
||||
_decision_wiring_stage(
|
||||
stage_id="playbook_candidate_selection",
|
||||
display_name="Allowlisted PlayBook candidate selected",
|
||||
evidence_sources=["automation_operation_log:ansible_candidate_matched"],
|
||||
total=candidate_total,
|
||||
recent=candidate_recent,
|
||||
required_for_decision_wiring=True,
|
||||
feeds_next_stage="check_mode_dry_run",
|
||||
next_action_if_missing="candidate_backfill_worker_enqueue_allowlisted_playbook",
|
||||
),
|
||||
_decision_wiring_stage(
|
||||
stage_id="check_mode_dry_run",
|
||||
display_name="Check-mode / dry-run receipt recorded",
|
||||
evidence_sources=["automation_operation_log:ansible_check_mode_executed"],
|
||||
total=check_mode_total,
|
||||
recent=check_mode_recent,
|
||||
required_for_decision_wiring=True,
|
||||
feeds_next_stage="controlled_apply_boundary",
|
||||
next_action_if_missing="ansible_check_mode_worker_claims_candidate",
|
||||
),
|
||||
_decision_wiring_stage(
|
||||
stage_id="controlled_apply_boundary",
|
||||
display_name="Controlled apply / rollback boundary recorded",
|
||||
evidence_sources=[
|
||||
"automation_operation_log:ansible_apply_executed",
|
||||
"automation_operation_log:ansible_rollback_executed",
|
||||
],
|
||||
total=apply_total + rollback_total,
|
||||
recent=apply_recent,
|
||||
required_for_decision_wiring=True,
|
||||
feeds_next_stage="post_apply_verifier",
|
||||
next_action_if_missing="controlled_apply_worker_waits_for_check_mode_success",
|
||||
),
|
||||
_decision_wiring_stage(
|
||||
stage_id="post_apply_verifier",
|
||||
display_name="Post-apply verifier receipt recorded",
|
||||
evidence_sources=["incident_evidence"],
|
||||
total=verifier_total,
|
||||
recent=verifier_recent,
|
||||
required_for_decision_wiring=True,
|
||||
feeds_next_stage="learning_writeback",
|
||||
next_action_if_missing="post_apply_verifier_writes_incident_evidence",
|
||||
),
|
||||
]
|
||||
missing_required = [
|
||||
str(stage["stage_id"])
|
||||
for stage in stages
|
||||
if stage["required_for_decision_wiring"] is True and stage["present"] is not True
|
||||
]
|
||||
present_required_count = sum(
|
||||
1
|
||||
for stage in stages
|
||||
if stage["required_for_decision_wiring"] is True and stage["present"] is True
|
||||
)
|
||||
required_count = sum(1 for stage in stages if stage["required_for_decision_wiring"] is True)
|
||||
closed_loop_observed = bool(
|
||||
loop_ledger.get("closed") is True
|
||||
or latest_flow_closure.get("closed") is True
|
||||
)
|
||||
return {
|
||||
"schema_version": "ai_agent_decision_wiring_readback_v1",
|
||||
"status": "completed" if not missing_required else "in_progress",
|
||||
"stages": stages,
|
||||
"missing_required_stage_ids": missing_required,
|
||||
"runtime_switches": {
|
||||
"candidate_backfill_worker_enabled": bool(settings.ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER),
|
||||
"check_mode_worker_enabled": bool(settings.ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER),
|
||||
"controlled_apply_enabled": bool(settings.ENABLE_AWOOOP_ANSIBLE_CONTROLLED_APPLY),
|
||||
"allowed_risk_levels": _allowed_risk_levels(),
|
||||
},
|
||||
"closed_loop_observed": closed_loop_observed,
|
||||
"public_safety": {
|
||||
"stores_raw_logs": False,
|
||||
"stores_secret_values": False,
|
||||
"executes_on_read": False,
|
||||
"critical_break_glass_still_required": True,
|
||||
},
|
||||
"rollups": {
|
||||
"stage_count": len(stages),
|
||||
"required_stage_count": required_count,
|
||||
"required_stage_present_count": present_required_count,
|
||||
"required_stage_missing_count": len(missing_required),
|
||||
"evidence_event_total": evidence_total,
|
||||
"rag_context_total": rag_context_total,
|
||||
"candidate_total": candidate_total,
|
||||
"check_mode_total": check_mode_total,
|
||||
"controlled_apply_total": apply_total,
|
||||
"rollback_total": rollback_total,
|
||||
"verifier_total": verifier_total,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _build_work_item_progress(
|
||||
*,
|
||||
trace_ledger: Mapping[str, Any],
|
||||
log_integration_taxonomy: Mapping[str, Any],
|
||||
agent_decision_wiring: Mapping[str, Any],
|
||||
db_read_status: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Build ordered work items that the UI and agent can keep advancing."""
|
||||
@@ -614,6 +800,16 @@ def _build_work_item_progress(
|
||||
missing_required = trace_ledger.get("missing_required_stage_ids")
|
||||
if not isinstance(missing_required, list):
|
||||
missing_required = []
|
||||
decision_rollups = agent_decision_wiring.get("rollups")
|
||||
if not isinstance(decision_rollups, Mapping):
|
||||
decision_rollups = {}
|
||||
decision_wiring_missing = _int_value(decision_rollups.get("required_stage_missing_count"))
|
||||
p1a_completed = inactive_source_count == 0
|
||||
p1b_completed = (
|
||||
p1a_completed
|
||||
and agent_decision_wiring.get("schema_version") == "ai_agent_decision_wiring_readback_v1"
|
||||
and decision_wiring_missing == 0
|
||||
)
|
||||
deployed_readback_complete = (
|
||||
db_read_status == "ok"
|
||||
and trace_ledger.get("schema_version") == "ai_agent_autonomous_trace_ledger_v1"
|
||||
@@ -661,7 +857,7 @@ def _build_work_item_progress(
|
||||
"work_item_id": "P1-A-ingestion-coverage",
|
||||
"priority": "P1-A",
|
||||
"title": "Collector and sanitizer coverage for all source families",
|
||||
"status": "completed" if inactive_source_count == 0 else "in_progress",
|
||||
"status": "completed" if p1a_completed else "in_progress",
|
||||
"exit_criteria": "all source families have active sanitized classified events",
|
||||
"remaining_source_family_count": inactive_source_count,
|
||||
},
|
||||
@@ -669,8 +865,9 @@ def _build_work_item_progress(
|
||||
"work_item_id": "P1-B-agent-decision-wiring",
|
||||
"priority": "P1-B",
|
||||
"title": "RAG retrieval to PlayBook select/repair/check-mode/apply/verifier",
|
||||
"status": "pending",
|
||||
"status": "completed" if p1b_completed else "in_progress" if p1a_completed else "pending",
|
||||
"exit_criteria": "AI Agent consumes labeled evidence and emits target selector, dry-run, apply, verifier, rollback",
|
||||
"remaining_decision_wiring_stage_count": decision_wiring_missing,
|
||||
},
|
||||
{
|
||||
"work_item_id": "P1-C-learning-loop",
|
||||
@@ -1543,9 +1740,23 @@ def build_runtime_receipt_readback_from_rows(
|
||||
timeline_summary=timeline_summary,
|
||||
playbook_trust_summary=playbook_trust_summary,
|
||||
)
|
||||
agent_decision_wiring = _build_agent_decision_wiring(
|
||||
operation_summary=operation_summary,
|
||||
verifier_summary=verifier_summary,
|
||||
km_summary=km_summary,
|
||||
mcp_gateway_summary=mcp_gateway_summary,
|
||||
legacy_mcp_summary=legacy_mcp_summary,
|
||||
service_log_summary=service_log_summary,
|
||||
timeline_summary=timeline_summary,
|
||||
playbook_trust_summary=playbook_trust_summary,
|
||||
log_integration_taxonomy=log_integration_taxonomy,
|
||||
loop_ledger=loop_ledger,
|
||||
latest_flow_closure=latest_closure,
|
||||
)
|
||||
work_item_progress = _build_work_item_progress(
|
||||
trace_ledger=trace_ledger,
|
||||
log_integration_taxonomy=log_integration_taxonomy,
|
||||
agent_decision_wiring=agent_decision_wiring,
|
||||
db_read_status=db_read_status,
|
||||
)
|
||||
apply_summary = operation_summary.get("ansible_apply_executed") or {}
|
||||
@@ -1667,6 +1878,7 @@ def build_runtime_receipt_readback_from_rows(
|
||||
"autonomous_execution_loop_ledger": loop_ledger,
|
||||
"trace_ledger": trace_ledger,
|
||||
"log_integration_taxonomy": log_integration_taxonomy,
|
||||
"agent_decision_wiring": agent_decision_wiring,
|
||||
"work_item_progress": work_item_progress,
|
||||
}
|
||||
if error_type:
|
||||
@@ -1770,6 +1982,26 @@ def _attach_runtime_receipt_readback(
|
||||
"recent_classified_event_total"
|
||||
)
|
||||
),
|
||||
"live_agent_decision_wiring_stage_count": _int_value(
|
||||
((readback.get("agent_decision_wiring") or {}).get("rollups") or {}).get(
|
||||
"stage_count"
|
||||
)
|
||||
),
|
||||
"live_agent_decision_wiring_required_present_count": _int_value(
|
||||
((readback.get("agent_decision_wiring") or {}).get("rollups") or {}).get(
|
||||
"required_stage_present_count"
|
||||
)
|
||||
),
|
||||
"live_agent_decision_wiring_required_missing_count": _int_value(
|
||||
((readback.get("agent_decision_wiring") or {}).get("rollups") or {}).get(
|
||||
"required_stage_missing_count"
|
||||
)
|
||||
),
|
||||
"live_agent_decision_wiring_complete_count": (
|
||||
1
|
||||
if (readback.get("agent_decision_wiring") or {}).get("status") == "completed"
|
||||
else 0
|
||||
),
|
||||
"live_work_item_count": _int_value(
|
||||
((readback.get("work_item_progress") or {}).get("rollups") or {}).get(
|
||||
"work_item_count"
|
||||
|
||||
@@ -395,6 +395,27 @@ def test_runtime_receipt_readback_summarizes_live_executor_closure_rows():
|
||||
assert taxonomy["rollups"]["classified_event_total"] > 0
|
||||
assert taxonomy["public_safety"]["raw_secret_collection_allowed"] is False
|
||||
assert taxonomy["public_safety"]["unredacted_payload_storage_allowed"] is False
|
||||
decision_wiring = readback["agent_decision_wiring"]
|
||||
assert decision_wiring["schema_version"] == "ai_agent_decision_wiring_readback_v1"
|
||||
assert decision_wiring["status"] == "completed"
|
||||
assert decision_wiring["missing_required_stage_ids"] == []
|
||||
assert decision_wiring["closed_loop_observed"] is True
|
||||
assert {
|
||||
stage["stage_id"]
|
||||
for stage in decision_wiring["stages"]
|
||||
if stage["required_for_decision_wiring"]
|
||||
} == {
|
||||
"labeled_evidence_sources",
|
||||
"rag_context_retrieval",
|
||||
"playbook_candidate_selection",
|
||||
"check_mode_dry_run",
|
||||
"controlled_apply_boundary",
|
||||
"post_apply_verifier",
|
||||
}
|
||||
assert decision_wiring["rollups"]["candidate_total"] == 1
|
||||
assert decision_wiring["rollups"]["check_mode_total"] == 1
|
||||
assert decision_wiring["rollups"]["controlled_apply_total"] == 1
|
||||
assert decision_wiring["rollups"]["required_stage_missing_count"] == 0
|
||||
progress = readback["work_item_progress"]
|
||||
assert progress["schema_version"] == "ai_agent_automation_work_item_progress_v1"
|
||||
ordered_ids = [item["work_item_id"] for item in progress["ordered_items"]]
|
||||
@@ -413,10 +434,11 @@ def test_runtime_receipt_readback_summarizes_live_executor_closure_rows():
|
||||
]
|
||||
assert progress["ordered_items"][4]["status"] == "completed"
|
||||
assert progress["ordered_items"][5]["status"] == "completed"
|
||||
assert progress["ordered_items"][6]["status"] == "completed"
|
||||
assert progress["source_family_items"]
|
||||
assert {item["status"] for item in progress["source_family_items"]} == {"completed"}
|
||||
assert progress["rollups"]["source_family_work_item_count"] == 10
|
||||
assert progress["rollups"]["pending_count"] >= 5
|
||||
assert progress["rollups"]["pending_count"] >= 4
|
||||
|
||||
|
||||
def test_runtime_receipt_readback_classifies_closed_failed_apply_as_ai_repair():
|
||||
|
||||
@@ -11345,6 +11345,7 @@
|
||||
"trace": "Trace",
|
||||
"mcp": "MCP",
|
||||
"logs": "Logs",
|
||||
"decision": "Decision",
|
||||
"apply": "Apply",
|
||||
"receipt": "Receipt",
|
||||
"verifier": "Verifier",
|
||||
|
||||
@@ -11345,6 +11345,7 @@
|
||||
"trace": "Trace",
|
||||
"mcp": "MCP",
|
||||
"logs": "Logs",
|
||||
"decision": "Decision",
|
||||
"apply": "Apply",
|
||||
"receipt": "Receipt",
|
||||
"verifier": "Verifier",
|
||||
|
||||
@@ -88,6 +88,15 @@ type RuntimeReceiptReadback = {
|
||||
learning_source_family_count?: number | null;
|
||||
} | null;
|
||||
} | null;
|
||||
agent_decision_wiring?: {
|
||||
status?: string | null;
|
||||
missing_required_stage_ids?: string[] | null;
|
||||
rollups?: {
|
||||
stage_count?: number | null;
|
||||
required_stage_present_count?: number | null;
|
||||
required_stage_missing_count?: number | null;
|
||||
} | null;
|
||||
} | null;
|
||||
work_item_progress?: {
|
||||
rollups?: {
|
||||
work_item_count?: number | null;
|
||||
@@ -217,6 +226,8 @@ export function AutonomousRuntimeReceiptPanel({
|
||||
const traceLedger = readback?.trace_ledger;
|
||||
const logTaxonomy = readback?.log_integration_taxonomy;
|
||||
const logRollups = logTaxonomy?.rollups ?? {};
|
||||
const decisionRollups = readback?.agent_decision_wiring?.rollups ?? {};
|
||||
const decisionMissing = readback?.agent_decision_wiring?.missing_required_stage_ids ?? [];
|
||||
const workItemRollups = readback?.work_item_progress?.rollups ?? {};
|
||||
const latestFlow = readback?.latest_flow_closure;
|
||||
const rollups = payload?.rollups ?? {};
|
||||
@@ -270,6 +281,31 @@ export function AutonomousRuntimeReceiptPanel({
|
||||
+ toNumber(readback?.executor_log_projection?.recent),
|
||||
icon: Activity,
|
||||
},
|
||||
{
|
||||
key: "decision",
|
||||
label: t("metrics.decision"),
|
||||
value: toNumber(
|
||||
rollups.live_agent_decision_wiring_required_present_count
|
||||
?? decisionRollups.required_stage_present_count
|
||||
),
|
||||
recent: toNumber(
|
||||
rollups.live_agent_decision_wiring_stage_count
|
||||
?? decisionRollups.stage_count
|
||||
),
|
||||
icon: Bot,
|
||||
caption: t("traceCaption", {
|
||||
count: numberValue(
|
||||
rollups.live_agent_decision_wiring_stage_count
|
||||
?? decisionRollups.stage_count
|
||||
?? 0
|
||||
),
|
||||
missing: numberValue(
|
||||
rollups.live_agent_decision_wiring_required_missing_count
|
||||
?? decisionRollups.required_stage_missing_count
|
||||
?? decisionMissing.length
|
||||
),
|
||||
}),
|
||||
},
|
||||
{
|
||||
key: "apply",
|
||||
label: t("metrics.apply"),
|
||||
@@ -313,7 +349,18 @@ export function AutonomousRuntimeReceiptPanel({
|
||||
icon: Send,
|
||||
},
|
||||
],
|
||||
[logRollups.classified_event_total, missingStages.length, readback, rollups, t, traceLedger]
|
||||
[
|
||||
decisionMissing.length,
|
||||
decisionRollups.required_stage_missing_count,
|
||||
decisionRollups.required_stage_present_count,
|
||||
decisionRollups.stage_count,
|
||||
logRollups.classified_event_total,
|
||||
missingStages.length,
|
||||
readback,
|
||||
rollups,
|
||||
t,
|
||||
traceLedger,
|
||||
]
|
||||
);
|
||||
|
||||
const policy = payload?.current_policy;
|
||||
@@ -368,7 +415,7 @@ export function AutonomousRuntimeReceiptPanel({
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-px bg-[#e0ddd4] md:grid-cols-5 xl:grid-cols-10">
|
||||
<div className="grid grid-cols-2 gap-px bg-[#e0ddd4] md:grid-cols-5 xl:grid-cols-11">
|
||||
<div className="bg-white px-4 py-3">
|
||||
<p className="text-xs font-semibold text-[#77736a]">{t("metrics.loop")}</p>
|
||||
<div className="mt-2 flex items-center gap-2">
|
||||
|
||||
Reference in New Issue
Block a user