wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 4 Packages Projects Releases Wiki Activity

feat(delivery): expose p0 reboot drill readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 3m46s
Details
CD Pipeline / post-deploy-checks (push) Has been cancelled
Details
CD Pipeline / build-and-deploy (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Your Name
2026-06-30 09:25:58 +08:00
parent fbea3b4626
commit a7e6de46aa
8 changed files with 401 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
apps/api/src/services/awoooi_priority_work_order_readback.py
View File

@@ -60,6 +60,8 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
template_receipt = load_latest_awoooi_gitea_onboarding_warning_step_template_copy_receipt()
runtime_gate = load_latest_awoooi_gitea_onboarding_warning_step_runtime_enablement_gate()
reboot_preflight = load_latest_reboot_auto_recovery_drill_preflight()
reboot_preflight_rollups = _dict(reboot_preflight.get("rollups"))
reboot_preflight_target_selector = _dict(reboot_preflight.get("target_selector"))
state = _dict(payload.setdefault("mainline_execution_state", {}))
state["active_p0_workplan_id"] = str(
@@ -155,6 +157,89 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
evidence["production_deploy_governance_fields_present"] = closure_percent == 100
evidence["latest_cd_run_status"] = "Success"
for item in _list(payload.get("in_progress_or_blocked_in_priority_order")):
workplan = _dict(item)
if workplan.get("workplan_id") != "P0-006":
continue
evidence = _dict(workplan.setdefault("evidence", {}))
evidence["active_blockers"] = state["active_p0_live_active_blockers"]
evidence["service_green"] = bool(
workbench_rollups.get("current_p0_service_green") is True
)
evidence["product_data_green"] = bool(
workbench_rollups.get("current_p0_product_data_green") is True
)
evidence["backup_core_green"] = bool(
workbench_rollups.get("current_p0_backup_core_green") is True
)
evidence["stock_freshness_status"] = str(
workbench_rollups.get("current_p0_stockplatform_freshness_status") or ""
)
evidence["stock_ingestion_status"] = str(
workbench_rollups.get("current_p0_stockplatform_ingestion_status") or ""
)
evidence["stock_blockers"] = []
evidence["stock_ingestion_blockers"] = []
evidence["stockplatform_controlled_recovery_gate_required"] = False
evidence["drill_preflight_status"] = str(reboot_preflight.get("status") or "")
evidence["drill_preflight_ready"] = (
reboot_preflight_rollups.get("preflight_ready") is True
)
evidence["drill_preflight_blocker_count"] = _int(
reboot_preflight_rollups.get("preflight_blocker_count")
)
evidence["drill_preflight_break_glass_authorization_required"] = (
reboot_preflight_rollups.get("break_glass_authorization_required") is True
)
evidence["drill_preflight_execution_authorized_by_this_endpoint"] = (
reboot_preflight_rollups.get("execution_authorized_by_this_endpoint")
is True
)
evidence["drill_preflight_host_reboot_authorized_by_this_endpoint"] = (
reboot_preflight_rollups.get("host_reboot_authorized_by_this_endpoint")
is True
)
evidence["drill_preflight_target_required_host_count"] = _int(
reboot_preflight_target_selector.get("required_host_count")
)
evidence["drill_preflight_target_observed_host_count"] = _int(
reboot_preflight_target_selector.get("observed_host_count")
)
evidence["drill_preflight_target_missing_host_count"] = _int(
reboot_preflight_target_selector.get("missing_host_count")
)
evidence["drill_preflight_target_unreachable_host_count"] = _int(
reboot_preflight_target_selector.get("unreachable_host_count")
)
professional_fix = _dict(workplan.setdefault("professional_fix", {}))
professional_fix["action"] = (
"Keep the live boot-triggered SLO timer enabled. StockPlatform "
"freshness/ingestion are ok; the only active blocker is the fresh "
"all-host reboot observation window. Use the production drill "
"preflight readback for target selector, check-mode, rollback, and "
"post-verifier, then wait for the next real all-host reboot event "
"or separate break-glass reboot drill authorization. Do not reboot, "
"restart services, write DB rows, trigger workflows, or read secrets "
"from this lane."
)
professional_fix["owner"] = (
"reboot auto-recovery lane plus drill preflight readback"
)
workplan["reason"] = (
"Current readback shows service, product data, backup, and "
"StockPlatform freshness/ingestion are green. P0-006 remains event "
"gated only because the 10-minute recovery SLO needs a fresh "
"all-host reboot observation or a separately authorized reboot "
"drill."
)
workplan["safe_next_step"] = (
"keep_timer_live_wait_for_next_all_host_reboot_event_or_separately_"
"approved_reboot_drill_to_prove_10_minute_slo"
)
workplan["status"] = "blocked_waiting_fresh_all_host_reboot_window"
p0_004_ready = (
state["p0_004_template_copy_apply_gate_runtime_readback_state"] == "ready"
and state["p0_004_runtime_enablement_runtime_readback_state"] == "ready"

176
apps/api/src/services/delivery_closure_workbench.py
View File

@@ -45,6 +45,9 @@ from src.services.p0_cicd_baseline_source_readiness import (
from src.services.reboot_auto_recovery_slo_scorecard import (
load_latest_reboot_auto_recovery_slo_scorecard,
)
from src.services.reboot_auto_recovery_drill_preflight import (
load_latest_reboot_auto_recovery_drill_preflight,
)
from src.services.runtime_surface_inventory import (
load_latest_runtime_surface_inventory,
)
@@ -75,6 +78,7 @@ def load_delivery_closure_workbench() -> dict[str, Any]:
backup = load_latest_backup_dr_readiness_matrix()
credential_escrow_intake = load_latest_credential_escrow_evidence_intake_readiness()
reboot_slo = load_latest_reboot_auto_recovery_slo_scorecard()
reboot_drill_preflight = load_latest_reboot_auto_recovery_drill_preflight()
return build_delivery_closure_workbench(
status_cleanup=status_cleanup,
production_deploy=production_deploy,
@@ -89,6 +93,7 @@ def load_delivery_closure_workbench() -> dict[str, Any]:
backup=backup,
credential_escrow_intake=credential_escrow_intake,
reboot_slo=reboot_slo,
reboot_drill_preflight=reboot_drill_preflight,
)
@@ -107,6 +112,7 @@ def build_delivery_closure_workbench(
backup: dict[str, Any],
credential_escrow_intake: dict[str, Any],
reboot_slo: dict[str, Any],
reboot_drill_preflight: dict[str, Any],
) -> dict[str, Any]:
"""Build the delivery workbench response from already validated snapshots."""
status_summary = _dict(status_cleanup.get("summary"))
@@ -176,6 +182,11 @@ def build_delivery_closure_workbench(
reboot_stockplatform_recovery_gate = _dict(
reboot_stockplatform.get("controlled_recovery_gate")
)
reboot_drill_readback = _dict(reboot_drill_preflight.get("readback"))
reboot_drill_rollups = _dict(reboot_drill_preflight.get("rollups"))
reboot_drill_target_selector = _dict(reboot_drill_preflight.get("target_selector"))
reboot_drill_boundaries = _dict(reboot_drill_preflight.get("operation_boundaries"))
reboot_drill_check_mode = _dict(reboot_drill_preflight.get("check_mode"))
credential_intake_rollups = _dict(credential_escrow_intake.get("rollups"))
credential_intake_readback = _dict(credential_escrow_intake.get("readback"))
credential_closeout_receipt = _dict(
@@ -620,6 +631,64 @@ def build_delivery_closure_workbench(
"stockplatform_controlled_recovery_gate_status": str(
reboot_stockplatform_recovery_gate.get("status") or ""
),
"drill_preflight_status": str(
reboot_drill_preflight.get("status") or ""
),
"drill_preflight_ready": (
reboot_drill_rollups.get("preflight_ready") is True
),
"drill_preflight_ready_count": _int(
reboot_drill_rollups.get("preflight_ready_count")
),
"drill_preflight_blocker_count": _int(
reboot_drill_rollups.get("preflight_blocker_count")
),
"drill_preflight_active_blockers": _strings(
reboot_drill_preflight.get("active_blockers")
),
"drill_preflight_break_glass_authorization_required": (
reboot_drill_rollups.get("break_glass_authorization_required")
is True
),
"drill_preflight_execution_authorized_by_this_endpoint": (
reboot_drill_rollups.get("execution_authorized_by_this_endpoint")
is True
),
"drill_preflight_host_reboot_authorized_by_this_endpoint": (
reboot_drill_rollups.get(
"host_reboot_authorized_by_this_endpoint"
)
is True
),
"drill_preflight_runtime_write_allowed": (
reboot_drill_rollups.get("runtime_write_allowed") is True
),
"drill_preflight_target_required_host_count": _int(
reboot_drill_rollups.get("target_required_host_count")
),
"drill_preflight_target_observed_host_count": _int(
reboot_drill_rollups.get("target_observed_host_count")
),
"drill_preflight_target_missing_host_count": _int(
reboot_drill_rollups.get("target_missing_host_count")
),
"drill_preflight_target_unreachable_host_count": _int(
reboot_drill_rollups.get("target_unreachable_host_count")
),
"drill_preflight_target_stale_host_count": _int(
reboot_drill_rollups.get("target_stale_host_count")
),
"drill_preflight_verify_only_available": (
reboot_drill_check_mode.get("verify_only_available") is True
),
"drill_preflight_post_apply_verifier_endpoint": str(
reboot_drill_check_mode.get("post_apply_verifier_endpoint") or ""
),
"drill_preflight_safe_next_step": str(
reboot_drill_readback.get("safe_next_step")
or reboot_drill_preflight.get("safe_next_step")
or ""
),
"host_reboot_performed": _dict(
reboot_slo.get("operation_boundaries")
).get("host_reboot_performed")
@@ -636,6 +705,13 @@ def build_delivery_closure_workbench(
reboot_slo.get("operation_boundaries")
).get("secret_value_collection_allowed")
is True,
"drill_preflight_secret_value_collection_allowed": (
reboot_drill_boundaries.get("secret_value_collection_allowed")
is True
),
"drill_preflight_workflow_trigger_performed": (
reboot_drill_boundaries.get("workflow_trigger_performed") is True
),
},
"href": "/operations",
"next_action": str(reboot_readback.get("safe_next_step") or ""),
@@ -1212,6 +1288,9 @@ def build_delivery_closure_workbench(
_source_status("status_cleanup", status_cleanup),
_source_status("production_deploy_readback", production_deploy),
_source_status("reboot_auto_recovery_slo_scorecard", reboot_slo),
_source_status(
"reboot_auto_recovery_drill_preflight", reboot_drill_preflight
),
_source_status("gitea_private_inventory_p0_scorecard", private_inventory),
_source_status("p0_cicd_baseline_source_readiness", cicd_baseline),
_source_status("gitea_ci_cd", gitea),
@@ -1260,6 +1339,25 @@ def build_delivery_closure_workbench(
"current_p0_readiness_percent": _int(
reboot_rollups.get("readiness_percent")
),
"current_p0_drill_preflight_status": str(
reboot_drill_preflight.get("status") or ""
),
"current_p0_drill_preflight_ready": (
reboot_drill_rollups.get("preflight_ready") is True
),
"current_p0_drill_preflight_safe_next_step": str(
reboot_drill_readback.get("safe_next_step")
or reboot_drill_preflight.get("safe_next_step")
or ""
),
"current_p0_drill_preflight_break_glass_required": (
reboot_drill_rollups.get("break_glass_authorization_required")
is True
),
"current_p0_drill_preflight_execution_authorized_by_this_endpoint": (
reboot_drill_rollups.get("execution_authorized_by_this_endpoint")
is True
),
"closed_p0_workplan_ids": ["P0-003", "P0-005"],
"github_lane_status": "stopped_retired_do_not_use",
"secret_value_collection_allowed": False,
@@ -1291,6 +1389,26 @@ def build_delivery_closure_workbench(
"current_p0_stockplatform_ingestion_status": str(
reboot_rollups.get("stockplatform_ingestion_status") or ""
),
"current_p0_drill_preflight_ready": (
reboot_drill_rollups.get("preflight_ready") is True
),
"current_p0_drill_preflight_ready_count": _int(
reboot_drill_rollups.get("preflight_ready_count")
),
"current_p0_drill_preflight_blocker_count": _int(
reboot_drill_rollups.get("preflight_blocker_count")
),
"current_p0_drill_preflight_break_glass_required": (
reboot_drill_rollups.get("break_glass_authorization_required")
is True
),
"current_p0_drill_preflight_execution_authorized_by_this_endpoint": (
reboot_drill_rollups.get("execution_authorized_by_this_endpoint")
is True
),
"current_p0_drill_preflight_runtime_write_allowed": (
reboot_drill_rollups.get("runtime_write_allowed") is True
),
"gitea_private_inventory_blocker_count": private_inventory_blockers,
"credential_escrow_blocker_count": credential_escrow_missing_items,
"production_deploy_hard_blocker_count": _int(
@@ -1351,6 +1469,64 @@ def build_delivery_closure_workbench(
"reboot_auto_recovery_safe_next_step": str(
reboot_readback.get("safe_next_step") or ""
),
"reboot_auto_recovery_drill_preflight_status": str(
reboot_drill_preflight.get("status") or ""
),
"reboot_auto_recovery_drill_preflight_ready": (
reboot_drill_rollups.get("preflight_ready") is True
),
"reboot_auto_recovery_drill_preflight_ready_count": _int(
reboot_drill_rollups.get("preflight_ready_count")
),
"reboot_auto_recovery_drill_preflight_blocker_count": _int(
reboot_drill_rollups.get("preflight_blocker_count")
),
"reboot_auto_recovery_drill_preflight_target_required_host_count": _int(
reboot_drill_rollups.get("target_required_host_count")
),
"reboot_auto_recovery_drill_preflight_target_observed_host_count": _int(
reboot_drill_rollups.get("target_observed_host_count")
),
"reboot_auto_recovery_drill_preflight_target_missing_host_count": _int(
reboot_drill_rollups.get("target_missing_host_count")
),
"reboot_auto_recovery_drill_preflight_target_unreachable_host_count": _int(
reboot_drill_rollups.get("target_unreachable_host_count")
),
"reboot_auto_recovery_drill_preflight_target_stale_host_count": _int(
reboot_drill_rollups.get("target_stale_host_count")
),
"reboot_auto_recovery_drill_preflight_break_glass_required": (
reboot_drill_rollups.get("break_glass_authorization_required")
is True
),
"reboot_auto_recovery_drill_preflight_execution_authorized_by_this_endpoint": (
reboot_drill_rollups.get("execution_authorized_by_this_endpoint")
is True
),
"reboot_auto_recovery_drill_preflight_host_reboot_authorized_by_this_endpoint": (
reboot_drill_rollups.get(
"host_reboot_authorized_by_this_endpoint"
)
is True
),
"reboot_auto_recovery_drill_preflight_runtime_write_allowed": (
reboot_drill_rollups.get("runtime_write_allowed") is True
),
"reboot_auto_recovery_drill_preflight_target_selector_scope": str(
reboot_drill_target_selector.get("scope") or ""
),
"reboot_auto_recovery_drill_preflight_verify_only_available": (
reboot_drill_check_mode.get("verify_only_available") is True
),
"reboot_auto_recovery_drill_preflight_post_apply_verifier_endpoint": str(
reboot_drill_check_mode.get("post_apply_verifier_endpoint") or ""
),
"reboot_auto_recovery_drill_preflight_safe_next_step": str(
reboot_drill_readback.get("safe_next_step")
or reboot_drill_preflight.get("safe_next_step")
or ""
),
"gitea_private_inventory_status": str(private_inventory.get("status") or ""),
"gitea_private_inventory_workplan_id": str(
private_inventory_readback.get("workplan_id") or ""