fix(api): keep priority readback fail closed
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 31s
CD Pipeline / build-and-deploy (push) Failing after 2m33s
CD Pipeline / post-deploy-checks (push) Has been skipped
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 31s
CD Pipeline / build-and-deploy (push) Failing after 2m33s
CD Pipeline / post-deploy-checks (push) Has been skipped
This commit is contained in:
@@ -907,6 +907,31 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
|
||||
evidence["production_deploy_governance_fields_present"] = closure_percent == 100
|
||||
evidence["latest_cd_run_status"] = "Success"
|
||||
|
||||
stock_freshness_status = str(
|
||||
workbench_rollups.get("current_p0_stockplatform_freshness_status") or ""
|
||||
)
|
||||
stock_ingestion_status = str(
|
||||
workbench_rollups.get("current_p0_stockplatform_ingestion_status") or ""
|
||||
)
|
||||
stock_blockers = _stockplatform_status_blockers(
|
||||
freshness_status=stock_freshness_status,
|
||||
ingestion_status=stock_ingestion_status,
|
||||
)
|
||||
stockplatform_green = not stock_blockers
|
||||
service_green = workbench_rollups.get("current_p0_service_green") is True
|
||||
product_data_green = (
|
||||
workbench_rollups.get("current_p0_product_data_green") is True
|
||||
)
|
||||
backup_core_green = workbench_rollups.get("current_p0_backup_core_green") is True
|
||||
drill_preflight_ready = reboot_preflight_rollups.get("preflight_ready") is True
|
||||
base_readback_green = (
|
||||
service_green
|
||||
and product_data_green
|
||||
and backup_core_green
|
||||
and stockplatform_green
|
||||
and drill_preflight_ready
|
||||
)
|
||||
|
||||
for item in _list(payload.get("in_progress_or_blocked_in_priority_order")):
|
||||
workplan = _dict(item)
|
||||
if workplan.get("workplan_id") != "P0-006":
|
||||
@@ -914,28 +939,24 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
|
||||
|
||||
evidence = _dict(workplan.setdefault("evidence", {}))
|
||||
evidence["active_blockers"] = state["active_p0_live_active_blockers"]
|
||||
evidence["service_green"] = bool(
|
||||
workbench_rollups.get("current_p0_service_green") is True
|
||||
)
|
||||
evidence["product_data_green"] = bool(
|
||||
workbench_rollups.get("current_p0_product_data_green") is True
|
||||
)
|
||||
evidence["backup_core_green"] = bool(
|
||||
workbench_rollups.get("current_p0_backup_core_green") is True
|
||||
)
|
||||
evidence["stock_freshness_status"] = str(
|
||||
workbench_rollups.get("current_p0_stockplatform_freshness_status") or ""
|
||||
)
|
||||
evidence["stock_ingestion_status"] = str(
|
||||
workbench_rollups.get("current_p0_stockplatform_ingestion_status") or ""
|
||||
)
|
||||
evidence["stock_blockers"] = []
|
||||
evidence["stock_ingestion_blockers"] = []
|
||||
evidence["service_green"] = service_green
|
||||
evidence["product_data_green"] = product_data_green
|
||||
evidence["backup_core_green"] = backup_core_green
|
||||
evidence["stock_freshness_status"] = stock_freshness_status
|
||||
evidence["stock_ingestion_status"] = stock_ingestion_status
|
||||
evidence["stock_blockers"] = [
|
||||
blocker
|
||||
for blocker in stock_blockers
|
||||
if blocker.startswith("stockplatform_freshness_")
|
||||
]
|
||||
evidence["stock_ingestion_blockers"] = [
|
||||
blocker
|
||||
for blocker in stock_blockers
|
||||
if blocker.startswith("stockplatform_ingestion_")
|
||||
]
|
||||
evidence["stockplatform_controlled_recovery_gate_required"] = False
|
||||
evidence["drill_preflight_status"] = str(reboot_preflight.get("status") or "")
|
||||
evidence["drill_preflight_ready"] = (
|
||||
reboot_preflight_rollups.get("preflight_ready") is True
|
||||
)
|
||||
evidence["drill_preflight_ready"] = drill_preflight_ready
|
||||
evidence["drill_preflight_blocker_count"] = _int(
|
||||
reboot_preflight_rollups.get("preflight_blocker_count")
|
||||
)
|
||||
@@ -964,31 +985,57 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
|
||||
)
|
||||
|
||||
professional_fix = _dict(workplan.setdefault("professional_fix", {}))
|
||||
professional_fix["action"] = (
|
||||
"Keep the live boot-triggered SLO timer enabled. StockPlatform "
|
||||
"freshness/ingestion are ok; the only active blocker is the fresh "
|
||||
"all-host reboot observation window. Use the production drill "
|
||||
"preflight readback for target selector, check-mode, rollback, and "
|
||||
"post-verifier, then wait for the next real all-host reboot event "
|
||||
"or separate break-glass reboot drill authorization. Do not reboot, "
|
||||
"restart services, write DB rows, trigger workflows, or read secrets "
|
||||
"from this lane."
|
||||
)
|
||||
professional_fix["owner"] = (
|
||||
"reboot auto-recovery lane plus drill preflight readback"
|
||||
)
|
||||
|
||||
if base_readback_green:
|
||||
professional_fix["action"] = (
|
||||
"Keep the live boot-triggered SLO timer enabled. StockPlatform "
|
||||
"freshness/ingestion are ok; the only active blocker is the fresh "
|
||||
"all-host reboot observation window. Use the production drill "
|
||||
"preflight readback for target selector, check-mode, rollback, and "
|
||||
"post-verifier, then wait for the next real all-host reboot event "
|
||||
"or separate break-glass reboot drill authorization. Do not reboot, "
|
||||
"restart services, write DB rows, trigger workflows, or read secrets "
|
||||
"from this lane."
|
||||
)
|
||||
workplan["reason"] = (
|
||||
"Current readback shows service, product data, backup, and "
|
||||
"StockPlatform freshness/ingestion are green. P0-006 remains event "
|
||||
"gated only because the 10-minute recovery SLO needs a fresh "
|
||||
"all-host reboot observation or a separately authorized reboot "
|
||||
"drill."
|
||||
)
|
||||
workplan["safe_next_step"] = (
|
||||
"keep_timer_live_wait_for_next_all_host_reboot_event_or_separately_"
|
||||
"approved_reboot_drill_to_prove_10_minute_slo"
|
||||
)
|
||||
workplan["status"] = "blocked_waiting_fresh_all_host_reboot_window"
|
||||
continue
|
||||
|
||||
professional_fix["action"] = (
|
||||
"Keep the live boot-triggered SLO timer enabled, but do not claim "
|
||||
"reboot-window-only closure while service/data/backup/StockPlatform "
|
||||
"readback is not green. First run the controlled source freshness and "
|
||||
"Postgres contract readback, then rerun public health, freshness, "
|
||||
"ingestion, backup, and drill preflight verifiers. Do not reboot, "
|
||||
"restart services, write DB rows, fake freshness, trigger workflows, "
|
||||
"or read secrets from this lane."
|
||||
)
|
||||
workplan["reason"] = (
|
||||
"Current readback shows service, product data, backup, and "
|
||||
"StockPlatform freshness/ingestion are green. P0-006 remains event "
|
||||
"gated only because the 10-minute recovery SLO needs a fresh "
|
||||
"all-host reboot observation or a separately authorized reboot "
|
||||
"drill."
|
||||
"Current P0-006 readback is not green: service_green="
|
||||
f"{service_green}, product_data_green={product_data_green}, "
|
||||
f"backup_core_green={backup_core_green}, "
|
||||
f"stock_freshness_status={stock_freshness_status}, "
|
||||
f"stock_ingestion_status={stock_ingestion_status}, "
|
||||
f"drill_preflight_ready={drill_preflight_ready}."
|
||||
)
|
||||
workplan["safe_next_step"] = (
|
||||
"keep_timer_live_wait_for_next_all_host_reboot_event_or_separately_"
|
||||
"approved_reboot_drill_to_prove_10_minute_slo"
|
||||
"resolve_stockplatform_source_freshness_and_reboot_slo_readback_"
|
||||
"blockers_then_rerun_priority_work_order"
|
||||
)
|
||||
workplan["status"] = "blocked_waiting_fresh_all_host_reboot_window"
|
||||
workplan["status"] = "blocked_p0_006_readback_not_green"
|
||||
|
||||
p0_004_ready = (
|
||||
state["p0_004_template_copy_apply_gate_runtime_readback_state"] == "ready"
|
||||
@@ -1002,30 +1049,68 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
|
||||
if p0_004_ready and p0_006_event_gated
|
||||
else "mainline_readback_requires_attention"
|
||||
)
|
||||
payload["next_execution_order"] = [
|
||||
(
|
||||
"P0-006: service/data/backup/StockPlatform readback is green, but "
|
||||
"the 10-minute reboot SLO cannot be claimed until a fresh all-host "
|
||||
"reboot event or separately approved reboot drill; keep timer live "
|
||||
"and do not reboot/restart/DB-write from this lane."
|
||||
),
|
||||
(
|
||||
"P0-006-REBOOT-DRILL-PREFLIGHT-READBACK: production endpoint is 200 "
|
||||
"and preflight is ready for separate break-glass reboot drill "
|
||||
"authorization; do not reboot from this lane without that explicit "
|
||||
"drill authorization."
|
||||
),
|
||||
(
|
||||
"P0-004-TEMPLATE-COPY-APPLY-GATE-READBACK: production apply gate, "
|
||||
"template-copy receipt, and runtime enablement readbacks are ready; "
|
||||
"keep closed unless production readback regresses."
|
||||
),
|
||||
(
|
||||
"NEXT: keep this priority-order API as the source of truth before "
|
||||
"opening the next blocker-free mainline item; stale snapshots, old "
|
||||
"failed CD runs, and retired GitHub lanes must not reorder closed work."
|
||||
),
|
||||
]
|
||||
if p0_006_event_gated:
|
||||
payload["next_execution_order"] = [
|
||||
(
|
||||
"P0-006: service/data/backup/StockPlatform readback is green, but "
|
||||
"the 10-minute reboot SLO cannot be claimed until a fresh all-host "
|
||||
"reboot event or separately approved reboot drill; keep timer live "
|
||||
"and do not reboot/restart/DB-write from this lane."
|
||||
),
|
||||
(
|
||||
"P0-006-REBOOT-DRILL-PREFLIGHT-READBACK: production endpoint is 200 "
|
||||
"and preflight is ready for separate break-glass reboot drill "
|
||||
"authorization; do not reboot from this lane without that explicit "
|
||||
"drill authorization."
|
||||
),
|
||||
(
|
||||
"P0-004-TEMPLATE-COPY-APPLY-GATE-READBACK: production apply gate, "
|
||||
"template-copy receipt, and runtime enablement readbacks are ready; "
|
||||
"keep closed unless production readback regresses."
|
||||
),
|
||||
(
|
||||
"NEXT: keep this priority-order API as the source of truth before "
|
||||
"opening the next blocker-free mainline item; stale snapshots, old "
|
||||
"failed CD runs, and retired GitHub lanes must not reorder closed work."
|
||||
),
|
||||
]
|
||||
else:
|
||||
state["active_p0_state"] = "blocked_p0_006_readback_not_green"
|
||||
state["next_executable_mainline_workplan_id"] = (
|
||||
"P0-006-STOCKPLATFORM-DATA-DEPENDENCY-READBACK"
|
||||
if not stockplatform_green
|
||||
else "P0-006-REBOOT-SLO-READBACK-RECOVERY"
|
||||
)
|
||||
state["next_executable_mainline_state"] = (
|
||||
"controlled_stockplatform_source_freshness_and_postgres_contract_"
|
||||
"readback_required"
|
||||
if not stockplatform_green
|
||||
else "controlled_reboot_slo_readback_recovery_required"
|
||||
)
|
||||
payload["next_execution_order"] = [
|
||||
(
|
||||
"P0-006: service/data/backup/StockPlatform readback is not green; "
|
||||
"resolve controlled source freshness, Postgres contract, backup, "
|
||||
"and reboot SLO readback blockers before claiming reboot-window-only "
|
||||
"closure. Do not reboot/restart/DB-write/fake freshness from this lane."
|
||||
),
|
||||
(
|
||||
"P0-006-STOCKPLATFORM-DATA-DEPENDENCY-READBACK: StockPlatform "
|
||||
f"freshness={stock_freshness_status} and "
|
||||
f"ingestion={stock_ingestion_status}; run read-only source "
|
||||
"freshness/Postgres contract readback and controlled verifier."
|
||||
),
|
||||
(
|
||||
"P0-006-REBOOT-DRILL-PREFLIGHT-READBACK: keep preflight as "
|
||||
"evidence only while base service/data/backup readback is not green; "
|
||||
"do not reboot without separate break-glass drill authorization."
|
||||
),
|
||||
(
|
||||
"NEXT: keep this priority-order API as the source of truth before "
|
||||
"opening the next blocker-free mainline item; stale snapshots, old "
|
||||
"failed CD runs, and retired GitHub lanes must not reorder closed work."
|
||||
),
|
||||
]
|
||||
_set_rollups_and_summary(
|
||||
payload=payload,
|
||||
current_head=current_head,
|
||||
@@ -1249,6 +1334,23 @@ def _unique_strings(values: list[str]) -> list[str]:
|
||||
return unique
|
||||
|
||||
|
||||
def _stockplatform_status_blockers(
|
||||
*,
|
||||
freshness_status: str,
|
||||
ingestion_status: str,
|
||||
) -> list[str]:
|
||||
blockers: list[str] = []
|
||||
if freshness_status != "ok":
|
||||
blockers.append(
|
||||
f"stockplatform_freshness_{freshness_status or 'unknown'}"
|
||||
)
|
||||
if ingestion_status != "ok":
|
||||
blockers.append(
|
||||
f"stockplatform_ingestion_{ingestion_status or 'unknown'}"
|
||||
)
|
||||
return blockers
|
||||
|
||||
|
||||
def _is_sha(value: str) -> bool:
|
||||
return bool(_SHA_RE.fullmatch(value))
|
||||
|
||||
|
||||
@@ -43,21 +43,33 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order():
|
||||
assert payload["next_execution_order"][0].startswith("P0-006:")
|
||||
in_progress = payload["in_progress_or_blocked_in_priority_order"][0]
|
||||
assert in_progress["workplan_id"] == "P0-006"
|
||||
assert in_progress["evidence"]["stock_freshness_status"] == "ok"
|
||||
assert in_progress["evidence"]["stock_ingestion_status"] == "ok"
|
||||
assert in_progress["evidence"]["stock_blockers"] == []
|
||||
assert in_progress["evidence"]["drill_preflight_ready"] is True
|
||||
assert in_progress["evidence"]["drill_preflight_blocker_count"] == 0
|
||||
assert payload["mainline_execution_state"]["active_p0_state"] == (
|
||||
"blocked_p0_006_readback_not_green"
|
||||
)
|
||||
assert payload["mainline_execution_state"]["next_executable_mainline_workplan_id"] == (
|
||||
"P0-006-STOCKPLATFORM-DATA-DEPENDENCY-READBACK"
|
||||
)
|
||||
assert in_progress["status"] == "blocked_p0_006_readback_not_green"
|
||||
assert in_progress["evidence"]["stock_freshness_status"] == "not_configured"
|
||||
assert in_progress["evidence"]["stock_ingestion_status"] == "not_configured"
|
||||
assert in_progress["evidence"]["stock_blockers"] == [
|
||||
"stockplatform_freshness_not_configured"
|
||||
]
|
||||
assert in_progress["evidence"]["stock_ingestion_blockers"] == [
|
||||
"stockplatform_ingestion_not_configured"
|
||||
]
|
||||
assert in_progress["evidence"]["drill_preflight_ready"] is False
|
||||
assert in_progress["evidence"]["drill_preflight_blocker_count"] == 9
|
||||
assert (
|
||||
in_progress["evidence"][
|
||||
"drill_preflight_execution_authorized_by_this_endpoint"
|
||||
]
|
||||
is False
|
||||
)
|
||||
assert "StockPlatform freshness/ingestion are ok" in in_progress[
|
||||
assert "service/data/backup/StockPlatform readback is not green" in in_progress[
|
||||
"professional_fix"
|
||||
]["action"]
|
||||
assert "current service recovery blocker" not in in_progress[
|
||||
assert "Do not reboot" in in_progress[
|
||||
"professional_fix"
|
||||
]["action"]
|
||||
assert payload["operation_boundaries"]["github_api_used"] is False
|
||||
@@ -88,13 +100,17 @@ def test_awoooi_priority_work_order_readback_endpoint_returns_snapshot(
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "p0_006_event_gated_all_immediate_apply_gaps_closed"
|
||||
assert data["status"] == "mainline_readback_requires_attention"
|
||||
assert data["mainline_execution_state"]["active_p0_workplan_id"] == "P0-006"
|
||||
assert data["mainline_execution_state"]["p0_004_template_copy_apply_gate_runtime_readback_state"] == "ready"
|
||||
assert data["mainline_execution_state"]["reboot_drill_preflight_runtime_readback_state"] == "ready"
|
||||
assert data["mainline_execution_state"]["active_p0_state"] == (
|
||||
"blocked_p0_006_readback_not_green"
|
||||
)
|
||||
assert data["mainline_execution_state"]["reboot_drill_preflight_runtime_readback_state"] == "blocked"
|
||||
assert data["rollups"]["stockplatform_public_api_runtime_ready"] is True
|
||||
assert data["rollups"]["active_p0_event_gated_by_fresh_reboot_window_only"] is False
|
||||
assert data["next_execution_order"][0].startswith("P0-006:")
|
||||
assert "do not reboot" in data["next_execution_order"][0]
|
||||
assert "readback is not green" in data["next_execution_order"][0]
|
||||
|
||||
|
||||
def test_awoooi_priority_work_order_readback_overlays_harbor_deploy_blocker(
|
||||
|
||||
Reference in New Issue
Block a user