wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 4 Packages Projects Releases Wiki Activity

ops(reboot): classify stock eod freshness window
Some checks failed
Code Review / ai-code-review (push) Successful in 19s
Details
Ansible / Reboot Recovery Contract / validate (push) Has been cancelled
Details

Browse Source
This commit is contained in:
ogt
2026-06-26 18:24:28 +08:00
parent e7c368aa27
commit 6afa3e4f35
6 changed files with 226 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
scripts/reboot-recovery/post-reboot-declaration-guard.py
View File

@@ -161,6 +161,41 @@ def build_payload(summary: dict[str, str], proposed: list[str]) -> dict[str, Any
if overall_declaration != "unknown":
allowed.append(overall_declaration)
if (
service_green
and product_data_green
and backup_core_green
and not dr_escrow_blocked
and not host_188_hygiene_blocked
and wazuh_registry_accepted
and not runtime_authorized
):
allowed.append("FULL_STACK_GREEN")
else:
full_stack_reasons: list[str] = []
if not service_green:
full_stack_reasons.append("service_green_not_1")
if not product_data_green:
full_stack_reasons.append("product_data_green_not_1")
if not backup_core_green:
full_stack_reasons.append("backup_core_green_not_1")
if dr_escrow_blocked:
full_stack_reasons.append(
f"escrow_missing_count:{summary.get('ESCROW_MISSING_COUNT', 'unknown')}"
)
if host_188_hygiene_blocked:
full_stack_reasons.append("host_188_hygiene_blocked:1")
if not wazuh_registry_accepted:
full_stack_reasons.append("wazuh_manager_registry_accepted:0")
if runtime_authorized:
full_stack_reasons.append("runtime_action_authorized:1")
forbidden.append(
{
"declaration": "FULL_STACK_GREEN",
"reason": ",".join(full_stack_reasons) or "unknown",
}
)
if dr_escrow_blocked:
forbidden.append(
{
@@ -215,7 +250,12 @@ def build_payload(summary: dict[str, str], proposed: list[str]) -> dict[str, Any
if item not in allowed_set and item not in forbidden_map
]
status = "blocked_service_recovery" if not service_green else "allowed_with_boundary_blockers"
if not service_green:
status = "blocked_service_recovery"
elif not product_data_green:
status = "blocked_product_data_recovery"
else:
status = "allowed_with_boundary_blockers"
if rejected_proposed:
status = "blocked_false_green_proposal"
@@ -248,6 +288,21 @@ def build_payload(summary: dict[str, str], proposed: list[str]) -> dict[str, Any
"evidence": {
"service_green": summary.get("SERVICE_GREEN", "unknown"),
"product_data_green": summary.get("PRODUCT_DATA_GREEN", "unknown"),
"stock_freshness_status": summary.get("STOCK_FRESHNESS_STATUS", "unknown"),
"stock_latest_trading_date": summary.get(
"STOCK_LATEST_TRADING_DATE",
"unknown",
),
"stock_blockers": summary.get("STOCK_BLOCKERS", "unknown"),
"stock_eod_window_pending": summary.get(
"STOCK_EOD_WINDOW_PENDING",
"unknown",
),
"stock_eod_classification": summary.get(
"STOCK_EOD_CLASSIFICATION",
"unknown",
),
"stock_eod_next_action": summary.get("STOCK_EOD_NEXT_ACTION", "unknown"),
"backup_core_green": summary.get("BACKUP_CORE_GREEN", "unknown"),
"escrow_missing_count": summary.get("ESCROW_MISSING_COUNT", "unknown"),
"host_188_hygiene_blocked": summary.get("HOST_188_HYGIENE_BLOCKED", "unknown"),

42
scripts/reboot-recovery/post-reboot-next-gate-dispatch.sh
View File

@@ -76,8 +76,17 @@ value_for() {
}
service_green="$(value_for SERVICE_GREEN)"
product_data_green="$(value_for PRODUCT_DATA_GREEN)"
overall_declaration="$(value_for OVERALL_DECLARATION)"
next_required_gates="$(value_for NEXT_REQUIRED_GATES)"
stock_freshness_status="$(value_for STOCK_FRESHNESS_STATUS)"
stock_latest_trading_date="$(value_for STOCK_LATEST_TRADING_DATE)"
stock_blockers="$(value_for STOCK_BLOCKERS)"
stock_eod_window_pending="$(value_for STOCK_EOD_WINDOW_PENDING)"
stock_eod_classification="$(value_for STOCK_EOD_CLASSIFICATION)"
stock_eod_next_action="$(value_for STOCK_EOD_NEXT_ACTION)"
stock_eod_first_full_window_end="$(value_for STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL)"
stock_eod_final_retry_window_end="$(value_for STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL)"
escrow_missing_count="$(value_for ESCROW_MISSING_COUNT)"
host_188_hygiene_blocked="$(value_for HOST_188_HYGIENE_BLOCKED)"
wazuh_registry_accepted="$(value_for WAZUH_MANAGER_REGISTRY_ACCEPTED)"
@@ -110,6 +119,7 @@ echo "SUMMARY_FILE=$SUMMARY_FILE"
echo "SUMMARY_ARTIFACT_DIR=${summary_artifact_dir:-unknown}"
echo "SUMMARY_RC=$summary_rc"
echo "SERVICE_GREEN=${service_green:-unknown}"
echo "PRODUCT_DATA_GREEN=${product_data_green:-unknown}"
echo "OVERALL_DECLARATION=${overall_declaration:-unknown}"
echo "NEXT_REQUIRED_GATES=${next_required_gates:-unknown}"
echo "RUNTIME_ACTION_AUTHORIZED=0"
@@ -118,7 +128,9 @@ echo "REQUEST_SENT_COUNT=0"
echo "HOST_WRITE_AUTHORIZED=0"
echo "SECRET_VALUE_COLLECTION_ALLOWED=0"
if [[ "$service_green" != "1" ]]; then
if [[ "$service_green" != "1" ]] \
&& ! contains_gate "product_data_freshness_recovery" \
&& ! contains_gate "stockplatform_eod_window_completion"; then
echo
echo "BLOCKED_SERVICE_GREEN=0"
echo "NEXT_STEP=restore_service_before_boundary_dispatch"
@@ -127,6 +139,34 @@ fi
gate_count=0
if contains_gate "stockplatform_eod_window_completion"; then
gate_count=$((gate_count + 1))
print_gate_header "stockplatform_eod_window_completion" "StockPlatform scheduled EOD freshness completion"
echo "GATE_PRIORITY=P0"
echo "GATE_STATUS=scheduled_eod_window_pending"
echo "CURRENT_EVIDENCE=status:${stock_freshness_status:-unknown};latest_trading_date:${stock_latest_trading_date:-unknown};blockers:${stock_blockers:-unknown};pending:${stock_eod_window_pending:-unknown};classification:${stock_eod_classification:-unknown};next:${stock_eod_next_action:-unknown};first_full_window_end:${stock_eod_first_full_window_end:-unknown};final_retry_window_end:${stock_eod_final_retry_window_end:-unknown}"
echo "OWNER_GROUP=stockplatform_data_owner,market_data_owner,oncall_operator"
echo "REQUIRED_EVIDENCE=freshness_endpoint_after_19_15,cron_log_refs,daily_ingestion_ref,price_ref,chips_ref,margin_ref,ai_recommendation_ref"
echo "FORBIDDEN_PAYLOADS=raw_db_dump,secret_value,api_token,manual_fake_row,handwritten_freshness_override"
echo "FORBIDDEN_ACTIONS=manual_db_update,truncate_restore,fake_freshness_marker,skip_data_gate,declare_product_data_green_before_status_ok"
echo "ALLOWED_ACTION=wait_for_scheduled_cron_and_recheck_summary"
echo "DONE_CRITERIA=stock_freshness_status:ok,product_data_green:1,post_reboot_summary_recheck_after_eod_window"
fi
if contains_gate "product_data_freshness_recovery"; then
gate_count=$((gate_count + 1))
print_gate_header "product_data_freshness_recovery" "Product data freshness recovery"
echo "GATE_PRIORITY=P0"
echo "GATE_STATUS=data_freshness_recovery_required"
echo "CURRENT_EVIDENCE=status:${stock_freshness_status:-unknown};latest_trading_date:${stock_latest_trading_date:-unknown};blockers:${stock_blockers:-unknown};classification:${stock_eod_classification:-unknown};next:${stock_eod_next_action:-unknown}"
echo "OWNER_GROUP=stockplatform_data_owner,market_data_owner,rollback_owner"
echo "REQUIRED_EVIDENCE=source_of_truth_commit,cron_entrypoint_refs,cron_log_refs,official_source_status,db_count_readback,freshness_endpoint_after_recovery,rollback_plan"
echo "FORBIDDEN_PAYLOADS=raw_db_dump,secret_value,api_token,manual_fake_row,handwritten_freshness_override"
echo "FORBIDDEN_ACTIONS=manual_db_update,truncate_restore,fake_freshness_marker,skip_data_gate,declare_product_data_green_before_status_ok"
echo "ALLOWED_ACTION=diagnose_source_or_cron_failure_before_controlled_recovery"
echo "DONE_CRITERIA=stock_freshness_status:ok,product_data_green:1,post_reboot_summary_status:not_service_blocked"
fi
if contains_gate "credential_escrow_evidence"; then
gate_count=$((gate_count + 1))
print_gate_header "credential_escrow_evidence" "DR credential escrow non-secret evidence"

1
scripts/reboot-recovery/post-reboot-next-gate-owner-packets.py
View File

@@ -177,6 +177,7 @@ def build_packet(parsed: dict[str, Any]) -> dict[str, Any]:
"runtime_action_authorized_count": 0,
},
"no_false_green_rules": [
"product_route_green_does_not_equal_product_data_green",
"service_green_does_not_equal_dr_complete",
"backup_fresh_does_not_equal_credential_escrow_complete",
"host_188_service_green_does_not_equal_host_hygiene_green",

31
scripts/reboot-recovery/post-reboot-owner-packet-contract-guard.py
View File

@@ -26,9 +26,12 @@ EXPECTED_SCHEMA = "awoooi_post_reboot_next_gate_owner_packets_v1"
KNOWN_GATES = {
"credential_escrow_evidence",
"host_188_hygiene_maintenance_window",
"product_data_freshness_recovery",
"stockplatform_eod_window_completion",
"wazuh_manager_registry_export",
}
EXPECTED_NO_FALSE_GREEN_RULES = {
"product_route_green_does_not_equal_product_data_green",
"service_green_does_not_equal_dr_complete",
"backup_fresh_does_not_equal_credential_escrow_complete",
"host_188_service_green_does_not_equal_host_hygiene_green",
@@ -69,6 +72,20 @@ GATE_REQUIRED_FORBIDDEN_PAYLOADS = {
"password",
"authorization_header",
},
"product_data_freshness_recovery": {
"raw_db_dump",
"secret_value",
"api_token",
"manual_fake_row",
"handwritten_freshness_override",
},
"stockplatform_eod_window_completion": {
"raw_db_dump",
"secret_value",
"api_token",
"manual_fake_row",
"handwritten_freshness_override",
},
}
GATE_REQUIRED_FORBIDDEN_ACTIONS = {
@@ -94,6 +111,20 @@ GATE_REQUIRED_FORBIDDEN_ACTIONS = {
"host_write",
"kali_active_scan",
},
"product_data_freshness_recovery": {
"manual_db_update",
"truncate_restore",
"fake_freshness_marker",
"skip_data_gate",
"declare_product_data_green_before_status_ok",
},
"stockplatform_eod_window_completion": {
"manual_db_update",
"truncate_restore",
"fake_freshness_marker",
"skip_data_gate",
"declare_product_data_green_before_status_ok",
},
}
PACKET_FALSE_FIELDS = (

25
scripts/reboot-recovery/post-reboot-readiness-summary.sh
View File

@@ -114,6 +114,14 @@ if grep -q '^STOCK_FRESHNESS_STATUS ok$' "$post_start_log" \
&& grep -q '^DB_DAILY_FRESHNESS ' "$post_start_log"; then
product_data_green=1
fi
stock_freshness_status="$(awk '$1 == "STOCK_FRESHNESS_STATUS" {value=$2} END {print value}' "$post_start_log")"
stock_latest_trading_date="$(awk '$1 == "STOCK_LATEST_TRADING_DATE" {value=$2} END {print value}' "$post_start_log")"
stock_blockers="$(grep -E '^STOCK_BLOCKERS ' "$post_start_log" | tail -n 1 | cut -d' ' -f2- || true)"
stock_eod_window_pending="$(awk '$1 == "STOCK_EOD_WINDOW_PENDING" {value=$2} END {print value}' "$post_start_log")"
stock_eod_classification="$(awk '$1 == "STOCK_EOD_CLASSIFICATION" {value=$2} END {print value}' "$post_start_log")"
stock_eod_next_action="$(awk '$1 == "STOCK_EOD_NEXT_ACTION" {value=$2} END {print value}' "$post_start_log")"
stock_eod_first_full_window_end="$(awk '$1 == "STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL" {value=$2} END {print value}' "$post_start_log")"
stock_eod_final_retry_window_end="$(awk '$1 == "STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL" {value=$2} END {print value}' "$post_start_log")"
escrow_missing_count="$(grep -Eo 'escrow_missing=[0-9]+' "$post_start_log" | tail -n 1 | cut -d= -f2 || true)"
dr_escrow_blocked=0
@@ -176,6 +184,10 @@ fi
overall_declaration="GREEN"
if [[ "$service_green" != "1" ]]; then
overall_declaration="SERVICE_BLOCKED"
elif [[ "$product_data_green" != "1" && "$stock_eod_window_pending" == "1" ]]; then
overall_declaration="PRODUCT_DATA_PENDING_EOD_WINDOW"
elif [[ "$product_data_green" != "1" ]]; then
overall_declaration="PRODUCT_DATA_BLOCKED"
elif [[ "$dr_escrow_blocked" == "1" ]]; then
overall_declaration="FULL_STACK_GREEN_DR_ESCROW_BLOCKED"
elif [[ "$host_188_hygiene_blocked" == "1" ]]; then
@@ -187,6 +199,11 @@ elif [[ "$evidence_warn" != "0" && -n "$evidence_warn" ]]; then
fi
next_required_gates=()
if [[ "$product_data_green" != "1" && "$stock_eod_window_pending" == "1" ]]; then
next_required_gates+=("stockplatform_eod_window_completion")
elif [[ "$product_data_green" != "1" ]]; then
next_required_gates+=("product_data_freshness_recovery")
fi
[[ "$dr_escrow_blocked" == "1" ]] && next_required_gates+=("credential_escrow_evidence")
[[ "$host_188_hygiene_blocked" == "1" ]] && next_required_gates+=("host_188_hygiene_maintenance_window")
[[ "$wazuh_registry_accepted" == "0" ]] && next_required_gates+=("wazuh_manager_registry_export")
@@ -213,6 +230,14 @@ POST_START_BOUNDARY_WARNINGS=${boundary_warn:-unknown}
POST_START_EVIDENCE_WARNINGS=${evidence_warn:-unknown}
SERVICE_GREEN=$service_green
PRODUCT_DATA_GREEN=$product_data_green
STOCK_FRESHNESS_STATUS=${stock_freshness_status:-unknown}
STOCK_LATEST_TRADING_DATE=${stock_latest_trading_date:-unknown}
STOCK_BLOCKERS=${stock_blockers:-unknown}
STOCK_EOD_WINDOW_PENDING=${stock_eod_window_pending:-0}
STOCK_EOD_CLASSIFICATION=${stock_eod_classification:-unknown}
STOCK_EOD_NEXT_ACTION=${stock_eod_next_action:-unknown}
STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL=${stock_eod_first_full_window_end:-unknown}
STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL=${stock_eod_final_retry_window_end:-unknown}
BACKUP_CORE_GREEN=$backup_core_green
DR_ESCROW_BLOCKED=$dr_escrow_blocked
ESCROW_MISSING_COUNT=${escrow_missing_count:-unknown}

73
scripts/reboot-recovery/post-start-quick-check.sh
View File

@@ -22,6 +22,10 @@ COLD_START_PENDING_BLOCKERS=0
COLD_START_BLOCKED_SUMMARY=""
COLD_START_BLOCKED_LINES=""
ROUTE_SMOKE_BLOCKED=0
STOCK_EOD_WINDOW_PENDING=0
STOCK_EOD_CLASSIFICATION="not_evaluated"
STOCK_EOD_NEXT_ACTION="not_evaluated"
STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL="19:15"
PASS_COUNT=0
WARN_COUNT=0
@@ -334,6 +338,9 @@ with open(sys.argv[1], "r", encoding="utf-8") as fh:
PY
)"
if [[ "$stock_status" == "ok" ]]; then
printf 'STOCK_EOD_WINDOW_PENDING 0\n'
printf 'STOCK_EOD_CLASSIFICATION ok\n'
printf 'STOCK_EOD_NEXT_ACTION none\n'
ok "StockPlatform freshness is ok"
else
stock_blockers="$(python3 - "$stock_tmp" <<'PY'
@@ -344,7 +351,66 @@ with open(sys.argv[1], "r", encoding="utf-8") as fh:
print(",".join(json.load(fh).get("blockers") or []))
PY
)"
blocked "StockPlatform freshness is ${stock_status:-unknown}: ${stock_blockers:-no_blocker_list}"
stock_eod_context="$(python3 - "$stock_tmp" <<'PY'
import json
import sys
from datetime import datetime, time, timezone, timedelta
try:
from zoneinfo import ZoneInfo
except Exception: # pragma: no cover - old Python fallback
ZoneInfo = None
with open(sys.argv[1], "r", encoding="utf-8") as fh:
payload = json.load(fh)
tz = ZoneInfo("Asia/Taipei") if ZoneInfo else timezone(timedelta(hours=8))
now = datetime.now(tz)
today = now.date().isoformat()
latest = payload.get("latest_trading_date") or ""
status = payload.get("status") or "unknown"
blockers = payload.get("blockers") or []
first_full_window_end = time(19, 15)
final_retry_window_end = time(23, 35)
pending = 0
classification = "blocked_unknown"
next_action = "investigate_stockplatform_freshness"
if status == "ok":
classification = "ok"
next_action = "none"
elif latest == today and blockers and now.time() < first_full_window_end:
pending = 1
classification = "pending_first_eod_window"
next_action = "wait_for_18_20_19_10_cron_then_recheck"
elif latest == today and blockers and now.time() < final_retry_window_end:
classification = "after_first_eod_window_blocked"
next_action = "inspect_ingestion_logs_and_wait_retry_windows"
elif latest == today and blockers:
classification = "after_final_eod_window_blocked"
next_action = "open_stockplatform_data_recovery_gate"
elif blockers:
classification = "blocked_non_current_trading_day"
next_action = "inspect_trading_calendar_and_ingestion_logs"
print(f"STOCK_EOD_WINDOW_PENDING {pending}")
print(f"STOCK_EOD_CLASSIFICATION {classification}")
print(f"STOCK_EOD_NEXT_ACTION {next_action}")
print(f"STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL {first_full_window_end.strftime('%H:%M')}")
print(f"STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL {final_retry_window_end.strftime('%H:%M')}")
print(f"STOCK_EOD_OBSERVED_AT_LOCAL {now.isoformat(timespec='seconds')}")
PY
)"
printf '%s\n' "$stock_eod_context"
STOCK_EOD_WINDOW_PENDING="$(awk '$1 == "STOCK_EOD_WINDOW_PENDING" {print $2}' <<<"$stock_eod_context" | tail -n 1)"
STOCK_EOD_CLASSIFICATION="$(awk '$1 == "STOCK_EOD_CLASSIFICATION" {print $2}' <<<"$stock_eod_context" | tail -n 1)"
STOCK_EOD_NEXT_ACTION="$(awk '$1 == "STOCK_EOD_NEXT_ACTION" {print $2}' <<<"$stock_eod_context" | tail -n 1)"
if [[ "$STOCK_EOD_WINDOW_PENDING" == "1" ]]; then
evidence_warn "StockPlatform freshness pending scheduled EOD window: ${STOCK_EOD_CLASSIFICATION:-unknown}; next=${STOCK_EOD_NEXT_ACTION:-unknown}; blockers=${stock_blockers:-no_blocker_list}"
else
blocked "StockPlatform freshness is ${stock_status:-unknown}: ${stock_blockers:-no_blocker_list}; classification=${STOCK_EOD_CLASSIFICATION:-unknown}"
fi
fi
fi
rm -f "$stock_tmp"
@@ -460,6 +526,11 @@ if [[ "$SERVICE_WARN_COUNT" -gt 0 ]]; then
exit 1
fi
if [[ "$STOCK_EOD_WINDOW_PENDING" == "1" ]]; then
printf 'RESULT=PRODUCT_DATA_PENDING_EOD_WINDOW\n'
exit 0
fi
if [[ "$BOUNDARY_WARN_COUNT" -gt 0 ]]; then
printf 'RESULT=FULL_STACK_GREEN_DR_ESCROW_BLOCKED\n'
exit 0