From 6afa3e4f358350fe11ab651ca31ebce9094ff26c Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 26 Jun 2026 18:24:28 +0800 Subject: [PATCH] ops(reboot): classify stock eod freshness window --- .../post-reboot-declaration-guard.py | 57 ++++++++++++++- .../post-reboot-next-gate-dispatch.sh | 42 ++++++++++- .../post-reboot-next-gate-owner-packets.py | 1 + ...post-reboot-owner-packet-contract-guard.py | 31 ++++++++ .../post-reboot-readiness-summary.sh | 25 +++++++ .../reboot-recovery/post-start-quick-check.sh | 73 ++++++++++++++++++- 6 files changed, 226 insertions(+), 3 deletions(-) diff --git a/scripts/reboot-recovery/post-reboot-declaration-guard.py b/scripts/reboot-recovery/post-reboot-declaration-guard.py index 7c62c5de..6907ae9e 100755 --- a/scripts/reboot-recovery/post-reboot-declaration-guard.py +++ b/scripts/reboot-recovery/post-reboot-declaration-guard.py @@ -161,6 +161,41 @@ def build_payload(summary: dict[str, str], proposed: list[str]) -> dict[str, Any if overall_declaration != "unknown": allowed.append(overall_declaration) + if ( + service_green + and product_data_green + and backup_core_green + and not dr_escrow_blocked + and not host_188_hygiene_blocked + and wazuh_registry_accepted + and not runtime_authorized + ): + allowed.append("FULL_STACK_GREEN") + else: + full_stack_reasons: list[str] = [] + if not service_green: + full_stack_reasons.append("service_green_not_1") + if not product_data_green: + full_stack_reasons.append("product_data_green_not_1") + if not backup_core_green: + full_stack_reasons.append("backup_core_green_not_1") + if dr_escrow_blocked: + full_stack_reasons.append( + f"escrow_missing_count:{summary.get('ESCROW_MISSING_COUNT', 'unknown')}" + ) + if host_188_hygiene_blocked: + full_stack_reasons.append("host_188_hygiene_blocked:1") + if not wazuh_registry_accepted: + full_stack_reasons.append("wazuh_manager_registry_accepted:0") + if runtime_authorized: + full_stack_reasons.append("runtime_action_authorized:1") + forbidden.append( + { + "declaration": "FULL_STACK_GREEN", + "reason": ",".join(full_stack_reasons) or "unknown", + } + ) + if dr_escrow_blocked: forbidden.append( { @@ -215,7 +250,12 @@ def build_payload(summary: dict[str, str], proposed: list[str]) -> dict[str, Any if item not in allowed_set and item not in forbidden_map ] - status = "blocked_service_recovery" if not service_green else "allowed_with_boundary_blockers" + if not service_green: + status = "blocked_service_recovery" + elif not product_data_green: + status = "blocked_product_data_recovery" + else: + status = "allowed_with_boundary_blockers" if rejected_proposed: status = "blocked_false_green_proposal" @@ -248,6 +288,21 @@ def build_payload(summary: dict[str, str], proposed: list[str]) -> dict[str, Any "evidence": { "service_green": summary.get("SERVICE_GREEN", "unknown"), "product_data_green": summary.get("PRODUCT_DATA_GREEN", "unknown"), + "stock_freshness_status": summary.get("STOCK_FRESHNESS_STATUS", "unknown"), + "stock_latest_trading_date": summary.get( + "STOCK_LATEST_TRADING_DATE", + "unknown", + ), + "stock_blockers": summary.get("STOCK_BLOCKERS", "unknown"), + "stock_eod_window_pending": summary.get( + "STOCK_EOD_WINDOW_PENDING", + "unknown", + ), + "stock_eod_classification": summary.get( + "STOCK_EOD_CLASSIFICATION", + "unknown", + ), + "stock_eod_next_action": summary.get("STOCK_EOD_NEXT_ACTION", "unknown"), "backup_core_green": summary.get("BACKUP_CORE_GREEN", "unknown"), "escrow_missing_count": summary.get("ESCROW_MISSING_COUNT", "unknown"), "host_188_hygiene_blocked": summary.get("HOST_188_HYGIENE_BLOCKED", "unknown"), diff --git a/scripts/reboot-recovery/post-reboot-next-gate-dispatch.sh b/scripts/reboot-recovery/post-reboot-next-gate-dispatch.sh index 0451c6d4..604211b1 100755 --- a/scripts/reboot-recovery/post-reboot-next-gate-dispatch.sh +++ b/scripts/reboot-recovery/post-reboot-next-gate-dispatch.sh @@ -76,8 +76,17 @@ value_for() { } service_green="$(value_for SERVICE_GREEN)" +product_data_green="$(value_for PRODUCT_DATA_GREEN)" overall_declaration="$(value_for OVERALL_DECLARATION)" next_required_gates="$(value_for NEXT_REQUIRED_GATES)" +stock_freshness_status="$(value_for STOCK_FRESHNESS_STATUS)" +stock_latest_trading_date="$(value_for STOCK_LATEST_TRADING_DATE)" +stock_blockers="$(value_for STOCK_BLOCKERS)" +stock_eod_window_pending="$(value_for STOCK_EOD_WINDOW_PENDING)" +stock_eod_classification="$(value_for STOCK_EOD_CLASSIFICATION)" +stock_eod_next_action="$(value_for STOCK_EOD_NEXT_ACTION)" +stock_eod_first_full_window_end="$(value_for STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL)" +stock_eod_final_retry_window_end="$(value_for STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL)" escrow_missing_count="$(value_for ESCROW_MISSING_COUNT)" host_188_hygiene_blocked="$(value_for HOST_188_HYGIENE_BLOCKED)" wazuh_registry_accepted="$(value_for WAZUH_MANAGER_REGISTRY_ACCEPTED)" @@ -110,6 +119,7 @@ echo "SUMMARY_FILE=$SUMMARY_FILE" echo "SUMMARY_ARTIFACT_DIR=${summary_artifact_dir:-unknown}" echo "SUMMARY_RC=$summary_rc" echo "SERVICE_GREEN=${service_green:-unknown}" +echo "PRODUCT_DATA_GREEN=${product_data_green:-unknown}" echo "OVERALL_DECLARATION=${overall_declaration:-unknown}" echo "NEXT_REQUIRED_GATES=${next_required_gates:-unknown}" echo "RUNTIME_ACTION_AUTHORIZED=0" @@ -118,7 +128,9 @@ echo "REQUEST_SENT_COUNT=0" echo "HOST_WRITE_AUTHORIZED=0" echo "SECRET_VALUE_COLLECTION_ALLOWED=0" -if [[ "$service_green" != "1" ]]; then +if [[ "$service_green" != "1" ]] \ + && ! contains_gate "product_data_freshness_recovery" \ + && ! contains_gate "stockplatform_eod_window_completion"; then echo echo "BLOCKED_SERVICE_GREEN=0" echo "NEXT_STEP=restore_service_before_boundary_dispatch" @@ -127,6 +139,34 @@ fi gate_count=0 +if contains_gate "stockplatform_eod_window_completion"; then + gate_count=$((gate_count + 1)) + print_gate_header "stockplatform_eod_window_completion" "StockPlatform scheduled EOD freshness completion" + echo "GATE_PRIORITY=P0" + echo "GATE_STATUS=scheduled_eod_window_pending" + echo "CURRENT_EVIDENCE=status:${stock_freshness_status:-unknown};latest_trading_date:${stock_latest_trading_date:-unknown};blockers:${stock_blockers:-unknown};pending:${stock_eod_window_pending:-unknown};classification:${stock_eod_classification:-unknown};next:${stock_eod_next_action:-unknown};first_full_window_end:${stock_eod_first_full_window_end:-unknown};final_retry_window_end:${stock_eod_final_retry_window_end:-unknown}" + echo "OWNER_GROUP=stockplatform_data_owner,market_data_owner,oncall_operator" + echo "REQUIRED_EVIDENCE=freshness_endpoint_after_19_15,cron_log_refs,daily_ingestion_ref,price_ref,chips_ref,margin_ref,ai_recommendation_ref" + echo "FORBIDDEN_PAYLOADS=raw_db_dump,secret_value,api_token,manual_fake_row,handwritten_freshness_override" + echo "FORBIDDEN_ACTIONS=manual_db_update,truncate_restore,fake_freshness_marker,skip_data_gate,declare_product_data_green_before_status_ok" + echo "ALLOWED_ACTION=wait_for_scheduled_cron_and_recheck_summary" + echo "DONE_CRITERIA=stock_freshness_status:ok,product_data_green:1,post_reboot_summary_recheck_after_eod_window" +fi + +if contains_gate "product_data_freshness_recovery"; then + gate_count=$((gate_count + 1)) + print_gate_header "product_data_freshness_recovery" "Product data freshness recovery" + echo "GATE_PRIORITY=P0" + echo "GATE_STATUS=data_freshness_recovery_required" + echo "CURRENT_EVIDENCE=status:${stock_freshness_status:-unknown};latest_trading_date:${stock_latest_trading_date:-unknown};blockers:${stock_blockers:-unknown};classification:${stock_eod_classification:-unknown};next:${stock_eod_next_action:-unknown}" + echo "OWNER_GROUP=stockplatform_data_owner,market_data_owner,rollback_owner" + echo "REQUIRED_EVIDENCE=source_of_truth_commit,cron_entrypoint_refs,cron_log_refs,official_source_status,db_count_readback,freshness_endpoint_after_recovery,rollback_plan" + echo "FORBIDDEN_PAYLOADS=raw_db_dump,secret_value,api_token,manual_fake_row,handwritten_freshness_override" + echo "FORBIDDEN_ACTIONS=manual_db_update,truncate_restore,fake_freshness_marker,skip_data_gate,declare_product_data_green_before_status_ok" + echo "ALLOWED_ACTION=diagnose_source_or_cron_failure_before_controlled_recovery" + echo "DONE_CRITERIA=stock_freshness_status:ok,product_data_green:1,post_reboot_summary_status:not_service_blocked" +fi + if contains_gate "credential_escrow_evidence"; then gate_count=$((gate_count + 1)) print_gate_header "credential_escrow_evidence" "DR credential escrow non-secret evidence" diff --git a/scripts/reboot-recovery/post-reboot-next-gate-owner-packets.py b/scripts/reboot-recovery/post-reboot-next-gate-owner-packets.py index 4acf9574..305f1b35 100755 --- a/scripts/reboot-recovery/post-reboot-next-gate-owner-packets.py +++ b/scripts/reboot-recovery/post-reboot-next-gate-owner-packets.py @@ -177,6 +177,7 @@ def build_packet(parsed: dict[str, Any]) -> dict[str, Any]: "runtime_action_authorized_count": 0, }, "no_false_green_rules": [ + "product_route_green_does_not_equal_product_data_green", "service_green_does_not_equal_dr_complete", "backup_fresh_does_not_equal_credential_escrow_complete", "host_188_service_green_does_not_equal_host_hygiene_green", diff --git a/scripts/reboot-recovery/post-reboot-owner-packet-contract-guard.py b/scripts/reboot-recovery/post-reboot-owner-packet-contract-guard.py index 2ab61468..2ee3388a 100755 --- a/scripts/reboot-recovery/post-reboot-owner-packet-contract-guard.py +++ b/scripts/reboot-recovery/post-reboot-owner-packet-contract-guard.py @@ -26,9 +26,12 @@ EXPECTED_SCHEMA = "awoooi_post_reboot_next_gate_owner_packets_v1" KNOWN_GATES = { "credential_escrow_evidence", "host_188_hygiene_maintenance_window", + "product_data_freshness_recovery", + "stockplatform_eod_window_completion", "wazuh_manager_registry_export", } EXPECTED_NO_FALSE_GREEN_RULES = { + "product_route_green_does_not_equal_product_data_green", "service_green_does_not_equal_dr_complete", "backup_fresh_does_not_equal_credential_escrow_complete", "host_188_service_green_does_not_equal_host_hygiene_green", @@ -69,6 +72,20 @@ GATE_REQUIRED_FORBIDDEN_PAYLOADS = { "password", "authorization_header", }, + "product_data_freshness_recovery": { + "raw_db_dump", + "secret_value", + "api_token", + "manual_fake_row", + "handwritten_freshness_override", + }, + "stockplatform_eod_window_completion": { + "raw_db_dump", + "secret_value", + "api_token", + "manual_fake_row", + "handwritten_freshness_override", + }, } GATE_REQUIRED_FORBIDDEN_ACTIONS = { @@ -94,6 +111,20 @@ GATE_REQUIRED_FORBIDDEN_ACTIONS = { "host_write", "kali_active_scan", }, + "product_data_freshness_recovery": { + "manual_db_update", + "truncate_restore", + "fake_freshness_marker", + "skip_data_gate", + "declare_product_data_green_before_status_ok", + }, + "stockplatform_eod_window_completion": { + "manual_db_update", + "truncate_restore", + "fake_freshness_marker", + "skip_data_gate", + "declare_product_data_green_before_status_ok", + }, } PACKET_FALSE_FIELDS = ( diff --git a/scripts/reboot-recovery/post-reboot-readiness-summary.sh b/scripts/reboot-recovery/post-reboot-readiness-summary.sh index bb76aeb2..76ef5e90 100755 --- a/scripts/reboot-recovery/post-reboot-readiness-summary.sh +++ b/scripts/reboot-recovery/post-reboot-readiness-summary.sh @@ -114,6 +114,14 @@ if grep -q '^STOCK_FRESHNESS_STATUS ok$' "$post_start_log" \ && grep -q '^DB_DAILY_FRESHNESS ' "$post_start_log"; then product_data_green=1 fi +stock_freshness_status="$(awk '$1 == "STOCK_FRESHNESS_STATUS" {value=$2} END {print value}' "$post_start_log")" +stock_latest_trading_date="$(awk '$1 == "STOCK_LATEST_TRADING_DATE" {value=$2} END {print value}' "$post_start_log")" +stock_blockers="$(grep -E '^STOCK_BLOCKERS ' "$post_start_log" | tail -n 1 | cut -d' ' -f2- || true)" +stock_eod_window_pending="$(awk '$1 == "STOCK_EOD_WINDOW_PENDING" {value=$2} END {print value}' "$post_start_log")" +stock_eod_classification="$(awk '$1 == "STOCK_EOD_CLASSIFICATION" {value=$2} END {print value}' "$post_start_log")" +stock_eod_next_action="$(awk '$1 == "STOCK_EOD_NEXT_ACTION" {value=$2} END {print value}' "$post_start_log")" +stock_eod_first_full_window_end="$(awk '$1 == "STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL" {value=$2} END {print value}' "$post_start_log")" +stock_eod_final_retry_window_end="$(awk '$1 == "STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL" {value=$2} END {print value}' "$post_start_log")" escrow_missing_count="$(grep -Eo 'escrow_missing=[0-9]+' "$post_start_log" | tail -n 1 | cut -d= -f2 || true)" dr_escrow_blocked=0 @@ -176,6 +184,10 @@ fi overall_declaration="GREEN" if [[ "$service_green" != "1" ]]; then overall_declaration="SERVICE_BLOCKED" +elif [[ "$product_data_green" != "1" && "$stock_eod_window_pending" == "1" ]]; then + overall_declaration="PRODUCT_DATA_PENDING_EOD_WINDOW" +elif [[ "$product_data_green" != "1" ]]; then + overall_declaration="PRODUCT_DATA_BLOCKED" elif [[ "$dr_escrow_blocked" == "1" ]]; then overall_declaration="FULL_STACK_GREEN_DR_ESCROW_BLOCKED" elif [[ "$host_188_hygiene_blocked" == "1" ]]; then @@ -187,6 +199,11 @@ elif [[ "$evidence_warn" != "0" && -n "$evidence_warn" ]]; then fi next_required_gates=() +if [[ "$product_data_green" != "1" && "$stock_eod_window_pending" == "1" ]]; then + next_required_gates+=("stockplatform_eod_window_completion") +elif [[ "$product_data_green" != "1" ]]; then + next_required_gates+=("product_data_freshness_recovery") +fi [[ "$dr_escrow_blocked" == "1" ]] && next_required_gates+=("credential_escrow_evidence") [[ "$host_188_hygiene_blocked" == "1" ]] && next_required_gates+=("host_188_hygiene_maintenance_window") [[ "$wazuh_registry_accepted" == "0" ]] && next_required_gates+=("wazuh_manager_registry_export") @@ -213,6 +230,14 @@ POST_START_BOUNDARY_WARNINGS=${boundary_warn:-unknown} POST_START_EVIDENCE_WARNINGS=${evidence_warn:-unknown} SERVICE_GREEN=$service_green PRODUCT_DATA_GREEN=$product_data_green +STOCK_FRESHNESS_STATUS=${stock_freshness_status:-unknown} +STOCK_LATEST_TRADING_DATE=${stock_latest_trading_date:-unknown} +STOCK_BLOCKERS=${stock_blockers:-unknown} +STOCK_EOD_WINDOW_PENDING=${stock_eod_window_pending:-0} +STOCK_EOD_CLASSIFICATION=${stock_eod_classification:-unknown} +STOCK_EOD_NEXT_ACTION=${stock_eod_next_action:-unknown} +STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL=${stock_eod_first_full_window_end:-unknown} +STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL=${stock_eod_final_retry_window_end:-unknown} BACKUP_CORE_GREEN=$backup_core_green DR_ESCROW_BLOCKED=$dr_escrow_blocked ESCROW_MISSING_COUNT=${escrow_missing_count:-unknown} diff --git a/scripts/reboot-recovery/post-start-quick-check.sh b/scripts/reboot-recovery/post-start-quick-check.sh index e9cfadbd..77ceb6ed 100755 --- a/scripts/reboot-recovery/post-start-quick-check.sh +++ b/scripts/reboot-recovery/post-start-quick-check.sh @@ -22,6 +22,10 @@ COLD_START_PENDING_BLOCKERS=0 COLD_START_BLOCKED_SUMMARY="" COLD_START_BLOCKED_LINES="" ROUTE_SMOKE_BLOCKED=0 +STOCK_EOD_WINDOW_PENDING=0 +STOCK_EOD_CLASSIFICATION="not_evaluated" +STOCK_EOD_NEXT_ACTION="not_evaluated" +STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL="19:15" PASS_COUNT=0 WARN_COUNT=0 @@ -334,6 +338,9 @@ with open(sys.argv[1], "r", encoding="utf-8") as fh: PY )" if [[ "$stock_status" == "ok" ]]; then + printf 'STOCK_EOD_WINDOW_PENDING 0\n' + printf 'STOCK_EOD_CLASSIFICATION ok\n' + printf 'STOCK_EOD_NEXT_ACTION none\n' ok "StockPlatform freshness is ok" else stock_blockers="$(python3 - "$stock_tmp" <<'PY' @@ -344,7 +351,66 @@ with open(sys.argv[1], "r", encoding="utf-8") as fh: print(",".join(json.load(fh).get("blockers") or [])) PY )" - blocked "StockPlatform freshness is ${stock_status:-unknown}: ${stock_blockers:-no_blocker_list}" + stock_eod_context="$(python3 - "$stock_tmp" <<'PY' +import json +import sys +from datetime import datetime, time, timezone, timedelta + +try: + from zoneinfo import ZoneInfo +except Exception: # pragma: no cover - old Python fallback + ZoneInfo = None + +with open(sys.argv[1], "r", encoding="utf-8") as fh: + payload = json.load(fh) + +tz = ZoneInfo("Asia/Taipei") if ZoneInfo else timezone(timedelta(hours=8)) +now = datetime.now(tz) +today = now.date().isoformat() +latest = payload.get("latest_trading_date") or "" +status = payload.get("status") or "unknown" +blockers = payload.get("blockers") or [] +first_full_window_end = time(19, 15) +final_retry_window_end = time(23, 35) + +pending = 0 +classification = "blocked_unknown" +next_action = "investigate_stockplatform_freshness" + +if status == "ok": + classification = "ok" + next_action = "none" +elif latest == today and blockers and now.time() < first_full_window_end: + pending = 1 + classification = "pending_first_eod_window" + next_action = "wait_for_18_20_19_10_cron_then_recheck" +elif latest == today and blockers and now.time() < final_retry_window_end: + classification = "after_first_eod_window_blocked" + next_action = "inspect_ingestion_logs_and_wait_retry_windows" +elif latest == today and blockers: + classification = "after_final_eod_window_blocked" + next_action = "open_stockplatform_data_recovery_gate" +elif blockers: + classification = "blocked_non_current_trading_day" + next_action = "inspect_trading_calendar_and_ingestion_logs" + +print(f"STOCK_EOD_WINDOW_PENDING {pending}") +print(f"STOCK_EOD_CLASSIFICATION {classification}") +print(f"STOCK_EOD_NEXT_ACTION {next_action}") +print(f"STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL {first_full_window_end.strftime('%H:%M')}") +print(f"STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL {final_retry_window_end.strftime('%H:%M')}") +print(f"STOCK_EOD_OBSERVED_AT_LOCAL {now.isoformat(timespec='seconds')}") +PY +)" + printf '%s\n' "$stock_eod_context" + STOCK_EOD_WINDOW_PENDING="$(awk '$1 == "STOCK_EOD_WINDOW_PENDING" {print $2}' <<<"$stock_eod_context" | tail -n 1)" + STOCK_EOD_CLASSIFICATION="$(awk '$1 == "STOCK_EOD_CLASSIFICATION" {print $2}' <<<"$stock_eod_context" | tail -n 1)" + STOCK_EOD_NEXT_ACTION="$(awk '$1 == "STOCK_EOD_NEXT_ACTION" {print $2}' <<<"$stock_eod_context" | tail -n 1)" + if [[ "$STOCK_EOD_WINDOW_PENDING" == "1" ]]; then + evidence_warn "StockPlatform freshness pending scheduled EOD window: ${STOCK_EOD_CLASSIFICATION:-unknown}; next=${STOCK_EOD_NEXT_ACTION:-unknown}; blockers=${stock_blockers:-no_blocker_list}" + else + blocked "StockPlatform freshness is ${stock_status:-unknown}: ${stock_blockers:-no_blocker_list}; classification=${STOCK_EOD_CLASSIFICATION:-unknown}" + fi fi fi rm -f "$stock_tmp" @@ -460,6 +526,11 @@ if [[ "$SERVICE_WARN_COUNT" -gt 0 ]]; then exit 1 fi +if [[ "$STOCK_EOD_WINDOW_PENDING" == "1" ]]; then + printf 'RESULT=PRODUCT_DATA_PENDING_EOD_WINDOW\n' + exit 0 +fi + if [[ "$BOUNDARY_WARN_COUNT" -gt 0 ]]; then printf 'RESULT=FULL_STACK_GREEN_DR_ESCROW_BLOCKED\n' exit 0