fix(reboot): preserve post-start summary on slo timeout
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m0s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled

This commit is contained in:
Your Name
2026-07-03 03:39:56 +08:00
parent c3bf930bcd
commit b38337b5f2
2 changed files with 111 additions and 1 deletions

View File

@@ -16,7 +16,7 @@ LOCK_FILE="${LOCK_FILE:-${LOG_DIR}/reboot_auto_recovery_slo.lock}"
STOCK_FRESHNESS_URL="${STOCK_FRESHNESS_URL:-https://stock.wooo.work/api/v1/system/freshness}"
STOCK_INGESTION_URL="${STOCK_INGESTION_URL:-https://stock.wooo.work/api/v1/system/ingestion}"
STOCK_READBACK_TIMEOUT_SECONDS="${STOCK_READBACK_TIMEOUT_SECONDS:-10}"
POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-60}"
POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-120}"
PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS="${PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS:-8}"
PUBLIC_MAINTENANCE_URLS="${PUBLIC_MAINTENANCE_URLS:-https://awoooi.wooo.work/api/v1/health https://awoooi.wooo.work/ https://stock.wooo.work/api/v1/system/freshness https://mo.wooo.work/health https://bitan.wooo.work/ https://www.tsenyang.com/}"
WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS="${WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS:-900}"
@@ -45,6 +45,109 @@ windows99_management_file="$artifact_dir/windows99-management-channel.json"
windows99_vmware_file="$artifact_dir/windows99-vmware-verify.txt"
reboot_event_state_file="${REBOOT_EVENT_STATE_FILE:-${LOG_DIR}/reboot-event-state.json}"
extract_summary_number() {
local label="$1"
local line="$2"
sed -nE "s/.*${label}=([0-9]+).*/\\1/p" <<<"$line" | tail -n 1
}
extract_named_token() {
local name="$1"
local line="$2"
tr ' ' '\n' <<<"$line" | awk -F= -v name="$name" '$1 == name {print $2; found=1} END {if (!found) print ""}'
}
write_partial_post_reboot_summary_from_post_start() {
local post_start_log="$artifact_dir/post-reboot-readiness/post-start-quick-check.log"
local post_summary
local post_warning_summary
local post_result
local post_pass
local post_warn
local post_blocked
local service_warn
local boundary_warn
local evidence_warn
local service_green=0
local backup_core_green=0
local product_data_green=0
local stock_freshness_status
local stock_latest_trading_date
local stock_blockers
[ -s "$post_start_log" ] || return 0
post_summary="$(grep -E '^POST_START_QUICK_CHECK PASS=[0-9]+ WARN=[0-9]+ BLOCKED=[0-9]+' "$post_start_log" | tail -n 1 || true)"
[ -n "$post_summary" ] || return 0
post_warning_summary="$(grep -E '^POST_START_QUICK_CHECK_WARNINGS SERVICE=[0-9]+ BOUNDARY=[0-9]+ EVIDENCE=[0-9]+' "$post_start_log" | tail -n 1 || true)"
post_result="$(grep -E '^RESULT=' "$post_start_log" | tail -n 1 | cut -d= -f2- || true)"
post_pass="$(extract_summary_number PASS "$post_summary")"
post_warn="$(extract_summary_number WARN "$post_summary")"
post_blocked="$(extract_summary_number BLOCKED "$post_summary")"
service_warn="$(extract_named_token SERVICE "$post_warning_summary")"
boundary_warn="$(extract_named_token BOUNDARY "$post_warning_summary")"
evidence_warn="$(extract_named_token EVIDENCE "$post_warning_summary")"
if [[ "${post_blocked:-999}" == "0" && "${service_warn:-999}" == "0" ]]; then
service_green=1
fi
grep -q 'OK backup core blockers are 0' "$post_start_log" && backup_core_green=1
if grep -q '^STOCK_FRESHNESS_STATUS ok$' "$post_start_log" \
&& grep -q '^DB_DAILY_FRESHNESS ' "$post_start_log"; then
product_data_green=1
fi
stock_freshness_status="$(awk '$1 == "STOCK_FRESHNESS_STATUS" {value=$2} END {print value}' "$post_start_log")"
stock_latest_trading_date="$(awk '$1 == "STOCK_LATEST_TRADING_DATE" {value=$2} END {print value}' "$post_start_log")"
stock_blockers="$(grep -E '^STOCK_BLOCKERS ' "$post_start_log" | tail -n 1 | cut -d' ' -f2- || true)"
cat >"$summary_file" <<SUMMARY
AWOOOI_POST_REBOOT_READINESS_SUMMARY=1
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S %Z')
ARTIFACT_DIR=$artifact_dir/post-reboot-readiness
POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1
POST_REBOOT_READINESS_PARTIAL_FROM_POST_START=1
POST_START_RC=timeout
POST_START_RESULT=${post_result:-unknown}
POST_START_PASS=${post_pass:-0}
POST_START_WARN=${post_warn:-0}
POST_START_BLOCKED=${post_blocked:-999}
POST_START_SERVICE_WARNINGS=${service_warn:-999}
POST_START_BOUNDARY_WARNINGS=${boundary_warn:-999}
POST_START_EVIDENCE_WARNINGS=${evidence_warn:-999}
SERVICE_GREEN=$service_green
PRODUCT_DATA_GREEN=$product_data_green
STOCK_FRESHNESS_STATUS=${stock_freshness_status:-unknown}
STOCK_LATEST_TRADING_DATE=${stock_latest_trading_date:-unknown}
STOCK_BLOCKERS=${stock_blockers:-unknown}
STOCK_EOD_WINDOW_PENDING=unknown
STOCK_EOD_CLASSIFICATION=unknown
STOCK_EOD_NEXT_ACTION=post_reboot_summary_timeout_partial
STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL=unknown
STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL=unknown
BACKUP_CORE_GREEN=$backup_core_green
DR_ESCROW_BLOCKED=unknown
DR_ESCROW_EVIDENCE_UNKNOWN=1
ESCROW_MISSING_COUNT=unknown
HOST_188_SERVICE_GREEN=unknown
HOST_188_HYGIENE_BLOCKED=unknown
HOST_188_CHECK_RC=timeout
HOST_188_RESULT=unknown_timeout_after_post_start
WAZUH_ROUTE_CODE=
WAZUH_TRANSPORT_COUNT=
WAZUH_COVERAGE_SCOPE=
WAZUH_DIRECT_ACTIVE=
WAZUH_NO_TRANSPORT=
WAZUH_SSH_BLOCKED=
WAZUH_DASHBOARD_DEGRADED=
WAZUH_DASHBOARD_API_CONNECTION=
WAZUH_DASHBOARD_INDEX_OK=
WAZUH_MANAGER_REGISTRY_ACCEPTED=
WAZUH_RUNTIME_GATE=
RUNTIME_ACTION_AUTHORIZED=0
OVERALL_DECLARATION=PARTIAL_POST_START_GREEN_SUMMARY_TIMEOUT
NEXT_REQUIRED_GATES=post_reboot_readiness_full_summary_timeout
SUMMARY
}
bash "$ROOT_DIR/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh" >"$host_probe" 2>&1 || true
python3 "$ROOT_DIR/scripts/reboot-recovery/reboot-event-detector.py" \
--host-probe-file "$host_probe" \
@@ -72,6 +175,9 @@ else
POST_REBOOT_SKIP_RUNNER_GUARD=1 \
"${post_reboot_summary_command[@]}" >"$summary_file" 2>&1 || true
fi
if ! grep -q '^AWOOOI_POST_REBOOT_READINESS_SUMMARY=1$' "$summary_file" 2>/dev/null; then
write_partial_post_reboot_summary_from_post_start
fi
if command -v curl >/dev/null 2>&1; then
curl -fsS --max-time "$STOCK_READBACK_TIMEOUT_SECONDS" \

View File

@@ -113,8 +113,12 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None:
assert "windows99_vmware_verify_fallback_max_age_seconds" in text
assert "find \"$LOG_DIR\" -mindepth 2 -maxdepth 2" in text
assert "POST_REBOOT_READINESS_TIMEOUT_SECONDS" in text
assert 'POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-120}"' in text
assert "POST_REBOOT_SKIP_RUNNER_GUARD=1" in text
assert "POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1" in text
assert "POST_REBOOT_READINESS_PARTIAL_FROM_POST_START=1" in text
assert "PARTIAL_POST_START_GREEN_SUMMARY_TIMEOUT" in text
assert "write_partial_post_reboot_summary_from_post_start" in text
assert "awoooi_windows99_vmware_missing_vmx_alias" in text
assert "awoooi_windows99_vmware_powered_off_alias" in text
assert "active_blocker_metrics" in text