From b38337b5f253fe069b92575a1db9560ff02b3e9b Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 3 Jul 2026 03:39:56 +0800 Subject: [PATCH] fix(reboot): preserve post-start summary on slo timeout --- .../reboot-auto-recovery-slo-exporter.sh | 108 +++++++++++++++++- ...test_reboot_auto_recovery_slo_installer.py | 4 + 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index 040f8a90f..b849fe796 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -16,7 +16,7 @@ LOCK_FILE="${LOCK_FILE:-${LOG_DIR}/reboot_auto_recovery_slo.lock}" STOCK_FRESHNESS_URL="${STOCK_FRESHNESS_URL:-https://stock.wooo.work/api/v1/system/freshness}" STOCK_INGESTION_URL="${STOCK_INGESTION_URL:-https://stock.wooo.work/api/v1/system/ingestion}" STOCK_READBACK_TIMEOUT_SECONDS="${STOCK_READBACK_TIMEOUT_SECONDS:-10}" -POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-60}" +POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-120}" PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS="${PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS:-8}" PUBLIC_MAINTENANCE_URLS="${PUBLIC_MAINTENANCE_URLS:-https://awoooi.wooo.work/api/v1/health https://awoooi.wooo.work/ https://stock.wooo.work/api/v1/system/freshness https://mo.wooo.work/health https://bitan.wooo.work/ https://www.tsenyang.com/}" WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS="${WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS:-900}" @@ -45,6 +45,109 @@ windows99_management_file="$artifact_dir/windows99-management-channel.json" windows99_vmware_file="$artifact_dir/windows99-vmware-verify.txt" reboot_event_state_file="${REBOOT_EVENT_STATE_FILE:-${LOG_DIR}/reboot-event-state.json}" +extract_summary_number() { + local label="$1" + local line="$2" + sed -nE "s/.*${label}=([0-9]+).*/\\1/p" <<<"$line" | tail -n 1 +} + +extract_named_token() { + local name="$1" + local line="$2" + tr ' ' '\n' <<<"$line" | awk -F= -v name="$name" '$1 == name {print $2; found=1} END {if (!found) print ""}' +} + +write_partial_post_reboot_summary_from_post_start() { + local post_start_log="$artifact_dir/post-reboot-readiness/post-start-quick-check.log" + local post_summary + local post_warning_summary + local post_result + local post_pass + local post_warn + local post_blocked + local service_warn + local boundary_warn + local evidence_warn + local service_green=0 + local backup_core_green=0 + local product_data_green=0 + local stock_freshness_status + local stock_latest_trading_date + local stock_blockers + + [ -s "$post_start_log" ] || return 0 + post_summary="$(grep -E '^POST_START_QUICK_CHECK PASS=[0-9]+ WARN=[0-9]+ BLOCKED=[0-9]+' "$post_start_log" | tail -n 1 || true)" + [ -n "$post_summary" ] || return 0 + post_warning_summary="$(grep -E '^POST_START_QUICK_CHECK_WARNINGS SERVICE=[0-9]+ BOUNDARY=[0-9]+ EVIDENCE=[0-9]+' "$post_start_log" | tail -n 1 || true)" + post_result="$(grep -E '^RESULT=' "$post_start_log" | tail -n 1 | cut -d= -f2- || true)" + post_pass="$(extract_summary_number PASS "$post_summary")" + post_warn="$(extract_summary_number WARN "$post_summary")" + post_blocked="$(extract_summary_number BLOCKED "$post_summary")" + service_warn="$(extract_named_token SERVICE "$post_warning_summary")" + boundary_warn="$(extract_named_token BOUNDARY "$post_warning_summary")" + evidence_warn="$(extract_named_token EVIDENCE "$post_warning_summary")" + + if [[ "${post_blocked:-999}" == "0" && "${service_warn:-999}" == "0" ]]; then + service_green=1 + fi + grep -q 'OK backup core blockers are 0' "$post_start_log" && backup_core_green=1 + if grep -q '^STOCK_FRESHNESS_STATUS ok$' "$post_start_log" \ + && grep -q '^DB_DAILY_FRESHNESS ' "$post_start_log"; then + product_data_green=1 + fi + stock_freshness_status="$(awk '$1 == "STOCK_FRESHNESS_STATUS" {value=$2} END {print value}' "$post_start_log")" + stock_latest_trading_date="$(awk '$1 == "STOCK_LATEST_TRADING_DATE" {value=$2} END {print value}' "$post_start_log")" + stock_blockers="$(grep -E '^STOCK_BLOCKERS ' "$post_start_log" | tail -n 1 | cut -d' ' -f2- || true)" + + cat >"$summary_file" <"$host_probe" 2>&1 || true python3 "$ROOT_DIR/scripts/reboot-recovery/reboot-event-detector.py" \ --host-probe-file "$host_probe" \ @@ -72,6 +175,9 @@ else POST_REBOOT_SKIP_RUNNER_GUARD=1 \ "${post_reboot_summary_command[@]}" >"$summary_file" 2>&1 || true fi +if ! grep -q '^AWOOOI_POST_REBOOT_READINESS_SUMMARY=1$' "$summary_file" 2>/dev/null; then + write_partial_post_reboot_summary_from_post_start +fi if command -v curl >/dev/null 2>&1; then curl -fsS --max-time "$STOCK_READBACK_TIMEOUT_SECONDS" \ diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py index cad26b9ca..30f193967 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py @@ -113,8 +113,12 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None: assert "windows99_vmware_verify_fallback_max_age_seconds" in text assert "find \"$LOG_DIR\" -mindepth 2 -maxdepth 2" in text assert "POST_REBOOT_READINESS_TIMEOUT_SECONDS" in text + assert 'POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-120}"' in text assert "POST_REBOOT_SKIP_RUNNER_GUARD=1" in text assert "POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1" in text + assert "POST_REBOOT_READINESS_PARTIAL_FROM_POST_START=1" in text + assert "PARTIAL_POST_START_GREEN_SUMMARY_TIMEOUT" in text + assert "write_partial_post_reboot_summary_from_post_start" in text assert "awoooi_windows99_vmware_missing_vmx_alias" in text assert "awoooi_windows99_vmware_powered_off_alias" in text assert "active_blocker_metrics" in text