fix(reboot): preserve post-start summary on slo timeout
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m0s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m0s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
This commit is contained in:
@@ -16,7 +16,7 @@ LOCK_FILE="${LOCK_FILE:-${LOG_DIR}/reboot_auto_recovery_slo.lock}"
|
||||
STOCK_FRESHNESS_URL="${STOCK_FRESHNESS_URL:-https://stock.wooo.work/api/v1/system/freshness}"
|
||||
STOCK_INGESTION_URL="${STOCK_INGESTION_URL:-https://stock.wooo.work/api/v1/system/ingestion}"
|
||||
STOCK_READBACK_TIMEOUT_SECONDS="${STOCK_READBACK_TIMEOUT_SECONDS:-10}"
|
||||
POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-60}"
|
||||
POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-120}"
|
||||
PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS="${PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS:-8}"
|
||||
PUBLIC_MAINTENANCE_URLS="${PUBLIC_MAINTENANCE_URLS:-https://awoooi.wooo.work/api/v1/health https://awoooi.wooo.work/ https://stock.wooo.work/api/v1/system/freshness https://mo.wooo.work/health https://bitan.wooo.work/ https://www.tsenyang.com/}"
|
||||
WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS="${WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS:-900}"
|
||||
@@ -45,6 +45,109 @@ windows99_management_file="$artifact_dir/windows99-management-channel.json"
|
||||
windows99_vmware_file="$artifact_dir/windows99-vmware-verify.txt"
|
||||
reboot_event_state_file="${REBOOT_EVENT_STATE_FILE:-${LOG_DIR}/reboot-event-state.json}"
|
||||
|
||||
extract_summary_number() {
|
||||
local label="$1"
|
||||
local line="$2"
|
||||
sed -nE "s/.*${label}=([0-9]+).*/\\1/p" <<<"$line" | tail -n 1
|
||||
}
|
||||
|
||||
extract_named_token() {
|
||||
local name="$1"
|
||||
local line="$2"
|
||||
tr ' ' '\n' <<<"$line" | awk -F= -v name="$name" '$1 == name {print $2; found=1} END {if (!found) print ""}'
|
||||
}
|
||||
|
||||
write_partial_post_reboot_summary_from_post_start() {
|
||||
local post_start_log="$artifact_dir/post-reboot-readiness/post-start-quick-check.log"
|
||||
local post_summary
|
||||
local post_warning_summary
|
||||
local post_result
|
||||
local post_pass
|
||||
local post_warn
|
||||
local post_blocked
|
||||
local service_warn
|
||||
local boundary_warn
|
||||
local evidence_warn
|
||||
local service_green=0
|
||||
local backup_core_green=0
|
||||
local product_data_green=0
|
||||
local stock_freshness_status
|
||||
local stock_latest_trading_date
|
||||
local stock_blockers
|
||||
|
||||
[ -s "$post_start_log" ] || return 0
|
||||
post_summary="$(grep -E '^POST_START_QUICK_CHECK PASS=[0-9]+ WARN=[0-9]+ BLOCKED=[0-9]+' "$post_start_log" | tail -n 1 || true)"
|
||||
[ -n "$post_summary" ] || return 0
|
||||
post_warning_summary="$(grep -E '^POST_START_QUICK_CHECK_WARNINGS SERVICE=[0-9]+ BOUNDARY=[0-9]+ EVIDENCE=[0-9]+' "$post_start_log" | tail -n 1 || true)"
|
||||
post_result="$(grep -E '^RESULT=' "$post_start_log" | tail -n 1 | cut -d= -f2- || true)"
|
||||
post_pass="$(extract_summary_number PASS "$post_summary")"
|
||||
post_warn="$(extract_summary_number WARN "$post_summary")"
|
||||
post_blocked="$(extract_summary_number BLOCKED "$post_summary")"
|
||||
service_warn="$(extract_named_token SERVICE "$post_warning_summary")"
|
||||
boundary_warn="$(extract_named_token BOUNDARY "$post_warning_summary")"
|
||||
evidence_warn="$(extract_named_token EVIDENCE "$post_warning_summary")"
|
||||
|
||||
if [[ "${post_blocked:-999}" == "0" && "${service_warn:-999}" == "0" ]]; then
|
||||
service_green=1
|
||||
fi
|
||||
grep -q 'OK backup core blockers are 0' "$post_start_log" && backup_core_green=1
|
||||
if grep -q '^STOCK_FRESHNESS_STATUS ok$' "$post_start_log" \
|
||||
&& grep -q '^DB_DAILY_FRESHNESS ' "$post_start_log"; then
|
||||
product_data_green=1
|
||||
fi
|
||||
stock_freshness_status="$(awk '$1 == "STOCK_FRESHNESS_STATUS" {value=$2} END {print value}' "$post_start_log")"
|
||||
stock_latest_trading_date="$(awk '$1 == "STOCK_LATEST_TRADING_DATE" {value=$2} END {print value}' "$post_start_log")"
|
||||
stock_blockers="$(grep -E '^STOCK_BLOCKERS ' "$post_start_log" | tail -n 1 | cut -d' ' -f2- || true)"
|
||||
|
||||
cat >"$summary_file" <<SUMMARY
|
||||
AWOOOI_POST_REBOOT_READINESS_SUMMARY=1
|
||||
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S %Z')
|
||||
ARTIFACT_DIR=$artifact_dir/post-reboot-readiness
|
||||
POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1
|
||||
POST_REBOOT_READINESS_PARTIAL_FROM_POST_START=1
|
||||
POST_START_RC=timeout
|
||||
POST_START_RESULT=${post_result:-unknown}
|
||||
POST_START_PASS=${post_pass:-0}
|
||||
POST_START_WARN=${post_warn:-0}
|
||||
POST_START_BLOCKED=${post_blocked:-999}
|
||||
POST_START_SERVICE_WARNINGS=${service_warn:-999}
|
||||
POST_START_BOUNDARY_WARNINGS=${boundary_warn:-999}
|
||||
POST_START_EVIDENCE_WARNINGS=${evidence_warn:-999}
|
||||
SERVICE_GREEN=$service_green
|
||||
PRODUCT_DATA_GREEN=$product_data_green
|
||||
STOCK_FRESHNESS_STATUS=${stock_freshness_status:-unknown}
|
||||
STOCK_LATEST_TRADING_DATE=${stock_latest_trading_date:-unknown}
|
||||
STOCK_BLOCKERS=${stock_blockers:-unknown}
|
||||
STOCK_EOD_WINDOW_PENDING=unknown
|
||||
STOCK_EOD_CLASSIFICATION=unknown
|
||||
STOCK_EOD_NEXT_ACTION=post_reboot_summary_timeout_partial
|
||||
STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL=unknown
|
||||
STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL=unknown
|
||||
BACKUP_CORE_GREEN=$backup_core_green
|
||||
DR_ESCROW_BLOCKED=unknown
|
||||
DR_ESCROW_EVIDENCE_UNKNOWN=1
|
||||
ESCROW_MISSING_COUNT=unknown
|
||||
HOST_188_SERVICE_GREEN=unknown
|
||||
HOST_188_HYGIENE_BLOCKED=unknown
|
||||
HOST_188_CHECK_RC=timeout
|
||||
HOST_188_RESULT=unknown_timeout_after_post_start
|
||||
WAZUH_ROUTE_CODE=
|
||||
WAZUH_TRANSPORT_COUNT=
|
||||
WAZUH_COVERAGE_SCOPE=
|
||||
WAZUH_DIRECT_ACTIVE=
|
||||
WAZUH_NO_TRANSPORT=
|
||||
WAZUH_SSH_BLOCKED=
|
||||
WAZUH_DASHBOARD_DEGRADED=
|
||||
WAZUH_DASHBOARD_API_CONNECTION=
|
||||
WAZUH_DASHBOARD_INDEX_OK=
|
||||
WAZUH_MANAGER_REGISTRY_ACCEPTED=
|
||||
WAZUH_RUNTIME_GATE=
|
||||
RUNTIME_ACTION_AUTHORIZED=0
|
||||
OVERALL_DECLARATION=PARTIAL_POST_START_GREEN_SUMMARY_TIMEOUT
|
||||
NEXT_REQUIRED_GATES=post_reboot_readiness_full_summary_timeout
|
||||
SUMMARY
|
||||
}
|
||||
|
||||
bash "$ROOT_DIR/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh" >"$host_probe" 2>&1 || true
|
||||
python3 "$ROOT_DIR/scripts/reboot-recovery/reboot-event-detector.py" \
|
||||
--host-probe-file "$host_probe" \
|
||||
@@ -72,6 +175,9 @@ else
|
||||
POST_REBOOT_SKIP_RUNNER_GUARD=1 \
|
||||
"${post_reboot_summary_command[@]}" >"$summary_file" 2>&1 || true
|
||||
fi
|
||||
if ! grep -q '^AWOOOI_POST_REBOOT_READINESS_SUMMARY=1$' "$summary_file" 2>/dev/null; then
|
||||
write_partial_post_reboot_summary_from_post_start
|
||||
fi
|
||||
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
curl -fsS --max-time "$STOCK_READBACK_TIMEOUT_SECONDS" \
|
||||
|
||||
@@ -113,8 +113,12 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None:
|
||||
assert "windows99_vmware_verify_fallback_max_age_seconds" in text
|
||||
assert "find \"$LOG_DIR\" -mindepth 2 -maxdepth 2" in text
|
||||
assert "POST_REBOOT_READINESS_TIMEOUT_SECONDS" in text
|
||||
assert 'POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-120}"' in text
|
||||
assert "POST_REBOOT_SKIP_RUNNER_GUARD=1" in text
|
||||
assert "POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1" in text
|
||||
assert "POST_REBOOT_READINESS_PARTIAL_FROM_POST_START=1" in text
|
||||
assert "PARTIAL_POST_START_GREEN_SUMMARY_TIMEOUT" in text
|
||||
assert "write_partial_post_reboot_summary_from_post_start" in text
|
||||
assert "awoooi_windows99_vmware_missing_vmx_alias" in text
|
||||
assert "awoooi_windows99_vmware_powered_off_alias" in text
|
||||
assert "active_blocker_metrics" in text
|
||||
|
||||
Reference in New Issue
Block a user