ops(reboot): add post-reboot readiness summary [skip ci]
This commit is contained in:
ogt
229
scripts/reboot-recovery/post-reboot-readiness-summary.sh
Executable file
229
scripts/reboot-recovery/post-reboot-readiness-summary.sh
Executable file
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env bash
|
||||
# AWOOOI post-reboot machine-readable readiness summary.
|
||||
# Read-only by design: this script never restarts, reloads, repairs, imports,
|
||||
# deletes, patches, or writes remote runtime state.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
NO_COLOR_FLAG=0
|
||||
SHOW_LOGS=0
|
||||
RUN_188_HYGIENE=1
|
||||
RUN_WAZUH_GATES=1
|
||||
ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/awoooi-post-reboot-readiness-$(date '+%Y%m%d-%H%M%S')}"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: post-reboot-readiness-summary.sh [options]
|
||||
|
||||
Runs the existing read-only post-reboot checks and prints a stable key/value
|
||||
summary for operators and AI agents.
|
||||
|
||||
Options:
|
||||
--no-color Disable color in delegated checks.
|
||||
--show-logs Print delegated check logs after the summary.
|
||||
--skip-188-hygiene Skip the 188 host hygiene boundary checklist.
|
||||
--skip-wazuh-gates Skip repo-side Wazuh no-false-green gates.
|
||||
-h, --help Show this help.
|
||||
|
||||
Exit codes:
|
||||
0 = core services are green. Boundary blockers may still be present.
|
||||
2 = core service blocker observed.
|
||||
|
||||
This script stores read-only evidence logs under ARTIFACT_DIR.
|
||||
USAGE
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--no-color)
|
||||
NO_COLOR_FLAG=1
|
||||
;;
|
||||
--show-logs)
|
||||
SHOW_LOGS=1
|
||||
;;
|
||||
--skip-188-hygiene)
|
||||
RUN_188_HYGIENE=0
|
||||
;;
|
||||
--skip-wazuh-gates)
|
||||
RUN_WAZUH_GATES=0
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
printf 'Unknown argument: %s\n' "$1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
mkdir -p "$ARTIFACT_DIR"
|
||||
|
||||
extract_key() {
|
||||
local key="$1"
|
||||
local file="$2"
|
||||
awk -F= -v key="$key" '$1 == key {value=$2; found=1} END {if (found) print value; else print ""}' "$file"
|
||||
}
|
||||
|
||||
extract_summary_number() {
|
||||
local label="$1"
|
||||
local line="$2"
|
||||
sed -nE "s/.*${label}=([0-9]+).*/\\1/p" <<<"$line" | tail -n 1
|
||||
}
|
||||
|
||||
extract_named_token() {
|
||||
local name="$1"
|
||||
local line="$2"
|
||||
tr ' ' '\n' <<<"$line" | awk -F= -v name="$name" '$1 == name {print $2; found=1} END {if (!found) print ""}'
|
||||
}
|
||||
|
||||
post_start_log="$ARTIFACT_DIR/post-start-quick-check.log"
|
||||
post_start_args=(--no-color)
|
||||
if [[ "$NO_COLOR_FLAG" -eq 1 ]]; then
|
||||
post_start_args=(--no-color)
|
||||
fi
|
||||
|
||||
post_start_rc=0
|
||||
bash "$ROOT_DIR/scripts/reboot-recovery/post-start-quick-check.sh" "${post_start_args[@]}" >"$post_start_log" 2>&1 || post_start_rc=$?
|
||||
|
||||
post_summary="$(grep -E '^POST_START_QUICK_CHECK PASS=[0-9]+ WARN=[0-9]+ BLOCKED=[0-9]+' "$post_start_log" | tail -n 1 || true)"
|
||||
post_warning_summary="$(grep -E '^POST_START_QUICK_CHECK_WARNINGS SERVICE=[0-9]+ BOUNDARY=[0-9]+ EVIDENCE=[0-9]+' "$post_start_log" | tail -n 1 || true)"
|
||||
post_result="$(grep -E '^RESULT=' "$post_start_log" | tail -n 1 | cut -d= -f2- || true)"
|
||||
|
||||
post_pass="$(extract_summary_number PASS "$post_summary")"
|
||||
post_warn="$(extract_summary_number WARN "$post_summary")"
|
||||
post_blocked="$(extract_summary_number BLOCKED "$post_summary")"
|
||||
service_warn="$(extract_named_token SERVICE "$post_warning_summary")"
|
||||
boundary_warn="$(extract_named_token BOUNDARY "$post_warning_summary")"
|
||||
evidence_warn="$(extract_named_token EVIDENCE "$post_warning_summary")"
|
||||
|
||||
service_green=0
|
||||
if [[ "${post_blocked:-999}" == "0" && "${service_warn:-999}" == "0" ]]; then
|
||||
service_green=1
|
||||
fi
|
||||
|
||||
backup_core_green=0
|
||||
grep -q 'OK backup core blockers are 0' "$post_start_log" && backup_core_green=1
|
||||
|
||||
product_data_green=0
|
||||
if grep -q '^STOCK_FRESHNESS_STATUS ok$' "$post_start_log" \
|
||||
&& grep -q '^DB_DAILY_FRESHNESS ' "$post_start_log"; then
|
||||
product_data_green=1
|
||||
fi
|
||||
|
||||
escrow_missing_count="$(grep -Eo 'escrow_missing=[0-9]+' "$post_start_log" | tail -n 1 | cut -d= -f2 || true)"
|
||||
dr_escrow_blocked=0
|
||||
if [[ -n "$escrow_missing_count" && "$escrow_missing_count" != "0" ]]; then
|
||||
dr_escrow_blocked=1
|
||||
elif [[ "$post_result" == "FULL_STACK_GREEN_DR_ESCROW_BLOCKED" ]]; then
|
||||
dr_escrow_blocked=1
|
||||
fi
|
||||
|
||||
host_188_hygiene_blocked="unknown"
|
||||
host_188_service_green="unknown"
|
||||
host_188_runtime_action="0"
|
||||
host_188_result="skipped"
|
||||
host_188_rc=0
|
||||
if [[ "$RUN_188_HYGIENE" -eq 1 ]]; then
|
||||
host_188_log="$ARTIFACT_DIR/188-host-hygiene.log"
|
||||
bash "$ROOT_DIR/scripts/reboot-recovery/188-host-hygiene-maintenance-checklist.sh" --no-color >"$host_188_log" 2>&1 || host_188_rc=$?
|
||||
host_188_hygiene_blocked="$(extract_key HOST_HYGIENE_BLOCKED "$host_188_log")"
|
||||
host_188_service_green="$(extract_key SERVICE_GREEN "$host_188_log")"
|
||||
host_188_runtime_action="$(extract_key RUNTIME_ACTION_AUTHORIZED "$host_188_log")"
|
||||
host_188_result="$(grep -E '^Result:' "$host_188_log" | tail -n 1 | sed 's/^Result: //')"
|
||||
fi
|
||||
|
||||
wazuh_registry_accepted="unknown"
|
||||
wazuh_route_code="unknown"
|
||||
wazuh_transport_count="unknown"
|
||||
wazuh_dashboard_degraded="unknown"
|
||||
wazuh_runtime_gate="0"
|
||||
if [[ "$RUN_WAZUH_GATES" -eq 1 ]]; then
|
||||
wazuh_coverage_log="$ARTIFACT_DIR/wazuh-managed-host-coverage.log"
|
||||
wazuh_runtime_log="$ARTIFACT_DIR/wazuh-runtime-gate.log"
|
||||
python3 "$ROOT_DIR/scripts/security/wazuh-managed-host-coverage-gate.py" --root "$ROOT_DIR" >"$wazuh_coverage_log" 2>&1 || true
|
||||
python3 "$ROOT_DIR/scripts/security/wazuh-agent-visibility-runtime-gate.py" --root "$ROOT_DIR" >"$wazuh_runtime_log" 2>&1 || true
|
||||
coverage_line="$(tail -n 1 "$wazuh_coverage_log" || true)"
|
||||
runtime_line="$(tail -n 1 "$wazuh_runtime_log" || true)"
|
||||
wazuh_registry_accepted="$(extract_named_token registry "$coverage_line")"
|
||||
wazuh_route_code="$(extract_named_token route "$runtime_line")"
|
||||
wazuh_transport_count="$(extract_named_token transport "$runtime_line")"
|
||||
wazuh_dashboard_degraded="$(extract_named_token dashboard_degraded "$runtime_line")"
|
||||
wazuh_runtime_gate="$(extract_named_token runtime_gate "$runtime_line")"
|
||||
fi
|
||||
|
||||
runtime_action_authorized=0
|
||||
if [[ "$host_188_runtime_action" =~ ^[1-9][0-9]*$ || "$wazuh_runtime_gate" =~ ^[1-9][0-9]*$ ]]; then
|
||||
runtime_action_authorized=1
|
||||
fi
|
||||
|
||||
overall_declaration="GREEN"
|
||||
if [[ "$service_green" != "1" ]]; then
|
||||
overall_declaration="SERVICE_BLOCKED"
|
||||
elif [[ "$dr_escrow_blocked" == "1" ]]; then
|
||||
overall_declaration="FULL_STACK_GREEN_DR_ESCROW_BLOCKED"
|
||||
elif [[ "$host_188_hygiene_blocked" == "1" ]]; then
|
||||
overall_declaration="FULL_STACK_GREEN_HOST_HYGIENE_BLOCKED"
|
||||
elif [[ "$wazuh_registry_accepted" == "0" ]]; then
|
||||
overall_declaration="FULL_STACK_GREEN_SECURITY_REGISTRY_BLOCKED"
|
||||
elif [[ "$evidence_warn" != "0" && -n "$evidence_warn" ]]; then
|
||||
overall_declaration="GREEN_WITH_EVIDENCE_WARNINGS"
|
||||
fi
|
||||
|
||||
next_required_gates=()
|
||||
[[ "$dr_escrow_blocked" == "1" ]] && next_required_gates+=("credential_escrow_evidence")
|
||||
[[ "$host_188_hygiene_blocked" == "1" ]] && next_required_gates+=("host_188_hygiene_maintenance_window")
|
||||
[[ "$wazuh_registry_accepted" == "0" ]] && next_required_gates+=("wazuh_manager_registry_export")
|
||||
if [[ "${#next_required_gates[@]}" -eq 0 ]]; then
|
||||
next_required_gates_csv="none"
|
||||
else
|
||||
next_required_gates_csv="$(IFS=,; echo "${next_required_gates[*]}")"
|
||||
fi
|
||||
|
||||
cat <<SUMMARY
|
||||
AWOOOI_POST_REBOOT_READINESS_SUMMARY=1
|
||||
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S %Z')
|
||||
ARTIFACT_DIR=$ARTIFACT_DIR
|
||||
POST_START_RC=$post_start_rc
|
||||
POST_START_RESULT=${post_result:-unknown}
|
||||
POST_START_PASS=${post_pass:-unknown}
|
||||
POST_START_WARN=${post_warn:-unknown}
|
||||
POST_START_BLOCKED=${post_blocked:-unknown}
|
||||
POST_START_SERVICE_WARNINGS=${service_warn:-unknown}
|
||||
POST_START_BOUNDARY_WARNINGS=${boundary_warn:-unknown}
|
||||
POST_START_EVIDENCE_WARNINGS=${evidence_warn:-unknown}
|
||||
SERVICE_GREEN=$service_green
|
||||
PRODUCT_DATA_GREEN=$product_data_green
|
||||
BACKUP_CORE_GREEN=$backup_core_green
|
||||
DR_ESCROW_BLOCKED=$dr_escrow_blocked
|
||||
ESCROW_MISSING_COUNT=${escrow_missing_count:-unknown}
|
||||
HOST_188_SERVICE_GREEN=$host_188_service_green
|
||||
HOST_188_HYGIENE_BLOCKED=$host_188_hygiene_blocked
|
||||
HOST_188_CHECK_RC=$host_188_rc
|
||||
HOST_188_RESULT=$host_188_result
|
||||
WAZUH_ROUTE_CODE=$wazuh_route_code
|
||||
WAZUH_TRANSPORT_COUNT=$wazuh_transport_count
|
||||
WAZUH_DASHBOARD_DEGRADED=$wazuh_dashboard_degraded
|
||||
WAZUH_MANAGER_REGISTRY_ACCEPTED=$wazuh_registry_accepted
|
||||
WAZUH_RUNTIME_GATE=$wazuh_runtime_gate
|
||||
RUNTIME_ACTION_AUTHORIZED=$runtime_action_authorized
|
||||
OVERALL_DECLARATION=$overall_declaration
|
||||
NEXT_REQUIRED_GATES=$next_required_gates_csv
|
||||
SUMMARY
|
||||
|
||||
if [[ "$SHOW_LOGS" -eq 1 ]]; then
|
||||
for log_file in "$ARTIFACT_DIR"/*.log; do
|
||||
printf '\n=== %s ===\n' "$log_file"
|
||||
cat "$log_file"
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ "$service_green" != "1" ]]; then
|
||||
exit 2
|
||||
fi
|
||||
|
||||
exit 0
|
||||
Reference in New Issue
Block a user