fix(reboot): collect windows99 vmware verifier in memory
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
set -u
|
||||
|
||||
MODE="check"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TARGET_HOST="${WINDOWS99_HOST:-192.168.0.99}"
|
||||
CONNECT_TIMEOUT="${WINDOWS99_CONNECT_TIMEOUT:-3}"
|
||||
SSH_TIMEOUT="${WINDOWS99_SSH_TIMEOUT:-3}"
|
||||
@@ -12,7 +13,8 @@ if [[ -n "${WINDOWS99_MAX_AUTH_USERS:-}" ]]; then
|
||||
MAX_AUTH_USERS_EXPLICIT=1
|
||||
fi
|
||||
KNOWN_HOSTS_FILE="${WINDOWS99_KNOWN_HOSTS_FILE:-/tmp/awoooi-windows99-known_hosts}"
|
||||
REMOTE_VERIFY_COMMAND="${WINDOWS99_REMOTE_VERIFY_COMMAND:-powershell -NoProfile -ExecutionPolicy Bypass -File .\\windows99-vmware-autostart.ps1 -Mode Verify}"
|
||||
LOCAL_VERIFY_SCRIPT="${WINDOWS99_LOCAL_VERIFY_SCRIPT:-${SCRIPT_DIR}/windows99-vmware-autostart.ps1}"
|
||||
REMOTE_VERIFY_COMMAND="${WINDOWS99_REMOTE_VERIFY_COMMAND:-powershell -NoProfile -ExecutionPolicy Bypass -Command \"& ([scriptblock]::Create([Console]::In.ReadToEnd())) -Mode Verify\"}"
|
||||
SSH_USERS=(ogt wooo ooo administrator Administrator)
|
||||
SSH_USERS_EXPLICIT=0
|
||||
|
||||
@@ -214,6 +216,10 @@ REMOTE_VERIFY_EXIT_STATUS="not_attempted"
|
||||
VERIFY_COLLECTION_STATUS="blocked_ssh_publickey_auth_missing"
|
||||
SAFE_NEXT_STEP="select_existing_authorized_public_key_user_or_set_WINDOWS99_SSH_USERS_then_rerun_collector_no_password"
|
||||
REMOTE_VERIFY_OUTPUT=""
|
||||
LOCAL_VERIFY_SCRIPT_PRESENT=0
|
||||
if [[ -f "${LOCAL_VERIFY_SCRIPT}" ]]; then
|
||||
LOCAL_VERIFY_SCRIPT_PRESENT=1
|
||||
fi
|
||||
PROCESS_EXIT_STATUS=0
|
||||
|
||||
if [[ "${PORT_22_OPEN}" != "1" ]]; then
|
||||
@@ -231,17 +237,28 @@ elif [[ "${SSH_BATCHMODE_AUTH_READY}" != "1" ]]; then
|
||||
elif [[ "${MODE}" == "check" ]]; then
|
||||
VERIFY_COLLECTION_STATUS="ready_ssh_batchmode_auth_probe_only"
|
||||
SAFE_NEXT_STEP="rerun_collector_with_collect_then_commit_no_secret_verify_artifact_and_scorecard_rerun"
|
||||
elif [[ "${LOCAL_VERIFY_SCRIPT_PRESENT}" != "1" ]]; then
|
||||
DRY_RUN="false"
|
||||
VERIFY_COLLECTION_STATUS="blocked_local_verify_script_missing"
|
||||
SAFE_NEXT_STEP="restore_windows99_vmware_autostart_ps1_source_then_rerun_collector_no_secret_no_remote_write"
|
||||
PROCESS_EXIT_STATUS=75
|
||||
else
|
||||
DRY_RUN="false"
|
||||
REMOTE_VERIFY_ATTEMPTED=1
|
||||
if REMOTE_VERIFY_OUTPUT="$(run_ssh "${SSH_AUTHENTICATED_USER}" "${REMOTE_VERIFY_COMMAND}" 2>&1)"; then
|
||||
if REMOTE_VERIFY_OUTPUT="$(run_ssh "${SSH_AUTHENTICATED_USER}" "${REMOTE_VERIFY_COMMAND}" <"${LOCAL_VERIFY_SCRIPT}" 2>&1)"; then
|
||||
REMOTE_VERIFY_EXIT_STATUS=0
|
||||
VERIFY_COLLECTION_STATUS="collected_windows99_vmware_verify_stdout"
|
||||
SAFE_NEXT_STEP="commit_no_secret_verify_artifact_then_rerun_reboot_auto_recovery_slo_scorecard"
|
||||
if grep -q '^AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1$' <<<"${REMOTE_VERIFY_OUTPUT}" && grep -q '^MODE=Verify$' <<<"${REMOTE_VERIFY_OUTPUT}"; then
|
||||
VERIFY_COLLECTION_STATUS="collected_windows99_vmware_verify_stdout"
|
||||
SAFE_NEXT_STEP="commit_no_secret_verify_artifact_then_rerun_reboot_auto_recovery_slo_scorecard"
|
||||
else
|
||||
VERIFY_COLLECTION_STATUS="blocked_remote_verify_output_invalid"
|
||||
SAFE_NEXT_STEP="inspect_no_secret_verify_stdout_then_fix_in_memory_verify_script_and_rerun_collector"
|
||||
PROCESS_EXIT_STATUS=75
|
||||
fi
|
||||
else
|
||||
REMOTE_VERIFY_EXIT_STATUS=$?
|
||||
VERIFY_COLLECTION_STATUS="blocked_remote_verify_command_failed"
|
||||
SAFE_NEXT_STEP="inspect_no_secret_verify_stdout_then_fix_verify_script_or_path_and_rerun_collector"
|
||||
SAFE_NEXT_STEP="inspect_no_secret_verify_stdout_then_fix_in_memory_verify_script_and_rerun_collector"
|
||||
PROCESS_EXIT_STATUS=75
|
||||
fi
|
||||
fi
|
||||
@@ -266,6 +283,8 @@ printf '%s\n' "ssh_batchmode_auth_ready=${SSH_BATCHMODE_AUTH_READY}"
|
||||
printf '%s\n' "ssh_authenticated_user=${SSH_AUTHENTICATED_USER}"
|
||||
printf '%s\n' "ssh_auth_probe_exit_status=${SSH_AUTH_PROBE_EXIT_STATUS}"
|
||||
printf '%s\n' "ssh_auth_probe_stdout_present=${SSH_AUTH_PROBE_STDOUT_PRESENT}"
|
||||
printf '%s\n' "remote_verify_mode=in_memory_stdin_scriptblock"
|
||||
printf '%s\n' "local_verify_script_present=${LOCAL_VERIFY_SCRIPT_PRESENT}"
|
||||
printf '%s\n' "remote_verify_attempted=${REMOTE_VERIFY_ATTEMPTED}"
|
||||
printf '%s\n' "remote_verify_exit_status=${REMOTE_VERIFY_EXIT_STATUS}"
|
||||
printf '%s\n' "verify_collection_status=${VERIFY_COLLECTION_STATUS}"
|
||||
|
||||
@@ -40,6 +40,7 @@ stock_ingestion_file="$artifact_dir/stock-ingestion.json"
|
||||
public_maintenance_file="$artifact_dir/public-maintenance-fallback.json"
|
||||
public_maintenance_prom="$artifact_dir/public-maintenance-fallback.prom"
|
||||
windows99_management_file="$artifact_dir/windows99-management-channel.json"
|
||||
windows99_vmware_file="$artifact_dir/windows99-vmware-verify.txt"
|
||||
reboot_event_state_file="${REBOOT_EVENT_STATE_FILE:-${LOG_DIR}/reboot-event-state.json}"
|
||||
|
||||
bash "$ROOT_DIR/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh" >"$host_probe" 2>&1 || true
|
||||
@@ -80,6 +81,12 @@ if [ -f "$ROOT_DIR/scripts/reboot-recovery/windows99-management-channel-probe.py
|
||||
--output "$windows99_management_file" >"$artifact_dir/windows99-management-channel.stdout" 2>"$artifact_dir/windows99-management-channel.err" \
|
||||
|| rm -f "$windows99_management_file"
|
||||
fi
|
||||
if [ -x "$ROOT_DIR/scripts/reboot-recovery/collect-windows99-vmware-verify.sh" ]; then
|
||||
WINDOWS99_MAX_AUTH_USERS="${WINDOWS99_MAX_AUTH_USERS:-5}" \
|
||||
bash "$ROOT_DIR/scripts/reboot-recovery/collect-windows99-vmware-verify.sh" --collect \
|
||||
>"$windows99_vmware_file" 2>"$artifact_dir/windows99-vmware-verify.err" \
|
||||
|| true
|
||||
fi
|
||||
|
||||
scorecard_args=(
|
||||
"$ROOT_DIR/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py"
|
||||
@@ -103,6 +110,9 @@ fi
|
||||
if [ -s "$windows99_management_file" ]; then
|
||||
scorecard_args+=(--windows99-management-file "$windows99_management_file")
|
||||
fi
|
||||
if [ -s "$windows99_vmware_file" ] && grep -q '^AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1$' "$windows99_vmware_file"; then
|
||||
scorecard_args+=(--windows99-vmware-file "$windows99_vmware_file")
|
||||
fi
|
||||
|
||||
python3 "${scorecard_args[@]}" || true
|
||||
|
||||
|
||||
@@ -102,6 +102,11 @@ def test_exporter_carries_stockplatform_readbacks_into_scorecard() -> None:
|
||||
def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None:
|
||||
text = EXPORTER.read_text(encoding="utf-8")
|
||||
|
||||
assert "windows99-vmware-verify.txt" in text
|
||||
assert "collect-windows99-vmware-verify.sh\" --collect" in text
|
||||
assert "WINDOWS99_MAX_AUTH_USERS=\"${WINDOWS99_MAX_AUTH_USERS:-5}\"" in text
|
||||
assert "AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1" in text
|
||||
assert "scorecard_args+=(--windows99-vmware-file" in text
|
||||
assert "active_blocker_metrics" in text
|
||||
assert 'for blocker in payload.get("active_blockers") or []' in text
|
||||
assert "awoooi_reboot_auto_recovery_slo_active_blocker" in text
|
||||
|
||||
@@ -50,6 +50,9 @@ def test_collector_contract_forbids_secret_and_runtime_actions() -> None:
|
||||
assert "NumberOfPasswordPrompts=0" in text
|
||||
assert "ConnectionAttempts=1" in text
|
||||
assert "GSSAPIAuthentication=no" in text
|
||||
assert "in_memory_stdin_scriptblock" in text
|
||||
assert "[Console]::In.ReadToEnd()" in text
|
||||
assert ".\\\\windows99-vmware-autostart.ps1 -Mode Verify" not in text
|
||||
for forbidden in [
|
||||
"sshpass",
|
||||
"PasswordAuthentication=yes",
|
||||
@@ -208,6 +211,8 @@ def test_check_mode_auth_ready_does_not_run_remote_verify(tmp_path: Path) -> Non
|
||||
assert result.returncode == 0
|
||||
values = _key_values(result.stdout)
|
||||
assert values["ssh_batchmode_auth_ready"] == "1"
|
||||
assert values["remote_verify_mode"] == "in_memory_stdin_scriptblock"
|
||||
assert values["local_verify_script_present"] == "1"
|
||||
assert values["remote_verify_attempted"] == "0"
|
||||
assert values["verify_collection_status"] == "ready_ssh_batchmode_auth_probe_only"
|
||||
assert "VMRUN_PRESENT=1" not in result.stdout
|
||||
@@ -254,3 +259,38 @@ def test_collect_mode_runs_readonly_remote_verify_when_auth_ready(tmp_path: Path
|
||||
assert "VMRUN_PRESENT=1" in result.stdout
|
||||
assert "VMWARE_AUTOSTART_VERIFY_READY=1" in result.stdout
|
||||
assert "remote_verify_output_end" in result.stdout
|
||||
|
||||
|
||||
def test_collect_mode_rejects_success_exit_without_verify_marker(tmp_path: Path) -> None:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
_write_executable(
|
||||
fake_bin / "nc",
|
||||
"""
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "ssh",
|
||||
"""
|
||||
#!/usr/bin/env bash
|
||||
args="$*"
|
||||
if [[ "$args" == *"powershell"* ]]; then
|
||||
printf '%s\n' 'Windows PowerShell 5.1'
|
||||
printf '%s\n' "-File argument '.\\windows99-vmware-autostart.ps1' does not exist"
|
||||
exit 0
|
||||
fi
|
||||
printf '%s\n' 'AWOOOI_WINDOWS99_SSH_READY'
|
||||
exit 0
|
||||
""",
|
||||
)
|
||||
|
||||
result = _run_collector(fake_bin, "--collect")
|
||||
|
||||
assert result.returncode == 75
|
||||
values = _key_values(result.stdout)
|
||||
assert values["remote_verify_attempted"] == "1"
|
||||
assert values["remote_verify_exit_status"] == "0"
|
||||
assert values["verify_collection_status"] == "blocked_remote_verify_output_invalid"
|
||||
assert "AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1" not in result.stdout
|
||||
|
||||
Reference in New Issue
Block a user