diff --git a/scripts/reboot-recovery/collect-windows99-vmware-verify.sh b/scripts/reboot-recovery/collect-windows99-vmware-verify.sh index 4eea4f11f..50e6499ab 100755 --- a/scripts/reboot-recovery/collect-windows99-vmware-verify.sh +++ b/scripts/reboot-recovery/collect-windows99-vmware-verify.sh @@ -2,6 +2,7 @@ set -u MODE="check" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TARGET_HOST="${WINDOWS99_HOST:-192.168.0.99}" CONNECT_TIMEOUT="${WINDOWS99_CONNECT_TIMEOUT:-3}" SSH_TIMEOUT="${WINDOWS99_SSH_TIMEOUT:-3}" @@ -12,7 +13,8 @@ if [[ -n "${WINDOWS99_MAX_AUTH_USERS:-}" ]]; then MAX_AUTH_USERS_EXPLICIT=1 fi KNOWN_HOSTS_FILE="${WINDOWS99_KNOWN_HOSTS_FILE:-/tmp/awoooi-windows99-known_hosts}" -REMOTE_VERIFY_COMMAND="${WINDOWS99_REMOTE_VERIFY_COMMAND:-powershell -NoProfile -ExecutionPolicy Bypass -File .\\windows99-vmware-autostart.ps1 -Mode Verify}" +LOCAL_VERIFY_SCRIPT="${WINDOWS99_LOCAL_VERIFY_SCRIPT:-${SCRIPT_DIR}/windows99-vmware-autostart.ps1}" +REMOTE_VERIFY_COMMAND="${WINDOWS99_REMOTE_VERIFY_COMMAND:-powershell -NoProfile -ExecutionPolicy Bypass -Command \"& ([scriptblock]::Create([Console]::In.ReadToEnd())) -Mode Verify\"}" SSH_USERS=(ogt wooo ooo administrator Administrator) SSH_USERS_EXPLICIT=0 @@ -214,6 +216,10 @@ REMOTE_VERIFY_EXIT_STATUS="not_attempted" VERIFY_COLLECTION_STATUS="blocked_ssh_publickey_auth_missing" SAFE_NEXT_STEP="select_existing_authorized_public_key_user_or_set_WINDOWS99_SSH_USERS_then_rerun_collector_no_password" REMOTE_VERIFY_OUTPUT="" +LOCAL_VERIFY_SCRIPT_PRESENT=0 +if [[ -f "${LOCAL_VERIFY_SCRIPT}" ]]; then + LOCAL_VERIFY_SCRIPT_PRESENT=1 +fi PROCESS_EXIT_STATUS=0 if [[ "${PORT_22_OPEN}" != "1" ]]; then @@ -231,17 +237,28 @@ elif [[ "${SSH_BATCHMODE_AUTH_READY}" != "1" ]]; then elif [[ "${MODE}" == "check" ]]; then VERIFY_COLLECTION_STATUS="ready_ssh_batchmode_auth_probe_only" SAFE_NEXT_STEP="rerun_collector_with_collect_then_commit_no_secret_verify_artifact_and_scorecard_rerun" +elif [[ "${LOCAL_VERIFY_SCRIPT_PRESENT}" != "1" ]]; then + DRY_RUN="false" + VERIFY_COLLECTION_STATUS="blocked_local_verify_script_missing" + SAFE_NEXT_STEP="restore_windows99_vmware_autostart_ps1_source_then_rerun_collector_no_secret_no_remote_write" + PROCESS_EXIT_STATUS=75 else DRY_RUN="false" REMOTE_VERIFY_ATTEMPTED=1 - if REMOTE_VERIFY_OUTPUT="$(run_ssh "${SSH_AUTHENTICATED_USER}" "${REMOTE_VERIFY_COMMAND}" 2>&1)"; then + if REMOTE_VERIFY_OUTPUT="$(run_ssh "${SSH_AUTHENTICATED_USER}" "${REMOTE_VERIFY_COMMAND}" <"${LOCAL_VERIFY_SCRIPT}" 2>&1)"; then REMOTE_VERIFY_EXIT_STATUS=0 - VERIFY_COLLECTION_STATUS="collected_windows99_vmware_verify_stdout" - SAFE_NEXT_STEP="commit_no_secret_verify_artifact_then_rerun_reboot_auto_recovery_slo_scorecard" + if grep -q '^AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1$' <<<"${REMOTE_VERIFY_OUTPUT}" && grep -q '^MODE=Verify$' <<<"${REMOTE_VERIFY_OUTPUT}"; then + VERIFY_COLLECTION_STATUS="collected_windows99_vmware_verify_stdout" + SAFE_NEXT_STEP="commit_no_secret_verify_artifact_then_rerun_reboot_auto_recovery_slo_scorecard" + else + VERIFY_COLLECTION_STATUS="blocked_remote_verify_output_invalid" + SAFE_NEXT_STEP="inspect_no_secret_verify_stdout_then_fix_in_memory_verify_script_and_rerun_collector" + PROCESS_EXIT_STATUS=75 + fi else REMOTE_VERIFY_EXIT_STATUS=$? VERIFY_COLLECTION_STATUS="blocked_remote_verify_command_failed" - SAFE_NEXT_STEP="inspect_no_secret_verify_stdout_then_fix_verify_script_or_path_and_rerun_collector" + SAFE_NEXT_STEP="inspect_no_secret_verify_stdout_then_fix_in_memory_verify_script_and_rerun_collector" PROCESS_EXIT_STATUS=75 fi fi @@ -266,6 +283,8 @@ printf '%s\n' "ssh_batchmode_auth_ready=${SSH_BATCHMODE_AUTH_READY}" printf '%s\n' "ssh_authenticated_user=${SSH_AUTHENTICATED_USER}" printf '%s\n' "ssh_auth_probe_exit_status=${SSH_AUTH_PROBE_EXIT_STATUS}" printf '%s\n' "ssh_auth_probe_stdout_present=${SSH_AUTH_PROBE_STDOUT_PRESENT}" +printf '%s\n' "remote_verify_mode=in_memory_stdin_scriptblock" +printf '%s\n' "local_verify_script_present=${LOCAL_VERIFY_SCRIPT_PRESENT}" printf '%s\n' "remote_verify_attempted=${REMOTE_VERIFY_ATTEMPTED}" printf '%s\n' "remote_verify_exit_status=${REMOTE_VERIFY_EXIT_STATUS}" printf '%s\n' "verify_collection_status=${VERIFY_COLLECTION_STATUS}" diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index 4663c1aa3..ab5faac91 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -40,6 +40,7 @@ stock_ingestion_file="$artifact_dir/stock-ingestion.json" public_maintenance_file="$artifact_dir/public-maintenance-fallback.json" public_maintenance_prom="$artifact_dir/public-maintenance-fallback.prom" windows99_management_file="$artifact_dir/windows99-management-channel.json" +windows99_vmware_file="$artifact_dir/windows99-vmware-verify.txt" reboot_event_state_file="${REBOOT_EVENT_STATE_FILE:-${LOG_DIR}/reboot-event-state.json}" bash "$ROOT_DIR/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh" >"$host_probe" 2>&1 || true @@ -80,6 +81,12 @@ if [ -f "$ROOT_DIR/scripts/reboot-recovery/windows99-management-channel-probe.py --output "$windows99_management_file" >"$artifact_dir/windows99-management-channel.stdout" 2>"$artifact_dir/windows99-management-channel.err" \ || rm -f "$windows99_management_file" fi +if [ -x "$ROOT_DIR/scripts/reboot-recovery/collect-windows99-vmware-verify.sh" ]; then + WINDOWS99_MAX_AUTH_USERS="${WINDOWS99_MAX_AUTH_USERS:-5}" \ + bash "$ROOT_DIR/scripts/reboot-recovery/collect-windows99-vmware-verify.sh" --collect \ + >"$windows99_vmware_file" 2>"$artifact_dir/windows99-vmware-verify.err" \ + || true +fi scorecard_args=( "$ROOT_DIR/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py" @@ -103,6 +110,9 @@ fi if [ -s "$windows99_management_file" ]; then scorecard_args+=(--windows99-management-file "$windows99_management_file") fi +if [ -s "$windows99_vmware_file" ] && grep -q '^AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1$' "$windows99_vmware_file"; then + scorecard_args+=(--windows99-vmware-file "$windows99_vmware_file") +fi python3 "${scorecard_args[@]}" || true diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py index 593f981a9..1483c21fa 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py @@ -102,6 +102,11 @@ def test_exporter_carries_stockplatform_readbacks_into_scorecard() -> None: def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None: text = EXPORTER.read_text(encoding="utf-8") + assert "windows99-vmware-verify.txt" in text + assert "collect-windows99-vmware-verify.sh\" --collect" in text + assert "WINDOWS99_MAX_AUTH_USERS=\"${WINDOWS99_MAX_AUTH_USERS:-5}\"" in text + assert "AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1" in text + assert "scorecard_args+=(--windows99-vmware-file" in text assert "active_blocker_metrics" in text assert 'for blocker in payload.get("active_blockers") or []' in text assert "awoooi_reboot_auto_recovery_slo_active_blocker" in text diff --git a/scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py b/scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py index 56adfbb7f..4c39f4abb 100644 --- a/scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py +++ b/scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py @@ -50,6 +50,9 @@ def test_collector_contract_forbids_secret_and_runtime_actions() -> None: assert "NumberOfPasswordPrompts=0" in text assert "ConnectionAttempts=1" in text assert "GSSAPIAuthentication=no" in text + assert "in_memory_stdin_scriptblock" in text + assert "[Console]::In.ReadToEnd()" in text + assert ".\\\\windows99-vmware-autostart.ps1 -Mode Verify" not in text for forbidden in [ "sshpass", "PasswordAuthentication=yes", @@ -208,6 +211,8 @@ def test_check_mode_auth_ready_does_not_run_remote_verify(tmp_path: Path) -> Non assert result.returncode == 0 values = _key_values(result.stdout) assert values["ssh_batchmode_auth_ready"] == "1" + assert values["remote_verify_mode"] == "in_memory_stdin_scriptblock" + assert values["local_verify_script_present"] == "1" assert values["remote_verify_attempted"] == "0" assert values["verify_collection_status"] == "ready_ssh_batchmode_auth_probe_only" assert "VMRUN_PRESENT=1" not in result.stdout @@ -254,3 +259,38 @@ def test_collect_mode_runs_readonly_remote_verify_when_auth_ready(tmp_path: Path assert "VMRUN_PRESENT=1" in result.stdout assert "VMWARE_AUTOSTART_VERIFY_READY=1" in result.stdout assert "remote_verify_output_end" in result.stdout + + +def test_collect_mode_rejects_success_exit_without_verify_marker(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + _write_executable( + fake_bin / "nc", + """ + #!/usr/bin/env bash + exit 0 + """, + ) + _write_executable( + fake_bin / "ssh", + """ + #!/usr/bin/env bash + args="$*" + if [[ "$args" == *"powershell"* ]]; then + printf '%s\n' 'Windows PowerShell 5.1' + printf '%s\n' "-File argument '.\\windows99-vmware-autostart.ps1' does not exist" + exit 0 + fi + printf '%s\n' 'AWOOOI_WINDOWS99_SSH_READY' + exit 0 + """, + ) + + result = _run_collector(fake_bin, "--collect") + + assert result.returncode == 75 + values = _key_values(result.stdout) + assert values["remote_verify_attempted"] == "1" + assert values["remote_verify_exit_status"] == "0" + assert values["verify_collection_status"] == "blocked_remote_verify_output_invalid" + assert "AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1" not in result.stdout