diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 5bf01d02e..7a383b278 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,20 @@ +## 2026-07-02 — 11:28 110 cold-start SSH 探測誤判修正 + +**完成內容**: +- 修正 `ops/runner/read-public-gitea-actions-queue.py`:Gitea/act log 會先列印 shell script 內容,不能把 script body 裡的 `echo "BLOCKED ..."` 當成 runtime blocker;若後續已有 `harbor_110_remote_ssh_probe_attempt=... result=success`、local recovery package 被呼叫且 job succeeded,則以最後 runtime 成功覆蓋前段 transient timeout。 +- 修正 `scripts/reboot-recovery/full-stack-cold-start-check.sh`:110 read-only SSH check 與 110 schedule/readiness check 均加入 bounded retry,避免全主機重啟後 sshd / PAM / session 尚在暖機時,一次 timeout 就把 cold-start scorecard 判成 hard blocker 或 false WARN。 +- 新增 regression tests,鎖住「前段 timeout、後段成功」不得留下 `harbor_110_remote_ssh_publickey_auth_stalled` false blocker。 + +**驗證**: +- `python3.11 -m pytest ops/runner/test_read_public_gitea_actions_queue.py scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py -q`:`56 passed`。 +- `python3.11 ops/runner/guard-gitea-runner-pressure.py --root .`:通過,`auto_branch_events_on_110=0`、`generic_runner_labels=0`。 +- 修正後 public queue readback:`harbor_110_repair_blocked=false`、`remote_ssh_command_path_ready=true`、`remote_control_channel_unavailable=false`。 +- 修正後 full-stack cold-start scorecard:`PASS=96 WARN=0 BLOCKED=0`,`Result: GREEN. Full stack is ready for controlled runner/CD release.`。 + +**仍維持**: +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth;沒有讀 `.runner` 內容。 +- 沒有重啟主機,沒有 Docker / Nginx / K3s / DB / firewall restart,沒有 workflow_dispatch,沒有 runner registration。 + ## 2026-07-02 — 11:01 Gitea CD B5 profile 跨 step 漂移修正 **照主線修正的問題**: diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index 87fbf6a62..6a55c5c71 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -81,6 +81,9 @@ _HARBOR_CONTROLLED_REPAIR_STATUS_RE = re.compile( _HARBOR_110_REMOTE_SSH_REACHABLE_RE = re.compile( r"harbor_110_remote_ssh_reachable=(?Ptrue|false)" ) +_HARBOR_110_REMOTE_SSH_PROBE_SUCCESS_RE = re.compile( + r"harbor_110_remote_ssh_probe_attempt=\d+\s+result=success" +) _HARBOR_110_REMOTE_SSH_TIMEOUT_RE = re.compile( r"(Connection to 192\.168\.0\.110 port 22 timed out|" r"ssh: connect to host 192\.168\.0\.110 port 22: Operation timed out)" @@ -1532,6 +1535,13 @@ def classify_cd_build_log(text: str) -> dict[str, Any]: def classify_harbor_110_repair_log(text: str) -> dict[str, Any]: + job_succeeded = "Job succeeded" in text + remote_ssh_probe_succeeded = ( + _HARBOR_110_REMOTE_SSH_PROBE_SUCCESS_RE.search(text) is not None + ) + local_recovery_invoked = ( + "AWOOOI_110_CONTROL_PATH_AND_HARBOR_LOCAL_RECOVERY" in text + ) ssh_reachable_matches = list(_HARBOR_110_REMOTE_SSH_REACHABLE_RE.finditer(text)) remote_ssh_reachable: bool | None = None if ssh_reachable_matches: @@ -1569,6 +1579,11 @@ def classify_harbor_110_repair_log(text: str) -> dict[str, Any]: re.MULTILINE, ) is not None + or ( + job_succeeded + and remote_ssh_probe_succeeded + and (remote_ssh_reachable is True or local_recovery_invoked) + ) ) remote_ssh_publickey_offer_timeout_raw = ( "classification=publickey_offer_timeout" in text @@ -1658,6 +1673,8 @@ def classify_harbor_110_repair_log(text: str) -> dict[str, Any]: "remote_control_channel_unavailable": remote_control_channel_unavailable, "remote_ssh_reachable": remote_ssh_reachable, "remote_ssh_command_path_ready": remote_ssh_command_path_ready, + "remote_ssh_probe_succeeded": remote_ssh_probe_succeeded, + "job_succeeded": job_succeeded, "bounded_ssh_timeout_seen": bounded_ssh_timeout_seen, "remote_ssh_tcp_connected": remote_ssh_tcp_connected, "remote_ssh_banner_seen": remote_ssh_banner_seen, diff --git a/ops/runner/test_read_public_gitea_actions_queue.py b/ops/runner/test_read_public_gitea_actions_queue.py index 3ca7aba9c..a641cc292 100644 --- a/ops/runner/test_read_public_gitea_actions_queue.py +++ b/ops/runner/test_read_public_gitea_actions_queue.py @@ -355,6 +355,27 @@ harbor_110_remote_ssh_reachable=false """ +def _harbor_110_repair_transient_timeout_then_success_log() -> str: + return """ +echo "BLOCKED harbor_110_remote_control_channel_unavailable target=${AWOOOI_110_SSH_TARGET}" +echo "harbor_110_remote_ssh_publickey_auth_stalled=true" +echo "BLOCKED harbor_110_remote_ssh_publickey_auth_stalled target=${AWOOOI_110_SSH_TARGET}" +Timeout, server 192.168.0.110 not responding. +harbor_110_remote_ssh_probe_attempt=1 result=failure rc=255 +Timeout, server 192.168.0.110 not responding. +harbor_110_remote_ssh_probe_attempt=2 result=failure rc=255 +Timeout, server 192.168.0.110 not responding. +harbor_110_remote_ssh_probe_attempt=3 result=failure rc=255 +harbor_110_remote_ssh_probe_attempt=4 result=success +harbor_110_remote_ssh_reachable=true +harbor_110_remote_ssh_probe_attempt=1 result=success +AWOOOI_110_CONTROL_PATH_AND_HARBOR_LOCAL_RECOVERY mode=check requested_mode=--check target_user=wooo +ssh_publickey_auth_stall_recovery_supported=true +harbor_110_remote_repair_skipped=already_ready +🏁 Job succeeded +""" + + def _harbor_110_repair_success_jobs() -> dict: return { "total_count": 2, @@ -710,6 +731,21 @@ def test_harbor_ssh_command_path_ready_overrides_raw_publickey_stall() -> None: assert classifier["failure_classifier"] == "" +def test_harbor_ssh_transient_timeouts_are_cleared_by_successful_job() -> None: + module = _load_module() + classifier = module.classify_harbor_110_repair_log( + _harbor_110_repair_transient_timeout_then_success_log() + ) + + assert classifier["job_succeeded"] is True + assert classifier["remote_ssh_probe_succeeded"] is True + assert classifier["remote_ssh_command_path_ready"] is True + assert classifier["remote_ssh_publickey_auth_stalled_raw"] is True + assert classifier["remote_ssh_publickey_auth_stalled"] is False + assert classifier["remote_control_channel_unavailable"] is False + assert classifier["failure_classifier"] == "" + + def test_latest_cd_success_makes_old_harbor_repair_failure_historical() -> None: module = _load_module() payload = module.build_readback( diff --git a/scripts/reboot-recovery/full-stack-cold-start-check.sh b/scripts/reboot-recovery/full-stack-cold-start-check.sh index bca0b6da2..53ba684b1 100755 --- a/scripts/reboot-recovery/full-stack-cold-start-check.sh +++ b/scripts/reboot-recovery/full-stack-cold-start-check.sh @@ -5,6 +5,8 @@ set -uo pipefail SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-45}" +SSH_110_CHECK_ATTEMPTS="${SSH_110_CHECK_ATTEMPTS:-4}" +SSH_110_CHECK_SLEEP_SECONDS="${SSH_110_CHECK_SLEEP_SECONDS:-5}" SSH_STRICT_HOST_KEY_CHECKING="${SSH_STRICT_HOST_KEY_CHECKING:-accept-new}" REGISTRY_HTTPS_URL="${REGISTRY_HTTPS_URL:-https://192.168.0.110:5000/v2/}" REGISTRY_HTTP_URL="${REGISTRY_HTTP_URL:-http://192.168.0.110:5000/v2/}" @@ -190,6 +192,27 @@ host_cmd() { ssh_cmd "$user_host" "$cmd" } +host_cmd_retry() { + local user_host="$1" + local cmd="$2" + local attempts="$3" + local sleep_seconds="$4" + local attempt rc out + for attempt in $(seq 1 "$attempts"); do + out="$(host_cmd "$user_host" "$cmd" 2>&1)" + rc=$? + if [ "$rc" -eq 0 ]; then + [ "$attempt" -gt 1 ] && echo "SSH_RETRY_RECOVERED user_host=$user_host attempt=$attempt/$attempts" + echo "$out" + return 0 + fi + echo "SSH_RETRY user_host=$user_host attempt=$attempt/$attempts rc=$rc" + echo "$out" + [ "$attempt" -lt "$attempts" ] && sleep "$sleep_seconds" + done + return "$rc" +} + probe_http_code() { local url="$1" local attempt code @@ -315,7 +338,7 @@ check_110() { fail "110 registry external /v2 not reachable" fi - if ! out=$(host_cmd "wooo@192.168.0.110" ' + if ! out=$(host_cmd_retry "wooo@192.168.0.110" ' sc() { if command -v timeout >/dev/null 2>&1; then timeout 3 systemctl "$@" 2>/dev/null || true @@ -451,7 +474,7 @@ for p in /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real- echo "$kind" | grep -qi "ELF" && echo "RUNNER_FAILCLOSED_BINARY_ELF $p" done docker ps --format "DOCKER {{.Names}}\t{{.Status}}" | head -120 -' 2>&1); then +' "$SSH_110_CHECK_ATTEMPTS" "$SSH_110_CHECK_SLEEP_SECONDS" 2>&1); then fail "ssh 110 read-only check" echo "SSH_110_BLOCKER remote_control_channel_unavailable" echo "SSH_110_NEXT_ACTION local_console_run_recover_110_control_path_and_harbor_local_check" @@ -845,7 +868,7 @@ printf "%s\n" "$momo_drive_source_probe" echo "$out" fi - if out=$(host_cmd "wooo@192.168.0.110" ' + if out=$(host_cmd_retry "wooo@192.168.0.110" ' now=$(date +%s) echo "CRON_110 $(systemctl is-active cron 2>/dev/null || systemctl is-active crond 2>/dev/null || true)" echo "FAILED_UNITS_110 $(systemctl --failed --no-legend --plain 2>/dev/null | wc -l)" @@ -865,7 +888,7 @@ fi if [ -f /home/wooo/node_exporter_textfiles/backup_health.prom ]; then awk "/^awoooi_backup_job_fresh/ {total++; if (int(\$2) == 0) stale++} /^awoooi_backup_job_configured/ {if (int(\$2) == 0) missing_cron++} /^awoooi_backup_script_present/ {if (int(\$2) == 0) missing_script++} /^awoooi_backup_last_run_failed_count/ {if (\$0 ~ /(exported_job|job)=\"backup_all\"/) failed=int(\$2)} /^awoooi_backup_config_capture_critical_failed_count/ {config_failed=int(\$2)} /^awoooi_backup_integrity_fresh/ {integrity_total++; if (int(\$2) == 0) integrity_stale++} END {printf \"BACKUP_HEALTH_110 total=%d stale=%d missing_cron=%d missing_script=%d failed_count=%d config_failed=%d integrity_total=%d integrity_stale=%d\\n\", total+0, stale+0, missing_cron+0, missing_script+0, failed+0, config_failed+0, integrity_total+0, integrity_stale+0}" /home/wooo/node_exporter_textfiles/backup_health.prom fi -' 2>&1); then +' "$SSH_110_CHECK_ATTEMPTS" "$SSH_110_CHECK_SLEEP_SECONDS" 2>&1); then echo "$out" grep -q "CRON_110 active" <<<"$out" && ok "110 cron active" || warn "110 cron not confirmed" grep -q "FAILED_UNITS_110 0" <<<"$out" && ok "110 systemd has no failed units" || warn "110 systemd failed units remain" diff --git a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py index e829af36a..95cb2f235 100644 --- a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py +++ b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py @@ -35,10 +35,16 @@ def test_full_stack_cold_start_check_bounds_ssh_probes() -> None: text = COLD_START_CHECK.read_text(encoding="utf-8") assert 'SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-45}"' in text + assert 'SSH_110_CHECK_ATTEMPTS="${SSH_110_CHECK_ATTEMPTS:-4}"' in text + assert 'SSH_110_CHECK_SLEEP_SECONDS="${SSH_110_CHECK_SLEEP_SECONDS:-5}"' in text assert "-o ConnectionAttempts=1" in text assert "-o ServerAliveInterval=5" in text assert "-o ServerAliveCountMax=1" in text assert "timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc" in text + assert "host_cmd_retry()" in text + assert "SSH_RETRY_RECOVERED user_host=$user_host attempt=$attempt/$attempts" in text + assert '"$SSH_110_CHECK_ATTEMPTS" "$SSH_110_CHECK_SLEEP_SECONDS"' in text + assert text.count('host_cmd_retry "wooo@192.168.0.110"') >= 2 assert "printf -v quoted_cmd '%q' \"$cmd\"" in text assert ( 'MOMO_SOURCE_PREFLIGHT_SCRIPT="${MOMO_SOURCE_PREFLIGHT_SCRIPT:-$SCRIPT_DIR/momo-drive-token-source-recovery-preflight.sh}"'