fix(reboot): clear transient 110 ssh false blockers

docs/LOGBOOK.md

@@ -1,3 +1,20 @@
+## 2026-07-02 — 11:28 110 cold-start SSH 探測誤判修正
+
+**完成內容**：
+- 修正 `ops/runner/read-public-gitea-actions-queue.py`：Gitea/act log 會先列印 shell script 內容，不能把 script body 裡的 `echo "BLOCKED ..."` 當成 runtime blocker；若後續已有 `harbor_110_remote_ssh_probe_attempt=... result=success`、local recovery package 被呼叫且 job succeeded，則以最後 runtime 成功覆蓋前段 transient timeout。
+- 修正 `scripts/reboot-recovery/full-stack-cold-start-check.sh`：110 read-only SSH check 與 110 schedule/readiness check 均加入 bounded retry，避免全主機重啟後 sshd / PAM / session 尚在暖機時，一次 timeout 就把 cold-start scorecard 判成 hard blocker 或 false WARN。
+- 新增 regression tests，鎖住「前段 timeout、後段成功」不得留下 `harbor_110_remote_ssh_publickey_auth_stalled` false blocker。
+
+**驗證**：
+- `python3.11 -m pytest ops/runner/test_read_public_gitea_actions_queue.py scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py -q`：`56 passed`。
+- `python3.11 ops/runner/guard-gitea-runner-pressure.py --root .`：通過，`auto_branch_events_on_110=0`、`generic_runner_labels=0`。
+- 修正後 public queue readback：`harbor_110_repair_blocked=false`、`remote_ssh_command_path_ready=true`、`remote_control_channel_unavailable=false`。
+- 修正後 full-stack cold-start scorecard：`PASS=96 WARN=0 BLOCKED=0`，`Result: GREEN. Full stack is ready for controlled runner/CD release.`。
+
+**仍維持**：
+- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth；沒有讀 `.runner` 內容。
+- 沒有重啟主機，沒有 Docker / Nginx / K3s / DB / firewall restart，沒有 workflow_dispatch，沒有 runner registration。
+
 ## 2026-07-02 — 11:01 Gitea CD B5 profile 跨 step 漂移修正
 
 **照主線修正的問題**：

ops/runner/read-public-gitea-actions-queue.py

@@ -81,6 +81,9 @@ _HARBOR_CONTROLLED_REPAIR_STATUS_RE = re.compile(
 _HARBOR_110_REMOTE_SSH_REACHABLE_RE = re.compile(
     r"harbor_110_remote_ssh_reachable=(?P<reachable>true|false)"
 )
+_HARBOR_110_REMOTE_SSH_PROBE_SUCCESS_RE = re.compile(
+    r"harbor_110_remote_ssh_probe_attempt=\d+\s+result=success"
+)
 _HARBOR_110_REMOTE_SSH_TIMEOUT_RE = re.compile(
     r"(Connection to 192\.168\.0\.110 port 22 timed out|"
     r"ssh: connect to host 192\.168\.0\.110 port 22: Operation timed out)"
@@ -1532,6 +1535,13 @@ def classify_cd_build_log(text: str) -> dict[str, Any]:
 
 
 def classify_harbor_110_repair_log(text: str) -> dict[str, Any]:
+    job_succeeded = "Job succeeded" in text
+    remote_ssh_probe_succeeded = (
+        _HARBOR_110_REMOTE_SSH_PROBE_SUCCESS_RE.search(text) is not None
+    )
+    local_recovery_invoked = (
+        "AWOOOI_110_CONTROL_PATH_AND_HARBOR_LOCAL_RECOVERY" in text
+    )
     ssh_reachable_matches = list(_HARBOR_110_REMOTE_SSH_REACHABLE_RE.finditer(text))
     remote_ssh_reachable: bool | None = None
     if ssh_reachable_matches:
@@ -1569,6 +1579,11 @@ def classify_harbor_110_repair_log(text: str) -> dict[str, Any]:
             re.MULTILINE,
         )
         is not None
+        or (
+            job_succeeded
+            and remote_ssh_probe_succeeded
+            and (remote_ssh_reachable is True or local_recovery_invoked)
+        )
     )
     remote_ssh_publickey_offer_timeout_raw = (
         "classification=publickey_offer_timeout" in text
@@ -1658,6 +1673,8 @@ def classify_harbor_110_repair_log(text: str) -> dict[str, Any]:
         "remote_control_channel_unavailable": remote_control_channel_unavailable,
         "remote_ssh_reachable": remote_ssh_reachable,
         "remote_ssh_command_path_ready": remote_ssh_command_path_ready,
+        "remote_ssh_probe_succeeded": remote_ssh_probe_succeeded,
+        "job_succeeded": job_succeeded,
         "bounded_ssh_timeout_seen": bounded_ssh_timeout_seen,
         "remote_ssh_tcp_connected": remote_ssh_tcp_connected,
         "remote_ssh_banner_seen": remote_ssh_banner_seen,

ops/runner/test_read_public_gitea_actions_queue.py

@@ -355,6 +355,27 @@ harbor_110_remote_ssh_reachable=false
 """
 
 
+def _harbor_110_repair_transient_timeout_then_success_log() -> str:
+    return """
+echo "BLOCKED harbor_110_remote_control_channel_unavailable target=${AWOOOI_110_SSH_TARGET}"
+echo "harbor_110_remote_ssh_publickey_auth_stalled=true"
+echo "BLOCKED harbor_110_remote_ssh_publickey_auth_stalled target=${AWOOOI_110_SSH_TARGET}"
+Timeout, server 192.168.0.110 not responding.
+harbor_110_remote_ssh_probe_attempt=1 result=failure rc=255
+Timeout, server 192.168.0.110 not responding.
+harbor_110_remote_ssh_probe_attempt=2 result=failure rc=255
+Timeout, server 192.168.0.110 not responding.
+harbor_110_remote_ssh_probe_attempt=3 result=failure rc=255
+harbor_110_remote_ssh_probe_attempt=4 result=success
+harbor_110_remote_ssh_reachable=true
+harbor_110_remote_ssh_probe_attempt=1 result=success
+AWOOOI_110_CONTROL_PATH_AND_HARBOR_LOCAL_RECOVERY mode=check requested_mode=--check target_user=wooo
+ssh_publickey_auth_stall_recovery_supported=true
+harbor_110_remote_repair_skipped=already_ready
+🏁  Job succeeded
+"""
+
+
 def _harbor_110_repair_success_jobs() -> dict:
     return {
         "total_count": 2,
@@ -710,6 +731,21 @@ def test_harbor_ssh_command_path_ready_overrides_raw_publickey_stall() -> None:
     assert classifier["failure_classifier"] == ""
 
 
+def test_harbor_ssh_transient_timeouts_are_cleared_by_successful_job() -> None:
+    module = _load_module()
+    classifier = module.classify_harbor_110_repair_log(
+        _harbor_110_repair_transient_timeout_then_success_log()
+    )
+
+    assert classifier["job_succeeded"] is True
+    assert classifier["remote_ssh_probe_succeeded"] is True
+    assert classifier["remote_ssh_command_path_ready"] is True
+    assert classifier["remote_ssh_publickey_auth_stalled_raw"] is True
+    assert classifier["remote_ssh_publickey_auth_stalled"] is False
+    assert classifier["remote_control_channel_unavailable"] is False
+    assert classifier["failure_classifier"] == ""
+
+
 def test_latest_cd_success_makes_old_harbor_repair_failure_historical() -> None:
     module = _load_module()
     payload = module.build_readback(

scripts/reboot-recovery/full-stack-cold-start-check.sh

@@ -5,6 +5,8 @@
 set -uo pipefail
 
 SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-45}"
+SSH_110_CHECK_ATTEMPTS="${SSH_110_CHECK_ATTEMPTS:-4}"
+SSH_110_CHECK_SLEEP_SECONDS="${SSH_110_CHECK_SLEEP_SECONDS:-5}"
 SSH_STRICT_HOST_KEY_CHECKING="${SSH_STRICT_HOST_KEY_CHECKING:-accept-new}"
 REGISTRY_HTTPS_URL="${REGISTRY_HTTPS_URL:-https://192.168.0.110:5000/v2/}"
 REGISTRY_HTTP_URL="${REGISTRY_HTTP_URL:-http://192.168.0.110:5000/v2/}"
@@ -190,6 +192,27 @@ host_cmd() {
   ssh_cmd "$user_host" "$cmd"
 }
 
+host_cmd_retry() {
+  local user_host="$1"
+  local cmd="$2"
+  local attempts="$3"
+  local sleep_seconds="$4"
+  local attempt rc out
+  for attempt in $(seq 1 "$attempts"); do
+    out="$(host_cmd "$user_host" "$cmd" 2>&1)"
+    rc=$?
+    if [ "$rc" -eq 0 ]; then
+      [ "$attempt" -gt 1 ] && echo "SSH_RETRY_RECOVERED user_host=$user_host attempt=$attempt/$attempts"
+      echo "$out"
+      return 0
+    fi
+    echo "SSH_RETRY user_host=$user_host attempt=$attempt/$attempts rc=$rc"
+    echo "$out"
+    [ "$attempt" -lt "$attempts" ] && sleep "$sleep_seconds"
+  done
+  return "$rc"
+}
+
 probe_http_code() {
   local url="$1"
   local attempt code
@@ -315,7 +338,7 @@ check_110() {
     fail "110 registry external /v2 not reachable"
   fi
 
-  if ! out=$(host_cmd "wooo@192.168.0.110" '
+  if ! out=$(host_cmd_retry "wooo@192.168.0.110" '
 sc() {
   if command -v timeout >/dev/null 2>&1; then
     timeout 3 systemctl "$@" 2>/dev/null || true
@@ -451,7 +474,7 @@ for p in /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-
   echo "$kind" | grep -qi "ELF" && echo "RUNNER_FAILCLOSED_BINARY_ELF $p"
 done
 docker ps --format "DOCKER {{.Names}}\t{{.Status}}" | head -120
-' 2>&1); then
+' "$SSH_110_CHECK_ATTEMPTS" "$SSH_110_CHECK_SLEEP_SECONDS" 2>&1); then
     fail "ssh 110 read-only check"
     echo "SSH_110_BLOCKER remote_control_channel_unavailable"
     echo "SSH_110_NEXT_ACTION local_console_run_recover_110_control_path_and_harbor_local_check"
@@ -845,7 +868,7 @@ printf "%s\n" "$momo_drive_source_probe"
     echo "$out"
   fi
 
-  if out=$(host_cmd "wooo@192.168.0.110" '
+  if out=$(host_cmd_retry "wooo@192.168.0.110" '
 now=$(date +%s)
 echo "CRON_110 $(systemctl is-active cron 2>/dev/null || systemctl is-active crond 2>/dev/null || true)"
 echo "FAILED_UNITS_110 $(systemctl --failed --no-legend --plain 2>/dev/null | wc -l)"
@@ -865,7 +888,7 @@ fi
 if [ -f /home/wooo/node_exporter_textfiles/backup_health.prom ]; then
   awk "/^awoooi_backup_job_fresh/ {total++; if (int(\$2) == 0) stale++} /^awoooi_backup_job_configured/ {if (int(\$2) == 0) missing_cron++} /^awoooi_backup_script_present/ {if (int(\$2) == 0) missing_script++} /^awoooi_backup_last_run_failed_count/ {if (\$0 ~ /(exported_job|job)=\"backup_all\"/) failed=int(\$2)} /^awoooi_backup_config_capture_critical_failed_count/ {config_failed=int(\$2)} /^awoooi_backup_integrity_fresh/ {integrity_total++; if (int(\$2) == 0) integrity_stale++} END {printf \"BACKUP_HEALTH_110 total=%d stale=%d missing_cron=%d missing_script=%d failed_count=%d config_failed=%d integrity_total=%d integrity_stale=%d\\n\", total+0, stale+0, missing_cron+0, missing_script+0, failed+0, config_failed+0, integrity_total+0, integrity_stale+0}" /home/wooo/node_exporter_textfiles/backup_health.prom
 fi
-' 2>&1); then
+' "$SSH_110_CHECK_ATTEMPTS" "$SSH_110_CHECK_SLEEP_SECONDS" 2>&1); then
     echo "$out"
     grep -q "CRON_110 active" <<<"$out" && ok "110 cron active" || warn "110 cron not confirmed"
     grep -q "FAILED_UNITS_110 0" <<<"$out" && ok "110 systemd has no failed units" || warn "110 systemd failed units remain"

scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py

@@ -35,10 +35,16 @@ def test_full_stack_cold_start_check_bounds_ssh_probes() -> None:
     text = COLD_START_CHECK.read_text(encoding="utf-8")
 
     assert 'SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-45}"' in text
+    assert 'SSH_110_CHECK_ATTEMPTS="${SSH_110_CHECK_ATTEMPTS:-4}"' in text
+    assert 'SSH_110_CHECK_SLEEP_SECONDS="${SSH_110_CHECK_SLEEP_SECONDS:-5}"' in text
     assert "-o ConnectionAttempts=1" in text
     assert "-o ServerAliveInterval=5" in text
     assert "-o ServerAliveCountMax=1" in text
     assert "timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc" in text
+    assert "host_cmd_retry()" in text
+    assert "SSH_RETRY_RECOVERED user_host=$user_host attempt=$attempt/$attempts" in text
+    assert '"$SSH_110_CHECK_ATTEMPTS" "$SSH_110_CHECK_SLEEP_SECONDS"' in text
+    assert text.count('host_cmd_retry "wooo@192.168.0.110"') >= 2
     assert "printf -v quoted_cmd '%q' \"$cmd\"" in text
     assert (
         'MOMO_SOURCE_PREFLIGHT_SCRIPT="${MOMO_SOURCE_PREFLIGHT_SCRIPT:-$SCRIPT_DIR/momo-drive-token-source-recovery-preflight.sh}"'