This commit is contained in:
@@ -27,6 +27,7 @@ from sqlalchemy import text
|
||||
from services.logger_manager import SystemLogger
|
||||
from services.ai_automation_metrics import record_autoheal_action
|
||||
from database.manager import get_session
|
||||
from utils.ssh_helper import run_ssh_command
|
||||
|
||||
logger = SystemLogger("AutoHealService").get_logger()
|
||||
|
||||
@@ -98,17 +99,6 @@ def _load_escalation(trigger_type: str) -> Optional[int]:
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
# ---- SSH helper ----
|
||||
def _ensure_key_permissions(key_path: str) -> None:
|
||||
if not os.path.exists(key_path):
|
||||
logger.warning("SSH key not found: %s", key_path)
|
||||
return
|
||||
try:
|
||||
os.chmod(key_path, 0o600)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to secure SSH key: %s", e)
|
||||
|
||||
|
||||
def _ssh_exec(
|
||||
jump_host: str,
|
||||
jump_user: str,
|
||||
@@ -121,45 +111,29 @@ def _ssh_exec(
|
||||
Execute command on target_host via SSH jump host.
|
||||
command must be a list (argv) to avoid shell injection.
|
||||
"""
|
||||
import subprocess
|
||||
|
||||
safe_key = key_path or SSH_KEY_PATH
|
||||
_ensure_key_permissions(safe_key)
|
||||
|
||||
full_cmd = [
|
||||
"ssh",
|
||||
"-p", str(SSH_PORT),
|
||||
"-i", safe_key,
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", f"ConnectTimeout={SSH_CONNECT_TIMEOUT}",
|
||||
"-o", "ServerAliveInterval=15",
|
||||
"-o", "ServerAliveCountMax=3",
|
||||
"-J", f"{jump_user}@{jump_host}",
|
||||
f"{target_user}@{target_host}",
|
||||
"--",
|
||||
*command,
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(
|
||||
full_cmd,
|
||||
shell=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=SSH_COMMAND_TIMEOUT,
|
||||
)
|
||||
return {
|
||||
"success": result.returncode == 0,
|
||||
"exit_code": result.returncode,
|
||||
"stdout": result.stdout.strip(),
|
||||
"stderr": result.stderr.strip(),
|
||||
"command": command,
|
||||
}
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"success": False, "exit_code": -1, "stdout": "", "stderr": "SSH timeout", "command": command}
|
||||
except Exception as e:
|
||||
logger.warning("SSH exec error: %s", e)
|
||||
return {"success": False, "exit_code": -1, "stdout": "", "stderr": str(e), "command": command}
|
||||
result = run_ssh_command(
|
||||
host=target_host,
|
||||
user=target_user,
|
||||
command=command,
|
||||
port=SSH_PORT,
|
||||
key_path=safe_key,
|
||||
connect_timeout=SSH_CONNECT_TIMEOUT,
|
||||
command_timeout=SSH_COMMAND_TIMEOUT,
|
||||
jump_host=jump_host,
|
||||
jump_user=jump_user,
|
||||
batch_mode=True,
|
||||
server_alive_interval=15,
|
||||
server_alive_count_max=3,
|
||||
logger=logger,
|
||||
)
|
||||
return {
|
||||
"success": result.success,
|
||||
"exit_code": result.returncode,
|
||||
"stdout": result.stdout,
|
||||
"stderr": "SSH timeout" if result.stderr.startswith("SSH timeout after ") else result.stderr,
|
||||
"command": command,
|
||||
}
|
||||
|
||||
|
||||
# ---- PlayBook ----
|
||||
|
||||
111
tests/test_ssh_helper.py
Normal file
111
tests/test_ssh_helper.py
Normal file
@@ -0,0 +1,111 @@
|
||||
from utils.ssh_helper import SshExecResult, build_ssh_command
|
||||
|
||||
|
||||
def test_build_ssh_command_supports_jump_host_and_argv_command():
|
||||
argv = build_ssh_command(
|
||||
host="192.168.0.188",
|
||||
user="ollama",
|
||||
command=["docker", "ps"],
|
||||
port=22,
|
||||
key_path="/tmp/autoheal.key",
|
||||
connect_timeout=10,
|
||||
jump_host="192.168.0.110",
|
||||
jump_user="wooo",
|
||||
batch_mode=True,
|
||||
server_alive_interval=15,
|
||||
server_alive_count_max=3,
|
||||
)
|
||||
|
||||
assert argv == [
|
||||
"ssh",
|
||||
"-p",
|
||||
"22",
|
||||
"-i",
|
||||
"/tmp/autoheal.key",
|
||||
"-o",
|
||||
"StrictHostKeyChecking=no",
|
||||
"-o",
|
||||
"BatchMode=yes",
|
||||
"-o",
|
||||
"ConnectTimeout=10",
|
||||
"-o",
|
||||
"ServerAliveInterval=15",
|
||||
"-o",
|
||||
"ServerAliveCountMax=3",
|
||||
"-J",
|
||||
"wooo@192.168.0.110",
|
||||
"ollama@192.168.0.188",
|
||||
"--",
|
||||
"docker",
|
||||
"ps",
|
||||
]
|
||||
|
||||
|
||||
def test_build_ssh_command_preserves_direct_shell_command():
|
||||
argv = build_ssh_command(
|
||||
host="192.168.0.110",
|
||||
user="wooo",
|
||||
command="cd /home/wooo/ewoooc && git status",
|
||||
port=2222,
|
||||
key_path="/tmp/heal.key",
|
||||
connect_timeout=5,
|
||||
)
|
||||
|
||||
assert "--" not in argv
|
||||
assert argv[-2:] == [
|
||||
"wooo@192.168.0.110",
|
||||
"cd /home/wooo/ewoooc && git status",
|
||||
]
|
||||
|
||||
|
||||
def test_auto_heal_ssh_wrapper_keeps_legacy_dict_contract(monkeypatch):
|
||||
from services import auto_heal_service
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_run_ssh_command(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return SshExecResult(0, "ok", "", ["ssh"])
|
||||
|
||||
monkeypatch.setattr(auto_heal_service, "run_ssh_command", fake_run_ssh_command)
|
||||
|
||||
result = auto_heal_service._ssh_exec(
|
||||
jump_host="192.168.0.110",
|
||||
jump_user="wooo",
|
||||
target_host="192.168.0.188",
|
||||
target_user="ollama",
|
||||
command=["df", "-h"],
|
||||
key_path="/tmp/autoheal.key",
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"success": True,
|
||||
"exit_code": 0,
|
||||
"stdout": "ok",
|
||||
"stderr": "",
|
||||
"command": ["df", "-h"],
|
||||
}
|
||||
assert captured["jump_host"] == "192.168.0.110"
|
||||
assert captured["jump_user"] == "wooo"
|
||||
assert captured["batch_mode"] is True
|
||||
assert captured["server_alive_interval"] == 15
|
||||
|
||||
|
||||
def test_aider_heal_ssh_wrapper_keeps_tuple_contract(monkeypatch):
|
||||
from services import aider_heal_executor
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_run_ssh_command(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return SshExecResult(0, "ready", "", ["ssh"])
|
||||
|
||||
monkeypatch.setattr(aider_heal_executor, "run_ssh_command", fake_run_ssh_command)
|
||||
|
||||
rc, out, err = aider_heal_executor._ssh_exec("echo ready", timeout=5)
|
||||
|
||||
assert (rc, out, err) == (0, "ready", "")
|
||||
assert captured["host"] == aider_heal_executor.HEAL_SSH_HOST
|
||||
assert captured["user"] == aider_heal_executor.HEAL_SSH_USER
|
||||
assert captured["command"] == "echo ready"
|
||||
assert captured["command_timeout"] == 5
|
||||
144
utils/ssh_helper.py
Normal file
144
utils/ssh_helper.py
Normal file
@@ -0,0 +1,144 @@
|
||||
"""
|
||||
Shared SSH command helpers for AutoHeal and AiderHeal.
|
||||
|
||||
The service layer owns allowlists and action semantics; this module only
|
||||
builds and runs the SSH command consistently.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, List, Optional, Sequence, Union
|
||||
|
||||
|
||||
RemoteCommand = Union[str, Sequence[Any]]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SshExecResult:
|
||||
returncode: int
|
||||
stdout: str
|
||||
stderr: str
|
||||
argv: List[str]
|
||||
|
||||
@property
|
||||
def success(self) -> bool:
|
||||
return self.returncode == 0
|
||||
|
||||
|
||||
def ensure_ssh_key_permissions(key_path: Optional[str], logger: Optional[Any] = None) -> None:
|
||||
if not key_path:
|
||||
return
|
||||
safe_key = os.path.expanduser(key_path)
|
||||
if not os.path.exists(safe_key):
|
||||
if logger:
|
||||
logger.warning("SSH key not found: %s", safe_key)
|
||||
return
|
||||
try:
|
||||
os.chmod(safe_key, 0o600)
|
||||
except Exception as exc:
|
||||
if logger:
|
||||
logger.warning("Failed to secure SSH key: %s", exc)
|
||||
|
||||
|
||||
def build_ssh_command(
|
||||
*,
|
||||
host: str,
|
||||
user: str,
|
||||
command: RemoteCommand,
|
||||
port: int = 22,
|
||||
key_path: Optional[str] = None,
|
||||
connect_timeout: int = 10,
|
||||
jump_host: Optional[str] = None,
|
||||
jump_user: Optional[str] = None,
|
||||
strict_host_key_checking: str = "no",
|
||||
batch_mode: bool = False,
|
||||
server_alive_interval: Optional[int] = None,
|
||||
server_alive_count_max: Optional[int] = None,
|
||||
) -> List[str]:
|
||||
argv = [
|
||||
"ssh",
|
||||
"-p",
|
||||
str(port),
|
||||
]
|
||||
if key_path:
|
||||
argv.extend(["-i", os.path.expanduser(key_path)])
|
||||
argv.extend(["-o", f"StrictHostKeyChecking={strict_host_key_checking}"])
|
||||
if batch_mode:
|
||||
argv.extend(["-o", "BatchMode=yes"])
|
||||
argv.extend(["-o", f"ConnectTimeout={connect_timeout}"])
|
||||
if server_alive_interval is not None:
|
||||
argv.extend(["-o", f"ServerAliveInterval={server_alive_interval}"])
|
||||
if server_alive_count_max is not None:
|
||||
argv.extend(["-o", f"ServerAliveCountMax={server_alive_count_max}"])
|
||||
if jump_host and jump_user:
|
||||
argv.extend(["-J", f"{jump_user}@{jump_host}"])
|
||||
argv.append(f"{user}@{host}")
|
||||
|
||||
if isinstance(command, str):
|
||||
argv.append(command)
|
||||
else:
|
||||
argv.append("--")
|
||||
argv.extend(str(part) for part in command)
|
||||
return argv
|
||||
|
||||
|
||||
def run_ssh_command(
|
||||
*,
|
||||
host: str,
|
||||
user: str,
|
||||
command: RemoteCommand,
|
||||
port: int = 22,
|
||||
key_path: Optional[str] = None,
|
||||
connect_timeout: int = 10,
|
||||
command_timeout: int = 60,
|
||||
jump_host: Optional[str] = None,
|
||||
jump_user: Optional[str] = None,
|
||||
strict_host_key_checking: str = "no",
|
||||
batch_mode: bool = False,
|
||||
server_alive_interval: Optional[int] = None,
|
||||
server_alive_count_max: Optional[int] = None,
|
||||
cwd: Optional[str] = None,
|
||||
logger: Optional[Any] = None,
|
||||
) -> SshExecResult:
|
||||
ensure_ssh_key_permissions(key_path, logger=logger)
|
||||
argv = build_ssh_command(
|
||||
host=host,
|
||||
user=user,
|
||||
command=command,
|
||||
port=port,
|
||||
key_path=key_path,
|
||||
connect_timeout=connect_timeout,
|
||||
jump_host=jump_host,
|
||||
jump_user=jump_user,
|
||||
strict_host_key_checking=strict_host_key_checking,
|
||||
batch_mode=batch_mode,
|
||||
server_alive_interval=server_alive_interval,
|
||||
server_alive_count_max=server_alive_count_max,
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
argv,
|
||||
shell=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=cwd,
|
||||
timeout=command_timeout,
|
||||
)
|
||||
return SshExecResult(
|
||||
returncode=result.returncode,
|
||||
stdout=result.stdout.strip(),
|
||||
stderr=result.stderr.strip(),
|
||||
argv=argv,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return SshExecResult(
|
||||
returncode=-1,
|
||||
stdout="",
|
||||
stderr=f"SSH timeout after {command_timeout}s",
|
||||
argv=argv,
|
||||
)
|
||||
except Exception as exc:
|
||||
if logger:
|
||||
logger.warning("SSH exec error: %s", exc)
|
||||
return SshExecResult(returncode=-1, stdout="", stderr=str(exc), argv=argv)
|
||||
@@ -303,4 +303,27 @@
|
||||
.momo-app:not(.momo-observability-mode) .daily-sales-page .kpi-card__value {
|
||||
font-size: 1.32rem !important;
|
||||
}
|
||||
|
||||
.momo-app:not(.momo-observability-mode) .edm-page :is(
|
||||
.campaign-switcher,
|
||||
.campaign-slot-tabs,
|
||||
.campaign-filterbar
|
||||
) {
|
||||
display: grid !important;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr)) !important;
|
||||
gap: 8px !important;
|
||||
overflow: visible !important;
|
||||
}
|
||||
|
||||
.momo-app:not(.momo-observability-mode) .edm-page :is(
|
||||
.campaign-tab,
|
||||
.campaign-slot-tab,
|
||||
.campaign-filter-chip
|
||||
) {
|
||||
width: 100% !important;
|
||||
min-width: 0 !important;
|
||||
justify-content: center !important;
|
||||
text-align: center !important;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user