1ac8808607
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 33s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
CD Pipeline / build-and-deploy (push) Has been skipped
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Failing after 1m41s
CD Pipeline / post-deploy-checks (push) Has been skipped
699 lines
25 KiB
Python
699 lines
25 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
SCRIPT_ROOT = Path(__file__).resolve().parents[1]
|
|
EXPORTER_PATH = SCRIPT_ROOT / "host-runaway-process-exporter.py"
|
|
REMEDIATION_PATH = SCRIPT_ROOT / "host-runaway-process-remediation.py"
|
|
CONTROLLER_PATH = SCRIPT_ROOT / "host-sustained-load-controller.py"
|
|
EVIDENCE_PATH = SCRIPT_ROOT / "host-sustained-load-evidence.py"
|
|
|
|
|
|
def load_exporter():
|
|
spec = importlib.util.spec_from_file_location("host_runaway_process_exporter", EXPORTER_PATH)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def test_classifies_orphan_stockplatform_headless_group() -> None:
|
|
exporter = load_exporter()
|
|
rows = exporter.parse_ps_rows(
|
|
"""
|
|
100 1 100 100 7200 65.0 S chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa
|
|
101 100 100 100 7190 55.0 S chromium /opt/chrome/chromium --type=renderer /tmp/stockplatform-review-bulk-ux-aa
|
|
200 10 200 200 600 90.0 S node pnpm --filter @awoooi/web build
|
|
"""
|
|
)
|
|
|
|
groups = exporter.classify_groups(rows, min_age_seconds=1800, min_cpu_percent=50)
|
|
|
|
assert len(groups) == 1
|
|
assert groups[0].rule_id == "stockplatform_headless_smoke"
|
|
assert groups[0].pgid == 100
|
|
assert groups[0].orphan_reason == "ppid_1"
|
|
assert groups[0].cpu_percent == 120.0
|
|
assert len(groups[0].rows) == 2
|
|
|
|
|
|
def test_ignores_non_orphan_or_young_browser_processes() -> None:
|
|
exporter = load_exporter()
|
|
rows = exporter.parse_ps_rows(
|
|
"""
|
|
100 99 100 100 7200 65.0 S chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa
|
|
101 100 100 100 7190 55.0 S chromium /opt/chrome/chromium /tmp/stockplatform-review-bulk-ux-aa
|
|
300 1 300 300 60 120.0 S chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-bb
|
|
"""
|
|
)
|
|
|
|
assert exporter.classify_groups(rows, min_age_seconds=1800, min_cpu_percent=50) == []
|
|
|
|
|
|
def test_parses_bsd_elapsed_time_for_local_smoke() -> None:
|
|
exporter = load_exporter()
|
|
rows = exporter.parse_ps_rows(
|
|
"""
|
|
100 1 100 100 01:00:00 65.0 S chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa
|
|
101 100 100 100 2-00:00:10 55.0 S chromium /opt/chrome/chromium /tmp/stockplatform-review-bulk-ux-aa
|
|
"""
|
|
)
|
|
|
|
assert rows[0].etimes == 3600
|
|
assert rows[1].etimes == 172810
|
|
|
|
|
|
def test_renders_ci_load_and_swap_without_authorizing_repair(tmp_path: Path) -> None:
|
|
exporter = load_exporter()
|
|
groups = exporter.classify_groups(
|
|
exporter.parse_ps_rows(
|
|
"100 1 100 100 7200 65.0 S chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa"
|
|
),
|
|
min_age_seconds=1800,
|
|
min_cpu_percent=50,
|
|
)
|
|
metrics = exporter.render_metrics(
|
|
host="110",
|
|
groups=groups,
|
|
active_action_containers=3,
|
|
active_action_process_load=exporter.ActiveCiLoad(
|
|
group_count=2,
|
|
process_count=4,
|
|
cpu_percent=188.5,
|
|
oldest_age_seconds=240,
|
|
),
|
|
min_age_seconds=1800,
|
|
min_cpu_percent=50,
|
|
now=123,
|
|
load_ratio=1.25,
|
|
swap_ratio=1.0,
|
|
)
|
|
|
|
assert 'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1' in metrics
|
|
assert 'awoooi_host_gitea_actions_active_container_count{host="110"} 3' in metrics
|
|
assert 'awoooi_host_gitea_actions_active_process_group_count{host="110"} 2' in metrics
|
|
assert 'awoooi_host_gitea_actions_active_process_count{host="110"} 4' in metrics
|
|
assert 'awoooi_host_gitea_actions_active_process_cpu_percent{host="110"} 188.500000' in metrics
|
|
assert 'awoooi_host_gitea_actions_active_process_oldest_age_seconds{host="110"} 240' in metrics
|
|
assert 'awoooi_host_swap_used_ratio{host="110"} 1.000000' in metrics
|
|
assert 'awoooi_host_runaway_process_remediation_authorized{host="110"} 0' in metrics
|
|
assert 'rule="stockplatform_headless_smoke"' in metrics
|
|
|
|
|
|
def test_counts_modern_gitea_action_container_names(tmp_path: Path) -> None:
|
|
exporter = load_exporter()
|
|
docker_file = tmp_path / "docker.txt"
|
|
docker_file.write_text(
|
|
"\n".join(
|
|
[
|
|
"GITEA-ACTIONS-TASK-123",
|
|
"awoooi-cd-5901-1-e2e-smoke",
|
|
"awoooi-cd-5873-1-source-link-smoke",
|
|
"awoooi-code-review-3323-1-ai-code-review",
|
|
"gitea",
|
|
"stockplatform-v2-api-1",
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
assert exporter.active_gitea_action_containers(docker_file) == 4
|
|
|
|
|
|
def test_counts_buildkit_runner_process_load() -> None:
|
|
exporter = load_exporter()
|
|
rows = exporter.parse_ps_rows(
|
|
"""
|
|
100 10 100 100 240 0.0 S bash bash --noprofile --norc -e -o pipefail /home/wooo/.cache/act/14cc/act/workflow/8.sh
|
|
101 100 100 100 239 1.0 S docker docker build -f apps/web/Dockerfile .
|
|
102 101 100 100 239 2.0 S docker-buildx /home/wooo/.docker/cli-plugins/docker-buildx buildx build -f apps/web/Dockerfile .
|
|
200 150 200 200 210 12.5 S turbo turbo build --filter=@awoooi/web --concurrency=1
|
|
201 200 200 200 200 145.0 S node node /app/apps/web/node_modules/.bin/../next/dist/bin/next build
|
|
300 1 300 300 9999 0.1 S act_runner act_runner daemon --config /config.yaml
|
|
400 1 400 400 120 30.0 S node node apps/web/server.js
|
|
"""
|
|
)
|
|
|
|
load = exporter.active_gitea_action_process_load(rows)
|
|
|
|
assert load.group_count == 2
|
|
assert load.process_count == 5
|
|
assert load.cpu_percent == 160.5
|
|
assert load.oldest_age_seconds == 240
|
|
|
|
|
|
def test_ignores_the_host_pressure_gate_process_group() -> None:
|
|
exporter = load_exporter()
|
|
rows = exporter.parse_ps_rows(
|
|
"""
|
|
100 10 100 100 240 0.0 S bash bash --noprofile --norc -e -o pipefail /home/wooo/.cache/act/14cc/act/workflow/2.sh
|
|
101 100 100 100 239 0.0 S bash bash scripts/ci/wait-host-web-build-pressure.sh
|
|
102 101 100 100 238 0.0 S sleep sleep 10
|
|
200 150 200 200 210 12.5 S turbo turbo build --filter=@awoooi/web --concurrency=1
|
|
"""
|
|
)
|
|
|
|
load = exporter.active_gitea_action_process_load(rows)
|
|
|
|
assert load.group_count == 1
|
|
assert load.process_count == 1
|
|
assert load.cpu_percent == 12.5
|
|
assert load.oldest_age_seconds == 210
|
|
|
|
|
|
def test_remediation_defaults_to_dry_run(tmp_path: Path) -> None:
|
|
ps_file = tmp_path / "ps.txt"
|
|
ps_file.write_text(
|
|
"999999 1 999999 999999 7200 65.0 S chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(REMEDIATION_PATH),
|
|
"--ps-file",
|
|
str(ps_file),
|
|
"--rule",
|
|
"stockplatform_headless_smoke",
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert '"mode": "dry_run"' in result.stdout
|
|
assert '"runtime_gate": 0' in result.stdout
|
|
assert '"action": "dry_run"' in result.stdout
|
|
|
|
|
|
def test_remediation_refuses_apply_without_gates(tmp_path: Path) -> None:
|
|
ps_file = tmp_path / "ps.txt"
|
|
ps_file.write_text(
|
|
"999999 1 999999 999999 7200 65.0 S chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(REMEDIATION_PATH),
|
|
"--ps-file",
|
|
str(ps_file),
|
|
"--apply",
|
|
"--rule",
|
|
"stockplatform_headless_smoke",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert result.returncode != 0
|
|
assert "Refusing apply" in result.stderr
|
|
assert "--controlled-apply-id" in result.stderr
|
|
assert "--confirm-apply" in result.stderr
|
|
assert "--post-apply-verifier" in result.stderr
|
|
|
|
|
|
def test_remediation_accepts_controlled_apply_gate_without_owner_gate(tmp_path: Path) -> None:
|
|
ps_file = tmp_path / "ps.txt"
|
|
ps_file.write_text(
|
|
"100 1 1 100 7200 65.0 S chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(REMEDIATION_PATH),
|
|
"--ps-file",
|
|
str(ps_file),
|
|
"--apply",
|
|
"--confirm-apply",
|
|
"--rule",
|
|
"stockplatform_headless_smoke",
|
|
"--controlled-apply-id",
|
|
"CAP-20260701-HOSTLOAD",
|
|
"--evidence-ref",
|
|
"HostLoadAverageSustainedHigh:110",
|
|
"--post-apply-verifier",
|
|
"scripts/ops/host-sustained-load-controller.py --host 110 --json",
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert '"mode": "apply_sigterm"' in result.stdout
|
|
assert '"runtime_gate": 1' in result.stdout
|
|
assert '"controlled_apply_id": "CAP-20260701-HOSTLOAD"' in result.stdout
|
|
assert '"owner_approval_id": ""' in result.stdout
|
|
assert '"blocked_reason": "unsafe_pgid"' in result.stdout
|
|
assert '"missing_process_group_count": 0' in result.stdout
|
|
assert '"signal_error_count": 0' in result.stdout
|
|
assert '"signaled_process_group_count": 0' in result.stdout
|
|
|
|
|
|
def test_sustained_load_controller_routes_orphan_browser_to_controlled_remediation(tmp_path: Path) -> None:
|
|
metrics_file = tmp_path / "host.prom"
|
|
metrics_file.write_text(
|
|
"\n".join(
|
|
[
|
|
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
|
|
'awoooi_host_load5_per_core{host="110"} 2.2',
|
|
'awoooi_host_swap_used_ratio{host="110"} 0.1',
|
|
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
|
|
'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
|
|
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
|
|
'awoooi_host_runaway_browser_orphan_group_count{host="110",rule="stockplatform_headless_smoke",min_age_seconds="1800",min_cpu_percent="50"} 1',
|
|
'awoooi_host_runaway_browser_orphan_cpu_percent{host="110",rule="stockplatform_headless_smoke",min_age_seconds="1800",min_cpu_percent="50"} 155.5',
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(CONTROLLER_PATH),
|
|
"--host",
|
|
"110",
|
|
"--metrics-file",
|
|
str(metrics_file),
|
|
"--json",
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
payload = json.loads(result.stdout)
|
|
assert payload["classification"] == "controlled_orphan_browser_remediation_ready"
|
|
assert payload["controlled_apply_allowed"] is True
|
|
assert "host-runaway-process-remediation.py --rule stockplatform_headless_smoke" in payload["commands"]["dry_run"]
|
|
assert "--controlled-apply-id" in payload["commands"]["controlled_apply"]
|
|
assert payload["operation_boundaries"]["process_signal_performed"] is False
|
|
|
|
|
|
def test_sustained_load_controller_keeps_ci_saturation_on_runner_path(tmp_path: Path) -> None:
|
|
metrics_file = tmp_path / "host.prom"
|
|
metrics_file.write_text(
|
|
"\n".join(
|
|
[
|
|
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
|
|
'awoooi_host_load5_per_core{host="110"} 2.0',
|
|
'awoooi_host_swap_used_ratio{host="110"} 0.1',
|
|
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
|
|
'awoooi_host_gitea_actions_active_container_count{host="110"} 2',
|
|
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 1',
|
|
'awoooi_host_gitea_actions_active_process_cpu_percent{host="110"} 180.0',
|
|
'awoooi_host_gitea_actions_active_process_oldest_age_seconds{host="110"} 1900',
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(CONTROLLER_PATH),
|
|
"--host",
|
|
"110",
|
|
"--metrics-file",
|
|
str(metrics_file),
|
|
"--json",
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
payload = json.loads(result.stdout)
|
|
assert payload["classification"] == "controlled_ci_runner_saturation_guarded"
|
|
assert payload["controlled_apply_allowed"] is True
|
|
assert "fail_closed" in payload["commands"]["controlled_apply"]
|
|
assert "process_kill" not in payload["commands"]["controlled_apply"]
|
|
|
|
|
|
def test_sustained_load_controller_blocks_monitor_authority_violation(tmp_path: Path) -> None:
|
|
metrics_file = tmp_path / "host.prom"
|
|
metrics_file.write_text(
|
|
"\n".join(
|
|
[
|
|
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
|
|
'awoooi_host_load5_per_core{host="110"} 2.0',
|
|
'awoooi_host_runaway_process_remediation_authorized{host="110"} 1',
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(CONTROLLER_PATH),
|
|
"--host",
|
|
"110",
|
|
"--metrics-file",
|
|
str(metrics_file),
|
|
"--json",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert result.returncode == 75
|
|
payload = json.loads(result.stdout)
|
|
assert payload["classification"] == "blocked_monitor_authority_violation"
|
|
assert payload["controlled_apply_allowed"] is False
|
|
|
|
|
|
def test_sustained_load_controller_routes_gitea_backlog_from_docker_metrics(tmp_path: Path) -> None:
|
|
metrics_file = tmp_path / "host.prom"
|
|
metrics_file.write_text(
|
|
"\n".join(
|
|
[
|
|
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
|
|
'awoooi_host_load5_per_core{host="110"} 2.5',
|
|
'awoooi_host_swap_used_ratio{host="110"} 0.1',
|
|
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
|
|
'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
|
|
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
|
|
'awoooi_host_runaway_browser_orphan_group_count{host="110",rule="stockplatform_headless_smoke",min_age_seconds="1800",min_cpu_percent="50"} 0',
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
docker_file = tmp_path / "docker.prom"
|
|
docker_file.write_text(
|
|
"\n".join(
|
|
[
|
|
'docker_container_cpu_cores{host="110",container_name="gitea"} 3.4',
|
|
'docker_container_cpu_cores{host="110",container_name="redis"} 0.2',
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(CONTROLLER_PATH),
|
|
"--host",
|
|
"110",
|
|
"--metrics-file",
|
|
str(metrics_file),
|
|
"--docker-stats-file",
|
|
str(docker_file),
|
|
"--json",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert result.returncode == 75
|
|
payload = json.loads(result.stdout)
|
|
assert payload["classification"] == "blocked_gitea_queue_or_hook_backlog_requires_playbook"
|
|
assert payload["readback"]["top_container_cpu"]["container_name"] == "gitea"
|
|
assert payload["controlled_apply_allowed"] is False
|
|
assert "host-sustained-load-evidence.py" in payload["commands"]["dry_run"]
|
|
|
|
|
|
def test_sustained_load_controller_ignores_stale_docker_stats_attribution(tmp_path: Path) -> None:
|
|
metrics_file = tmp_path / "host.prom"
|
|
metrics_file.write_text(
|
|
"\n".join(
|
|
[
|
|
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
|
|
'awoooi_host_load5_per_core{host="110"} 2.5',
|
|
'awoooi_host_swap_used_ratio{host="110"} 0.1',
|
|
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
|
|
'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
|
|
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
|
|
'awoooi_host_runaway_browser_orphan_group_count{host="110",rule="stockplatform_headless_smoke",min_age_seconds="1800",min_cpu_percent="50"} 0',
|
|
'node_textfile_mtime_seconds{file="/host/home/wooo/node_exporter_textfiles/docker_stats.prom"} 1000',
|
|
'node_time_seconds 5000',
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
docker_file = tmp_path / "docker.prom"
|
|
docker_file.write_text(
|
|
"\n".join(
|
|
[
|
|
'docker_container_cpu_cores{host="110",container_name="gitea"} 3.4',
|
|
'docker_container_cpu_cores{host="110",container_name="redis"} 0.2',
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(CONTROLLER_PATH),
|
|
"--host",
|
|
"110",
|
|
"--metrics-file",
|
|
str(metrics_file),
|
|
"--docker-stats-file",
|
|
str(docker_file),
|
|
"--json",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert result.returncode == 75
|
|
payload = json.loads(result.stdout)
|
|
assert payload["classification"] == "blocked_unknown_sustained_load_requires_source_specific_playbook"
|
|
assert payload["readback"]["docker_stats"]["fresh"] is False
|
|
assert payload["readback"]["top_container_cpu"] is None
|
|
assert payload["readback"]["top_container_cpu_untrusted"]["container_name"] == "gitea"
|
|
assert payload["controlled_apply_allowed"] is False
|
|
|
|
|
|
def test_sustained_load_controller_routes_unknown_load_to_sanitized_evidence(tmp_path: Path) -> None:
|
|
metrics_file = tmp_path / "host.prom"
|
|
metrics_file.write_text(
|
|
"\n".join(
|
|
[
|
|
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
|
|
'awoooi_host_load5_per_core{host="110"} 2.0',
|
|
'awoooi_host_swap_used_ratio{host="110"} 0.1',
|
|
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
|
|
'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
|
|
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
|
|
'awoooi_host_runaway_browser_orphan_group_count{host="110",rule="stockplatform_headless_smoke",min_age_seconds="1800",min_cpu_percent="50"} 0',
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(CONTROLLER_PATH),
|
|
"--host",
|
|
"110",
|
|
"--metrics-file",
|
|
str(metrics_file),
|
|
"--json",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert result.returncode == 75
|
|
payload = json.loads(result.stdout)
|
|
assert payload["classification"] == "blocked_unknown_sustained_load_requires_source_specific_playbook"
|
|
assert payload["controlled_apply_allowed"] is False
|
|
assert "host-sustained-load-evidence.py" in payload["commands"]["dry_run"]
|
|
assert payload["operation_boundaries"]["process_signal_performed"] is False
|
|
|
|
|
|
def test_sustained_load_evidence_emits_sanitized_gitea_recommendation(tmp_path: Path) -> None:
|
|
ps_file = tmp_path / "ps.txt"
|
|
ps_file.write_text(
|
|
"\n".join(
|
|
[
|
|
"100 1 100 7200 280.0 1.0 gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini",
|
|
"200 1 200 180 15.0 0.5 systemd systemctl show gitea-act-runner-host.service",
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
docker_file = tmp_path / "docker.prom"
|
|
docker_file.write_text(
|
|
'docker_container_cpu_cores{host="110",container_name="gitea"} 3.4\n',
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(EVIDENCE_PATH),
|
|
"--host",
|
|
"110",
|
|
"--ps-file",
|
|
str(ps_file),
|
|
"--docker-stats-file",
|
|
str(docker_file),
|
|
"--json",
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
payload = json.loads(result.stdout)
|
|
assert payload["schema_version"] == "host_sustained_load_sanitized_evidence_v1"
|
|
assert payload["recommendation"] == "gitea_queue_or_hook_backlog_playbook"
|
|
assert payload["redaction"]["raw_command_lines_emitted"] is False
|
|
assert payload["operation_boundaries"]["host_write_performed"] is False
|
|
assert "/home/wooo" not in result.stdout
|
|
|
|
|
|
def test_sustained_load_evidence_keeps_stale_container_samples_untrusted(tmp_path: Path) -> None:
|
|
metrics_file = tmp_path / "host.prom"
|
|
metrics_file.write_text(
|
|
"\n".join(
|
|
[
|
|
'node_textfile_mtime_seconds{file="/host/home/wooo/node_exporter_textfiles/docker_stats.prom"} 1000',
|
|
'node_time_seconds 5000',
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
docker_file = tmp_path / "docker.prom"
|
|
docker_file.write_text(
|
|
'docker_container_cpu_cores{host="110",container_name="gitea"} 3.4\n',
|
|
encoding="utf-8",
|
|
)
|
|
ps_file = tmp_path / "ps.txt"
|
|
ps_file.write_text(
|
|
"100 1 100 120 5.0 1.0 python python monitor.py\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(EVIDENCE_PATH),
|
|
"--host",
|
|
"110",
|
|
"--metrics-file",
|
|
str(metrics_file),
|
|
"--ps-file",
|
|
str(ps_file),
|
|
"--docker-stats-file",
|
|
str(docker_file),
|
|
"--json",
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
payload = json.loads(result.stdout)
|
|
assert payload["recommendation"] != "gitea_queue_or_hook_backlog_playbook"
|
|
assert payload["readback"]["docker_stats"]["fresh"] is False
|
|
assert payload["top_containers"] == []
|
|
assert payload["top_containers_untrusted"][0]["container_name"] == "gitea"
|
|
assert payload["operation_boundaries"]["host_write_performed"] is False
|
|
|
|
|
|
def test_sustained_load_controller_routes_unknown_load_to_sanitized_evidence(tmp_path: Path) -> None:
|
|
metrics_file = tmp_path / "host.prom"
|
|
metrics_file.write_text(
|
|
"\n".join(
|
|
[
|
|
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
|
|
'awoooi_host_load5_per_core{host="110"} 2.4',
|
|
'awoooi_host_swap_used_ratio{host="110"} 0.1',
|
|
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(CONTROLLER_PATH),
|
|
"--host",
|
|
"110",
|
|
"--metrics-file",
|
|
str(metrics_file),
|
|
"--json",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert result.returncode == 75
|
|
payload = json.loads(result.stdout)
|
|
assert (
|
|
payload["classification"]
|
|
== "blocked_unknown_sustained_load_requires_source_specific_playbook"
|
|
)
|
|
assert payload["controlled_apply_allowed"] is False
|
|
assert "host-sustained-load-evidence.py" in payload["commands"]["dry_run"]
|
|
assert payload["operation_boundaries"]["host_write_performed"] is False
|
|
|
|
|
|
def test_sustained_load_evidence_sanitizes_process_details(tmp_path: Path) -> None:
|
|
ps_file = tmp_path / "ps.txt"
|
|
ps_file.write_text(
|
|
"\n".join(
|
|
[
|
|
"101 1 101 7200 65.0 2.5 chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa --url=https://example.invalid/token",
|
|
"102 1 102 3600 20.0 1.0 node node /srv/private/app/server.js --api-key=SECRET",
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
docker_stats_file = tmp_path / "docker.prom"
|
|
docker_stats_file.write_text(
|
|
'docker_container_cpu_cores{host="110",container_name="gitea"} 3.2\n',
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(EVIDENCE_PATH),
|
|
"--host",
|
|
"110",
|
|
"--ps-file",
|
|
str(ps_file),
|
|
"--docker-stats-file",
|
|
str(docker_stats_file),
|
|
"--json",
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
payload = json.loads(result.stdout)
|
|
assert payload["schema_version"] == "host_sustained_load_sanitized_evidence_v1"
|
|
assert payload["recommendation"] == "gitea_queue_or_hook_backlog_playbook"
|
|
assert payload["redaction"]["raw_command_lines_emitted"] is False
|
|
assert payload["redaction"]["workspace_paths_emitted"] is False
|
|
assert payload["redaction"]["urls_emitted"] is False
|
|
assert payload["operation_boundaries"]["host_write_performed"] is False
|
|
assert "https://example.invalid/token" not in result.stdout
|
|
assert "/tmp/stockplatform-review-bulk-ux-aa" not in result.stdout
|
|
assert "SECRET" not in result.stdout
|
|
assert {item["family"] for item in payload["top_process_families"]} >= {
|
|
"headless_browser",
|
|
"node_service",
|
|
}
|