wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(ops): honor host load controller readback paths
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 57s
Details
CD Pipeline / post-deploy-checks (push) Has been cancelled
Details
CD Pipeline / build-and-deploy (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Your Name
2026-07-03 01:17:01 +08:00
parent 8caafdd06a
commit 2a83ec01b8
2 changed files with 76 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
scripts/ops/host-sustained-load-controller.py
View File

@@ -379,6 +379,8 @@ def build_packet(
docker_samples: list[dict[str, Any]],
docker_stats_status: dict[str, Any],
process_summary: dict[str, Any],
metrics_file: Path,
docker_stats_file: Path,
load5_per_core_threshold: float,
container_cpu_threshold: float,
hot_container_cpu_threshold: float,
@@ -454,7 +456,8 @@ def build_packet(
remediation_script = script_dir / "host-runaway-process-remediation.py"
verifier_command = (
f"{controller_script} "
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE}"
f"--host {host} --metrics-file {metrics_file}"
f" --docker-stats-file {docker_stats_file}"
)
if monitor_up != 1:
@@ -478,7 +481,8 @@ def build_packet(
"--evidence-ref ${EVIDENCE_REF} "
"--post-apply-verifier "
f"'{controller_script} --host "
f"{host} --metrics-file {DEFAULT_METRICS_FILE}' "
f"{host} --metrics-file {metrics_file} "
f"--docker-stats-file {docker_stats_file}' "
"--wait-seconds 10"
)
next_action = "run_orphan_browser_remediation_dry_run_then_controlled_sigterm"
@@ -518,8 +522,8 @@ def build_packet(
)
dry_run_command = (
f"{evidence_script} "
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
f"--host {host} --metrics-file {metrics_file} "
f"--docker-stats-file {docker_stats_file} --json"
)
next_action = "run_stockplatform_hot_query_or_api_pressure_playbook_check_mode"
elif (
@@ -534,8 +538,8 @@ def build_packet(
)
dry_run_command = (
f"{gitea_playbook_script} "
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
f"--host {host} --metrics-file {metrics_file} "
f"--docker-stats-file {docker_stats_file} --json"
)
next_action = "run_gitea_queue_or_hook_backlog_playbook_check_mode"
elif (
@@ -549,8 +553,8 @@ def build_packet(
severity = "critical" if load5_per_core > load5_per_core_threshold else "warning"
dry_run_command = (
f"{evidence_script} "
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
f"--host {host} --metrics-file {metrics_file} "
f"--docker-stats-file {docker_stats_file} --json"
)
next_action = "run_stockplatform_hot_query_or_api_pressure_playbook_check_mode"
elif (
@@ -560,8 +564,8 @@ def build_packet(
severity = "critical" if load5_per_core > load5_per_core_threshold else "warning"
dry_run_command = (
f"{gitea_playbook_script} "
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
f"--host {host} --metrics-file {metrics_file} "
f"--docker-stats-file {docker_stats_file} --json"
)
next_action = "run_gitea_queue_or_hook_backlog_playbook_check_mode"
elif control_plane_cpu >= process_family_cpu_threshold:
@@ -569,8 +573,8 @@ def build_packet(
severity = "critical" if load5_per_core > load5_per_core_threshold else "warning"
dry_run_command = (
f"{evidence_script} "
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
f"--host {host} --metrics-file {metrics_file} "
f"--docker-stats-file {docker_stats_file} --json"
)
next_action = "run_control_plane_saturation_playbook_check_mode"
elif load5_per_core > load5_per_core_threshold and swap_used_ratio >= 0.85:
@@ -582,8 +586,8 @@ def build_packet(
severity = "critical"
dry_run_command = (
f"{evidence_script} "
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
f"--host {host} --metrics-file {metrics_file} "
f"--docker-stats-file {docker_stats_file} --json"
)
next_action = "collect_sanitized_top_process_and_container_stats_then_select_playbook"
@@ -602,6 +606,8 @@ def build_packet(
"container_cpu_threshold": container_cpu_threshold,
"hot_container_cpu_threshold": hot_container_cpu_threshold,
"process_family_cpu_threshold": process_family_cpu_threshold,
"metrics_file": str(metrics_file),
"docker_stats_file": str(docker_stats_file),
"script_dir": str(script_dir),
"swap_used_ratio": round(swap_used_ratio, 6),
"remediation_authorized": remediation_authorized,
@@ -670,6 +676,8 @@ def main() -> int:
max_age_seconds=args.docker_stats_max_age_seconds,
),
process_summary=_summarize_processes(_parse_ps_text(_collect_ps_text(args.ps_file)), top_n=args.top_n),
metrics_file=args.metrics_file,
docker_stats_file=args.docker_stats_file,
load5_per_core_threshold=args.load5_per_core_threshold,
container_cpu_threshold=args.container_cpu_threshold,
hot_container_cpu_threshold=args.hot_container_cpu_threshold,