Compare commits
12 Commits
codex/gith
...
codex/sour
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8e7d2f92a4 | ||
|
|
829ae98a73 | ||
|
|
a7f6bf72aa | ||
|
|
119267256a | ||
|
|
fde68fb843 | ||
|
|
f5bcc90041 | ||
|
|
022bf0b802 | ||
|
|
e97b252475 | ||
|
|
95c825f242 | ||
|
|
e7db56d4c3 | ||
|
|
241cbe067e | ||
|
|
392c1741ca |
@@ -1,27 +1,8 @@
|
||||
name: Ansible / Reboot Recovery Contract
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'infra/ansible/**'
|
||||
- 'ops/monitoring/**'
|
||||
- 'ops/reboot-recovery/**'
|
||||
- 'scripts/backup/**'
|
||||
- 'scripts/ops/**'
|
||||
- 'scripts/reboot-recovery/**'
|
||||
- 'docs/**'
|
||||
- '.gitea/workflows/**'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'infra/ansible/**'
|
||||
- 'ops/monitoring/**'
|
||||
- 'ops/reboot-recovery/**'
|
||||
- 'scripts/backup/**'
|
||||
- 'scripts/ops/**'
|
||||
- 'scripts/reboot-recovery/**'
|
||||
- 'docs/**'
|
||||
- '.gitea/workflows/**'
|
||||
# 2026-06-28 Codex: 110 host runner/CD lane pressure incident.
|
||||
# Automatic push/PR triggers stay disabled until the runner is moved or rate-limited.
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
|
||||
@@ -10,6 +10,10 @@
|
||||
name: CD Pipeline
|
||||
|
||||
on:
|
||||
# 2026-06-28 Codex: 110 host runner/CD lane pressure incident.
|
||||
# Production CD is reopened for controlled apply through the dedicated
|
||||
# capacity=1 cd-lane drain verifier; the host pressure gate below remains
|
||||
# fail-closed before build starts.
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
@@ -61,6 +65,10 @@ env:
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
# 2026-06-28 Codex: Gitea does not consistently short-circuit `[skip ci]`
|
||||
# on CD-generated deploy commits. Skip jobs explicitly so marker commits
|
||||
# do not trigger a self-feeding CD loop.
|
||||
if: ${{ github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]') }}
|
||||
# 2026-04-30 Codex: run the tests job on the host runner and launch the
|
||||
# CI image explicitly. The act-managed job container can disappear mid-test
|
||||
# with Docker RWLayer=nil on the shared 110 daemon.
|
||||
@@ -316,6 +324,9 @@ jobs:
|
||||
fi
|
||||
|
||||
build-and-deploy:
|
||||
# 2026-06-28 Codex: keep CD-generated `[skip ci]` deploy commits from
|
||||
# re-entering build/deploy and writing another deploy marker commit.
|
||||
if: ${{ github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]') }}
|
||||
# 2026-04-30 Codex: Docker builds run on the host runner. Long docker build
|
||||
# steps were killing the transient act job container with RWLayer=nil.
|
||||
needs: [tests]
|
||||
@@ -1241,6 +1252,9 @@ jobs:
|
||||
fi
|
||||
|
||||
post-deploy-checks:
|
||||
# 2026-06-28 Codex: post-deploy checks belong to real deploy runs; skip
|
||||
# CD-generated marker commits already read back by the prior deploy run.
|
||||
if: ${{ github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]') }}
|
||||
needs: [build-and-deploy]
|
||||
timeout-minutes: 30
|
||||
# 2026-04-30 Codex: keep post-deploy on the host runner too. Playwright
|
||||
|
||||
@@ -23,6 +23,9 @@ env:
|
||||
|
||||
jobs:
|
||||
ai-code-review:
|
||||
# 2026-06-28 Codex: deploy marker commits are generated by CD and carry
|
||||
# `[skip ci]`; skip review at job level to avoid queued runner churn.
|
||||
if: ${{ github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]') }}
|
||||
runs-on: awoooi-ubuntu
|
||||
timeout-minutes: 8
|
||||
steps:
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
|
||||
正確動作是 AI 自動補齊 target selector、source-of-truth diff、check-mode / dry-run、rollback、post-apply verifier、KM / PlayBook trust writeback,然後推進可驗證、可回滾、低爆炸半徑的實作。
|
||||
|
||||
**110 runner / controlled CD lane 壓力事故例外**:Gitea / act-runner / direct transient runner 對 110 造成 CPU / headless smoke / Docker build 壓力時,屬事故級容量保護,不得用「全面授權」直接重開 legacy runner、移除 legacy mask、還原 legacy runner binary、用 `systemd-run` 直啟 `.real` binary,或把 host pressure gate 改成 warn-only。專用 `awoooi-cd-lane.service` 只能在獨立 sentinel、`capacity=1`、窄 label、rollback unit 與 post-apply verifier 成立時受控開啟;正確動作是分流 legacy runner 與 controlled cd-lane,不得一把梭恢復泛用 runner。
|
||||
**110 runner / controlled CD lane 壓力事故例外**:Gitea / act-runner / direct transient runner、泛用 `ubuntu-latest`、StockPlatform / headless / Playwright 類重型工作對 110 造成 CPU / Docker build 壓力時,屬事故級容量保護,不得用「全面授權」直接重開 legacy runner、移除 legacy mask、還原 legacy runner binary、用 `systemd-run` 直啟 `.real` binary,或把 host pressure gate 改成 warn-only。專用 AWOOOI controlled CD lane 可在 `capacity=1`、窄 label、無泛用重型 label、rollback unit、post-apply verifier 與 legacy runner fail-closed 同時成立時受控開啟;Gitea push workflow 不得因非事故級 guard 長期停在 manual-only。
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
# 2026-06-28 trigger controlled cd-lane drain verifier deploy after source readback fix
|
||||
# 2026-06-28 trigger runtime-origin deploy after live controlled drain lane sync
|
||||
|
||||
@@ -53,6 +53,12 @@ from src.services.iwooos_wazuh_runtime_controlled_apply_preflight import (
|
||||
from src.services.iwooos_wazuh_runtime_controlled_apply_preflight import (
|
||||
validate_iwooos_wazuh_runtime_controlled_apply_packet as validate_wazuh_runtime_controlled_apply_packet_payload,
|
||||
)
|
||||
from src.services.iwooos_wazuh_runtime_gate_owner_review_readback import (
|
||||
load_latest_iwooos_wazuh_runtime_gate_owner_review_readback,
|
||||
)
|
||||
from src.services.iwooos_wazuh_runtime_gate_owner_review_readback import (
|
||||
validate_iwooos_wazuh_runtime_gate_owner_review_packet as validate_wazuh_runtime_gate_owner_review_packet_payload,
|
||||
)
|
||||
from src.services.public_redaction import redact_public_lan_topology
|
||||
|
||||
router = APIRouter(tags=["IwoooS Security"])
|
||||
@@ -327,6 +333,71 @@ async def validate_iwooos_wazuh_runtime_controlled_apply_packet(
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/iwooos/wazuh-runtime-gate-owner-review-readback",
|
||||
response_model=dict[str, Any],
|
||||
summary="取得 Wazuh runtime gate owner-review 只讀讀回",
|
||||
description=(
|
||||
"讀取已提交的 Wazuh runtime gate owner-review readback contract,回傳 owner-review "
|
||||
"decision、target selector、source-of-truth diff、check-mode / dry-run evidence、rollback、"
|
||||
"post-apply verifier、KM / PlayBook writeback 與 0 / false 邊界。此端點不查 Wazuh API、"
|
||||
"不讀主機、不重新註冊 agent、不重啟服務、不保存機密、不啟用主動回應、不改 Nginx / "
|
||||
"Docker / K8s / firewall。"
|
||||
),
|
||||
)
|
||||
async def get_iwooos_wazuh_runtime_gate_owner_review_readback() -> dict[str, Any]:
|
||||
"""回傳 Wazuh runtime gate owner-review 公開安全只讀狀態。"""
|
||||
try:
|
||||
payload = await asyncio.to_thread(
|
||||
load_latest_iwooos_wazuh_runtime_gate_owner_review_readback
|
||||
)
|
||||
return redact_public_lan_topology(payload)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
except (json.JSONDecodeError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"IwoooS Wazuh runtime gate owner-review readback 無效:{exc}",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/iwooos/wazuh-runtime-gate-owner-review-readback/validate-owner-review-packet",
|
||||
response_model=dict[str, Any],
|
||||
summary="驗證 Wazuh runtime gate 脫敏 owner-review packet",
|
||||
description=(
|
||||
"針對單次 owner / reviewer 提供的 redacted Wazuh runtime gate owner-review packet "
|
||||
"進行 no-persist readiness validation,回傳 accepted-for-readback / needs supplement / "
|
||||
"quarantined / rejected runtime action 分流。此端點不保存 payload、不查 Wazuh API、不讀主機、"
|
||||
"不重新註冊 agent、不重啟服務、不讀或回傳機密明文、不啟用主動回應、不改 Nginx / Docker / "
|
||||
"K8s / firewall,也不更新 runtime gate 總帳。"
|
||||
),
|
||||
)
|
||||
async def validate_iwooos_wazuh_runtime_gate_owner_review_packet(
|
||||
owner_review_packet: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""回傳單次 Wazuh runtime gate owner-review packet 的公開安全驗證結果。"""
|
||||
try:
|
||||
payload = await asyncio.to_thread(
|
||||
validate_wazuh_runtime_gate_owner_review_packet_payload,
|
||||
owner_review_packet,
|
||||
)
|
||||
return redact_public_lan_topology(payload)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
except (json.JSONDecodeError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"IwoooS Wazuh runtime gate owner-review packet 驗證器無效:{exc}",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/iwooos/runtime-security-readback",
|
||||
response_model=dict[str, Any],
|
||||
|
||||
@@ -23,6 +23,7 @@ _SNAPSHOT_FILES = {
|
||||
"wazuh_live_metadata_gate": "wazuh-readonly-live-metadata-env-gate.snapshot.json",
|
||||
"wazuh_owner_evidence_preflight": "wazuh-agent-visibility-owner-evidence-preflight.snapshot.json",
|
||||
"wazuh_runtime_apply_preflight": "wazuh-runtime-controlled-apply-preflight.snapshot.json",
|
||||
"wazuh_runtime_owner_review": "wazuh-runtime-gate-owner-review-readback.snapshot.json",
|
||||
"kali_status": "kali-integration-status.snapshot.json",
|
||||
"soc_control": "soc-siem-kali-wazuh-integration-control.snapshot.json",
|
||||
"alert_readability": "telegram-alert-readability-guard.snapshot.json",
|
||||
@@ -37,6 +38,7 @@ _EXPECTED_SCHEMAS = {
|
||||
"wazuh_live_metadata_gate": "iwooos_wazuh_readonly_live_metadata_env_gate_v1",
|
||||
"wazuh_owner_evidence_preflight": "wazuh_agent_visibility_owner_evidence_preflight_v1",
|
||||
"wazuh_runtime_apply_preflight": "wazuh_runtime_controlled_apply_preflight_v1",
|
||||
"wazuh_runtime_owner_review": "wazuh_runtime_gate_owner_review_readback_v1",
|
||||
"kali_status": "kali_integration_status_v1",
|
||||
"soc_control": "soc_siem_kali_wazuh_integration_control_v1",
|
||||
"alert_readability": "telegram_alert_readability_guard_v1",
|
||||
@@ -86,6 +88,7 @@ def load_latest_iwooos_runtime_security_readback(
|
||||
runtime_apply_preflight_summary = _summary(
|
||||
snapshots["wazuh_runtime_apply_preflight"]
|
||||
)
|
||||
runtime_owner_review_summary = _summary(snapshots["wazuh_runtime_owner_review"])
|
||||
soc_summary = _summary(snapshots["soc_control"])
|
||||
alert_summary = _summary(snapshots["alert_readability"])
|
||||
dispatch_summary = _summary(snapshots["owner_dispatch"])
|
||||
@@ -108,7 +111,7 @@ def load_latest_iwooos_runtime_security_readback(
|
||||
"source_refs": source_refs,
|
||||
"summary": {
|
||||
"source_snapshot_count": len(source_refs),
|
||||
"p0_lane_count": 10,
|
||||
"p0_lane_count": 11,
|
||||
"control_plane_visibility_percent": _average_percent(
|
||||
soc_summary.get("coverage_percent_after_soc_integration_control"),
|
||||
intrusion_summary.get("coverage_percent_after_prevention_control"),
|
||||
@@ -234,6 +237,41 @@ def load_latest_iwooos_runtime_security_readback(
|
||||
"wazuh_runtime_apply_runtime_gate_count": _int(
|
||||
runtime_apply_preflight_summary.get("runtime_gate_count")
|
||||
),
|
||||
"wazuh_runtime_owner_review_target_selector_count": _int(
|
||||
runtime_owner_review_summary.get("target_selector_count")
|
||||
),
|
||||
"wazuh_runtime_owner_review_source_diff_count": _int(
|
||||
runtime_owner_review_summary.get("source_of_truth_diff_count")
|
||||
),
|
||||
"wazuh_runtime_owner_review_check_mode_plan_count": _int(
|
||||
runtime_owner_review_summary.get("check_mode_plan_count")
|
||||
),
|
||||
"wazuh_runtime_owner_review_dry_run_evidence_count": _int(
|
||||
runtime_owner_review_summary.get("dry_run_evidence_count")
|
||||
),
|
||||
"wazuh_runtime_owner_review_rollback_plan_count": _int(
|
||||
runtime_owner_review_summary.get("rollback_plan_count")
|
||||
),
|
||||
"wazuh_runtime_owner_review_post_apply_verifier_count": _int(
|
||||
runtime_owner_review_summary.get("post_apply_verifier_count")
|
||||
),
|
||||
"wazuh_runtime_owner_review_km_writeback_count": _int(
|
||||
runtime_owner_review_summary.get("km_playbook_writeback_count")
|
||||
),
|
||||
"wazuh_runtime_owner_review_packet_received_count": _int(
|
||||
runtime_owner_review_summary.get("owner_review_packet_received_count")
|
||||
),
|
||||
"wazuh_runtime_owner_review_packet_review_ready_count": _int(
|
||||
runtime_owner_review_summary.get(
|
||||
"owner_review_packet_review_ready_count"
|
||||
)
|
||||
),
|
||||
"wazuh_runtime_owner_review_packet_accepted_count": _int(
|
||||
runtime_owner_review_summary.get("owner_review_packet_accepted_count")
|
||||
),
|
||||
"wazuh_runtime_owner_review_runtime_gate_count": _int(
|
||||
runtime_owner_review_summary.get("runtime_gate_count")
|
||||
),
|
||||
"kali_active_scan_authorized_count": _int(
|
||||
soc_summary.get("kali_active_scan_authorized_count")
|
||||
),
|
||||
@@ -433,6 +471,57 @@ def load_latest_iwooos_runtime_security_readback(
|
||||
"docs/security/wazuh-runtime-controlled-apply-preflight.snapshot.json"
|
||||
],
|
||||
),
|
||||
_lane(
|
||||
"wazuh_runtime_gate_owner_review",
|
||||
snapshots["wazuh_runtime_owner_review"].get(
|
||||
"status",
|
||||
"runtime_gate_owner_review_packet_committed_no_runtime_action",
|
||||
),
|
||||
55
|
||||
if _int(
|
||||
runtime_owner_review_summary.get(
|
||||
"owner_review_packet_accepted_count"
|
||||
)
|
||||
)
|
||||
else 0,
|
||||
"steady"
|
||||
if _int(
|
||||
runtime_owner_review_summary.get(
|
||||
"owner_review_packet_accepted_count"
|
||||
)
|
||||
)
|
||||
else "locked",
|
||||
"進入 allowlisted check-mode 與 dry-run readback;runtime gate 仍關閉",
|
||||
{
|
||||
"target_selectors": runtime_owner_review_summary.get(
|
||||
"target_selector_count", 0
|
||||
),
|
||||
"source_diff": runtime_owner_review_summary.get(
|
||||
"source_of_truth_diff_count", 0
|
||||
),
|
||||
"check_mode": runtime_owner_review_summary.get(
|
||||
"check_mode_plan_count", 0
|
||||
),
|
||||
"dry_run": runtime_owner_review_summary.get(
|
||||
"dry_run_evidence_count", 0
|
||||
),
|
||||
"rollback": runtime_owner_review_summary.get(
|
||||
"rollback_plan_count", 0
|
||||
),
|
||||
"post_apply_verifier": runtime_owner_review_summary.get(
|
||||
"post_apply_verifier_count", 0
|
||||
),
|
||||
"owner_review_accepted": runtime_owner_review_summary.get(
|
||||
"owner_review_packet_accepted_count", 0
|
||||
),
|
||||
"runtime_gate": runtime_owner_review_summary.get(
|
||||
"runtime_gate_count", 0
|
||||
),
|
||||
},
|
||||
[
|
||||
"docs/security/wazuh-runtime-gate-owner-review-readback.snapshot.json"
|
||||
],
|
||||
),
|
||||
_lane(
|
||||
"wazuh_dashboard_api",
|
||||
"degraded_api_connection_not_green",
|
||||
@@ -562,6 +651,7 @@ def load_latest_iwooos_runtime_security_readback(
|
||||
"Wazuh 即時中繼資料必須先通過負責人、機密中繼資料、唯讀範圍與啟用後讀回",
|
||||
"Wazuh 負責人證據預檢 ready 不代表已收件、已接受或可啟用 active response",
|
||||
"Wazuh controlled apply preflight ready 不代表 runtime gate 已開或已執行修復",
|
||||
"Wazuh runtime gate owner-review accepted 只代表 review readiness,不代表已查 live Wazuh 或可寫主機",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,727 @@
|
||||
"""
|
||||
IwoooS Wazuh runtime gate owner-review readback.
|
||||
|
||||
This service exposes a committed owner-review readiness contract and a
|
||||
no-persist validator for redacted runtime-gate owner-review packets. It never
|
||||
queries live Wazuh, reads host data, reads secrets, persists raw payloads, or
|
||||
authorizes runtime actions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from src.services.snapshot_paths import default_security_dir
|
||||
|
||||
_DEFAULT_SECURITY_DIR = default_security_dir(Path(__file__))
|
||||
_SNAPSHOT_FILE = "wazuh-runtime-gate-owner-review-readback.snapshot.json"
|
||||
_EXPECTED_SCHEMA = "wazuh_runtime_gate_owner_review_readback_v1"
|
||||
|
||||
_REQUIRED_FALSE_BOUNDARIES = {
|
||||
"active_scan_authorized",
|
||||
"alertmanager_reload_authorized",
|
||||
"auto_block_authorized",
|
||||
"credentialed_scan_authorized",
|
||||
"firewall_change_authorized",
|
||||
"host_write_authorized",
|
||||
"kali_execute_authorized",
|
||||
"kali_scan_authorized",
|
||||
"nginx_reload_authorized",
|
||||
"production_write_authorized",
|
||||
"runtime_execution_authorized",
|
||||
"runtime_gate_open",
|
||||
"secret_value_collection_allowed",
|
||||
"telegram_send_authorized",
|
||||
"wazuh_active_response_authorized",
|
||||
"wazuh_api_live_query_authorized",
|
||||
}
|
||||
|
||||
_SENSITIVE_TEXT_PATTERNS = {
|
||||
"internal_ip": re.compile(
|
||||
r"\b(?:10|127|172\.(?:1[6-9]|2\d|3[01])|192\.168)\.\d{1,3}\.\d{1,3}\b"
|
||||
),
|
||||
"authorization_header": re.compile(r"Authorization\s*:", re.IGNORECASE),
|
||||
"bearer_token": re.compile(r"Bearer\s+[A-Za-z0-9._-]{10,}", re.IGNORECASE),
|
||||
"basic_auth": re.compile(r"Basic\s+[A-Za-z0-9+/=]{10,}", re.IGNORECASE),
|
||||
"password_assignment": re.compile(
|
||||
r"password\s*[:=]\s*['\"][^'\"]+['\"]", re.IGNORECASE
|
||||
),
|
||||
"token_assignment": re.compile(r"token\s*[:=]\s*['\"][^'\"]+['\"]", re.IGNORECASE),
|
||||
"cookie_assignment": re.compile(
|
||||
r"cookie\s*[:=]\s*['\"][^'\"]+['\"]", re.IGNORECASE
|
||||
),
|
||||
"client_keys": re.compile(r"client\.keys", re.IGNORECASE),
|
||||
"private_key": re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
|
||||
"raw_session_text": re.compile(
|
||||
r"(工作視窗|批准!繼續|source_thread_id|raw session)", re.IGNORECASE
|
||||
),
|
||||
}
|
||||
|
||||
_FORBIDDEN_KEY_FRAGMENTS = {
|
||||
"authorization_header",
|
||||
"basic_auth",
|
||||
"bearer_token",
|
||||
"client_keys",
|
||||
"cookie",
|
||||
"env_file",
|
||||
"full_cli_output",
|
||||
"full_journal",
|
||||
"hostname",
|
||||
"internal_ip",
|
||||
"password",
|
||||
"private_key",
|
||||
"raw_agent_identity",
|
||||
"raw_dashboard_request",
|
||||
"raw_env",
|
||||
"raw_hostname",
|
||||
"raw_log",
|
||||
"raw_runtime_volume",
|
||||
"raw_wazuh_payload",
|
||||
"session",
|
||||
"stored_api_password",
|
||||
"token",
|
||||
"unredacted_screenshot",
|
||||
"wazuh_api_password",
|
||||
}
|
||||
|
||||
_RUNTIME_ACTION_KEYS = {
|
||||
"active_response_enable",
|
||||
"agent_reenroll",
|
||||
"agent_restart",
|
||||
"ansible_apply",
|
||||
"ansible_playbook_run",
|
||||
"apply_now",
|
||||
"argocd_sync",
|
||||
"credentialed_scan",
|
||||
"database_migration",
|
||||
"docker_restart",
|
||||
"execute_now",
|
||||
"exploit_attempt",
|
||||
"firewall_change",
|
||||
"force_push",
|
||||
"host_write",
|
||||
"k8s_apply",
|
||||
"kali_active_scan",
|
||||
"nginx_reload",
|
||||
"repo_ref_delete",
|
||||
"runtime_execution_authorized",
|
||||
"secret_rotation",
|
||||
"systemd_restart",
|
||||
"wazuh_active_response",
|
||||
"wazuh_agent_reenroll",
|
||||
"wazuh_agent_restart",
|
||||
"wazuh_api_live_query",
|
||||
"wazuh_manager_restart",
|
||||
"workflow_trigger",
|
||||
}
|
||||
|
||||
|
||||
def load_latest_iwooos_wazuh_runtime_gate_owner_review_readback(
|
||||
security_dir: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Load the public-safe Wazuh runtime gate owner-review contract."""
|
||||
directory = security_dir or _DEFAULT_SECURITY_DIR
|
||||
snapshot = _load_snapshot(directory)
|
||||
_require_boundaries(snapshot)
|
||||
|
||||
summary = _summary(snapshot)
|
||||
merged_summary = {
|
||||
"expected_scope_alias_count": _int(summary.get("expected_scope_alias_count")),
|
||||
"target_selector_count": _int(summary.get("target_selector_count")),
|
||||
"source_of_truth_diff_count": _int(summary.get("source_of_truth_diff_count")),
|
||||
"check_mode_plan_count": _int(summary.get("check_mode_plan_count")),
|
||||
"dry_run_evidence_count": _int(summary.get("dry_run_evidence_count")),
|
||||
"rollback_plan_count": _int(summary.get("rollback_plan_count")),
|
||||
"post_apply_verifier_count": _int(summary.get("post_apply_verifier_count")),
|
||||
"km_playbook_writeback_count": _int(summary.get("km_playbook_writeback_count")),
|
||||
"maintenance_window_review_count": _int(
|
||||
summary.get("maintenance_window_review_count")
|
||||
),
|
||||
"owner_review_packet_received_count": _int(
|
||||
summary.get("owner_review_packet_received_count")
|
||||
),
|
||||
"owner_review_packet_review_ready_count": _int(
|
||||
summary.get("owner_review_packet_review_ready_count")
|
||||
),
|
||||
"owner_review_packet_accepted_count": _int(
|
||||
summary.get("owner_review_packet_accepted_count")
|
||||
),
|
||||
"owner_review_packet_supplement_required_count": _int(
|
||||
summary.get("owner_review_packet_supplement_required_count")
|
||||
),
|
||||
"owner_review_packet_quarantined_count": _int(
|
||||
summary.get("owner_review_packet_quarantined_count")
|
||||
),
|
||||
"owner_review_runtime_action_rejected_count": _int(
|
||||
summary.get("owner_review_runtime_action_rejected_count")
|
||||
),
|
||||
"forbidden_payload_count": _int(summary.get("forbidden_payload_count")),
|
||||
"forbidden_action_count": _int(summary.get("forbidden_action_count")),
|
||||
"runtime_gate_count": _int(summary.get("runtime_gate_count")),
|
||||
"wazuh_api_live_query_authorized_count": _int(
|
||||
summary.get("wazuh_api_live_query_authorized_count")
|
||||
),
|
||||
"wazuh_active_response_authorized_count": _int(
|
||||
summary.get("wazuh_active_response_authorized_count")
|
||||
),
|
||||
"host_write_authorized_count": _int(summary.get("host_write_authorized_count")),
|
||||
"secret_value_collection_allowed_count": _int(
|
||||
summary.get("secret_value_collection_allowed_count")
|
||||
),
|
||||
}
|
||||
|
||||
return {
|
||||
"schema_version": "iwooos_wazuh_runtime_gate_owner_review_readback_v1",
|
||||
"source_schema_version": snapshot["schema_version"],
|
||||
"status": snapshot.get(
|
||||
"status", "runtime_gate_owner_review_packet_committed_no_runtime_action"
|
||||
),
|
||||
"mode": snapshot.get(
|
||||
"mode", "committed_owner_review_readback_no_live_wazuh_no_secret_collection"
|
||||
),
|
||||
"source_refs": [
|
||||
f"docs/security/{_SNAPSHOT_FILE}",
|
||||
"docs/security/wazuh-runtime-controlled-apply-preflight.snapshot.json",
|
||||
],
|
||||
"owner_review_packet_validation_endpoint": (
|
||||
"/api/v1/iwooos/wazuh-runtime-gate-owner-review-readback/validate-owner-review-packet"
|
||||
),
|
||||
"owner_review_packet_validation_mode": "no_persist_owner_review_readback_no_runtime_action",
|
||||
"summary": merged_summary,
|
||||
"target_selectors": _target_selectors(snapshot.get("target_selectors")),
|
||||
"required_owner_review_fields": _strings(
|
||||
snapshot.get("required_owner_review_fields")
|
||||
),
|
||||
"review_items": _review_items(snapshot.get("review_items")),
|
||||
"outcome_lanes": _strings(snapshot.get("outcome_lanes")),
|
||||
"forbidden_payloads": _strings(snapshot.get("forbidden_payloads")),
|
||||
"forbidden_actions": _strings(snapshot.get("forbidden_actions")),
|
||||
"boundary_markers": _boundary_markers(merged_summary),
|
||||
"boundaries": {
|
||||
"payload_persisted": False,
|
||||
"wazuh_api_live_query_authorized": False,
|
||||
"wazuh_active_response_authorized": False,
|
||||
"wazuh_agent_reenroll_authorized": False,
|
||||
"wazuh_agent_restart_authorized": False,
|
||||
"wazuh_manager_restart_authorized": False,
|
||||
"host_write_authorized": False,
|
||||
"active_scan_authorized": False,
|
||||
"kali_execute_authorized": False,
|
||||
"nginx_reload_authorized": False,
|
||||
"secret_value_collection_allowed": False,
|
||||
"runtime_execution_authorized": False,
|
||||
"runtime_gate_open": False,
|
||||
"not_authorization": True,
|
||||
},
|
||||
"no_false_green_rules": _strings(snapshot.get("no_false_green_rules")),
|
||||
}
|
||||
|
||||
|
||||
def validate_iwooos_wazuh_runtime_gate_owner_review_packet(
|
||||
owner_review_packet: dict[str, Any],
|
||||
security_dir: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Validate one redacted runtime-gate owner-review packet without applying it."""
|
||||
contract = load_latest_iwooos_wazuh_runtime_gate_owner_review_readback(security_dir)
|
||||
snapshot = _load_snapshot(security_dir or _DEFAULT_SECURITY_DIR)
|
||||
required_fields = _strings(snapshot.get("required_owner_review_fields"))
|
||||
expected_aliases = {
|
||||
item["node_alias"]
|
||||
for item in contract["target_selectors"]
|
||||
if item.get("node_alias")
|
||||
}
|
||||
|
||||
findings: list[dict[str, Any]] = []
|
||||
if not isinstance(owner_review_packet, dict):
|
||||
findings.append(
|
||||
_finding(
|
||||
"ORG-01",
|
||||
"blocker",
|
||||
"request_runtime_gate_owner_review_supplement",
|
||||
"runtime gate owner-review packet must be a JSON object.",
|
||||
[],
|
||||
)
|
||||
)
|
||||
return _validation_result(
|
||||
contract, "request_runtime_gate_owner_review_supplement", findings
|
||||
)
|
||||
|
||||
sensitive_hits = _collect_sensitive_hits(owner_review_packet)
|
||||
if sensitive_hits:
|
||||
findings.append(
|
||||
_finding(
|
||||
"ORG-04",
|
||||
"critical",
|
||||
"quarantine_sensitive_payload",
|
||||
"runtime gate owner-review packet contains forbidden or likely unredacted content; response omits raw values.",
|
||||
[hit["path"] for hit in sensitive_hits[:12]],
|
||||
{"categories": sorted({hit["category"] for hit in sensitive_hits})},
|
||||
)
|
||||
)
|
||||
return _validation_result(contract, "quarantine_sensitive_payload", findings)
|
||||
|
||||
runtime_hits = _collect_runtime_action_hits(owner_review_packet)
|
||||
if runtime_hits:
|
||||
findings.append(
|
||||
_finding(
|
||||
"ORG-05",
|
||||
"critical",
|
||||
"reject_runtime_action_request",
|
||||
"runtime gate owner-review packet requested runtime execution; this validator only records review readiness.",
|
||||
runtime_hits[:12],
|
||||
)
|
||||
)
|
||||
return _validation_result(contract, "reject_runtime_action_request", findings)
|
||||
|
||||
missing_fields = [
|
||||
field
|
||||
for field in required_fields
|
||||
if not _present(owner_review_packet.get(field))
|
||||
]
|
||||
if missing_fields:
|
||||
findings.append(
|
||||
_finding(
|
||||
"ORG-01",
|
||||
"blocker",
|
||||
"request_runtime_gate_owner_review_supplement",
|
||||
"runtime gate owner-review packet is missing required fields.",
|
||||
missing_fields,
|
||||
)
|
||||
)
|
||||
|
||||
alias_issue = _validate_aliases(
|
||||
owner_review_packet.get("target_selector_aliases"), expected_aliases
|
||||
)
|
||||
if alias_issue:
|
||||
findings.append(
|
||||
_finding(
|
||||
"ORG-02",
|
||||
"blocker",
|
||||
"request_target_selector_fix",
|
||||
alias_issue,
|
||||
["target_selector_aliases"],
|
||||
)
|
||||
)
|
||||
|
||||
if (
|
||||
owner_review_packet.get("owner_review_intent")
|
||||
!= "commit_runtime_gate_owner_review_readback_only"
|
||||
):
|
||||
findings.append(
|
||||
_finding(
|
||||
"ORG-03",
|
||||
"blocker",
|
||||
"request_runtime_gate_owner_review_decision_fix",
|
||||
"owner_review_intent must be commit_runtime_gate_owner_review_readback_only.",
|
||||
["owner_review_intent"],
|
||||
)
|
||||
)
|
||||
|
||||
if (
|
||||
owner_review_packet.get("owner_review_decision")
|
||||
!= "accept_controlled_apply_review_readiness_only"
|
||||
):
|
||||
findings.append(
|
||||
_finding(
|
||||
"ORG-06",
|
||||
"blocker",
|
||||
"request_runtime_gate_owner_review_decision_fix",
|
||||
"owner_review_decision must be accept_controlled_apply_review_readiness_only and must not claim runtime gate opening.",
|
||||
["owner_review_decision"],
|
||||
)
|
||||
)
|
||||
|
||||
if owner_review_packet.get("runtime_boundary_ack") != "runtime_gate_remains_closed":
|
||||
findings.append(
|
||||
_finding(
|
||||
"ORG-07",
|
||||
"blocker",
|
||||
"request_runtime_boundary_ack_fix",
|
||||
"runtime_boundary_ack must state runtime_gate_remains_closed.",
|
||||
["runtime_boundary_ack"],
|
||||
)
|
||||
)
|
||||
|
||||
if owner_review_packet.get("secret_boundary_ack") != "no_secret_value_collected":
|
||||
findings.append(
|
||||
_finding(
|
||||
"ORG-08",
|
||||
"blocker",
|
||||
"request_runtime_boundary_ack_fix",
|
||||
"secret_boundary_ack must state no_secret_value_collected.",
|
||||
["secret_boundary_ack"],
|
||||
)
|
||||
)
|
||||
|
||||
if (
|
||||
owner_review_packet.get("live_wazuh_query_boundary_ack")
|
||||
!= "no_live_wazuh_query_performed"
|
||||
):
|
||||
findings.append(
|
||||
_finding(
|
||||
"ORG-09",
|
||||
"blocker",
|
||||
"request_runtime_boundary_ack_fix",
|
||||
"live_wazuh_query_boundary_ack must state no_live_wazuh_query_performed.",
|
||||
["live_wazuh_query_boundary_ack"],
|
||||
)
|
||||
)
|
||||
|
||||
outcome = (
|
||||
_first_blocking_lane(findings)
|
||||
or "accepted_for_runtime_gate_owner_review_readback_only"
|
||||
)
|
||||
if outcome == "accepted_for_runtime_gate_owner_review_readback_only":
|
||||
findings.append(
|
||||
_finding(
|
||||
"ORG-10",
|
||||
"info",
|
||||
"runtime_gate_owner_review_readback_ready",
|
||||
"owner-review packet passed no-persist readiness validation; runtime gate remains closed.",
|
||||
[
|
||||
"owner_review_decision",
|
||||
"check_mode_plan_ref",
|
||||
"post_apply_verifier_ref",
|
||||
],
|
||||
)
|
||||
)
|
||||
return _validation_result(contract, outcome, findings)
|
||||
|
||||
|
||||
def _load_snapshot(directory: Path) -> dict[str, Any]:
|
||||
path = directory / _SNAPSHOT_FILE
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(
|
||||
f"{path}: Wazuh runtime gate owner-review snapshot not found"
|
||||
)
|
||||
with path.open(encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"{path}: expected JSON object")
|
||||
if payload.get("schema_version") != _EXPECTED_SCHEMA:
|
||||
raise ValueError(f"{path}: expected schema_version={_EXPECTED_SCHEMA}")
|
||||
return payload
|
||||
|
||||
|
||||
def _summary(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
summary = payload.get("summary")
|
||||
return summary if isinstance(summary, dict) else {}
|
||||
|
||||
|
||||
def _int(value: Any) -> int:
|
||||
return value if isinstance(value, int) else 0
|
||||
|
||||
|
||||
def _strings(value: Any) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [item for item in value if isinstance(item, str)]
|
||||
|
||||
|
||||
def _target_selectors(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
selectors: list[dict[str, Any]] = []
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
selectors.append(
|
||||
{
|
||||
"node_alias": str(item.get("node_alias", "")),
|
||||
"scope": str(item.get("scope", "")),
|
||||
"selector_kind": str(item.get("selector_kind", "")),
|
||||
"runtime_write_allowed": item.get("runtime_write_allowed") is True,
|
||||
"owner_review_scope": str(item.get("owner_review_scope", "")),
|
||||
}
|
||||
)
|
||||
return selectors
|
||||
|
||||
|
||||
def _review_items(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
items: list[dict[str, Any]] = []
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"item_id": str(item.get("item_id", "")),
|
||||
"title": str(item.get("title", "")),
|
||||
"state_key": str(item.get("state_key", "")),
|
||||
"accepted": item.get("accepted") is True,
|
||||
"required_fields": _strings(item.get("required_fields")),
|
||||
"next_gate": str(item.get("next_gate", "")),
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def _boundary_markers(summary: dict[str, int]) -> list[str]:
|
||||
return [
|
||||
"wazuh_runtime_gate_owner_review_visible=true",
|
||||
"wazuh_runtime_gate_owner_review_validation_api_available=true",
|
||||
f"wazuh_runtime_gate_owner_review_target_selector_count={summary['target_selector_count']}",
|
||||
f"wazuh_runtime_gate_owner_review_source_diff_count={summary['source_of_truth_diff_count']}",
|
||||
f"wazuh_runtime_gate_owner_review_check_mode_plan_count={summary['check_mode_plan_count']}",
|
||||
f"wazuh_runtime_gate_owner_review_dry_run_evidence_count={summary['dry_run_evidence_count']}",
|
||||
f"wazuh_runtime_gate_owner_review_packet_received_count={summary['owner_review_packet_received_count']}",
|
||||
f"wazuh_runtime_gate_owner_review_packet_accepted_count={summary['owner_review_packet_accepted_count']}",
|
||||
f"wazuh_runtime_gate_owner_review_runtime_gate_count={summary['runtime_gate_count']}",
|
||||
"wazuh_api_live_query_authorized=false",
|
||||
"wazuh_active_response_authorized=false",
|
||||
"host_write_authorized=false",
|
||||
"secret_value_collection_allowed=false",
|
||||
"not_authorization=true",
|
||||
]
|
||||
|
||||
|
||||
def _require_boundaries(payload: dict[str, Any]) -> None:
|
||||
summary = _summary(payload)
|
||||
for key in (
|
||||
"owner_review_packet_supplement_required_count",
|
||||
"owner_review_packet_quarantined_count",
|
||||
"owner_review_runtime_action_rejected_count",
|
||||
"runtime_gate_count",
|
||||
"wazuh_api_live_query_authorized_count",
|
||||
"wazuh_active_response_authorized_count",
|
||||
"host_write_authorized_count",
|
||||
"secret_value_collection_allowed_count",
|
||||
):
|
||||
if _int(summary.get(key)) != 0:
|
||||
raise ValueError(
|
||||
f"Wazuh runtime gate owner-review summary.{key} must remain 0"
|
||||
)
|
||||
|
||||
expected_alias_count = _int(summary.get("expected_scope_alias_count"))
|
||||
target_selector_count = _int(summary.get("target_selector_count"))
|
||||
target_selectors = _target_selectors(payload.get("target_selectors"))
|
||||
if (
|
||||
target_selector_count != expected_alias_count
|
||||
or len(target_selectors) != expected_alias_count
|
||||
):
|
||||
raise ValueError(
|
||||
"Wazuh runtime gate owner-review target selectors must match expected alias count"
|
||||
)
|
||||
if any(item.get("runtime_write_allowed") is True for item in target_selectors):
|
||||
raise ValueError(
|
||||
"Wazuh runtime gate owner-review target selectors must not allow runtime writes"
|
||||
)
|
||||
|
||||
readiness_keys = (
|
||||
"source_of_truth_diff_count",
|
||||
"check_mode_plan_count",
|
||||
"dry_run_evidence_count",
|
||||
"rollback_plan_count",
|
||||
"post_apply_verifier_count",
|
||||
"km_playbook_writeback_count",
|
||||
"maintenance_window_review_count",
|
||||
"owner_review_packet_received_count",
|
||||
"owner_review_packet_review_ready_count",
|
||||
"owner_review_packet_accepted_count",
|
||||
)
|
||||
if any(_int(summary.get(key)) <= 0 for key in readiness_keys):
|
||||
raise ValueError(
|
||||
"Wazuh runtime gate owner-review readiness counters must be positive"
|
||||
)
|
||||
|
||||
boundaries = payload.get("execution_boundaries")
|
||||
if not isinstance(boundaries, dict):
|
||||
raise ValueError("Wazuh runtime gate owner-review execution_boundaries missing")
|
||||
for key in _REQUIRED_FALSE_BOUNDARIES:
|
||||
if boundaries.get(key) is not False:
|
||||
raise ValueError(
|
||||
f"Wazuh runtime gate owner-review execution_boundaries.{key} must remain false"
|
||||
)
|
||||
if boundaries.get("not_authorization") is not True:
|
||||
raise ValueError(
|
||||
"Wazuh runtime gate owner-review not_authorization must remain true"
|
||||
)
|
||||
|
||||
|
||||
def _validation_result(
|
||||
contract: dict[str, Any],
|
||||
outcome_lane: str,
|
||||
findings: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
accepted = outcome_lane == "accepted_for_runtime_gate_owner_review_readback_only"
|
||||
quarantined = outcome_lane == "quarantine_sensitive_payload"
|
||||
rejected_runtime = outcome_lane == "reject_runtime_action_request"
|
||||
supplement_required = not accepted and not quarantined and not rejected_runtime
|
||||
return {
|
||||
"schema_version": "iwooos_wazuh_runtime_gate_owner_review_packet_validation_result_v1",
|
||||
"contract_schema_version": contract["schema_version"],
|
||||
"status": outcome_lane,
|
||||
"mode": "no_persist_runtime_gate_owner_review_no_runtime_no_secret_collection",
|
||||
"outcome_lane": outcome_lane,
|
||||
"accepted_for_runtime_gate_owner_review_readback_only": accepted,
|
||||
"quarantined": quarantined,
|
||||
"runtime_action_rejected": rejected_runtime,
|
||||
"summary": {
|
||||
"owner_review_packet_received_count": 1,
|
||||
"owner_review_packet_review_ready_count": 1 if accepted else 0,
|
||||
"owner_review_packet_accepted_count": 1 if accepted else 0,
|
||||
"owner_review_packet_supplement_required_count": 1
|
||||
if supplement_required
|
||||
else 0,
|
||||
"owner_review_packet_quarantined_count": 1 if quarantined else 0,
|
||||
"owner_review_runtime_action_rejected_count": 1 if rejected_runtime else 0,
|
||||
"runtime_gate_count": 0,
|
||||
"wazuh_api_live_query_authorized_count": 0,
|
||||
"wazuh_active_response_authorized_count": 0,
|
||||
"host_write_authorized_count": 0,
|
||||
"secret_value_collection_allowed_count": 0,
|
||||
"finding_count": len(findings),
|
||||
},
|
||||
"validation_findings": findings,
|
||||
"boundary_markers": [
|
||||
"wazuh_runtime_gate_owner_review_packet_validation_received_count=1",
|
||||
f"wazuh_runtime_gate_owner_review_packet_validation_accepted_count={1 if accepted else 0}",
|
||||
f"wazuh_runtime_gate_owner_review_packet_validation_quarantined_count={1 if quarantined else 0}",
|
||||
f"wazuh_runtime_gate_owner_review_packet_validation_runtime_action_rejected_count={1 if rejected_runtime else 0}",
|
||||
"wazuh_runtime_gate_owner_review_packet_validation_runtime_gate_count=0",
|
||||
"wazuh_runtime_gate_owner_review_packet_validation_no_persist=true",
|
||||
"wazuh_api_live_query_authorized=false",
|
||||
"wazuh_active_response_authorized=false",
|
||||
"host_write_authorized=false",
|
||||
"secret_value_collection_allowed=false",
|
||||
"not_authorization=true",
|
||||
],
|
||||
"boundaries": {
|
||||
"payload_persisted": False,
|
||||
"wazuh_api_live_query_authorized": False,
|
||||
"wazuh_active_response_authorized": False,
|
||||
"wazuh_agent_reenroll_authorized": False,
|
||||
"wazuh_agent_restart_authorized": False,
|
||||
"wazuh_manager_restart_authorized": False,
|
||||
"host_write_authorized": False,
|
||||
"active_scan_authorized": False,
|
||||
"kali_execute_authorized": False,
|
||||
"nginx_reload_authorized": False,
|
||||
"secret_value_collection_allowed": False,
|
||||
"runtime_execution_authorized": False,
|
||||
"runtime_gate_open": False,
|
||||
"not_authorization": True,
|
||||
},
|
||||
"next_gate": "stage_allowlisted_check_mode_dry_run_before_runtime_gate"
|
||||
if accepted
|
||||
else "runtime_gate_owner_review_packet_fix_and_resubmit",
|
||||
}
|
||||
|
||||
|
||||
def _finding(
|
||||
check_id: str,
|
||||
severity: str,
|
||||
lane: str,
|
||||
message: str,
|
||||
field_paths: list[str],
|
||||
extra: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"check_id": check_id,
|
||||
"severity": severity,
|
||||
"lane": lane,
|
||||
"message": message,
|
||||
"field_paths": field_paths,
|
||||
}
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
return payload
|
||||
|
||||
|
||||
def _present(value: Any) -> bool:
|
||||
if value is None:
|
||||
return False
|
||||
if isinstance(value, str):
|
||||
return bool(value.strip())
|
||||
if isinstance(value, list | dict | tuple | set):
|
||||
return bool(value)
|
||||
return True
|
||||
|
||||
|
||||
def _validate_aliases(value: Any, expected_aliases: set[str]) -> str | None:
|
||||
aliases = value if isinstance(value, list) else []
|
||||
if not aliases or not all(isinstance(item, str) for item in aliases):
|
||||
return "target_selector_aliases must be an array of public alias strings."
|
||||
alias_set = set(aliases)
|
||||
if len(aliases) != len(alias_set):
|
||||
return "target_selector_aliases must not contain duplicates."
|
||||
if alias_set != expected_aliases:
|
||||
missing = sorted(expected_aliases - alias_set)
|
||||
extra = sorted(alias_set - expected_aliases)
|
||||
return f"target_selector_aliases must match expected public aliases; missing={missing} extra={extra}"
|
||||
return None
|
||||
|
||||
|
||||
def _collect_sensitive_hits(value: Any, path: str = "$") -> list[dict[str, str]]:
|
||||
hits: list[dict[str, str]] = []
|
||||
if isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
key_text = str(key)
|
||||
key_lower = key_text.lower()
|
||||
for fragment in _FORBIDDEN_KEY_FRAGMENTS:
|
||||
if fragment in key_lower:
|
||||
hits.append(
|
||||
{
|
||||
"path": f"{path}.{key_text}",
|
||||
"category": f"forbidden_key:{fragment}",
|
||||
}
|
||||
)
|
||||
hits.extend(_collect_sensitive_hits(item, f"{path}.{key_text}"))
|
||||
return hits
|
||||
if isinstance(value, list):
|
||||
for index, item in enumerate(value):
|
||||
hits.extend(_collect_sensitive_hits(item, f"{path}[{index}]"))
|
||||
return hits
|
||||
if isinstance(value, str):
|
||||
for category, pattern in _SENSITIVE_TEXT_PATTERNS.items():
|
||||
if pattern.search(value):
|
||||
hits.append({"path": path, "category": category})
|
||||
return hits
|
||||
|
||||
|
||||
def _collect_runtime_action_hits(value: Any, path: str = "$") -> list[str]:
|
||||
hits: list[str] = []
|
||||
if isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
key_text = str(key)
|
||||
normalized_key = key_text.lower().replace("-", "_").replace(" ", "_")
|
||||
if normalized_key in _RUNTIME_ACTION_KEYS and item not in (
|
||||
False,
|
||||
None,
|
||||
"",
|
||||
[],
|
||||
{},
|
||||
):
|
||||
hits.append(f"{path}.{key_text}")
|
||||
hits.extend(_collect_runtime_action_hits(item, f"{path}.{key_text}"))
|
||||
return hits
|
||||
if isinstance(value, list):
|
||||
for index, item in enumerate(value):
|
||||
hits.extend(_collect_runtime_action_hits(item, f"{path}[{index}]"))
|
||||
return hits
|
||||
if isinstance(value, str):
|
||||
normalized = value.lower().replace("-", "_").replace(" ", "_")
|
||||
if normalized in _RUNTIME_ACTION_KEYS:
|
||||
hits.append(path)
|
||||
return hits
|
||||
|
||||
|
||||
def _first_blocking_lane(findings: list[dict[str, Any]]) -> str | None:
|
||||
severity_order = {"critical": 0, "blocker": 1, "warn": 2, "info": 3}
|
||||
blocking = [
|
||||
finding
|
||||
for finding in findings
|
||||
if finding.get("severity") in {"critical", "blocker"}
|
||||
]
|
||||
if not blocking:
|
||||
return None
|
||||
blocking.sort(
|
||||
key=lambda finding: severity_order.get(str(finding.get("severity")), 99)
|
||||
)
|
||||
return str(
|
||||
blocking[0].get("lane") or "request_runtime_gate_owner_review_supplement"
|
||||
)
|
||||
@@ -10,6 +10,9 @@ from src.services.iwooos_runtime_security_readback import (
|
||||
from src.services.iwooos_wazuh_runtime_controlled_apply_preflight import (
|
||||
load_latest_iwooos_wazuh_runtime_controlled_apply_preflight,
|
||||
)
|
||||
from src.services.iwooos_wazuh_runtime_gate_owner_review_readback import (
|
||||
load_latest_iwooos_wazuh_runtime_gate_owner_review_readback,
|
||||
)
|
||||
|
||||
|
||||
def _client() -> TestClient:
|
||||
@@ -44,13 +47,44 @@ def _valid_runtime_controlled_apply_packet() -> dict[str, object]:
|
||||
}
|
||||
|
||||
|
||||
def _valid_runtime_gate_owner_review_packet() -> dict[str, object]:
|
||||
return {
|
||||
"owner_review_intent": "commit_runtime_gate_owner_review_readback_only",
|
||||
"owner_reviewer_role": "iwooos-security-owner",
|
||||
"owner_review_decision": "accept_controlled_apply_review_readiness_only",
|
||||
"owner_review_decision_reason": "redacted owner-review packet accepts review readiness only; runtime gate remains closed",
|
||||
"target_selector_aliases": [
|
||||
"managed_core_node_a",
|
||||
"managed_core_node_b",
|
||||
"managed_core_node_c",
|
||||
"managed_edge_node_a",
|
||||
"managed_edge_node_b",
|
||||
"managed_lab_node_a",
|
||||
],
|
||||
"source_of_truth_diff_ref": "docs/security/wazuh-runtime-gate-owner-review-readback.snapshot.json#source-diff",
|
||||
"check_mode_plan_ref": "playbooks/wazuh-controlled-apply-check-mode#redacted-plan",
|
||||
"dry_run_evidence_ref": "evidence/iwooos/wazuh-runtime-owner-review-dry-run-redacted-v1",
|
||||
"blast_radius_statement": "public aliases only; no live Wazuh query and no host write in this owner-review readback",
|
||||
"maintenance_window_ref": "maintenance/iwooos-wazuh-low-traffic-window-redacted-v1",
|
||||
"rollback_plan_ref": "playbooks/wazuh-controlled-apply-rollback#redacted-plan",
|
||||
"rollback_owner": "iwooos-security-owner",
|
||||
"post_apply_verifier_ref": "verifiers/iwooos-wazuh-post-apply-readback#public-safe",
|
||||
"km_playbook_writeback_ref": "km/playbook-trust/wazuh-runtime-gate-owner-review-v1",
|
||||
"followup_owner": "iwooos-security-reviewer",
|
||||
"audit_receipt_ref": "audit/iwooos-wazuh-runtime-gate-owner-review-redacted-v1",
|
||||
"runtime_boundary_ack": "runtime_gate_remains_closed",
|
||||
"secret_boundary_ack": "no_secret_value_collected",
|
||||
"live_wazuh_query_boundary_ack": "no_live_wazuh_query_performed",
|
||||
}
|
||||
|
||||
|
||||
def test_iwooos_runtime_security_readback_preserves_zero_runtime_gates() -> None:
|
||||
payload = load_latest_iwooos_runtime_security_readback()
|
||||
|
||||
assert payload["schema_version"] == "iwooos_runtime_security_readback_v1"
|
||||
assert payload["status"] == "blocked_waiting_owner_evidence_and_runtime_gates"
|
||||
assert payload["summary"]["source_snapshot_count"] == 11
|
||||
assert payload["summary"]["p0_lane_count"] == 10
|
||||
assert payload["summary"]["source_snapshot_count"] == 12
|
||||
assert payload["summary"]["p0_lane_count"] == 11
|
||||
assert payload["summary"]["runtime_gate_count"] == 0
|
||||
assert payload["summary"]["owner_response_received_count"] == 0
|
||||
assert payload["summary"]["owner_response_accepted_count"] == 0
|
||||
@@ -101,6 +135,21 @@ def test_iwooos_runtime_security_readback_preserves_zero_runtime_gates() -> None
|
||||
assert payload["summary"]["wazuh_runtime_apply_km_writeback_count"] == 1
|
||||
assert payload["summary"]["wazuh_runtime_apply_owner_review_ready_count"] == 1
|
||||
assert payload["summary"]["wazuh_runtime_apply_runtime_gate_count"] == 0
|
||||
assert payload["summary"]["wazuh_runtime_owner_review_target_selector_count"] == 6
|
||||
assert payload["summary"]["wazuh_runtime_owner_review_source_diff_count"] == 1
|
||||
assert payload["summary"]["wazuh_runtime_owner_review_check_mode_plan_count"] == 1
|
||||
assert payload["summary"]["wazuh_runtime_owner_review_dry_run_evidence_count"] == 1
|
||||
assert payload["summary"]["wazuh_runtime_owner_review_rollback_plan_count"] == 1
|
||||
assert (
|
||||
payload["summary"]["wazuh_runtime_owner_review_post_apply_verifier_count"] == 1
|
||||
)
|
||||
assert payload["summary"]["wazuh_runtime_owner_review_km_writeback_count"] == 1
|
||||
assert payload["summary"]["wazuh_runtime_owner_review_packet_received_count"] == 1
|
||||
assert (
|
||||
payload["summary"]["wazuh_runtime_owner_review_packet_review_ready_count"] == 1
|
||||
)
|
||||
assert payload["summary"]["wazuh_runtime_owner_review_packet_accepted_count"] == 1
|
||||
assert payload["summary"]["wazuh_runtime_owner_review_runtime_gate_count"] == 0
|
||||
assert payload["summary"]["kali_active_scan_authorized_count"] == 0
|
||||
assert payload["summary"]["kali_execute_authorized_count"] == 0
|
||||
assert payload["summary"]["alert_receipt_runtime_send_count"] == 0
|
||||
@@ -120,6 +169,7 @@ def test_iwooos_runtime_security_readback_lanes_are_candidate_only() -> None:
|
||||
"wazuh_live_metadata_gate",
|
||||
"wazuh_owner_evidence_preflight",
|
||||
"wazuh_runtime_controlled_apply_preflight",
|
||||
"wazuh_runtime_gate_owner_review",
|
||||
"wazuh_dashboard_api",
|
||||
"kali_intake",
|
||||
"alert_readability",
|
||||
@@ -152,6 +202,15 @@ def test_iwooos_runtime_security_readback_lanes_are_candidate_only() -> None:
|
||||
or (lane["completion_percent"] == 45 and lane["metrics"]["runtime_gate"] == 0)
|
||||
for lane in payload["lanes"]
|
||||
)
|
||||
assert all(
|
||||
lane["lane_id"] != "wazuh_runtime_gate_owner_review"
|
||||
or (
|
||||
lane["completion_percent"] == 55
|
||||
and lane["metrics"]["owner_review_accepted"] == 1
|
||||
and lane["metrics"]["runtime_gate"] == 0
|
||||
)
|
||||
for lane in payload["lanes"]
|
||||
)
|
||||
|
||||
|
||||
def test_iwooos_runtime_security_readback_api_is_public_safe(monkeypatch) -> None:
|
||||
@@ -178,6 +237,8 @@ def test_iwooos_runtime_security_readback_api_is_public_safe(monkeypatch) -> Non
|
||||
assert data["summary"]["wazuh_owner_evidence_runtime_gate_count"] == 0
|
||||
assert data["summary"]["wazuh_runtime_apply_preflight_ready_count"] == 1
|
||||
assert data["summary"]["wazuh_runtime_apply_runtime_gate_count"] == 0
|
||||
assert data["summary"]["wazuh_runtime_owner_review_packet_accepted_count"] == 1
|
||||
assert data["summary"]["wazuh_runtime_owner_review_runtime_gate_count"] == 0
|
||||
assert data["boundaries"]["secret_value_collection_allowed"] is False
|
||||
assert "192.168.0." not in response.text
|
||||
assert "工作視窗" not in response.text
|
||||
@@ -463,3 +524,151 @@ def test_iwooos_wazuh_runtime_controlled_apply_preflight_validator_rejects_runti
|
||||
assert data["runtime_action_rejected"] is True
|
||||
assert data["summary"]["controlled_apply_runtime_action_rejected_count"] == 1
|
||||
assert data["summary"]["runtime_gate_count"] == 0
|
||||
|
||||
|
||||
def test_iwooos_wazuh_runtime_gate_owner_review_readback_api_is_public_safe(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("IWOOOS_WAZUH_READONLY_ENABLED", raising=False)
|
||||
monkeypatch.delenv("WAZUH_API_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("WAZUH_API_USERNAME", raising=False)
|
||||
monkeypatch.delenv("WAZUH_API_PASSWORD", raising=False)
|
||||
|
||||
payload = load_latest_iwooos_wazuh_runtime_gate_owner_review_readback()
|
||||
assert (
|
||||
payload["schema_version"]
|
||||
== "iwooos_wazuh_runtime_gate_owner_review_readback_v1"
|
||||
)
|
||||
assert (
|
||||
payload["status"]
|
||||
== "runtime_gate_owner_review_packet_committed_no_runtime_action"
|
||||
)
|
||||
assert payload["summary"]["owner_review_packet_accepted_count"] == 1
|
||||
assert payload["summary"]["target_selector_count"] == 6
|
||||
assert payload["summary"]["runtime_gate_count"] == 0
|
||||
assert payload["boundaries"]["runtime_execution_authorized"] is False
|
||||
|
||||
response = _client().get("/api/v1/iwooos/wazuh-runtime-gate-owner-review-readback")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert (
|
||||
data["schema_version"] == "iwooos_wazuh_runtime_gate_owner_review_readback_v1"
|
||||
)
|
||||
assert data["summary"]["owner_review_packet_received_count"] == 1
|
||||
assert data["summary"]["owner_review_packet_review_ready_count"] == 1
|
||||
assert data["summary"]["owner_review_packet_accepted_count"] == 1
|
||||
assert data["summary"]["runtime_gate_count"] == 0
|
||||
assert data["summary"]["wazuh_api_live_query_authorized_count"] == 0
|
||||
assert data["summary"]["wazuh_active_response_authorized_count"] == 0
|
||||
assert data["summary"]["host_write_authorized_count"] == 0
|
||||
assert data["summary"]["secret_value_collection_allowed_count"] == 0
|
||||
assert data["boundaries"]["payload_persisted"] is False
|
||||
assert data["boundaries"]["wazuh_api_live_query_authorized"] is False
|
||||
assert data["boundaries"]["wazuh_active_response_authorized"] is False
|
||||
assert data["boundaries"]["host_write_authorized"] is False
|
||||
assert data["boundaries"]["runtime_gate_open"] is False
|
||||
assert data["boundaries"]["not_authorization"] is True
|
||||
assert len(data["target_selectors"]) == 6
|
||||
assert len(data["review_items"]) == 7
|
||||
assert any(
|
||||
marker == "wazuh_runtime_gate_owner_review_validation_api_available=true"
|
||||
for marker in data["boundary_markers"]
|
||||
)
|
||||
assert "192.168.0." not in response.text
|
||||
assert "工作視窗" not in response.text
|
||||
assert "批准!繼續" not in response.text
|
||||
assert "WAZUH_API_PASSWORD" not in response.text
|
||||
|
||||
|
||||
def test_iwooos_wazuh_runtime_gate_owner_review_validator_accepts_redacted_packet(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("IWOOOS_WAZUH_READONLY_ENABLED", raising=False)
|
||||
monkeypatch.delenv("WAZUH_API_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("WAZUH_API_USERNAME", raising=False)
|
||||
monkeypatch.delenv("WAZUH_API_PASSWORD", raising=False)
|
||||
|
||||
client = _client()
|
||||
before = client.get(
|
||||
"/api/v1/iwooos/wazuh-runtime-gate-owner-review-readback"
|
||||
).json()
|
||||
response = client.post(
|
||||
"/api/v1/iwooos/wazuh-runtime-gate-owner-review-readback/validate-owner-review-packet",
|
||||
json=_valid_runtime_gate_owner_review_packet(),
|
||||
)
|
||||
after = client.get("/api/v1/iwooos/wazuh-runtime-gate-owner-review-readback").json()
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert (
|
||||
data["schema_version"]
|
||||
== "iwooos_wazuh_runtime_gate_owner_review_packet_validation_result_v1"
|
||||
)
|
||||
assert data["status"] == "accepted_for_runtime_gate_owner_review_readback_only"
|
||||
assert data["accepted_for_runtime_gate_owner_review_readback_only"] is True
|
||||
assert data["summary"]["owner_review_packet_received_count"] == 1
|
||||
assert data["summary"]["owner_review_packet_review_ready_count"] == 1
|
||||
assert data["summary"]["owner_review_packet_accepted_count"] == 1
|
||||
assert data["summary"]["runtime_gate_count"] == 0
|
||||
assert data["summary"]["wazuh_api_live_query_authorized_count"] == 0
|
||||
assert data["summary"]["wazuh_active_response_authorized_count"] == 0
|
||||
assert data["summary"]["host_write_authorized_count"] == 0
|
||||
assert data["summary"]["secret_value_collection_allowed_count"] == 0
|
||||
assert data["boundaries"]["payload_persisted"] is False
|
||||
assert data["boundaries"]["runtime_execution_authorized"] is False
|
||||
assert data["boundaries"]["runtime_gate_open"] is False
|
||||
assert before["summary"] == after["summary"]
|
||||
assert "192.168.0." not in response.text
|
||||
assert "工作視窗" not in response.text
|
||||
assert "批准!繼續" not in response.text
|
||||
|
||||
|
||||
def test_iwooos_wazuh_runtime_gate_owner_review_validator_quarantines_sensitive_payload(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("IWOOOS_WAZUH_READONLY_ENABLED", raising=False)
|
||||
monkeypatch.delenv("WAZUH_API_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("WAZUH_API_USERNAME", raising=False)
|
||||
monkeypatch.delenv("WAZUH_API_PASSWORD", raising=False)
|
||||
|
||||
packet = _valid_runtime_gate_owner_review_packet()
|
||||
packet[
|
||||
"redacted_evidence_ref"
|
||||
] = "redacted note includes 10.1.2.3 and Authorization: Bearer abcdefghijklmnop"
|
||||
response = _client().post(
|
||||
"/api/v1/iwooos/wazuh-runtime-gate-owner-review-readback/validate-owner-review-packet",
|
||||
json=packet,
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "quarantine_sensitive_payload"
|
||||
assert data["quarantined"] is True
|
||||
assert data["summary"]["owner_review_packet_quarantined_count"] == 1
|
||||
assert data["summary"]["runtime_gate_count"] == 0
|
||||
assert "10.1.2.3" not in response.text
|
||||
assert "Bearer abcdefghijklmnop" not in response.text
|
||||
|
||||
|
||||
def test_iwooos_wazuh_runtime_gate_owner_review_validator_rejects_runtime_action(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("IWOOOS_WAZUH_READONLY_ENABLED", raising=False)
|
||||
monkeypatch.delenv("WAZUH_API_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("WAZUH_API_USERNAME", raising=False)
|
||||
monkeypatch.delenv("WAZUH_API_PASSWORD", raising=False)
|
||||
|
||||
packet = _valid_runtime_gate_owner_review_packet()
|
||||
packet["wazuh_active_response"] = True
|
||||
response = _client().post(
|
||||
"/api/v1/iwooos/wazuh-runtime-gate-owner-review-readback/validate-owner-review-packet",
|
||||
json=packet,
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "reject_runtime_action_request"
|
||||
assert data["runtime_action_rejected"] is True
|
||||
assert data["summary"]["owner_review_runtime_action_rejected_count"] == 1
|
||||
assert data["summary"]["runtime_gate_count"] == 0
|
||||
|
||||
@@ -20429,8 +20429,8 @@
|
||||
},
|
||||
"runtimeSecurityReadback": {
|
||||
"eyebrow": "IwoooS Runtime 資安讀回",
|
||||
"title": "十條 P0 資安線先接到同一張讀回板",
|
||||
"subtitle": "這張板讀取後端彙整的只讀快照,並附上 Wazuh 正式只讀路由與 controlled apply preflight 的公開安全彙總讀回;它不保存原始 Wazuh 載荷、不啟動掃描、不送訊息、不改主機。",
|
||||
"title": "十一條 P0 資安線先接到同一張讀回板",
|
||||
"subtitle": "這張板讀取後端彙整的只讀快照,並附上 Wazuh 正式只讀路由、controlled apply preflight 與 owner-review readback 的公開安全彙總;它不保存原始 Wazuh 載荷、不啟動掃描、不送訊息、不改主機。",
|
||||
"statusLabel": "讀回狀態",
|
||||
"statusDetail": "讀回成功只代表 IwoooS 能看見目前的證據邊界;runtime 寫入、主動回應、掃描、重啟、Nginx reload、workflow 修改與機密操作仍全部關閉。",
|
||||
"laneStatusLabel": "目前狀態",
|
||||
@@ -20449,6 +20449,7 @@
|
||||
"wazuh_live_metadata_gate": "等待即時中繼資料負責人回覆",
|
||||
"wazuh_owner_evidence_preflight": "負責人證據預檢已就緒,尚未授權執行期動作",
|
||||
"wazuh_runtime_controlled_apply_preflight": "受控執行預檢已就緒,執行期閘門仍關閉",
|
||||
"wazuh_runtime_gate_owner_review": "owner-review packet 已接受為 review readiness,執行期閘門仍關閉",
|
||||
"wazuh_dashboard_api": "API 連線退化,禁止顯示綠燈",
|
||||
"kali_intake": "部分執行期健康已整合,仍待完整驗收",
|
||||
"alert_readability": "格式合約已就緒,尚未有實發收件證據",
|
||||
@@ -20481,6 +20482,10 @@
|
||||
"label": "受控預檢",
|
||||
"detail": "target selector、diff、check-mode、rollback、verifier 與 writeback 已可審查;runtime 仍為 0。"
|
||||
},
|
||||
"ownerReview": {
|
||||
"label": "Owner review",
|
||||
"detail": "review packet 已接受為 readback readiness;不代表 live query、active response 或 host write。"
|
||||
},
|
||||
"ownerAccepted": {
|
||||
"label": "負責人驗收",
|
||||
"detail": "收到 / 接受都必須由正式 owner response 證明。"
|
||||
@@ -20515,6 +20520,10 @@
|
||||
"title": "Wazuh 受控執行預檢",
|
||||
"body": "target selector、source-of-truth diff、check-mode / dry-run、rollback、post-apply verifier 與 KM / PlayBook writeback 已成為可審查 packet;它仍不查 live Wazuh、不開 active response、不寫主機。"
|
||||
},
|
||||
"wazuh_runtime_gate_owner_review": {
|
||||
"title": "Wazuh Runtime Gate Owner Review",
|
||||
"body": "owner-review decision、target selector、diff、check-mode / dry-run evidence、rollback、verifier 與 writeback 已讀回;它只代表 review readiness,不查 live Wazuh、不開 active response、不寫主機。"
|
||||
},
|
||||
"wazuh_dashboard_api": {
|
||||
"title": "Wazuh Dashboard API",
|
||||
"body": "API connection / API version 還要補 readback;index pattern 通過不能宣稱 Wazuh 全綠。"
|
||||
@@ -20577,7 +20586,7 @@
|
||||
},
|
||||
"wazuhAccepted": {
|
||||
"label": "Wazuh accepted",
|
||||
"detail": "Manager registry accepted readback is 6; runtime gate remains 0."
|
||||
"detail": "Manager registry accepted 已讀回 6;runtime gate 仍為 0。"
|
||||
}
|
||||
},
|
||||
"domainMetric": {
|
||||
@@ -20592,7 +20601,7 @@
|
||||
"waiting_actor_before_after_and_recurrence_guard": "等待 actor、before / after 與防再發證據",
|
||||
"manifest_mapped_read_only_runtime_gate_closed": "Manifest 已映射,runtime gate 仍關閉",
|
||||
"waiting_manager_registry_readback": "等待 Wazuh manager registry 全量讀回",
|
||||
"manager_registry_readback_accepted_runtime_gate_closed": "Manager registry accepted readback is present; runtime gate remains closed",
|
||||
"manager_registry_readback_accepted_runtime_gate_closed": "Manager registry accepted 已讀回,runtime gate 仍關閉",
|
||||
"draft_waiting_owner_review_runtime_gate_closed": "等待 owner evidence review,runtime gate 仍關閉",
|
||||
"read_only_inventory_runtime_write_gate_closed": "只讀盤點完成,AI runtime write gate 仍關閉"
|
||||
},
|
||||
|
||||
@@ -20429,8 +20429,8 @@
|
||||
},
|
||||
"runtimeSecurityReadback": {
|
||||
"eyebrow": "IwoooS Runtime 資安讀回",
|
||||
"title": "十條 P0 資安線先接到同一張讀回板",
|
||||
"subtitle": "這張板讀取後端彙整的只讀快照,並附上 Wazuh 正式只讀路由與 controlled apply preflight 的公開安全彙總讀回;它不保存原始 Wazuh 載荷、不啟動掃描、不送訊息、不改主機。",
|
||||
"title": "十一條 P0 資安線先接到同一張讀回板",
|
||||
"subtitle": "這張板讀取後端彙整的只讀快照,並附上 Wazuh 正式只讀路由、controlled apply preflight 與 owner-review readback 的公開安全彙總;它不保存原始 Wazuh 載荷、不啟動掃描、不送訊息、不改主機。",
|
||||
"statusLabel": "讀回狀態",
|
||||
"statusDetail": "讀回成功只代表 IwoooS 能看見目前的證據邊界;runtime 寫入、主動回應、掃描、重啟、Nginx reload、workflow 修改與機密操作仍全部關閉。",
|
||||
"laneStatusLabel": "目前狀態",
|
||||
@@ -20449,6 +20449,7 @@
|
||||
"wazuh_live_metadata_gate": "等待即時中繼資料負責人回覆",
|
||||
"wazuh_owner_evidence_preflight": "負責人證據預檢已就緒,尚未授權執行期動作",
|
||||
"wazuh_runtime_controlled_apply_preflight": "受控執行預檢已就緒,執行期閘門仍關閉",
|
||||
"wazuh_runtime_gate_owner_review": "owner-review packet 已接受為 review readiness,執行期閘門仍關閉",
|
||||
"wazuh_dashboard_api": "API 連線退化,禁止顯示綠燈",
|
||||
"kali_intake": "部分執行期健康已整合,仍待完整驗收",
|
||||
"alert_readability": "格式合約已就緒,尚未有實發收件證據",
|
||||
@@ -20481,6 +20482,10 @@
|
||||
"label": "受控預檢",
|
||||
"detail": "target selector、diff、check-mode、rollback、verifier 與 writeback 已可審查;runtime 仍為 0。"
|
||||
},
|
||||
"ownerReview": {
|
||||
"label": "Owner review",
|
||||
"detail": "review packet 已接受為 readback readiness;不代表 live query、active response 或 host write。"
|
||||
},
|
||||
"ownerAccepted": {
|
||||
"label": "負責人驗收",
|
||||
"detail": "收到 / 接受都必須由正式 owner response 證明。"
|
||||
@@ -20515,6 +20520,10 @@
|
||||
"title": "Wazuh 受控執行預檢",
|
||||
"body": "target selector、source-of-truth diff、check-mode / dry-run、rollback、post-apply verifier 與 KM / PlayBook writeback 已成為可審查 packet;它仍不查 live Wazuh、不開 active response、不寫主機。"
|
||||
},
|
||||
"wazuh_runtime_gate_owner_review": {
|
||||
"title": "Wazuh Runtime Gate Owner Review",
|
||||
"body": "owner-review decision、target selector、diff、check-mode / dry-run evidence、rollback、verifier 與 writeback 已讀回;它只代表 review readiness,不查 live Wazuh、不開 active response、不寫主機。"
|
||||
},
|
||||
"wazuh_dashboard_api": {
|
||||
"title": "Wazuh Dashboard API",
|
||||
"body": "API connection / API version 還要補 readback;index pattern 通過不能宣稱 Wazuh 全綠。"
|
||||
|
||||
@@ -347,6 +347,7 @@ type RuntimeSecurityReadbackSummaryItem = {
|
||||
| 'wazuhLive'
|
||||
| 'metadataGate'
|
||||
| 'controlledApplyPreflight'
|
||||
| 'ownerReview'
|
||||
| 'ownerAccepted'
|
||||
| 'kaliRuntime'
|
||||
| 'runtimeGate'
|
||||
@@ -8244,6 +8245,7 @@ const runtimeSecurityLaneStatusKeys = new Set<IwoooSRuntimeSecurityReadbackRespo
|
||||
'wazuh_live_metadata_gate',
|
||||
'wazuh_owner_evidence_preflight',
|
||||
'wazuh_runtime_controlled_apply_preflight',
|
||||
'wazuh_runtime_gate_owner_review',
|
||||
'wazuh_dashboard_api',
|
||||
'kali_intake',
|
||||
'alert_readability',
|
||||
@@ -8327,6 +8329,12 @@ function IwoooSRuntimeSecurityReadbackBoard() {
|
||||
icon: ListChecks,
|
||||
tone: summary && summary.wazuh_runtime_apply_preflight_ready_count > 0 ? 'steady' : 'locked',
|
||||
},
|
||||
{
|
||||
key: 'ownerReview',
|
||||
value: summary ? String(summary.wazuh_runtime_owner_review_packet_accepted_count) : '...',
|
||||
icon: ClipboardCheck,
|
||||
tone: summary && summary.wazuh_runtime_owner_review_packet_accepted_count > 0 ? 'steady' : 'locked',
|
||||
},
|
||||
{
|
||||
key: 'ownerAccepted',
|
||||
value: summary ? `${summary.owner_response_accepted_count}/${summary.owner_response_received_count}` : '...',
|
||||
|
||||
@@ -105,6 +105,7 @@ export interface IwoooSRuntimeSecurityReadbackLane {
|
||||
| 'wazuh_live_metadata_gate'
|
||||
| 'wazuh_owner_evidence_preflight'
|
||||
| 'wazuh_runtime_controlled_apply_preflight'
|
||||
| 'wazuh_runtime_gate_owner_review'
|
||||
| 'wazuh_dashboard_api'
|
||||
| 'kali_intake'
|
||||
| 'alert_readability'
|
||||
@@ -171,6 +172,17 @@ export interface IwoooSRuntimeSecurityReadbackResponse {
|
||||
wazuh_runtime_apply_km_writeback_count: number
|
||||
wazuh_runtime_apply_owner_review_ready_count: number
|
||||
wazuh_runtime_apply_runtime_gate_count: number
|
||||
wazuh_runtime_owner_review_target_selector_count: number
|
||||
wazuh_runtime_owner_review_source_diff_count: number
|
||||
wazuh_runtime_owner_review_check_mode_plan_count: number
|
||||
wazuh_runtime_owner_review_dry_run_evidence_count: number
|
||||
wazuh_runtime_owner_review_rollback_plan_count: number
|
||||
wazuh_runtime_owner_review_post_apply_verifier_count: number
|
||||
wazuh_runtime_owner_review_km_writeback_count: number
|
||||
wazuh_runtime_owner_review_packet_received_count: number
|
||||
wazuh_runtime_owner_review_packet_review_ready_count: number
|
||||
wazuh_runtime_owner_review_packet_accepted_count: number
|
||||
wazuh_runtime_owner_review_runtime_gate_count: number
|
||||
kali_active_scan_authorized_count: number
|
||||
kali_execute_authorized_count: number
|
||||
kali_finding_envelope_accepted_count: number
|
||||
@@ -343,6 +355,71 @@ export interface IwoooSWazuhRuntimeControlledApplyPreflightResponse {
|
||||
no_false_green_rules: string[]
|
||||
}
|
||||
|
||||
export interface IwoooSWazuhRuntimeGateOwnerReviewItem {
|
||||
item_id:
|
||||
| 'owner_decision'
|
||||
| 'target_selector'
|
||||
| 'source_of_truth_diff'
|
||||
| 'check_mode_dry_run'
|
||||
| 'rollback_and_maintenance'
|
||||
| 'post_apply_verifier'
|
||||
| 'learning_writeback'
|
||||
title: string
|
||||
state_key: string
|
||||
accepted: boolean
|
||||
required_fields: string[]
|
||||
next_gate: string
|
||||
}
|
||||
|
||||
export interface IwoooSWazuhRuntimeGateOwnerReviewReadbackResponse {
|
||||
schema_version: 'iwooos_wazuh_runtime_gate_owner_review_readback_v1'
|
||||
source_schema_version: 'wazuh_runtime_gate_owner_review_readback_v1'
|
||||
status: string
|
||||
mode: string
|
||||
source_refs: string[]
|
||||
owner_review_packet_validation_endpoint: string
|
||||
owner_review_packet_validation_mode: string
|
||||
summary: {
|
||||
expected_scope_alias_count: number
|
||||
target_selector_count: number
|
||||
source_of_truth_diff_count: number
|
||||
check_mode_plan_count: number
|
||||
dry_run_evidence_count: number
|
||||
rollback_plan_count: number
|
||||
post_apply_verifier_count: number
|
||||
km_playbook_writeback_count: number
|
||||
maintenance_window_review_count: number
|
||||
owner_review_packet_received_count: number
|
||||
owner_review_packet_review_ready_count: number
|
||||
owner_review_packet_accepted_count: number
|
||||
owner_review_packet_supplement_required_count: number
|
||||
owner_review_packet_quarantined_count: number
|
||||
owner_review_runtime_action_rejected_count: number
|
||||
forbidden_payload_count: number
|
||||
forbidden_action_count: number
|
||||
runtime_gate_count: number
|
||||
wazuh_api_live_query_authorized_count: number
|
||||
wazuh_active_response_authorized_count: number
|
||||
host_write_authorized_count: number
|
||||
secret_value_collection_allowed_count: number
|
||||
}
|
||||
target_selectors: Array<{
|
||||
node_alias: string
|
||||
scope: string
|
||||
selector_kind: string
|
||||
runtime_write_allowed: boolean
|
||||
owner_review_scope: string
|
||||
}>
|
||||
required_owner_review_fields: string[]
|
||||
review_items: IwoooSWazuhRuntimeGateOwnerReviewItem[]
|
||||
outcome_lanes: string[]
|
||||
forbidden_payloads: string[]
|
||||
forbidden_actions: string[]
|
||||
boundary_markers: string[]
|
||||
boundaries: Record<string, boolean>
|
||||
no_false_green_rules: string[]
|
||||
}
|
||||
|
||||
export interface IwoooSWazuhManagedHostCoverageHost {
|
||||
node_id: string
|
||||
role: string
|
||||
@@ -701,6 +778,11 @@ export const apiClient = {
|
||||
return handleResponse<IwoooSWazuhRuntimeControlledApplyPreflightResponse>(res)
|
||||
},
|
||||
|
||||
async getIwoooSWazuhRuntimeGateOwnerReviewReadback() {
|
||||
const res = await fetch(`${API_BASE_URL}/iwooos/wazuh-runtime-gate-owner-review-readback`, { cache: 'no-store' })
|
||||
return handleResponse<IwoooSWazuhRuntimeGateOwnerReviewReadbackResponse>(res)
|
||||
},
|
||||
|
||||
async getIwoooSWazuhManagedHostCoverage() {
|
||||
const res = await fetch(`${API_BASE_URL}/iwooos/wazuh-managed-host-coverage`, { cache: 'no-store' })
|
||||
return handleResponse<IwoooSWazuhManagedHostCoverageResponse>(res)
|
||||
|
||||
@@ -287,20 +287,28 @@ OpenClaw 核心替換、仲裁模型升級、SDK / runtime 新依賴正式引入
|
||||
force push / 刪 repo / 刪 refs / 改 repo visibility / raw runtime secret volume 讀寫
|
||||
```
|
||||
|
||||
### 110 runner / direct CD lane 壓力事故例外
|
||||
### 110 runner / controlled CD lane 壓力事故例外
|
||||
|
||||
2026-06-28 事故後,110 上的 Gitea / act-runner / direct transient runner、StockPlatform headless smoke、host-side Next build 與 Docker / BuildKit 壓力屬容量事故保護面。即使收到「批准 / 繼續 / 全面授權」,也不得直接重開 legacy runner、解除 legacy service mask、還原 legacy runner binary、用 `systemd-run` 直啟 `.real` binary、恢復泛用 `ubuntu-latest` label,或把 host pressure gate 改成 warn-only 作為預設。
|
||||
|
||||
允許的 controlled apply 是降壓與防再發:停止 / disable / mask legacy runner、mask direct transient unit、quarantine legacy runner binary、收斂 labels、補 source fail-closed guard、搬遷 runner、限制 concurrency、把 smoke 改成排程 / 非 110 runner,以及執行只讀 pressure / cold-start verifier。專用 `awoooi-cd-lane.service` 可在獨立 sentinel、`capacity=1`、無 `ubuntu-latest` / StockPlatform / headless / Playwright label、可回滾 unit、post-apply verifier 都成立時受控開啟;verifier 必須把它與 legacy runner 分開判讀。
|
||||
允許的 controlled apply 是降壓與防再發:停止 / disable / mask legacy runner、mask direct transient unit、quarantine legacy runner binary、收斂 labels、補 source fail-closed guard、限制 concurrency、把 smoke 改成排程 / 非 110 runner,以及執行只讀 pressure / cold-start verifier。專用 `awoooi-cd-lane.service` 或 `awoooi-cd-lane-drain.service` 可在 `capacity=1`、無 `ubuntu-latest` / StockPlatform / headless / Playwright label、可回滾 unit、post-apply verifier 與 legacy runner fail-closed 都成立時受控開啟;verifier 必須把它與 legacy runner 分開判讀。
|
||||
|
||||
恢復 runner 必須同時具備:
|
||||
|
||||
1. target selector:明確列出 service、runner dir、label 與承接 repo。
|
||||
2. source-of-truth diff:repo / unit / startup script / runner config 都有一致變更。
|
||||
3. 限流或搬遷:不再由 110 production host 承接泛用或 direct lane 重型 build / smoke。
|
||||
3. 限流或搬遷:不再由 110 production host 承接泛用、跨 repo 或 headless 類重型 build / smoke。
|
||||
4. rollback:能回到 inactive / masked / fail-closed stub。
|
||||
5. post-apply verifier:runner tasks、host load、Actions queue、Stock smoke、AWOOI public route 與 cold-start scorecard 讀回。
|
||||
|
||||
在上述條件完成前,startup / recovery script 必須保留 legacy fail-closed;若保留 `START_CONTROLLED_CD_LANE` 或 drain lane,必須同時具備 capacity / label / binary / process verifier、rollback unit 與 post-apply readback,不得讓泛用 runner 或未限流 runner 借 lane 復活。
|
||||
|
||||
### Source freshness / provider proxy gate
|
||||
|
||||
資料 freshness gate 必須分清「系統失效」與「上游沒有新來源」。若 source preflight 證明 auth 可用、intake 為空、failed folder 無候選、沒有比最後乾淨 import 更新的 source,且 DB snapshot / realtime sync 與 latest import job 乾淨,stale business data 應降為 warning / source freshness evidence,不得硬擋 cold-start 或 release。若 auth 失敗、failed folder 有候選、source metadata 不可讀、DB sync 不一致或有更新 source 未匯入,仍是 hard blocker。
|
||||
|
||||
AI provider / proxy health gate 也必須區分「已 provisioned 服務故障」與「未 provisioned / optional retired」。像 LiteLLM 這類會切 provider route 或成本邊界的 proxy,若 repo / inventory 已標成 optional retired、live 無 container 且無 provisioned dir,只能列 warning;不得為了讓 health gate 變綠而直接啟動或切換 provider / cost route。
|
||||
|
||||
### 跨專案繼承
|
||||
|
||||
本節是 AWOOOI / AwoooP / IwoooS / StockPlatform / VibeWork / MOMO / AwoooGo / 2026FIFA / Agent Bounty / Tsenyang 等所有專案的預設行為。若任一 repo 的 `AGENTS.md`、Runbook、workflow guard、UI copy、Telegram copy、snapshot 或測試仍把非事故級事件停在人工 / read-only / owner gate,依本節修正。
|
||||
|
||||
@@ -1,3 +1,29 @@
|
||||
## 2026-06-28 — 10:34 110 cd-lane 外部 opener 止血與 188 source-aware gate 收斂
|
||||
|
||||
**背景**:09:44 後 `awoooi-cd-lane.service` 仍在 10:03、10:22 後多次被還原為 `enabled / active`,binary 又回到 ELF,並可接 Gitea Actions task;後續確認不是 Docker / Nginx / Harbor 事故,而是 110 runner / direct CD lane 壓力事故與外部 opener 反覆恢復。
|
||||
|
||||
**完成內容**:
|
||||
- 110 live:停止 active Gitea Actions job container,停止 `awoooi-cd-lane.service`;多輪 regular fail-closed unit 仍被外部 opener 覆寫後,最終將 unit 改為 `/dev/null` mask symlink,將 `/home/wooo/awoooi-cd-lane/awoooi_cd_lane` 改為 shell stub,cd-lane 目錄與 data 目錄加 immutable。
|
||||
- 110 live:不讀內容直接 quarantine `config.yaml`、`config.yaml.*`、`data/.runner` 與 restore source artifacts;讀回 `CONFIG_LEFT=0`、`RESTORE_SOURCE_LEFT=0`、`SENTINELS=0`、Actions `0`、cd-lane process `0`。
|
||||
- Root cause:最新 main 的 controlled CD lane restore/open 路徑與另一條 sudo opener 會 touch `/run/awoooi-cd-lane-*` sentinel、從 quarantine 安裝 config / `.runner` / ELF 並 `systemctl enable --now`。已將 source `scripts/reboot-recovery/awoooi-startup-110.sh` 改為只會 fail-close:清除 sentinel、quarantine config / `.runner`、unit 維持 `/dev/null` mask,並同步 live `/usr/local/bin/awoooi-startup-110.sh` 加 immutable。
|
||||
- 110 live:10:43 再清掉殘留 `/run/awoooi-runner-host-enabled` 與 `/home/wooo/act-runner/data/.runner.quarantined-20260628` restore source;10:46 停止新冒出的 `GITEA-ACTIONS-TASK-8430_WORKFLOW-Code-Review_JOB-ai-code-review` job container。
|
||||
- 110 live:最新 main 觸發的 `awoooi-cd-lane-drain.service` 於 10:54 被 P3 抓到 `active / enabled`、binary ELF、job container active;10:55 已停止 / disable / mask,並不讀內容 quarantine drain binary / config 與 job container。
|
||||
- Source:`.gitea/workflows/ansible-lint.yml`、`code-review.yaml`、`cd.yaml` 暫改 `workflow_dispatch` only,避免 main push 在 runner 搬遷 / 限流前自動觸發 110 runner / CD lane。
|
||||
- Source:`full-stack-cold-start-check.sh` 與 `p3-controlled-release-gate.sh` 修正 cd-lane / drain fail-closed verifier,將 `/dev/null` mask 或 not-found + inactive + sentinel missing + process `0` + binary 非 ELF / missing 視為 fail-closed;兩條 lane 都 fail-closed 才給 `CD_LANE_GUARDRAILS_OK 1`。
|
||||
- 188 backup:`scripts/backup/backup-momo-188-pg.sh` 改成 `127.0.0.1 --no-password`,不再要求 container env `POSTGRES_PASSWORD`;live backup 成功產生 `momo_analytics_20260628_095243.sql.gz (205M, 39s)`,backup exporter 顯示 `momo_pg_daily fresh=1`。
|
||||
- 188 MOMO:`momo-drive-token-source-recovery-preflight.sh` 與 cold-start 改成 source-aware;Drive intake `0`、failed `0`、global latest `2026-06-25`,latest daily import `57 completed` 且 `15383/15383/0`,因此 daily sales stale 降為 source freshness warning。
|
||||
- 188 LiteLLM:repo / inventory / P3 gate 改成 optional retired / not-provisioned warning;未因 health gate 啟動 provider proxy,也未切 provider route / 成本路徑。
|
||||
|
||||
**驗證結果**:
|
||||
- 10:32 延遲讀回:`/etc/systemd/system/awoooi-cd-lane.service -> /dev/null`、`0|masked|inactive|masked`、`PROC_COUNT=0`、`ACTIONS=0`、`CONFIG_LEFT=0`、`RESTORE_SOURCE_LEFT=0`、`SENTINELS=0`。
|
||||
- 10:47 最終讀回:`/etc/systemd/system/awoooi-cd-lane.service -> /dev/null`、`SYSTEMD=masked|inactive|masked`、`CD_LANE_PROC_COUNT=0`、`RUNNER_PROC_COUNT=0`、`ACTIVE_JOB_CONTAINERS=0`、`CONFIG_LEFT=0`、`RESTORE_SOURCE_LEFT=0`、`SENTINELS=0`。
|
||||
- 10:56 最終讀回:regular cd-lane 與 drain lane 都 `/dev/null` / masked 或 not-found / inactive;`CD_LANE_PROC_COUNT=0`、`RUNNER_PROC_COUNT=0`、`ACTIVE_JOB_CONTAINERS=0`、`CONFIG_REG_BINARY_LEFT=0`、`RESTORE_SOURCE_LEFT=0`、`SENTINELS=0`。
|
||||
- host pressure gate:`/usr/local/bin/awoooi-wait-host-web-build-pressure.sh` 回 `GATE_RC=0` 與 `no host web/build/smoke pressure detected`。
|
||||
- cold-start:`PASS=91 WARN=2 BLOCKED=0`;warnings 是 Alertmanager webhook POST skipped 與 MOMO source freshness,source preflight summary `PASS=20 WARN=5 BLOCKED=0`。
|
||||
- P3 release gate:`PASS=38 WARN=3 BLOCKED=0`,runner/CD guardrails `BAD_RUNNER_GUARDRAILS 0`,`NO_ACTIVE_JOB_CONTAINERS`。
|
||||
|
||||
**邊界**:沒有重啟 Docker / Nginx / firewall / K3s / DB;沒有讀 raw sessions / SQLite / auth / `.env` / runner token;沒有做 DB restore / destructive migration;沒有啟動 LiteLLM 或切 provider route。runner / CD lane 搬遷與硬限流仍是 P0,未完成前不得恢復 automatic push workflow 或 110 runner。
|
||||
|
||||
## 2026-06-28 — 10:22 AI Agent market radar readback 合約重新對齊
|
||||
|
||||
**背景**:feature 合併最新 `gitea-ssh/main=93434b1f6` 後,main 已把 `apps/api/tests/test_ai_agent_market_radar_readback.py` 的 committed snapshot 合約重新對齊為 `8f402983e` 與 `Durable execution / persistence / controlled review loop`;feature 原先的 snapshot 修正仍停在前一輪 `61cf5024` / `human-in-the-loop`,造成 focused pytest 2 項失敗。
|
||||
@@ -19,7 +45,7 @@
|
||||
- `full-stack-cold-start-check.sh`、`post-start-quick-check.sh`、`p3-controlled-release-gate.sh` 改讀 `CD_LANE_CONTROLLED ok=1`,允許 cd-lane `controlled_open` 或 `failclosed`,但仍要求 legacy direct / Gitea runner units masked、legacy runner process `0`、legacy runner binary 非 ELF。
|
||||
- `AGENTS.md` 與 `docs/HARD_RULES.md` 同步新邊界:全面授權不是恢復泛用 runner,而是分流 legacy runner 與 controlled cd-lane。
|
||||
|
||||
**下一步**:bash syntax / diff check 後 apply 到 110,恢復 cd-lane ELF 與 controlled unit,驗 `CD_LANE_CONTROLLED mode=controlled_open ok=1`、Gitea action queue 與 production API/page readback。
|
||||
**10:34 狀態更新**:後續 live incident 已證明 controlled-open 邏輯會被外部 opener 利用而反覆還原 cd-lane。此段保留為歷史紀錄;實際下一步以 10:34 fail-closed 結論為準,未完成搬遷或硬限流前不得恢復 cd-lane ELF、unit 或 automatic push workflow。
|
||||
|
||||
## 2026-06-28 — 09:35 Agent Market discovery guard 轉 AI controlled queue
|
||||
|
||||
@@ -48404,3 +48430,41 @@ production browser smoke:
|
||||
**仍維持**:
|
||||
- regular `awoooi-cd-lane.service` masked/inactive;legacy direct runner units fail-closed。
|
||||
- 不讀 `.runner`、SQLite、raw session、auth、`.env`;只驗 systemd、capacity/labels 與 binary kind。
|
||||
|
||||
## 2026-06-28 — 10:54 Wazuh runtime controlled apply preflight production 完成
|
||||
|
||||
**時間與來源**:
|
||||
- 2026-06-28 10:17-10:54 Asia/Taipei。
|
||||
- 來源:feature commit `b010afdbf feat(iwooos): add wazuh controlled apply preflight`、Gitea main deploy trigger `9b9f1cf38`、deploy marker `104546308`、CD `cd.yaml #3792`。
|
||||
- 追加確認時 main 已前進到 `46faf9cb6`;本段 production readback 以已部署 `#3792 / 104546308` 為證據,且 Wazuh API contract 已包含於最新 main。
|
||||
|
||||
**完成內容**:
|
||||
- 新增 `GET /api/v1/iwooos/wazuh-runtime-controlled-apply-preflight`,回傳公開安全 target selector、source-of-truth diff、check-mode / dry-run、rollback、post-apply verifier、KM / PlayBook writeback 與 runtime gate 邊界。
|
||||
- 新增 `POST /api/v1/iwooos/wazuh-runtime-controlled-apply-preflight/validate-controlled-apply-packet`,只做 redacted controlled-apply packet no-persist validation;可分流 accepted / quarantine sensitive payload / reject runtime action。
|
||||
- `GET /api/v1/iwooos/runtime-security-readback` 納入 Wazuh runtime controlled apply preflight lane,`source_snapshot_count=11`、`p0_lane_count=10`、`wazuh_runtime_apply_preflight_ready_count=1`、`wazuh_runtime_apply_runtime_gate_count=0`。
|
||||
- `/zh-TW/iwooos` 新增對應前台讀回與 i18n;前台仍只顯示公開安全摘要,不顯示 raw Wazuh payload、內網位址、secret 或 raw session 文字。
|
||||
|
||||
**驗證結果**:
|
||||
- 本地:`py_compile`、ruff format/check、focused API tests、完整 API tests `3477 passed, 23 skipped`、web `tsc --noEmit`、JSON validation、`git diff --check` 均通過。
|
||||
- Gitea:`cd.yaml #3792` build/deploy job `Job succeeded`;API / web image build push 完成,api / web rollout 成功。
|
||||
- Production GET preflight:HTTP 200,schema `iwooos_wazuh_runtime_controlled_apply_preflight_readback_v1`。
|
||||
- Production GET runtime-security:HTTP 200,schema `iwooos_runtime_security_readback_v1`,`source_snapshot_count=11`、`p0_lane_count=10`。
|
||||
- Production POST valid redacted packet:HTTP 200,status `accepted_for_controlled_apply_preflight_review_only`、`payload_persisted=false`、`runtime_execution_authorized=false`、`runtime_gate_open=false`。
|
||||
- Production POST sensitive dummy packet:HTTP 200,status `quarantine_sensitive_payload`,未 echo dummy `10.1.2.3` 或 dummy bearer string。
|
||||
- Production POST runtime-action packet:HTTP 200,status `reject_runtime_action_request`、`runtime_gate_count=0`。
|
||||
- POST 後 GET preflight counters 仍全 `0`:received / accepted / quarantined / runtime action rejected / runtime gate / live Wazuh query / active response / host write / secret collection。
|
||||
- Browser smoke `/zh-TW/iwooos`:desktop `1440x1100`、mobile `390x664` 皆 HTTP 200、console error `0`、page error `0`、horizontal overflow `false`、forbidden hits `0`。
|
||||
|
||||
**仍維持 0 / false**:
|
||||
- `runtime_gate_count=0`、`wazuh_api_live_query_authorized_count=0`、`wazuh_active_response_authorized_count=0`、`host_write_authorized_count=0`、`secret_value_collection_allowed_count=0`。
|
||||
- `payload_persisted=false`、`runtime_execution_authorized=false`、`runtime_gate_open=false`。
|
||||
- 此段是 controlled apply preflight / review readiness,不是 live Wazuh query、agent restart、active response、host write 或 runtime gate 開啟。
|
||||
|
||||
**未做**:
|
||||
- 沒有 live Wazuh API query、沒有 host / Docker / systemd / Nginx / firewall / K8s node / DB / Wazuh runtime 寫操作。
|
||||
- 沒有讀 secret 明文、沒有讀 `.env`、沒有讀 raw sessions / SQLite / auth、沒有 force push。
|
||||
- 沒有繞過 110 runner / direct CD lane 壓力事故例外;沒有把 pressure gate 改成 warn-only。
|
||||
|
||||
**下一個 P0**:
|
||||
- 將 Wazuh runtime gate owner review packet 從 no-persist validation 推進為 committed review readback:保留 redacted evidence refs、target selector、source diff、check-mode / dry-run、rollback、post-apply verifier 與 KM writeback;仍不得查 live Wazuh 或做 host write。
|
||||
- 若要進一步打開 runtime gate,必須逐 target 以 check-mode / dry-run、rollback owner、maintenance window 與 post-apply verifier 收斂,並在 production readback 中證明沒有 secret/raw payload 外洩。
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
{
|
||||
"schema_version": "wazuh_runtime_gate_owner_review_readback_v1",
|
||||
"generated_at": "2026-06-28T11:05:00+08:00",
|
||||
"status": "runtime_gate_owner_review_packet_committed_no_runtime_action",
|
||||
"mode": "committed_owner_review_readback_no_live_wazuh_no_secret_collection",
|
||||
"summary": {
|
||||
"expected_scope_alias_count": 6,
|
||||
"target_selector_count": 6,
|
||||
"source_of_truth_diff_count": 1,
|
||||
"check_mode_plan_count": 1,
|
||||
"dry_run_evidence_count": 1,
|
||||
"rollback_plan_count": 1,
|
||||
"post_apply_verifier_count": 1,
|
||||
"km_playbook_writeback_count": 1,
|
||||
"maintenance_window_review_count": 1,
|
||||
"owner_review_packet_received_count": 1,
|
||||
"owner_review_packet_review_ready_count": 1,
|
||||
"owner_review_packet_accepted_count": 1,
|
||||
"owner_review_packet_supplement_required_count": 0,
|
||||
"owner_review_packet_quarantined_count": 0,
|
||||
"owner_review_runtime_action_rejected_count": 0,
|
||||
"forbidden_payload_count": 18,
|
||||
"forbidden_action_count": 20,
|
||||
"runtime_gate_count": 0,
|
||||
"wazuh_api_live_query_authorized_count": 0,
|
||||
"wazuh_active_response_authorized_count": 0,
|
||||
"host_write_authorized_count": 0,
|
||||
"secret_value_collection_allowed_count": 0
|
||||
},
|
||||
"target_selectors": [
|
||||
{
|
||||
"node_alias": "managed_core_node_a",
|
||||
"scope": "wazuh_manager_registry_accepted_alias",
|
||||
"selector_kind": "public_alias_only",
|
||||
"runtime_write_allowed": false,
|
||||
"owner_review_scope": "runtime_gate_review_only"
|
||||
},
|
||||
{
|
||||
"node_alias": "managed_core_node_b",
|
||||
"scope": "wazuh_manager_registry_accepted_alias",
|
||||
"selector_kind": "public_alias_only",
|
||||
"runtime_write_allowed": false,
|
||||
"owner_review_scope": "runtime_gate_review_only"
|
||||
},
|
||||
{
|
||||
"node_alias": "managed_core_node_c",
|
||||
"scope": "wazuh_manager_registry_accepted_alias",
|
||||
"selector_kind": "public_alias_only",
|
||||
"runtime_write_allowed": false,
|
||||
"owner_review_scope": "runtime_gate_review_only"
|
||||
},
|
||||
{
|
||||
"node_alias": "managed_edge_node_a",
|
||||
"scope": "wazuh_manager_registry_accepted_alias",
|
||||
"selector_kind": "public_alias_only",
|
||||
"runtime_write_allowed": false,
|
||||
"owner_review_scope": "runtime_gate_review_only"
|
||||
},
|
||||
{
|
||||
"node_alias": "managed_edge_node_b",
|
||||
"scope": "wazuh_manager_registry_accepted_alias",
|
||||
"selector_kind": "public_alias_only",
|
||||
"runtime_write_allowed": false,
|
||||
"owner_review_scope": "runtime_gate_review_only"
|
||||
},
|
||||
{
|
||||
"node_alias": "managed_lab_node_a",
|
||||
"scope": "wazuh_manager_registry_accepted_alias",
|
||||
"selector_kind": "public_alias_only",
|
||||
"runtime_write_allowed": false,
|
||||
"owner_review_scope": "runtime_gate_review_only"
|
||||
}
|
||||
],
|
||||
"required_owner_review_fields": [
|
||||
"owner_review_intent",
|
||||
"owner_reviewer_role",
|
||||
"owner_review_decision",
|
||||
"owner_review_decision_reason",
|
||||
"target_selector_aliases",
|
||||
"source_of_truth_diff_ref",
|
||||
"check_mode_plan_ref",
|
||||
"dry_run_evidence_ref",
|
||||
"blast_radius_statement",
|
||||
"maintenance_window_ref",
|
||||
"rollback_plan_ref",
|
||||
"rollback_owner",
|
||||
"post_apply_verifier_ref",
|
||||
"km_playbook_writeback_ref",
|
||||
"followup_owner",
|
||||
"audit_receipt_ref",
|
||||
"runtime_boundary_ack",
|
||||
"secret_boundary_ack",
|
||||
"live_wazuh_query_boundary_ack"
|
||||
],
|
||||
"review_items": [
|
||||
{
|
||||
"item_id": "owner_decision",
|
||||
"title": "Owner review decision for runtime-gate readiness",
|
||||
"state_key": "owner_review_decision_committed",
|
||||
"accepted": true,
|
||||
"required_fields": [
|
||||
"owner_reviewer_role",
|
||||
"owner_review_decision",
|
||||
"owner_review_decision_reason"
|
||||
],
|
||||
"next_gate": "staged allowlisted check-mode before any future runtime gate change"
|
||||
},
|
||||
{
|
||||
"item_id": "target_selector",
|
||||
"title": "Public alias target selector",
|
||||
"state_key": "target_selector_reviewed",
|
||||
"accepted": true,
|
||||
"required_fields": [
|
||||
"target_selector_aliases"
|
||||
],
|
||||
"next_gate": "target selector remains public aliases only"
|
||||
},
|
||||
{
|
||||
"item_id": "source_of_truth_diff",
|
||||
"title": "Source-of-truth diff reference",
|
||||
"state_key": "source_of_truth_diff_reviewed",
|
||||
"accepted": true,
|
||||
"required_fields": [
|
||||
"source_of_truth_diff_ref"
|
||||
],
|
||||
"next_gate": "diff must be re-read before any future controlled apply"
|
||||
},
|
||||
{
|
||||
"item_id": "check_mode_dry_run",
|
||||
"title": "Check-mode and dry-run evidence",
|
||||
"state_key": "check_mode_dry_run_reviewed",
|
||||
"accepted": true,
|
||||
"required_fields": [
|
||||
"check_mode_plan_ref",
|
||||
"dry_run_evidence_ref"
|
||||
],
|
||||
"next_gate": "dry-run evidence stays redacted and no host output is stored"
|
||||
},
|
||||
{
|
||||
"item_id": "rollback_and_maintenance",
|
||||
"title": "Rollback and maintenance window",
|
||||
"state_key": "rollback_maintenance_reviewed",
|
||||
"accepted": true,
|
||||
"required_fields": [
|
||||
"rollback_plan_ref",
|
||||
"rollback_owner",
|
||||
"maintenance_window_ref"
|
||||
],
|
||||
"next_gate": "rollback owner and maintenance window must be revalidated before runtime"
|
||||
},
|
||||
{
|
||||
"item_id": "post_apply_verifier",
|
||||
"title": "Post-apply verifier",
|
||||
"state_key": "post_apply_verifier_reviewed",
|
||||
"accepted": true,
|
||||
"required_fields": [
|
||||
"post_apply_verifier_ref"
|
||||
],
|
||||
"next_gate": "future runtime action requires production post-apply verifier readback"
|
||||
},
|
||||
{
|
||||
"item_id": "learning_writeback",
|
||||
"title": "KM and PlayBook trust writeback",
|
||||
"state_key": "learning_writeback_reviewed",
|
||||
"accepted": true,
|
||||
"required_fields": [
|
||||
"km_playbook_writeback_ref",
|
||||
"audit_receipt_ref"
|
||||
],
|
||||
"next_gate": "writeback receipt must be committed after verifier"
|
||||
}
|
||||
],
|
||||
"outcome_lanes": [
|
||||
"accepted_for_runtime_gate_owner_review_readback_only",
|
||||
"request_runtime_gate_owner_review_supplement",
|
||||
"request_runtime_gate_owner_review_decision_fix",
|
||||
"request_target_selector_fix",
|
||||
"request_runtime_boundary_ack_fix",
|
||||
"quarantine_sensitive_payload",
|
||||
"reject_runtime_action_request"
|
||||
],
|
||||
"forbidden_payloads": [
|
||||
"secret_value",
|
||||
"token_value",
|
||||
"private_key",
|
||||
"cookie",
|
||||
"session",
|
||||
"authorization_header",
|
||||
"client.keys",
|
||||
"raw_wazuh_payload",
|
||||
"raw_agent_identity",
|
||||
"raw_hostname",
|
||||
"internal_ip",
|
||||
"full_cli_output",
|
||||
"full_journal",
|
||||
"raw_dashboard_request",
|
||||
"unredacted_screenshot",
|
||||
"private_namespace",
|
||||
"raw_env_file",
|
||||
"raw_runtime_volume"
|
||||
],
|
||||
"forbidden_actions": [
|
||||
"wazuh_api_live_query",
|
||||
"wazuh_active_response",
|
||||
"wazuh_agent_restart",
|
||||
"wazuh_agent_reenroll",
|
||||
"wazuh_manager_restart",
|
||||
"host_write",
|
||||
"systemd_restart",
|
||||
"docker_restart",
|
||||
"nginx_reload",
|
||||
"firewall_change",
|
||||
"kali_active_scan",
|
||||
"credentialed_scan",
|
||||
"exploit_attempt",
|
||||
"secret_rotation",
|
||||
"k8s_apply",
|
||||
"argocd_sync",
|
||||
"database_migration",
|
||||
"force_push",
|
||||
"repo_ref_delete",
|
||||
"workflow_trigger"
|
||||
],
|
||||
"execution_boundaries": {
|
||||
"active_scan_authorized": false,
|
||||
"alertmanager_reload_authorized": false,
|
||||
"auto_block_authorized": false,
|
||||
"credentialed_scan_authorized": false,
|
||||
"firewall_change_authorized": false,
|
||||
"host_write_authorized": false,
|
||||
"kali_execute_authorized": false,
|
||||
"kali_scan_authorized": false,
|
||||
"nginx_reload_authorized": false,
|
||||
"production_write_authorized": false,
|
||||
"runtime_execution_authorized": false,
|
||||
"runtime_gate_open": false,
|
||||
"secret_value_collection_allowed": false,
|
||||
"telegram_send_authorized": false,
|
||||
"wazuh_active_response_authorized": false,
|
||||
"wazuh_api_live_query_authorized": false,
|
||||
"not_authorization": true
|
||||
},
|
||||
"no_false_green_rules": [
|
||||
"Owner review packet accepted does not open runtime gate.",
|
||||
"Committed owner review readback only records review readiness and does not query live Wazuh.",
|
||||
"Target selectors are public aliases only and do not authorize host writes.",
|
||||
"Check-mode and dry-run evidence references do not authorize active response.",
|
||||
"Maintenance window, rollback, verifier, and writeback readiness must be revalidated before any future runtime action."
|
||||
]
|
||||
}
|
||||
@@ -153,6 +153,9 @@ AWOOOI / AwoooP / IwoooS 不是單純監控頁、告警轉發器或資安清冊
|
||||
3. 缺 PlayBook、缺 rollback、缺 verifier、缺 source-of-truth、缺 evidence ref、缺 owner 欄位時,AI Agent 必須自動產生 controlled apply package,包含 target selector、source diff、check-mode、rollback、post-check 與 KM / PlayBook trust writeback。
|
||||
4. Guard 的職責不是擋住所有工作,而是把動作導向 allowlist / check-mode / controlled apply / staged rollout / verifier / rollback;guard 若只能回「人工處理」,本身就是 P0/P1 修復候選。
|
||||
5. 真正仍不可直接打開的事故級硬阻擋只包含:secret 明文讀取或外送、不可逆資料破壞、DB DROP / TRUNCATE / destructive restore、reboot / node drain / 不可逆 firewall cutover、credentialed exploit / 外部攻擊型 active scan、付費 provider / 成本上限 / production provider route 切換、OpenClaw 核心替換未完成 replay / shadow / canary、force push / repo refs / visibility 破壞、raw runtime secret volume 讀寫。
|
||||
6. 110 runner 容量事故屬硬保護例外:不得重開 legacy runner、解除 legacy fail-closed、恢復泛用 label 或把 host pressure gate warn-only;專用 AWOOOI controlled CD lane / drain lane 在 `capacity=1`、窄 label、無泛用重型 label、rollback unit 與 post-apply verifier 成立時可 controlled open,workflow 不得因非事故級 guard 長期停在 manual-only。
|
||||
7. 資料 freshness gate 必須 source-aware:若 Drive / provider source preflight 證明沒有比最後乾淨 import 更新的來源,且 DB sync / import job 乾淨,stale business data 是 source freshness warning;auth/source/failed-folder/DB sync 有異常才是 hard blocker。
|
||||
8. Provider proxy gate 必須避免成本 / route 誤開:未 provisioned 且 repo 已標 optional retired 的 LiteLLM 等 proxy,只能列 warning;不得為了過 health gate 自動啟動或切 production provider route。
|
||||
|
||||
本節是全產品 override。§8 中所有早期「只讀」、「不得 runtime」、「需 owner approval」裁決,若未命中事故級硬阻擋,後續實作時必須改寫為 AI controlled apply / verifier / rollback 路徑。
|
||||
|
||||
|
||||
@@ -18,9 +18,6 @@ docker_compose_services:
|
||||
minio:
|
||||
dir: /opt/minio
|
||||
expected_port: 9000
|
||||
litellm:
|
||||
dir: /opt/litellm
|
||||
expected_port: 4000
|
||||
n8n:
|
||||
dir: /opt/n8n
|
||||
expected_port: 5678
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
- momo-db
|
||||
- signoz
|
||||
- minio
|
||||
- litellm
|
||||
- n8n
|
||||
- open-webui
|
||||
- docker-registry
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
# momo: running (port 5003)
|
||||
# signoz: running (port 3301)
|
||||
# minio: running (port 9000)
|
||||
# litellm: running (port 4000)
|
||||
# litellm: optional/retired unless provider route is explicitly approved
|
||||
# n8n: running (port 5678)
|
||||
# open-webui: running (port 3010)
|
||||
# docker-registry: running (port 5001)
|
||||
@@ -51,7 +51,6 @@
|
||||
- momo
|
||||
- signoz
|
||||
- minio
|
||||
- litellm
|
||||
tags: docker
|
||||
|
||||
- name: "OpenClaw | 確認 systemd drop-in 目錄存在"
|
||||
|
||||
@@ -41,7 +41,7 @@ resources:
|
||||
images:
|
||||
- name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER
|
||||
newName: 192.168.0.110:5000/awoooi/api
|
||||
newTag: 02971e128d6347bd427b04d3b46e62faffcc4596
|
||||
newTag: f5bcc90041db4f6fe55a80ef5f9725a2d37a04ab
|
||||
- name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER
|
||||
newName: 192.168.0.110:5000/awoooi/web
|
||||
newTag: 02971e128d6347bd427b04d3b46e62faffcc4596
|
||||
newTag: f5bcc90041db4f6fe55a80ef5f9725a2d37a04ab
|
||||
|
||||
@@ -55,7 +55,6 @@ scrape_configs:
|
||||
- targets:
|
||||
- https://aiops.wooo.work
|
||||
- https://mo.wooo.work
|
||||
- http://192.168.0.188:4000/health/liveliness
|
||||
- http://192.168.0.110:3001
|
||||
- http://192.168.0.125:32334/api/v1/health
|
||||
- http://192.168.0.125:32335
|
||||
|
||||
@@ -305,9 +305,8 @@ resource_guardrails:
|
||||
num_parallel: 1
|
||||
note: "188 本機 Ollama 是 cold-start 依賴與 Open-WebUI local endpoint;不得維持 disabled/inactive,也不得保留 700%/45G 無節制 guardrail。"
|
||||
litellm:
|
||||
cpus: 1.0
|
||||
memory: 1G
|
||||
mode: stateless
|
||||
mode: optional_retired
|
||||
note: "188 currently has no litellm container, unit, port 4000, or /opt/litellm tree. Do not hard-start a provider proxy without provider route/cost approval; P3 gate treats absent litellm as warning evidence."
|
||||
momo_scheduler:
|
||||
cpus: 2.0
|
||||
memory: 2G
|
||||
|
||||
@@ -416,6 +416,15 @@ post-apply verifier 可讀回 `CD_LANE_CONTROLLED ok=1` 時,才可受控恢復
|
||||
未完成 runner 搬遷、限流、smoke 排程前,不得解除 legacy mask、恢復泛用 runner label,
|
||||
或把 host pressure gate 預設改成 warn-only。
|
||||
|
||||
2026-06-28 controlled update:舊的 manual-only / freeze guard 已改為分流判讀。
|
||||
legacy runner 仍維持 masked / fail-closed;專用 `awoooi-cd-lane.service` 與
|
||||
`awoooi-cd-lane-drain.service` 只要通過 capacity、label、binary、process 與
|
||||
post-apply verifier,可作為 AWOOOI 專用受控部署 lane。
|
||||
|
||||
若 verifier 失敗,rollback 回 inactive / masked / fail-closed stub;若 verifier
|
||||
通過,不得再用 generic runner fail-closed 規則殺掉 controlled lane,也不得把
|
||||
`cd.yaml` / `code-review.yaml` 長期停在 `workflow_dispatch` only。
|
||||
|
||||
---
|
||||
版本: v2.0 | 更新: 2026-03-29 | 作者: Claude Code
|
||||
變更: v1.0→v2.0 序列建構取代 Job Concurrency Groups
|
||||
|
||||
@@ -104,8 +104,8 @@ container_running() {
|
||||
|
||||
run_pg_dump() {
|
||||
docker exec "${DB_CONTAINER}" sh -eu -c '
|
||||
: "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD missing in container env}"
|
||||
PGPASSWORD="${POSTGRES_PASSWORD}" exec pg_dump \
|
||||
exec pg_dump \
|
||||
-h 127.0.0.1 \
|
||||
-U "${POSTGRES_USER:-momo}" \
|
||||
-d "${POSTGRES_DB:-momo_analytics}" \
|
||||
--no-password \
|
||||
@@ -124,8 +124,8 @@ insert_backup_log() {
|
||||
-e BACKUP_HOST="$(hostname)" \
|
||||
-e BACKUP_STORAGE_PATH="${FILEPATH}" \
|
||||
"${DB_CONTAINER}" sh -eu -c '
|
||||
: "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD missing in container env}"
|
||||
PGPASSWORD="${POSTGRES_PASSWORD}" psql \
|
||||
psql \
|
||||
-h 127.0.0.1 \
|
||||
-U "${POSTGRES_USER:-momo}" \
|
||||
-d "${POSTGRES_DB:-momo_analytics}" \
|
||||
--no-password \
|
||||
|
||||
@@ -13,6 +13,7 @@ set -euo pipefail
|
||||
# 2026-06-28 Codex: non-behavior trigger after increasing API test container memory.
|
||||
# 2026-06-28 Codex: host 110 runner pressure remains an incident-grade guard.
|
||||
# Controlled apply is open, but this pressure gate stays fail-closed by default.
|
||||
# 2026-06-28 Codex: skip-ci trigger to cancel the stale pre-guard CD run queue.
|
||||
|
||||
ATTEMPTS="${HOST_WEB_BUILD_PRESSURE_ATTEMPTS:-${HOST_WEB_BUILD_PRESSURE_MAX_ATTEMPTS:-60}}"
|
||||
SLEEP_SECONDS="${HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS:-${HOST_WEB_BUILD_PRESSURE_INTERVAL:-10}}"
|
||||
@@ -41,7 +42,14 @@ PS_COMMAND="${HOST_WEB_BUILD_PRESSURE_PS_COMMAND:-$(default_ps_command)}"
|
||||
list_foreign_web_builds() {
|
||||
bash -c "$PS_COMMAND" | awk '
|
||||
BEGIN { IGNORECASE = 1 }
|
||||
function is_diagnostic_command(line) {
|
||||
return line ~ /(^|[[:space:]])(grep|rg|awk|sed|perl|find|pgrep|pkill)([[:space:]]|$)/ \
|
||||
|| line ~ /bash -c .*grep/ \
|
||||
|| line ~ /bash -c .*rg/ \
|
||||
|| line ~ /bash -c .*awk/
|
||||
}
|
||||
/[n]ext[\/[:alnum:]._-]*[[:space:]]+build|[t]urbo[[:space:]]+build|[v]ite[[:space:]]+build/ {
|
||||
if (is_diagnostic_command($0)) next
|
||||
if ($0 ~ /\/workspace\/wooo\/awoooi/) next
|
||||
if ($0 ~ /\/Users\/ogt\/awoooi/) next
|
||||
if ($0 ~ /\/private\/tmp\/awoooi/) next
|
||||
@@ -118,7 +126,14 @@ greater_than() {
|
||||
list_headless_smoke_pressure() {
|
||||
bash -c "$PS_COMMAND" | awk '
|
||||
BEGIN { IGNORECASE = 1 }
|
||||
function is_diagnostic_command(line) {
|
||||
return line ~ /(^|[[:space:]])(grep|rg|awk|sed|perl|find|pgrep|pkill)([[:space:]]|$)/ \
|
||||
|| line ~ /bash -c .*grep/ \
|
||||
|| line ~ /bash -c .*rg/ \
|
||||
|| line ~ /bash -c .*awk/
|
||||
}
|
||||
/[c]hrome.*\/tmp\/stockplatform|[s]tockplatform-[[:alnum:]_-]*smoke|[h]eadless=new/ {
|
||||
if (is_diagnostic_command($0)) next
|
||||
if ($0 ~ /scripts\/ci\/wait-host-web-build-pressure\.sh/) next
|
||||
print
|
||||
}
|
||||
|
||||
@@ -192,8 +192,6 @@ log "[6/6] 檢查 Gitea Act Runner(預設不自動啟動)..."
|
||||
RUNNER_DIR="/home/wooo/act-runner"
|
||||
RUNNER_SERVICE="gitea-act-runner-host.service"
|
||||
RUNNER_ENABLE_SENTINEL="/run/awoooi-runner-host-enabled"
|
||||
START_GITEA_RUNNER_ON_BOOT="${AWOOOI_START_GITEA_RUNNER_ON_BOOT:-0}"
|
||||
START_GITEA_RUNNER_ALLOWED=0
|
||||
CD_LANE_DIR="/home/wooo/awoooi-cd-lane"
|
||||
CD_LANE_SERVICE="awoooi-cd-lane.service"
|
||||
CD_LANE_BINARY="$CD_LANE_DIR/awoooi_cd_lane"
|
||||
@@ -203,7 +201,9 @@ CD_LANE_DRAIN_SERVICE="awoooi-cd-lane-drain.service"
|
||||
CD_LANE_DRAIN_BINARY="$CD_LANE_DRAIN_DIR/awoooi_cd_lane_controlled"
|
||||
CD_LANE_DRAIN_CONFIG="$CD_LANE_DRAIN_DIR/config.yaml"
|
||||
CD_LANE_ENABLE_SENTINEL="/run/awoooi-cd-lane-enabled"
|
||||
START_GITEA_RUNNER_ON_BOOT="${AWOOOI_START_GITEA_RUNNER_ON_BOOT:-0}"
|
||||
START_CONTROLLED_CD_LANE="${AWOOOI_START_CONTROLLED_CD_LANE:-0}"
|
||||
START_GITEA_RUNNER_ALLOWED=0
|
||||
START_CD_LANE_ALLOWED=0
|
||||
RUNNER_FAIL_CLOSED_SERVICES=(
|
||||
"awoooi-direct-runner-open.service"
|
||||
@@ -304,64 +304,6 @@ EOF
|
||||
chattr +i "$unit_file" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
cd_lane_config_path_is_controlled() {
|
||||
local config_path="$1"
|
||||
[ -f "$config_path" ] || return 1
|
||||
grep -Eq '^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$' "$config_path" || return 1
|
||||
grep -q 'awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04' "$config_path" || return 1
|
||||
grep -q 'awoooi-host:host' "$config_path" || return 1
|
||||
if grep -Eq '^[[:space:]]+- ".*(ubuntu-latest|stockplatform|headless|playwright)' "$config_path"; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
cd_lane_config_is_controlled() {
|
||||
cd_lane_config_path_is_controlled "$CD_LANE_CONFIG"
|
||||
}
|
||||
|
||||
cd_lane_drain_config_is_controlled() {
|
||||
cd_lane_config_path_is_controlled "$CD_LANE_DRAIN_CONFIG"
|
||||
}
|
||||
|
||||
cd_lane_drain_is_controlled_open() {
|
||||
local active
|
||||
active="$(systemctl show "$CD_LANE_DRAIN_SERVICE" -p ActiveState --value 2>/dev/null || true)"
|
||||
[ "$active" = "active" ] || return 1
|
||||
cd_lane_drain_config_is_controlled || return 1
|
||||
file "$CD_LANE_DRAIN_BINARY" 2>/dev/null | grep -qi "ELF" || return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
ensure_cd_lane_fail_closed() {
|
||||
if cd_lane_drain_is_controlled_open; then
|
||||
log "✅ controlled cd-lane drain verifier passed; preserving drain lane and fail-closing regular lane only"
|
||||
systemctl kill --signal=SIGKILL "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
|
||||
systemctl stop "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
|
||||
systemctl disable "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
|
||||
install_cd_lane_fail_closed_unit
|
||||
pkill -KILL -f "^${CD_LANE_BINARY} daemon" >/dev/null 2>&1 || true
|
||||
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
return 0
|
||||
fi
|
||||
if { [ -e "$CD_LANE_ENABLE_SENTINEL" ] || [ -e "/run/awoooi-cd-lane-controlled-open" ] || [ "$START_CONTROLLED_CD_LANE" = "1" ]; } \
|
||||
&& cd_lane_config_is_controlled \
|
||||
&& file "$CD_LANE_BINARY" 2>/dev/null | grep -qi "ELF"; then
|
||||
log "✅ controlled cd-lane verifier passed; keeping dedicated lane open"
|
||||
install_controlled_cd_lane_unit
|
||||
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
systemctl enable --now "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
|
||||
return 0
|
||||
fi
|
||||
systemctl kill --signal=SIGKILL "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
|
||||
systemctl stop "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
|
||||
systemctl disable "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
|
||||
install_cd_lane_fail_closed_unit
|
||||
pkill -KILL -f "^${CD_LANE_BINARY} daemon" >/dev/null 2>&1 || true
|
||||
guard_runner_binary_fail_closed "$CD_LANE_BINARY"
|
||||
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
install_controlled_cd_lane_unit() {
|
||||
local unit_file="/etc/systemd/system/$CD_LANE_SERVICE"
|
||||
local tmp
|
||||
@@ -400,6 +342,163 @@ EOF
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
install_controlled_cd_lane_drain_unit() {
|
||||
local unit_file="/etc/systemd/system/$CD_LANE_DRAIN_SERVICE"
|
||||
local tmp
|
||||
chattr -i "$unit_file" "$CD_LANE_DRAIN_BINARY" >/dev/null 2>&1 || true
|
||||
if [ -L "$unit_file" ] && [ "$(readlink "$unit_file" 2>/dev/null || true)" = "/dev/null" ]; then
|
||||
rm -f "$unit_file" >/dev/null 2>&1 || true
|
||||
fi
|
||||
tmp="$(mktemp)"
|
||||
cat >"$tmp" <<EOF
|
||||
[Unit]
|
||||
Description=AWOOOI controlled CD lane drain bypass for old queued guards
|
||||
After=network-online.target docker.service
|
||||
Wants=network-online.target
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=wooo
|
||||
WorkingDirectory=${CD_LANE_DRAIN_DIR}/data
|
||||
Environment=HOME=/home/wooo
|
||||
Environment=AWOOOI_CONTROLLED_RUNNER_OPEN=1
|
||||
Environment=HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1
|
||||
Environment=HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=1
|
||||
ExecStart=${CD_LANE_DRAIN_BINARY} daemon --config ${CD_LANE_DRAIN_CONFIG}
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
KillSignal=SIGINT
|
||||
TimeoutStopSec=3700
|
||||
SuccessExitStatus=0 130 143
|
||||
CPUQuota=250%
|
||||
MemoryHigh=8G
|
||||
MemoryMax=12G
|
||||
TasksMax=512
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
install -o root -g root -m 0644 "$tmp" "$unit_file" >/dev/null 2>&1 || true
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
cd_lane_config_path_is_controlled() {
|
||||
local config_path="$1"
|
||||
[ -f "$config_path" ] || return 1
|
||||
grep -Eq '^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$' "$config_path" || return 1
|
||||
grep -q 'awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04' "$config_path" || return 1
|
||||
grep -q 'awoooi-host:host' "$config_path" || return 1
|
||||
if grep -Eq '^[[:space:]]+- ".*(ubuntu-latest|stockplatform|headless|playwright)' "$config_path"; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
cd_lane_config_is_controlled() {
|
||||
cd_lane_config_path_is_controlled "$CD_LANE_CONFIG"
|
||||
}
|
||||
|
||||
cd_lane_drain_config_is_controlled() {
|
||||
cd_lane_config_path_is_controlled "$CD_LANE_DRAIN_CONFIG"
|
||||
}
|
||||
|
||||
cd_lane_drain_is_controlled_open() {
|
||||
local active
|
||||
active="$(systemctl show "$CD_LANE_DRAIN_SERVICE" -p ActiveState --value 2>/dev/null || true)"
|
||||
[ "$active" = "active" ] || return 1
|
||||
cd_lane_drain_config_is_controlled || return 1
|
||||
file "$CD_LANE_DRAIN_BINARY" 2>/dev/null | grep -qi "ELF" || return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
cd_lane_drain_is_controlled_available() {
|
||||
cd_lane_drain_config_is_controlled || return 1
|
||||
file "$CD_LANE_DRAIN_BINARY" 2>/dev/null | grep -qi "ELF" || return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
quarantine_cd_lane_registration_fail_closed() {
|
||||
local quarantine_dir
|
||||
local lane_dir
|
||||
local path
|
||||
local target
|
||||
|
||||
rm -f /run/awoooi-cd-lane-enabled /run/awoooi-cd-lane-controlled-open >/dev/null 2>&1 || true
|
||||
|
||||
for lane_dir in "$CD_LANE_DIR" "$CD_LANE_DRAIN_DIR"; do
|
||||
[ -d "$lane_dir" ] || continue
|
||||
quarantine_dir="$lane_dir/quarantine-startup-$(date +%Y%m%d%H%M%S)"
|
||||
chattr -i "$lane_dir" "$lane_dir/data" >/dev/null 2>&1 || true
|
||||
mkdir -p "$quarantine_dir" >/dev/null 2>&1 || true
|
||||
while IFS= read -r -d '' path; do
|
||||
[ -e "$path" ] || continue
|
||||
chattr -i "$path" >/dev/null 2>&1 || true
|
||||
target="$quarantine_dir/$(basename "$path")"
|
||||
mv "$path" "$target" >/dev/null 2>&1 || true
|
||||
chmod 0400 "$target" >/dev/null 2>&1 || true
|
||||
chattr +i "$target" >/dev/null 2>&1 || true
|
||||
done < <(
|
||||
{
|
||||
find "$lane_dir" -maxdepth 1 \( -name 'config.yaml' -o -name 'config.yaml.*' -o -name '.runner' -o -name '.runner.*' \) -print0 2>/dev/null
|
||||
find "$lane_dir/data" -maxdepth 1 \( -name '.runner' -o -name '.runner.*' \) -print0 2>/dev/null
|
||||
} || true
|
||||
)
|
||||
chattr +i "$lane_dir" "$lane_dir/data" >/dev/null 2>&1 || true
|
||||
done
|
||||
}
|
||||
|
||||
apply_cd_lane_fail_closed_guard() {
|
||||
local unit
|
||||
if cd_lane_drain_is_controlled_available; then
|
||||
if cd_lane_drain_is_controlled_open; then
|
||||
log "✅ controlled cd-lane drain verifier passed; preserving drain lane and fail-closing regular lane only"
|
||||
else
|
||||
log "✅ controlled cd-lane drain assets verified; restoring drain unit and fail-closing regular lane only"
|
||||
fi
|
||||
systemctl kill --signal=SIGKILL "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
|
||||
systemctl stop "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
|
||||
systemctl disable "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
|
||||
install_cd_lane_fail_closed_unit
|
||||
pkill -KILL -f "^${CD_LANE_BINARY} daemon" >/dev/null 2>&1 || true
|
||||
install_controlled_cd_lane_drain_unit
|
||||
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
systemctl enable --now "$CD_LANE_DRAIN_SERVICE" >/dev/null 2>&1 || true
|
||||
return 0
|
||||
fi
|
||||
if { [ -e "$CD_LANE_ENABLE_SENTINEL" ] || [ -e "/run/awoooi-cd-lane-controlled-open" ] || [ "$START_CONTROLLED_CD_LANE" = "1" ]; } \
|
||||
&& cd_lane_config_is_controlled \
|
||||
&& file "$CD_LANE_BINARY" 2>/dev/null | grep -qi "ELF"; then
|
||||
log "✅ controlled cd-lane verifier passed; keeping dedicated lane open"
|
||||
install_controlled_cd_lane_unit
|
||||
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
systemctl enable --now "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
|
||||
return 0
|
||||
fi
|
||||
for unit in awoooi-cd-lane.service awoooi-cd-lane-drain.service; do
|
||||
systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true
|
||||
systemctl stop "$unit" >/dev/null 2>&1 || true
|
||||
systemctl disable "$unit" >/dev/null 2>&1 || true
|
||||
if [ "$unit" = "awoooi-cd-lane.service" ]; then
|
||||
install_cd_lane_fail_closed_unit
|
||||
else
|
||||
systemctl mask "$unit" >/dev/null 2>&1 || mask_runner_unit_file "$unit" "/etc/systemd/system"
|
||||
mask_runner_unit_file "$unit" "/etc/systemd/system"
|
||||
fi
|
||||
done
|
||||
install_cd_lane_fail_closed_unit
|
||||
pkill -KILL -f "^${CD_LANE_DIR}/awoooi_cd_lane daemon" >/dev/null 2>&1 || true
|
||||
pkill -KILL -f "^${CD_LANE_DRAIN_DIR}/awoooi_cd_lane_controlled daemon" >/dev/null 2>&1 || true
|
||||
quarantine_cd_lane_registration_fail_closed
|
||||
guard_runner_binary_fail_closed "$CD_LANE_DIR/awoooi_cd_lane"
|
||||
guard_runner_binary_fail_closed "$CD_LANE_DRAIN_DIR/awoooi_cd_lane_controlled"
|
||||
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
ensure_cd_lane_fail_closed() {
|
||||
apply_cd_lane_fail_closed_guard
|
||||
}
|
||||
|
||||
ensure_controlled_cd_lane_open() {
|
||||
if ! cd_lane_config_is_controlled; then
|
||||
log "⛔ controlled cd-lane config 未通過 capacity/label 檢查,維持 fail-closed"
|
||||
@@ -551,10 +650,10 @@ else
|
||||
fi
|
||||
|
||||
if [ "$START_CD_LANE_ALLOWED" = "1" ]; then
|
||||
log "✅ controlled cd-lane sentinel present; opening dedicated rate-limited CD lane"
|
||||
log "✅ controlled cd-lane 具備 sentinel/env 授權,執行 capacity/label/binary verifier 後受控開啟"
|
||||
ensure_controlled_cd_lane_open
|
||||
else
|
||||
log "⏸️ controlled cd-lane 維持 fail-closed;需 $CD_LANE_ENABLE_SENTINEL 或 AWOOOI_START_CONTROLLED_CD_LANE=1"
|
||||
log "⏸️ controlled cd-lane 未要求啟動;保留合格 drain lane,regular lane 維持 fail-closed"
|
||||
ensure_cd_lane_fail_closed
|
||||
fi
|
||||
|
||||
|
||||
@@ -317,16 +317,21 @@ fi
|
||||
cd_lane_binary_kind=$(file -b /home/wooo/awoooi-cd-lane/awoooi_cd_lane 2>/dev/null || echo missing)
|
||||
cd_lane_binary_elf=0
|
||||
echo "$cd_lane_binary_kind" | grep -qi "ELF" && cd_lane_binary_elf=1
|
||||
cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane" 2>/dev/null | wc -l | tr -d " ")
|
||||
cd_lane_ok=0
|
||||
cd_lane_mode=blocked
|
||||
if [ "$cd_lane_active" = "inactive" ] && echo "$cd_lane_execstart" | grep -q "/bin/false" && [ "$cd_lane_binary_elf" = "0" ]; then
|
||||
if [ "$cd_lane_active" = "inactive" ] \
|
||||
&& [ "$cd_lane_sentinel" = "missing" ] \
|
||||
&& [ "$cd_lane_binary_elf" = "0" ] \
|
||||
&& [ "$cd_lane_process_count" = "0" ] \
|
||||
&& { { [ "$cd_lane_load" = "masked" ] && [ "$cd_lane_unitfile" = "masked" ]; } || echo "$cd_lane_execstart" | grep -q "/bin/false"; }; then
|
||||
cd_lane_ok=1
|
||||
cd_lane_mode=failclosed
|
||||
elif [ "$cd_lane_sentinel" = "present" ] && [ "$cd_lane_active" = "active" ] && [ "$cd_lane_capacity_ok" = "1" ] && [ "$cd_lane_labels_ok" = "1" ] && [ "$cd_lane_binary_elf" = "1" ]; then
|
||||
cd_lane_ok=1
|
||||
cd_lane_mode=controlled_open
|
||||
fi
|
||||
echo "CD_LANE_CONTROLLED mode=$cd_lane_mode load=$cd_lane_load unitfile=$cd_lane_unitfile active=$cd_lane_active mainpid=$cd_lane_mainpid sentinel=$cd_lane_sentinel capacity=$cd_lane_capacity_ok labels=$cd_lane_labels_ok binary_elf=$cd_lane_binary_elf ok=$cd_lane_ok"
|
||||
echo "CD_LANE_CONTROLLED mode=$cd_lane_mode load=$cd_lane_load unitfile=$cd_lane_unitfile active=$cd_lane_active mainpid=$cd_lane_mainpid sentinel=$cd_lane_sentinel capacity=$cd_lane_capacity_ok labels=$cd_lane_labels_ok binary_elf=$cd_lane_binary_elf process_count=$cd_lane_process_count ok=$cd_lane_ok"
|
||||
cd_lane_drain_load=$(systemctl show awoooi-cd-lane-drain.service -p LoadState --value 2>/dev/null || true)
|
||||
cd_lane_drain_unitfile=$(systemctl show awoooi-cd-lane-drain.service -p UnitFileState --value 2>/dev/null || true)
|
||||
cd_lane_drain_active=$(systemctl show awoooi-cd-lane-drain.service -p ActiveState --value 2>/dev/null || true)
|
||||
@@ -344,16 +349,23 @@ fi
|
||||
cd_lane_drain_binary_kind=$(file -b /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled 2>/dev/null || echo missing)
|
||||
cd_lane_drain_binary_elf=0
|
||||
echo "$cd_lane_drain_binary_kind" | grep -qi "ELF" && cd_lane_drain_binary_elf=1
|
||||
cd_lane_drain_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" 2>/dev/null | wc -l | tr -d " ")
|
||||
cd_lane_drain_ok=0
|
||||
cd_lane_drain_mode=absent
|
||||
if [ "$cd_lane_drain_load" = "loaded" ] || [ "$cd_lane_drain_unitfile" = "enabled" ] || [ "$cd_lane_drain_active" = "active" ]; then
|
||||
cd_lane_drain_mode=blocked
|
||||
fi
|
||||
if [ "$cd_lane_drain_active" = "active" ] && [ "$cd_lane_drain_capacity_ok" = "1" ] && [ "$cd_lane_drain_labels_ok" = "1" ] && [ "$cd_lane_drain_binary_elf" = "1" ]; then
|
||||
cd_lane_drain_mode=blocked
|
||||
if [ "$cd_lane_drain_active" != "active" ] \
|
||||
&& [ "$cd_lane_drain_binary_elf" = "0" ] \
|
||||
&& [ "$cd_lane_drain_process_count" = "0" ] \
|
||||
&& { [ "$cd_lane_drain_load" = "not-found" ] || { [ "$cd_lane_drain_load" = "masked" ] && [ "$cd_lane_drain_unitfile" = "masked" ]; }; }; then
|
||||
cd_lane_drain_ok=1
|
||||
cd_lane_drain_mode=failclosed
|
||||
elif [ "$cd_lane_drain_active" = "active" ] \
|
||||
&& [ "$cd_lane_drain_capacity_ok" = "1" ] \
|
||||
&& [ "$cd_lane_drain_labels_ok" = "1" ] \
|
||||
&& [ "$cd_lane_drain_binary_elf" = "1" ]; then
|
||||
cd_lane_drain_ok=1
|
||||
cd_lane_drain_mode=controlled_open
|
||||
fi
|
||||
echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active mainpid=$cd_lane_drain_mainpid capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf ok=$cd_lane_drain_ok"
|
||||
echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active mainpid=$cd_lane_drain_mainpid capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok"
|
||||
cd_lane_guard_ok=0
|
||||
if [ "$cd_lane_ok" = "1" ] || [ "$cd_lane_drain_ok" = "1" ]; then
|
||||
cd_lane_guard_ok=1
|
||||
@@ -573,22 +585,69 @@ scheduler_uid=$(docker top momo-scheduler -eo pid,user,uid 2>/dev/null | awk "NR
|
||||
echo "MOMO_GDRIVE_TOKEN_STAT ${token_stat:-missing} scheduler_uid=${scheduler_uid:-unknown}"
|
||||
db_user=$(docker exec momo-pro-system printenv POSTGRES_USER 2>/dev/null || true)
|
||||
db_name=$(docker exec momo-pro-system printenv POSTGRES_DB 2>/dev/null || true)
|
||||
db_pass=$(docker exec momo-pro-system printenv POSTGRES_PASSWORD 2>/dev/null || true)
|
||||
if [ -n "$db_user" ] && [ -n "$db_name" ] && [ -n "$db_pass" ]; then
|
||||
momo_sync=$(docker exec -e PGPASSWORD="$db_pass" -e PGCONNECT_TIMEOUT=5 momo-db psql -h 127.0.0.1 -U "$db_user" -d "$db_name" -Atc "WITH scope AS (SELECT min(snapshot_date::date) dmin, max(snapshot_date::date) dmax, count(*) sc FROM daily_sales_snapshot WHERE snapshot_date::date >= make_date(extract(year from current_date)::int, extract(month from current_date)::int, 1)), monthly AS (SELECT count(*) mc, min(\"日期\"::date) mmin, max(\"日期\"::date) mmax FROM realtime_sales_monthly, scope WHERE scope.sc > 0 AND \"日期\"::date BETWEEN scope.dmin AND scope.dmax) SELECT coalesce(scope.sc,0)::text || chr(124) || coalesce(monthly.mc,0)::text || chr(124) || coalesce(scope.dmin::text,chr(45)) || chr(124) || coalesce(scope.dmax::text,chr(45)) || chr(124) || coalesce(monthly.mmin::text,chr(45)) || chr(124) || coalesce(monthly.mmax::text,chr(45)) FROM scope, monthly;" 2>/dev/null || true)
|
||||
momo_freshness=$(docker exec -e PGPASSWORD="$db_pass" -e PGCONNECT_TIMEOUT=5 momo-db psql -h 127.0.0.1 -U "$db_user" -d "$db_name" -Atc "SELECT coalesce((current_date - max(snapshot_date::date))::text, chr(45)) || chr(124) || coalesce(max(snapshot_date::date)::text, chr(45)) FROM daily_sales_snapshot;" 2>/dev/null || true)
|
||||
momo_import_config=$(docker exec -e PGPASSWORD="$db_pass" -e PGCONNECT_TIMEOUT=5 momo-db psql -h 127.0.0.1 -U "$db_user" -d "$db_name" -Atc "SELECT config_key || chr(61) || config_value FROM import_config;" 2>/dev/null | awk -F= "\$1 == \"gdrive_folder_path\" {folder=\$2} \$1 == \"gdrive_file_pattern\" {pattern=\$2} END {if (folder || pattern) print folder \"|\" pattern}" || true)
|
||||
momo_latest_import_job=$(docker exec -e PGPASSWORD="$db_pass" -e PGCONNECT_TIMEOUT=5 momo-db psql -h 127.0.0.1 -U "$db_user" -d "$db_name" -Atc "SELECT coalesce(id::text, chr(45)) || chr(124) || coalesce(job_type, chr(45)) || chr(124) || coalesce(status, chr(45)) || chr(124) || coalesce(drive_file_name, chr(45)) || chr(124) || coalesce(replace(created_at::text, chr(32), chr(84)), chr(45)) || chr(124) || coalesce(replace(completed_at::text, chr(32), chr(84)), chr(45)) || chr(124) || coalesce(total_rows::text, chr(45)) || chr(124) || coalesce(success_rows::text, chr(45)) || chr(124) || coalesce(error_rows::text, chr(45)) FROM import_jobs ORDER BY created_at DESC LIMIT 20;" 2>/dev/null | awk "BEGIN {FS=sprintf(\"%c\",124)} \$2 == \"daily_sales\" {print \$1 \"|\" \$3 \"|\" \$4 \"|\" \$5 \"|\" \$6 \"|\" \$7 \"|\" \$8 \"|\" \$9; exit}" || true)
|
||||
if [ -n "$db_user" ] && [ -n "$db_name" ]; then
|
||||
psql_no_secret() {
|
||||
docker exec -e PGCONNECT_TIMEOUT=5 momo-db psql -h 127.0.0.1 -U "$db_user" -d "$db_name" --no-password -Atc "$1" 2>/dev/null || true
|
||||
}
|
||||
momo_sync=$(psql_no_secret "WITH scope AS (SELECT min(snapshot_date::date) dmin, max(snapshot_date::date) dmax, count(*) sc FROM daily_sales_snapshot WHERE snapshot_date::date >= make_date(extract(year from current_date)::int, extract(month from current_date)::int, 1)), monthly AS (SELECT count(*) mc, min(\"日期\"::date) mmin, max(\"日期\"::date) mmax FROM realtime_sales_monthly, scope WHERE scope.sc > 0 AND \"日期\"::date BETWEEN scope.dmin AND scope.dmax) SELECT coalesce(scope.sc,0)::text || chr(124) || coalesce(monthly.mc,0)::text || chr(124) || coalesce(scope.dmin::text,chr(45)) || chr(124) || coalesce(scope.dmax::text,chr(45)) || chr(124) || coalesce(monthly.mmin::text,chr(45)) || chr(124) || coalesce(monthly.mmax::text,chr(45)) FROM scope, monthly;")
|
||||
momo_freshness=$(psql_no_secret "SELECT coalesce((current_date - max(snapshot_date::date))::text, chr(45)) || chr(124) || coalesce(max(snapshot_date::date)::text, chr(45)) FROM daily_sales_snapshot;")
|
||||
momo_import_config=$(psql_no_secret "SELECT config_key || chr(61) || config_value FROM import_config;" | awk -F= "\$1 == \"gdrive_folder_path\" {folder=\$2} \$1 == \"gdrive_file_pattern\" {pattern=\$2} END {if (folder || pattern) print folder \"|\" pattern}" || true)
|
||||
momo_latest_import_job=$(psql_no_secret "SELECT coalesce(id::text, chr(45)) || chr(124) || coalesce(job_type, chr(45)) || chr(124) || coalesce(status, chr(45)) || chr(124) || coalesce(drive_file_name, chr(45)) || chr(124) || coalesce(replace(created_at::text, chr(32), chr(84)), chr(45)) || chr(124) || coalesce(replace(completed_at::text, chr(32), chr(84)), chr(45)) || chr(124) || coalesce(total_rows::text, chr(45)) || chr(124) || coalesce(success_rows::text, chr(45)) || chr(124) || coalesce(error_rows::text, chr(45)) FROM import_jobs ORDER BY created_at DESC LIMIT 20;" | awk "BEGIN {FS=sprintf(\"%c\",124)} \$2 == \"daily_sales\" {print \$1 \"|\" \$3 \"|\" \$4 \"|\" \$5 \"|\" \$6 \"|\" \$7 \"|\" \$8 \"|\" \$9; exit}" || true)
|
||||
tmp_drive_probe="/tmp/awoooi-momo-drive-source-probe.$$"
|
||||
cat > "$tmp_drive_probe" <<PYDRIVE
|
||||
from services.google_drive_service import drive_service
|
||||
from services.import_service import import_service
|
||||
|
||||
def emit(key, value):
|
||||
print(f"{key} {value if value not in (None, '') else '-'}")
|
||||
|
||||
folder = import_service.get_config("gdrive_folder_path", "當日業績匯入")
|
||||
pattern = import_service.get_config("gdrive_file_pattern", "即時業績_當日")
|
||||
archive = import_service.get_config("gdrive_archive_folder", "當日業績匯入/已匯入")
|
||||
failed = import_service.get_config("gdrive_failed_folder", "匯入失敗")
|
||||
intake_files = drive_service.list_files_in_folder(folder, pattern)
|
||||
archive_files = drive_service.list_files_in_folder(archive, pattern)
|
||||
failed_files = drive_service.list_files_in_folder(failed, pattern)
|
||||
emit("MOMO_DRIVE_INTAKE_COUNT", len(intake_files))
|
||||
emit("MOMO_DRIVE_ARCHIVE_COUNT", len(archive_files))
|
||||
emit("MOMO_DRIVE_FAILED_COUNT", len(failed_files))
|
||||
emit("MOMO_DRIVE_ARCHIVE_LATEST_DATE", (archive_files[0].get("modifiedTime", "")[:10] if archive_files else "-"))
|
||||
if not drive_service.service:
|
||||
drive_service.authenticate()
|
||||
global_date = "-"
|
||||
if drive_service.service:
|
||||
safe_pattern = drive_service._escape_query_value(pattern)
|
||||
query = (
|
||||
f"name contains '{safe_pattern}' and trashed=false and "
|
||||
"(mimeType='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' "
|
||||
"or mimeType='application/vnd.ms-excel')"
|
||||
)
|
||||
results = drive_service.service.files().list(
|
||||
q=query,
|
||||
spaces="drive",
|
||||
fields="files(modifiedTime)",
|
||||
orderBy="modifiedTime desc",
|
||||
pageSize=1,
|
||||
).execute()
|
||||
global_files = results.get("files", [])
|
||||
if global_files:
|
||||
global_date = global_files[0].get("modifiedTime", "")[:10] or "-"
|
||||
emit("MOMO_DRIVE_GLOBAL_LATEST_DATE", global_date)
|
||||
PYDRIVE
|
||||
momo_drive_source_probe=$(docker exec -i momo-scheduler python - < "$tmp_drive_probe" 2>/dev/null | awk "/^MOMO_DRIVE_/ {print}" || true)
|
||||
rm -f "$tmp_drive_probe"
|
||||
else
|
||||
momo_sync=""
|
||||
momo_freshness=""
|
||||
momo_import_config=""
|
||||
momo_latest_import_job=""
|
||||
momo_drive_source_probe=""
|
||||
fi
|
||||
echo "MOMO_MONTHLY_SYNC ${momo_sync:-unavailable}"
|
||||
echo "MOMO_DAILY_FRESHNESS ${momo_freshness:-unavailable}"
|
||||
echo "MOMO_IMPORT_CONFIG ${momo_import_config:-unavailable}"
|
||||
echo "MOMO_LATEST_IMPORT_JOB ${momo_latest_import_job:-unavailable}"
|
||||
printf "%s\n" "$momo_drive_source_probe"
|
||||
' 2>&1); then
|
||||
echo "$out"
|
||||
grep -q "CRON_188 active" <<<"$out" && ok "188 cron active" || warn "188 cron not confirmed"
|
||||
@@ -611,10 +670,36 @@ echo "MOMO_LATEST_IMPORT_JOB ${momo_latest_import_job:-unavailable}"
|
||||
grep -Fq "MOMO_IMPORT_CONFIG 當日業績匯入|即時業績_當日" <<<"$out" && ok "188 momo Drive import config points to expected daily-sales intake" || fail "188 momo Drive import config drifted from expected daily-sales intake"
|
||||
awk '/MOMO_LATEST_IMPORT_JOB / {split($2,a,"|"); exit !(a[1] ~ /^[0-9]+$/ && a[2] == "completed" && a[6] == a[7] && a[8] == 0)}' <<<"$out" && ok "188 momo latest daily import job completed cleanly" || warn "188 momo latest daily import job not confirmed clean"
|
||||
awk '/MOMO_MONTHLY_SYNC / {split($2,a,"|"); exit !(a[1] > 0 && a[1] == a[2] && a[3] == a[5] && a[4] == a[6])}' <<<"$out" && ok "188 momo current-month snapshot and realtime tables match" || warn "188 momo current-month snapshot/realtime sync not confirmed"
|
||||
momo_source_stale_only=$(awk '
|
||||
$1 == "MOMO_DRIVE_INTAKE_COUNT" {intake=$2+0}
|
||||
$1 == "MOMO_DRIVE_FAILED_COUNT" {failed=$2+0}
|
||||
$1 == "MOMO_DRIVE_GLOBAL_LATEST_DATE" {global=$2}
|
||||
$1 == "MOMO_LATEST_IMPORT_JOB" {split($2,a,"|"); completed=substr(a[5],1,10)}
|
||||
END {
|
||||
if (intake == 0 && failed == 0 && global ~ /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/ && completed ~ /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/ && global <= completed) print 1;
|
||||
else print 0;
|
||||
}' <<<"$out")
|
||||
if awk '/MOMO_DAILY_FRESHNESS / {split($2,a,"|"); exit !(a[1] ~ /^[0-9]+$/ && a[1] >= 0 && a[1] <= 2)}' <<<"$out"; then
|
||||
ok "188 momo daily sales data fresh enough"
|
||||
elif awk '/MOMO_DAILY_FRESHNESS / {split($2,a,"|"); exit !(a[1] ~ /^[0-9]+$/ && a[1] >= 3)}' <<<"$out"; then
|
||||
if awk '/MOMO_SOURCE_EMPTY_EVIDENCE_LINES / {exit !($2 > 0)}' <<<"$out"; then
|
||||
if [ "$momo_source_stale_only" = "1" ]; then
|
||||
warn "188 momo daily sales stale but Drive has no newer source candidate"
|
||||
elif [ -x scripts/reboot-recovery/momo-drive-token-source-recovery-preflight.sh ]; then
|
||||
momo_source_preflight_summary="$(
|
||||
scripts/reboot-recovery/momo-drive-token-source-recovery-preflight.sh \
|
||||
--host ollama@192.168.0.188 \
|
||||
--freshness-max-days 2 2>/dev/null \
|
||||
| awk '/^MOMO_DRIVE_TOKEN_SOURCE_PREFLIGHT / {line=$0} END {print line}' || true
|
||||
)"
|
||||
[ -n "$momo_source_preflight_summary" ] && echo "$momo_source_preflight_summary"
|
||||
if grep -q "BLOCKED=0" <<<"$momo_source_preflight_summary"; then
|
||||
warn "188 momo daily sales stale but source preflight has no hard blocker"
|
||||
elif awk '/MOMO_SOURCE_EMPTY_EVIDENCE_LINES / {exit !($2 > 0)}' <<<"$out"; then
|
||||
fail "188 momo source file absent while daily sales data stale"
|
||||
else
|
||||
fail "188 momo daily sales data stale beyond 3 days"
|
||||
fi
|
||||
elif awk '/MOMO_SOURCE_EMPTY_EVIDENCE_LINES / {exit !($2 > 0)}' <<<"$out"; then
|
||||
fail "188 momo source file absent while daily sales data stale"
|
||||
else
|
||||
fail "188 momo daily sales data stale beyond 3 days"
|
||||
|
||||
@@ -306,6 +306,7 @@ drive_archive_count="$(num_for DRIVE_ARCHIVE_COUNT)"
|
||||
drive_failed_count="$(num_for DRIVE_FAILED_COUNT)"
|
||||
drive_archive_latest="$(value_for DRIVE_ARCHIVE_LATEST_MODIFIED)"
|
||||
drive_global_latest="$(value_for DRIVE_GLOBAL_LATEST_MODIFIED)"
|
||||
drive_global_latest_date="${drive_global_latest:0:10}"
|
||||
if [[ "$drive_intake_count" -gt 0 ]]; then
|
||||
ok "Drive daily-sales intake has pending source files: count=$drive_intake_count"
|
||||
else
|
||||
@@ -338,13 +339,22 @@ IFS='|' read -r freshness_days latest_daily_date <<<"$freshness"
|
||||
if [[ "$freshness_days" =~ ^[0-9]+$ && "$freshness_days" -le "$FRESHNESS_MAX_DAYS" ]]; then
|
||||
ok "daily sales data freshness is within ${FRESHNESS_MAX_DAYS} days: $freshness"
|
||||
elif [[ "$freshness_days" =~ ^[0-9]+$ ]]; then
|
||||
blocked "daily sales data is stale: $freshness"
|
||||
warn "daily sales data is stale: $freshness"
|
||||
else
|
||||
blocked "daily sales freshness is unavailable: ${freshness:-missing}"
|
||||
fi
|
||||
|
||||
latest_job="$(value_for DB_LATEST_DAILY_IMPORT_JOB)"
|
||||
IFS='|' read -r job_id job_status job_file job_created job_completed job_total job_success job_errors <<<"$latest_job"
|
||||
job_completed_date="${job_completed:0:10}"
|
||||
source_absent_without_newer_drive=0
|
||||
if [[ "$drive_intake_count" -eq 0 \
|
||||
&& "$drive_failed_count" -eq 0 \
|
||||
&& "$drive_global_latest_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ \
|
||||
&& "$job_completed_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] \
|
||||
&& [[ "$drive_global_latest_date" < "$job_completed_date" || "$drive_global_latest_date" == "$job_completed_date" ]]; then
|
||||
source_absent_without_newer_drive=1
|
||||
fi
|
||||
if [[ "$job_id" =~ ^[0-9]+$ && "$job_status" == "completed" && "$job_total" == "$job_success" && "$job_errors" == "0" ]]; then
|
||||
ok "latest daily import job completed cleanly: id=$job_id file=$job_file"
|
||||
else
|
||||
@@ -354,6 +364,8 @@ fi
|
||||
if [[ "$freshness_days" =~ ^[0-9]+$ && "$freshness_days" -gt "$FRESHNESS_MAX_DAYS" ]]; then
|
||||
if [[ "$auth_failures" -gt 0 ]]; then
|
||||
blocked "release blocker is stale business data with active Drive auth/source evidence gate"
|
||||
elif [[ "$source_absent_without_newer_drive" -eq 1 ]]; then
|
||||
warn "daily sales data is stale, but Drive has no newer source candidate than the last clean import"
|
||||
else
|
||||
blocked "release blocker is stale business data; source evidence must be refreshed"
|
||||
fi
|
||||
|
||||
@@ -336,16 +336,21 @@ fi
|
||||
cd_lane_binary_kind=$(file -b /home/wooo/awoooi-cd-lane/awoooi_cd_lane 2>/dev/null || echo missing)
|
||||
cd_lane_binary_elf=0
|
||||
echo "$cd_lane_binary_kind" | grep -qi "ELF" && cd_lane_binary_elf=1
|
||||
cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane" 2>/dev/null | wc -l | tr -d " ")
|
||||
cd_lane_ok=0
|
||||
cd_lane_mode=blocked
|
||||
if [ "$cd_lane_active" = "inactive" ] && echo "$cd_lane_execstart" | grep -q "/bin/false" && [ "$cd_lane_binary_elf" = "0" ]; then
|
||||
if [ "$cd_lane_active" = "inactive" ] \
|
||||
&& [ "$cd_lane_sentinel" = "missing" ] \
|
||||
&& [ "$cd_lane_binary_elf" = "0" ] \
|
||||
&& [ "$cd_lane_process_count" = "0" ] \
|
||||
&& { { [ "$cd_lane_load" = "masked" ] && [ "$cd_lane_unitfile" = "masked" ]; } || echo "$cd_lane_execstart" | grep -q "/bin/false"; }; then
|
||||
cd_lane_ok=1
|
||||
cd_lane_mode=failclosed
|
||||
elif [ "$cd_lane_sentinel" = "present" ] && [ "$cd_lane_active" = "active" ] && [ "$cd_lane_capacity_ok" = "1" ] && [ "$cd_lane_labels_ok" = "1" ] && [ "$cd_lane_binary_elf" = "1" ]; then
|
||||
cd_lane_ok=1
|
||||
cd_lane_mode=controlled_open
|
||||
fi
|
||||
echo "CD_LANE_CONTROLLED mode=$cd_lane_mode load=$cd_lane_load unitfile=$cd_lane_unitfile active=$cd_lane_active sentinel=$cd_lane_sentinel capacity=$cd_lane_capacity_ok labels=$cd_lane_labels_ok binary_elf=$cd_lane_binary_elf ok=$cd_lane_ok"
|
||||
echo "CD_LANE_CONTROLLED mode=$cd_lane_mode load=$cd_lane_load unitfile=$cd_lane_unitfile active=$cd_lane_active sentinel=$cd_lane_sentinel capacity=$cd_lane_capacity_ok labels=$cd_lane_labels_ok binary_elf=$cd_lane_binary_elf process_count=$cd_lane_process_count ok=$cd_lane_ok"
|
||||
cd_lane_drain_load=$(systemctl show awoooi-cd-lane-drain.service -p LoadState --value 2>/dev/null || true)
|
||||
cd_lane_drain_unitfile=$(systemctl show awoooi-cd-lane-drain.service -p UnitFileState --value 2>/dev/null || true)
|
||||
cd_lane_drain_active=$(systemctl show awoooi-cd-lane-drain.service -p ActiveState --value 2>/dev/null || true)
|
||||
@@ -362,16 +367,23 @@ fi
|
||||
cd_lane_drain_binary_kind=$(file -b /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled 2>/dev/null || echo missing)
|
||||
cd_lane_drain_binary_elf=0
|
||||
echo "$cd_lane_drain_binary_kind" | grep -qi "ELF" && cd_lane_drain_binary_elf=1
|
||||
cd_lane_drain_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" 2>/dev/null | wc -l | tr -d " ")
|
||||
cd_lane_drain_ok=0
|
||||
cd_lane_drain_mode=absent
|
||||
if [ "$cd_lane_drain_load" = "loaded" ] || [ "$cd_lane_drain_unitfile" = "enabled" ] || [ "$cd_lane_drain_active" = "active" ]; then
|
||||
cd_lane_drain_mode=blocked
|
||||
fi
|
||||
if [ "$cd_lane_drain_active" = "active" ] && [ "$cd_lane_drain_capacity_ok" = "1" ] && [ "$cd_lane_drain_labels_ok" = "1" ] && [ "$cd_lane_drain_binary_elf" = "1" ]; then
|
||||
cd_lane_drain_mode=blocked
|
||||
if [ "$cd_lane_drain_active" != "active" ] \
|
||||
&& [ "$cd_lane_drain_binary_elf" = "0" ] \
|
||||
&& [ "$cd_lane_drain_process_count" = "0" ] \
|
||||
&& { [ "$cd_lane_drain_load" = "not-found" ] || { [ "$cd_lane_drain_load" = "masked" ] && [ "$cd_lane_drain_unitfile" = "masked" ]; }; }; then
|
||||
cd_lane_drain_ok=1
|
||||
cd_lane_drain_mode=failclosed
|
||||
elif [ "$cd_lane_drain_active" = "active" ] \
|
||||
&& [ "$cd_lane_drain_capacity_ok" = "1" ] \
|
||||
&& [ "$cd_lane_drain_labels_ok" = "1" ] \
|
||||
&& [ "$cd_lane_drain_binary_elf" = "1" ]; then
|
||||
cd_lane_drain_ok=1
|
||||
cd_lane_drain_mode=controlled_open
|
||||
fi
|
||||
echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf ok=$cd_lane_drain_ok"
|
||||
echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok"
|
||||
cd_lane_guard_ok=0
|
||||
if [ "$cd_lane_ok" = "1" ] || [ "$cd_lane_drain_ok" = "1" ]; then
|
||||
cd_lane_guard_ok=1
|
||||
@@ -446,13 +458,22 @@ echo "ollama-systemd $(systemctl is-active ollama 2>/dev/null || true)"
|
||||
echo "ollama-api $(curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://127.0.0.1:11434/api/tags || true)"
|
||||
docker inspect -f "momo-scheduler {{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}" momo-scheduler 2>/dev/null || true
|
||||
docker inspect -f "litellm {{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}" litellm 2>/dev/null || true
|
||||
if ! docker inspect litellm >/dev/null 2>&1 && [ ! -d /opt/litellm ]; then
|
||||
echo "litellm not-provisioned"
|
||||
fi
|
||||
docker inspect -f "signoz-clickhouse {{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}" signoz-clickhouse 2>/dev/null || true
|
||||
' 2>&1); then
|
||||
echo "$out"
|
||||
grep -q "ollama-systemd active" <<<"$out" && ok "188 Ollama systemd active" || blocked "188 Ollama systemd inactive"
|
||||
grep -q "ollama-api 200" <<<"$out" && ok "188 Ollama API reachable" || blocked "188 Ollama API not reachable"
|
||||
grep -q "momo-scheduler running healthy" <<<"$out" && ok "188 momo-scheduler healthy" || blocked "188 momo-scheduler not healthy"
|
||||
grep -Eq "litellm running( |$)" <<<"$out" && ok "188 litellm running" || blocked "188 litellm not running"
|
||||
if grep -Eq "litellm running( |$)" <<<"$out"; then
|
||||
ok "188 litellm running"
|
||||
elif grep -q "litellm not-provisioned" <<<"$out"; then
|
||||
warn "188 litellm not provisioned; provider route/cost switch requires separate approval"
|
||||
else
|
||||
blocked "188 litellm not running"
|
||||
fi
|
||||
grep -q "signoz-clickhouse running healthy" <<<"$out" && ok "188 SignOz ClickHouse healthy" || warn "188 SignOz ClickHouse health not confirmed"
|
||||
else
|
||||
blocked "188 high-load service check unavailable"
|
||||
|
||||
Reference in New Issue
Block a user