825 lines
36 KiB
YAML
825 lines
36 KiB
YAML
# AWOOOI OpenClaw 告警規則匹配引擎
|
||
# ============================================================
|
||
# 格式說明:
|
||
# match.alertname : Prometheus alertname 完全匹配 (list = OR)
|
||
# match.alert_type : alert_type 關鍵字 (list = OR, 部分匹配)
|
||
# match.message : message 關鍵字 (list = OR, 部分匹配, 不分大小寫)
|
||
# response.* : 回應模板，支援變數 {target} {host} {container} {instance} {job} {namespace}
|
||
# responsibility : FE / BE / INFRA / DB / COLLAB
|
||
# risk : low / medium / critical
|
||
# confidence : 0.0 (規則匹配固定值，禁止偽造)
|
||
#
|
||
# 修改規則: 不需要重新部署，重啟 API Pod 即可熱載入
|
||
# 新增規則: 在 rules 清單末尾加入，priority 越小越優先
|
||
# 2026-04-09 ogt: 初版，從 openclaw.py _generate_mock_response 抽出
|
||
# ============================================================
|
||
|
||
version: "1.0.0"
|
||
updated_at: "2026-04-09"
|
||
|
||
rules:
|
||
# ── Docker / Host 層 ────────────────────────────────────────
|
||
|
||
- id: docker_container_unhealthy
|
||
priority: 10
|
||
description: Docker 容器 healthcheck 失敗
|
||
match:
|
||
alertname:
|
||
- DockerContainerUnhealthy
|
||
message:
|
||
- unhealthy
|
||
- health check
|
||
- healthcheck
|
||
response:
|
||
action_title: "檢查 Docker 容器 {container} 健康狀態"
|
||
description: "⚙️ 規則匹配: Docker 容器 {container} ({host}) healthcheck 失敗。常見原因: 應用程式啟動慢、healthcheck 指令錯誤、依賴服務未就緒。"
|
||
suggested_action: RESTART_DEPLOYMENT
|
||
kubectl_command: "ssh {host} 'docker inspect {container} --format=\"{{.State.Health.Status}}\" && docker restart {container}'"
|
||
estimated_downtime: "~30s"
|
||
risk: medium
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "Docker 容器健康檢查失敗屬基礎設施團隊責任，需確認 healthcheck 設定與容器狀態"
|
||
secondary_teams: [BE]
|
||
optimization:
|
||
- type: HEALTHCHECK
|
||
description: "確認 healthcheck 指令在容器內可執行 (mc/curl 是否存在)"
|
||
command: "ssh {host} 'docker exec {container} sh -c \"mc ready local 2>/dev/null || curl -sf http://localhost:9000/minio/health/live\"'"
|
||
reasoning: "[規則匹配] Docker healthcheck 失敗先 restart 恢復服務，同時確認 healthcheck 指令正確。"
|
||
|
||
- id: target_down
|
||
priority: 20
|
||
description: Prometheus scrape target 下線 — 自動重啟 exporter
|
||
match:
|
||
alertname:
|
||
- TargetDown
|
||
- InstanceDown
|
||
- NodeExporterDown
|
||
response:
|
||
action_title: "重啟 {job} exporter on {host}"
|
||
description: "⚙️ 規則匹配: Prometheus 無法抓取 {instance} ({job}) 指標。自動重啟主機上的 exporter container。"
|
||
suggested_action: RESTART_DEPLOYMENT
|
||
kubectl_command: "ssh {host} 'docker restart $(docker ps -a --filter name=exporter --format \"{{.Names}}\" | head -1) 2>/dev/null || systemctl restart node_exporter 2>/dev/null || systemctl restart prometheus-node-exporter'"
|
||
estimated_downtime: "~30s"
|
||
risk: medium
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "Prometheus scrape 目標下線屬基礎設施監控範疇，自動重啟 exporter"
|
||
secondary_teams: []
|
||
optimization:
|
||
- type: MONITORING
|
||
description: "確認 exporter 重啟後可被 Prometheus scrape"
|
||
command: "ssh {host} 'curl -s http://localhost:{port}/metrics | head -3'"
|
||
reasoning: "[規則匹配] Prometheus target 下線，SSH 到主機重啟 exporter container 或 systemd service。"
|
||
|
||
# ── K8s Pod 層 ──────────────────────────────────────────────
|
||
|
||
- id: oom_killed
|
||
priority: 30
|
||
description: Pod OOMKilled 記憶體不足
|
||
match:
|
||
# 2026-04-10 Claude Sonnet 4.6: Phase 2 飛輪修復 — 補齊 Prometheus alertname 變體
|
||
alertname:
|
||
- PodOOMKilled
|
||
- KubePodOOMKilled
|
||
- KubernetesMemoryPressure
|
||
- NodeMemoryUsageHigh
|
||
- HighMemoryUsage
|
||
alert_type:
|
||
- memory
|
||
message:
|
||
- oomkilled
|
||
- oom
|
||
- out of memory
|
||
response:
|
||
action_title: "刪除異常 Pod {target} (OOMKilled)"
|
||
description: "⚙️ 規則匹配: {target} 發生 OOMKilled，根因為 JVM Heap 配置與 K8s memory limit 不匹配或存在記憶體洩漏。"
|
||
suggested_action: DELETE_POD
|
||
kubectl_command: "kubectl delete pod {target} -n {namespace}"
|
||
estimated_downtime: "~30s"
|
||
risk: critical
|
||
responsibility: BE
|
||
responsibility_reasoning: "OOMKilled 通常源於應用程式記憶體配置不當，屬後端團隊責任範圍"
|
||
secondary_teams: [INFRA]
|
||
optimization:
|
||
- type: RESOURCE_LIMIT
|
||
description: "調整 memory limit 至 1Gi 並確保 JVM -Xmx 不超過 70%"
|
||
command: "kubectl set resources deployment/{target} -c {target} --limits=memory=1Gi -n {namespace}"
|
||
- type: HPA
|
||
description: "啟用基於記憶體的 HPA 自動擴展"
|
||
command: "kubectl autoscale deployment {target} --memory-percent=80 --min=2 --max=5 -n {namespace}"
|
||
reasoning: "[規則匹配] Pod OOMKilled 後 ReplicaSet 將自動重建，但需同步修正資源配置防止復發。"
|
||
|
||
# 2026-04-12 ogt: Host CPU 告警獨立規則 — node_exporter 告警無 pod/deployment label
|
||
# 2026-04-16 ogt + Claude Sonnet 4.6: 補齊主機層所有常見 Prometheus alertname
|
||
# 原則：主機層告警 = 只能通知 + 建議 SSH 排查，絕對禁止 kubectl restart
|
||
- id: host_resource_alert
|
||
priority: 45
|
||
description: Host 主機資源告警 (node_exporter — CPU/記憶體/負載/磁碟增長，非 K8s workload)
|
||
match:
|
||
alertname:
|
||
# CPU 相關
|
||
- HostHighCpuLoad
|
||
- NodeCPUUsageHigh
|
||
- NodeHighCpuLoad
|
||
# 負載相關
|
||
- HostHighLoadAverage
|
||
- NodeLoadAverageHigh
|
||
- HostLoadAverageHigh
|
||
# 記憶體相關
|
||
- HostOutOfMemory
|
||
- HostMemoryUnderMemoryPressure
|
||
- HostMemoryUsageHigh
|
||
- NodeMemoryPressure
|
||
# 磁碟 I/O 相關
|
||
- HostUnusualDiskReadLatency
|
||
- HostUnusualDiskWriteLatency
|
||
- HostUnusualDiskReadRate
|
||
- HostUnusualDiskWriteRate
|
||
- HostDiskWillFillIn24Hours
|
||
- HostOutOfDiskSpace
|
||
- HostDiskUsageHigh
|
||
- HostDiskUsageCritical
|
||
# 網路相關
|
||
- HostUnusualNetworkThroughputIn
|
||
- HostUnusualNetworkThroughputOut
|
||
# 系統服務
|
||
- HostSystemdServiceCrashed
|
||
- HostKernelVersionDeviations
|
||
- HostOomKillDetected
|
||
- HostEdacCorrectableErrors
|
||
- HostEdacUncorrectableErrors
|
||
- HostClockSkewDetected
|
||
- HostClockNotSynchronising
|
||
response:
|
||
action_title: "🔍 主機自動診斷 — SSH 收集根因"
|
||
description: "主機層告警（node_exporter）。自動 SSH 登入主機執行診斷指令，收集 CPU/記憶體/磁碟資訊後回報。"
|
||
# 2026-04-27 Claude Sonnet 4.6: 從 NO_ACTION 改為自動 SSH 診斷
|
||
# 根因：SSH_MCP_ALLOWED_HOSTS 空白導致全部降為人工審核（飛輪完全停轉）
|
||
# 修復：補 SSH_MCP_ALLOWED_HOSTS 白名單 + 改為自動診斷指令（收集不修改，安全）
|
||
# 診斷原則：只收集資訊，不做任何改動 → risk=low 且不在 _DESTRUCTIVE_PATTERNS 清單
|
||
suggested_action: SSH_DIAGNOSE
|
||
kubectl_command: "ssh {host} 'echo \"=== CPU TOP ===\"; ps aux --sort=-%cpu | head -15; echo \"=== MEMORY ===\"; free -h; echo \"=== DISK ===\"; df -h; echo \"=== LOAD ===\"; uptime'"
|
||
estimated_downtime: "N/A"
|
||
risk: low
|
||
responsibility: INFRA
|
||
reasoning: "[規則匹配] 主機層資源告警，自動 SSH 執行診斷指令（只讀，不修改），收集根因資訊後推送 Telegram 讓 SRE 決策。"
|
||
|
||
- id: high_cpu
|
||
priority: 40
|
||
description: K8s Pod/Deployment CPU 使用率過高
|
||
match:
|
||
# 2026-04-10 Claude Sonnet 4.6: Phase 2 飛輪修復 — 補齊 Prometheus alertname 變體
|
||
# 2026-04-12 ogt: 移除 HostHighCpuLoad/NodeCPUUsageHigh → 已獨立為 host_cpu_high 規則
|
||
alertname:
|
||
- HighCPUUsage
|
||
- ContainerCpuUsageSecondsTotal
|
||
- CPUThrottlingHigh
|
||
- KubeCPUOvercommit
|
||
alert_type:
|
||
- cpu
|
||
- high_cpu
|
||
response:
|
||
action_title: "擴展 {target} 副本數 + 啟用 HPA"
|
||
description: "⚙️ 規則匹配: {target} CPU 使用率過高，根因為流量突增或計算密集任務未配置自動擴展。"
|
||
suggested_action: SCALE_DEPLOYMENT
|
||
kubectl_command: "kubectl scale deployment {target} --replicas=3 -n {namespace}"
|
||
estimated_downtime: "0"
|
||
risk: medium
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "自動擴展策略未配置或閾值過高，屬基礎設施團隊責任"
|
||
secondary_teams: [BE]
|
||
optimization:
|
||
- type: RESOURCE_LIMIT
|
||
description: "增加 CPU request 確保 QoS 為 Guaranteed"
|
||
command: "kubectl set resources deployment/{target} --requests=cpu=500m --limits=cpu=2000m -n {namespace}"
|
||
reasoning: "[規則匹配] 水平擴展可即時分散負載，同時建議配置 HPA 防止復發。"
|
||
|
||
- id: http_5xx
|
||
priority: 50
|
||
description: HTTP 5xx 錯誤率過高
|
||
match:
|
||
alert_type:
|
||
- http
|
||
message:
|
||
- "5xx"
|
||
- "502"
|
||
- "503"
|
||
- "500"
|
||
response:
|
||
action_title: "重啟 {target} + 檢查上游服務"
|
||
description: "⚙️ 規則匹配: {target} 產生 HTTP 5xx 錯誤，可能為應用程式例外或上游服務不可達。"
|
||
suggested_action: RESTART_DEPLOYMENT
|
||
kubectl_command: "kubectl rollout restart deployment/{target} -n {namespace}"
|
||
estimated_downtime: "~1 min"
|
||
risk: critical
|
||
responsibility: COLLAB
|
||
responsibility_reasoning: "HTTP 5xx 可能源於前端路由、後端邏輯或基礎設施，需多團隊協同排查"
|
||
secondary_teams: [FE, BE, INFRA]
|
||
optimization:
|
||
- type: CIRCUIT_BREAKER
|
||
description: "配置熔斷器防止故障擴散"
|
||
command: "# Istio VirtualService outlierDetection 配置"
|
||
reasoning: "[規則匹配] HTTP 錯誤需協同排查，先重啟恢復服務同時通知相關團隊。"
|
||
|
||
- id: pod_crash
|
||
priority: 60
|
||
description: Pod CrashLoopBackOff
|
||
match:
|
||
# 2026-04-10 Claude Sonnet 4.6: Phase 2 飛輪修復 — 補齊 Prometheus alertname 變體
|
||
alertname:
|
||
- KubePodCrashLooping
|
||
- PodCrashLoopBackOff
|
||
- KubernetesPodCrashLooping
|
||
alert_type:
|
||
- pod_crash
|
||
- crash
|
||
message:
|
||
- crashloop
|
||
- crash
|
||
- backoff
|
||
response:
|
||
action_title: "診斷 {target} CrashLoop 根因"
|
||
description: "⚙️ 規則匹配: {target} 進入 CrashLoopBackOff，需檢查啟動錯誤日誌。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: "kubectl logs {target} -n {namespace} --previous --tail=50"
|
||
estimated_downtime: "依根因而定"
|
||
risk: critical
|
||
responsibility: BE
|
||
responsibility_reasoning: "Pod crash 通常源於應用程式啟動錯誤，屬後端團隊責任"
|
||
secondary_teams: [INFRA]
|
||
optimization:
|
||
- type: LIVENESS_PROBE
|
||
description: "調整 liveness probe 初始延遲防止誤殺"
|
||
command: "# 調整 initialDelaySeconds >= 應用啟動時間"
|
||
reasoning: "[規則匹配] 先查 previous log 確認 crash 原因，再決定修復策略。"
|
||
|
||
# ── 資料庫層 ─────────────────────────────────────────────────
|
||
|
||
# 2026-04-16 ogt + Claude Sonnet 4.6: PostgreSQL 監控告警 — 磁碟/資源類，絕對不能重啟
|
||
# 根因：PostgreSQLDiskGrowthRate 落 generic_fallback → 輸出 kubectl rollout restart postgresql（錯誤！）
|
||
- id: postgresql_disk_monitoring
|
||
priority: 68
|
||
description: PostgreSQL 磁碟/增長率/exporter 監控告警（不重啟資料庫）
|
||
match:
|
||
alertname:
|
||
- PostgreSQLDiskGrowthRate
|
||
- PostgreSQLDiskUsageHigh
|
||
- PostgreSQLDiskFull
|
||
- PostgresExporterDown
|
||
- PostgreSQLExporterDown
|
||
- PostgreSQLTableBloat
|
||
- PostgreSQLVacuumRequired
|
||
- PostgreSQLReplicationLag
|
||
- PostgreSQLTooManyConnections
|
||
response:
|
||
action_title: "⚠️ PostgreSQL 監控告警 — 需人工排查，禁止重啟"
|
||
description: "⚠️ PostgreSQL 資源/監控告警。磁碟增長過快或 exporter 異常，重啟資料庫會造成資料風險。請登入排查磁碟用量或 WAL 狀態。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: "kubectl exec -n {namespace} deployment/postgresql -- psql -U postgres -c 'SELECT pg_database_size(current_database()), pg_size_pretty(pg_database_size(current_database()));'"
|
||
estimated_downtime: "N/A"
|
||
risk: medium
|
||
responsibility: DB
|
||
responsibility_reasoning: "PostgreSQL 磁碟告警需 DBA 評估，自動重啟資料庫有資料丟失風險，必須人工確認"
|
||
secondary_teams: [INFRA]
|
||
reasoning: "[規則匹配] PostgreSQL 磁碟增長/監控告警，絕對禁止自動重啟資料庫。需 DBA 人工確認磁碟用量、WAL 清理、VACUUM 狀態。"
|
||
|
||
- id: postgresql_down
|
||
priority: 70
|
||
description: PostgreSQL 服務下線
|
||
match:
|
||
alertname:
|
||
- PostgreSQLDown
|
||
message:
|
||
- postgresql
|
||
- postgres
|
||
- pg down
|
||
response:
|
||
action_title: "重啟 PostgreSQL {target}"
|
||
description: "⚙️ 規則匹配: PostgreSQL ({instance}) 無法連線。常見原因: 程序崩潰、磁碟空間不足、連線數超限。"
|
||
suggested_action: RESTART_DEPLOYMENT
|
||
kubectl_command: "kubectl rollout restart deployment/postgresql -n {namespace}"
|
||
estimated_downtime: "~2 min"
|
||
risk: critical
|
||
responsibility: DB
|
||
responsibility_reasoning: "PostgreSQL 下線屬資料庫團隊責任，需立即確認資料完整性"
|
||
secondary_teams: [INFRA, BE]
|
||
optimization:
|
||
- type: HEALTH_CHECK
|
||
description: "確認 PostgreSQL 連線與資料完整性"
|
||
command: "kubectl exec -n {namespace} deployment/postgresql -- psql -U postgres -c 'SELECT 1'"
|
||
reasoning: "[規則匹配] PostgreSQL 下線影響所有依賴服務，優先重啟恢復，同時確認資料無損。"
|
||
|
||
- id: postgresql_connection_pool
|
||
priority: 75
|
||
description: PostgreSQL 連線池耗盡或接近上限
|
||
match:
|
||
alertname:
|
||
- PostgreSQLConnectionPoolNearLimit
|
||
- PostgreSQLConnectionPoolExhausted
|
||
message:
|
||
- connection pool
|
||
- connections
|
||
- pgbouncer
|
||
response:
|
||
action_title: "清理 PostgreSQL 閒置連線"
|
||
description: "⚙️ 規則匹配: PostgreSQL 連線池使用率過高，可能導致新請求被拒絕。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: "kubectl exec -n {namespace} deployment/postgresql -- psql -U postgres -c 'SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE state = ''idle'' AND state_change < NOW() - INTERVAL ''5 minutes'';'"
|
||
estimated_downtime: "0"
|
||
risk: critical
|
||
responsibility: DB
|
||
responsibility_reasoning: "連線池管理屬資料庫設定範疇"
|
||
secondary_teams: [BE]
|
||
optimization:
|
||
- type: CONNECTION_POOL
|
||
description: "調整 max_connections 或啟用 PgBouncer 連線池"
|
||
command: "kubectl exec -n {namespace} deployment/postgresql -- psql -U postgres -c 'SHOW max_connections;'"
|
||
reasoning: "[規則匹配] 清理閒置連線是最快恢復手段，同時需排查連線洩漏。"
|
||
|
||
- id: postgresql_slow_queries
|
||
priority: 80
|
||
description: PostgreSQL 慢查詢告警
|
||
match:
|
||
alertname:
|
||
- PostgreSQLSlowQueries
|
||
- PostgreSQLLockWaiting
|
||
message:
|
||
- slow query
|
||
- lock wait
|
||
- deadlock
|
||
response:
|
||
action_title: "診斷 PostgreSQL 慢查詢 + 索引優化"
|
||
description: "⚙️ 規則匹配: PostgreSQL 存在慢查詢或鎖等待，影響系統整體性能。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: "kubectl exec -n {namespace} deployment/postgresql -- psql -U postgres -c 'SELECT pid, query, state, wait_event_type, wait_event FROM pg_stat_activity WHERE state != ''idle'' ORDER BY query_start;'"
|
||
estimated_downtime: "0"
|
||
risk: medium
|
||
responsibility: DB
|
||
responsibility_reasoning: "慢查詢優化屬資料庫效能調優範疇"
|
||
secondary_teams: [BE]
|
||
optimization:
|
||
- type: INDEX
|
||
description: "使用 EXPLAIN ANALYZE 找出缺少索引的查詢"
|
||
command: "kubectl exec -n {namespace} deployment/postgresql -- psql -U postgres -c 'SELECT * FROM pg_stat_user_tables ORDER BY seq_scan DESC LIMIT 10;'"
|
||
reasoning: "[規則匹配] 先找出阻塞查詢，必要時 pg_terminate_backend 解除鎖定。"
|
||
|
||
# ── 基礎設施服務層 ──────────────────────────────────────────
|
||
|
||
- id: redis_down
|
||
priority: 85
|
||
description: Redis 服務下線
|
||
match:
|
||
alertname:
|
||
- RedisDown
|
||
message:
|
||
- redis
|
||
- cache down
|
||
response:
|
||
action_title: "重啟 Redis {target}"
|
||
description: "⚙️ 規則匹配: Redis ({instance}) 無法連線。影響 Session 管理、去重快取、AI Router 狀態。"
|
||
suggested_action: RESTART_DEPLOYMENT
|
||
kubectl_command: "kubectl rollout restart deployment/redis -n {namespace}"
|
||
estimated_downtime: "~30s"
|
||
risk: critical
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "Redis 屬基礎設施快取層，下線影響多個上層服務"
|
||
secondary_teams: [BE]
|
||
optimization:
|
||
- type: HEALTH_CHECK
|
||
description: "確認 Redis 連線"
|
||
command: "kubectl exec -n {namespace} deployment/redis -- redis-cli ping"
|
||
reasoning: "[規則匹配] Redis 下線會導致去重失效和 AI Router 狀態丟失，需立即重啟。"
|
||
|
||
- id: ollama_down
|
||
priority: 90
|
||
description: Ollama AI 服務下線
|
||
match:
|
||
alertname:
|
||
- OllamaDown
|
||
message:
|
||
- ollama
|
||
- llm down
|
||
- ai service
|
||
response:
|
||
action_title: "重啟 Ollama 服務 on {host}"
|
||
description: "⚙️ 規則匹配: Ollama ({instance}) 無法連線。影響 AI 規則自動生成和本地推理。"
|
||
suggested_action: RESTART_DEPLOYMENT
|
||
kubectl_command: "ssh {host} 'systemctl restart ollama || docker restart ollama'"
|
||
estimated_downtime: "~2 min (model reload)"
|
||
risk: medium
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "Ollama 屬 AI 推理基礎設施，由基礎設施團隊管理"
|
||
secondary_teams: []
|
||
optimization:
|
||
- type: HEALTH_CHECK
|
||
description: "確認 Ollama 狀態和已載入模型"
|
||
command: "curl -s http://{host}:11434/api/tags | jq '.models[].name'"
|
||
reasoning: "[規則匹配] Ollama 下線觸發 AI Router fallback 至 Gemini，重啟恢復本地推理能力。"
|
||
|
||
- id: minio_down
|
||
priority: 95
|
||
description: MinIO 物件儲存下線
|
||
match:
|
||
alertname:
|
||
- MinioDown
|
||
message:
|
||
- minio
|
||
- s3
|
||
- object storage
|
||
response:
|
||
action_title: "重啟 MinIO {target}"
|
||
description: "⚙️ 規則匹配: MinIO ({instance}) 無法連線。影響靜態資源和備份儲存。"
|
||
suggested_action: RESTART_DEPLOYMENT
|
||
kubectl_command: "ssh {host} 'docker restart minio'"
|
||
estimated_downtime: "~1 min"
|
||
risk: critical
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "MinIO 屬物件儲存基礎設施"
|
||
secondary_teams: []
|
||
optimization:
|
||
- type: DISK_CHECK
|
||
description: "確認磁碟空間充足"
|
||
command: "ssh {host} 'df -h /data/minio'"
|
||
reasoning: "[規則匹配] MinIO 下線需先確認磁碟空間，再重啟服務。"
|
||
|
||
- id: minio_disk_high
|
||
priority: 96
|
||
description: MinIO 磁碟使用率過高
|
||
match:
|
||
alertname:
|
||
- MinioDiskUsageHigh
|
||
- MinioDiskUsageCritical
|
||
message:
|
||
- disk usage
|
||
- disk full
|
||
- storage
|
||
response:
|
||
action_title: "清理 MinIO 過期資料 on {host}"
|
||
description: "⚙️ 規則匹配: MinIO 磁碟使用率過高，需清理舊資料或擴展儲存空間。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: "ssh {host} 'df -h /data/minio && du -sh /data/minio/* | sort -rh | head -10'"
|
||
estimated_downtime: "0"
|
||
risk: critical
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "磁碟空間管理屬基礎設施團隊責任"
|
||
secondary_teams: []
|
||
optimization:
|
||
- type: CLEANUP
|
||
description: "清理 MinIO 舊備份和 lifecycle policy"
|
||
command: "mc admin lifecycle add local --expiry-days 30"
|
||
reasoning: "[規則匹配] 磁碟滿會導致寫入失敗，需立即清理最大的目錄。"
|
||
|
||
- id: harbor_down
|
||
priority: 97
|
||
description: Harbor Registry 下線
|
||
match:
|
||
alertname:
|
||
- HarborDown
|
||
message:
|
||
- harbor
|
||
- registry
|
||
- docker registry
|
||
response:
|
||
action_title: "重啟 Harbor Registry on {host}"
|
||
description: "⚙️ 規則匹配: Harbor ({instance}) 無法連線。影響 CD 部署流程。"
|
||
suggested_action: RESTART_DEPLOYMENT
|
||
kubectl_command: "ssh {host} 'cd /data/harbor && docker-compose up -d'"
|
||
estimated_downtime: "~2 min"
|
||
risk: critical
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "Harbor 是 CD 部署的核心依賴，屬基礎設施團隊責任"
|
||
secondary_teams: []
|
||
optimization:
|
||
- type: HEALTH_CHECK
|
||
description: "確認 Harbor 各組件狀態"
|
||
command: "ssh {host} 'cd /data/harbor && docker-compose ps'"
|
||
reasoning: "[規則匹配] Harbor 下線會阻塞所有 CD 部署，需立即重啟。"
|
||
|
||
# ── K8s 叢集層 ──────────────────────────────────────────────
|
||
|
||
- id: k3s_node_down
|
||
priority: 100
|
||
description: K3s 節點下線
|
||
match:
|
||
alertname:
|
||
- K3sNodeDown
|
||
- K3sVIPDown
|
||
message:
|
||
- node down
|
||
- node not ready
|
||
- k3s
|
||
response:
|
||
action_title: "確認 K3s 節點 {target} 狀態"
|
||
description: "⚙️ 規則匹配: K3s 節點下線，影響叢集可用性和 Pod 調度。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: "kubectl get nodes -o wide && kubectl describe node {target}"
|
||
estimated_downtime: "依節點恢復時間"
|
||
risk: critical
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "K3s 叢集節點管理屬基礎設施團隊責任"
|
||
secondary_teams: []
|
||
optimization:
|
||
- type: NODE_DRAIN
|
||
description: "先 drain 節點確保 Pod 安全遷移"
|
||
command: "kubectl drain {target} --ignore-daemonsets --delete-emptydir-data"
|
||
reasoning: "[規則匹配] 節點下線需先確認主機可達性，必要時手動遷移 workload。"
|
||
|
||
- id: awoooi_api_down
|
||
priority: 105
|
||
description: AWOOOI API 服務下線
|
||
match:
|
||
alertname:
|
||
- AWOOOIApiDown
|
||
- OpenClawDown
|
||
message:
|
||
- awoooi api
|
||
- openclaw
|
||
- api down
|
||
response:
|
||
action_title: "重啟 AWOOOI API deployment"
|
||
description: "⚙️ 規則匹配: AWOOOI API 無法連線。影響所有告警處理和 AI 決策流程。"
|
||
suggested_action: RESTART_DEPLOYMENT
|
||
kubectl_command: "kubectl rollout restart deployment/awoooi-api -n awoooi"
|
||
estimated_downtime: "~1 min"
|
||
risk: critical
|
||
responsibility: BE
|
||
responsibility_reasoning: "AWOOOI API 是核心服務，屬後端團隊直接責任"
|
||
secondary_teams: [INFRA]
|
||
optimization:
|
||
- type: HEALTH_CHECK
|
||
description: "確認 API Pod 狀態和最近 log"
|
||
command: "kubectl get pods -n awoooi && kubectl logs -n awoooi deployment/awoooi-api --tail=50"
|
||
reasoning: "[規則匹配] AWOOOI API 下線需立即重啟，同時查 Pod log 確認根因。"
|
||
|
||
# ── 告警鏈路監控 ────────────────────────────────────────────
|
||
|
||
- id: alert_chain_broken
|
||
priority: 110
|
||
description: 告警鏈路中斷
|
||
match:
|
||
alertname:
|
||
- AlertChainBroken_Alertmanager
|
||
- AlertChainBroken_Sentry
|
||
- AlertChainBroken_SignOz
|
||
- AlertChainUnhealthy
|
||
- NoAlertsReceived2Hours
|
||
message:
|
||
- alert chain
|
||
- alertmanager
|
||
- no alerts
|
||
response:
|
||
action_title: "診斷告警鏈路中斷"
|
||
description: "⚙️ 規則匹配: 告警鏈路異常，可能導致真實告警無法送達 Telegram。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: "kubectl get pods -n monitoring && curl -s http://192.168.0.120:9093/api/v1/status | jq '.data.uptime'"
|
||
estimated_downtime: "監控盲區持續中"
|
||
risk: critical
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "告警鏈路屬基礎設施監控體系，需立即修復確保可觀測性"
|
||
secondary_teams: [BE]
|
||
optimization:
|
||
- type: E2E_TEST
|
||
description: "發送測試告警驗證整條鏈路"
|
||
command: "curl -X POST http://192.168.0.125:32334/api/v1/test-alert -H 'Content-Type: application/json' -d '{\"test\": true}'"
|
||
reasoning: "[規則匹配] 告警鏈路中斷等同監控失明，最高優先修復。"
|
||
|
||
# ── GPU / AI 基礎設施 ────────────────────────────────────────
|
||
|
||
- id: nvidia_circuit_breaker
|
||
priority: 115
|
||
description: NVIDIA/Nemotron 熔斷器開啟
|
||
match:
|
||
alertname:
|
||
- NvidiaCircuitBreakerOpen
|
||
- NvidiaToolCallingHighErrorRate
|
||
- NvidiaToolCallingHighLatency
|
||
message:
|
||
- circuit breaker
|
||
- nvidia
|
||
- nemotron
|
||
- tool calling
|
||
response:
|
||
action_title: "確認 NVIDIA API 熔斷狀態"
|
||
description: "⚙️ 規則匹配: NVIDIA/Nemotron 熔斷器開啟或錯誤率過高，AI Router 已自動降級。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: "curl -s http://192.168.0.125:32334/api/v1/ai-router/status | jq '.providers'"
|
||
estimated_downtime: "0 (已自動 fallback)"
|
||
risk: medium
|
||
responsibility: BE
|
||
responsibility_reasoning: "AI Provider 熔斷管理屬後端 AI Router 責任範圍"
|
||
secondary_teams: []
|
||
optimization:
|
||
- type: CIRCUIT_BREAKER_RESET
|
||
description: "等待熔斷器自動恢復 (half-open 狀態)"
|
||
command: "curl -s http://192.168.0.125:32334/api/v1/ai-router/reset -X POST"
|
||
reasoning: "[規則匹配] AI Router 已自動降級至備援 Provider，監控熔斷器恢復狀態即可。"
|
||
|
||
# ── E2E / Smoke Test 告警 ────────────────────────────────────
|
||
# 2026-04-09 Claude Sonnet 4.6: E2E test 假告警識別，僅記錄不修復
|
||
|
||
- id: e2e_smoke_test
|
||
priority: 120
|
||
description: E2E Smoke Test / 告警鏈路驗證假告警
|
||
match:
|
||
alertname:
|
||
- E2E_SMOKE_TEST
|
||
- E2E_FINAL_SMOKE_TEST
|
||
- SmokeTest
|
||
instance_prefix:
|
||
- e2e-final-
|
||
- e2e-test-
|
||
- test-host
|
||
- smoke-test-
|
||
message:
|
||
- e2e smoke test
|
||
- smoke test
|
||
- please ignore
|
||
- e2e test
|
||
- e2e-final
|
||
- e2e-test
|
||
- e2e_smoke
|
||
- alert chain smoke
|
||
response:
|
||
action_title: "告警鏈路驗證成功 (E2E)"
|
||
description: "✅ E2E Smoke Test 告警已收到，告警鏈路正常。此告警僅用於驗證，無需修復動作。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: ""
|
||
estimated_downtime: "N/A"
|
||
risk: low
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "E2E smoke test 假告警，告警鏈路驗證用途，系統自動識別跳過修復"
|
||
secondary_teams: []
|
||
optimization: []
|
||
reasoning: "[規則匹配] E2E Smoke Test 假告警，僅確認告警鏈路暢通，無實際服務異常。"
|
||
|
||
# ── 備份失敗 ────────────────────────────────────────────────
|
||
# 2026-04-11 Claude Sonnet 4.6: backup 類告警屬主機層，無 K8s deployment 可重啟
|
||
# → TYPE-1 純資訊通知，不應出現 [重啟] 按鈕
|
||
|
||
- id: host_backup_failed
|
||
priority: 50
|
||
description: 備份任務失敗 (rsync/velero/HostBackupFailed)
|
||
match:
|
||
alertname:
|
||
- HostBackupFailed
|
||
- VeleroBackupFailed
|
||
- VeleroBackupNotRun
|
||
- BackupJobFailed
|
||
response:
|
||
action_title: "🔍 備份失敗自動診斷 — SSH 收集備份與磁碟狀態"
|
||
description: "⚠️ 備份任務失敗。先自動 SSH 收集 backup log、last_success 與磁碟空間；若無法確認安全修復，立即升級緊急介入。"
|
||
suggested_action: SSH_DIAGNOSE
|
||
# 2026-05-02 ogt + Claude Sonnet 4.6: 補上 ps aux 讓 _ssh_execute 走 diagnostics 路徑（無阻擋）
|
||
kubectl_command: "ssh {host} 'ps aux --sort=-%cpu | head -15; echo \"=== BACKUP STATUS ===\"; ls -lah /home/ollama/backup/110 2>/dev/null || true; echo \"=== LAST SUCCESS ===\"; cat /home/ollama/backup/110/last_success 2>/dev/null || true; echo \"=== BACKUP LOG ===\"; tail -80 /home/ollama/backup/110/backup.log 2>/dev/null || true; echo \"=== DISK ===\"; df -h /home/ollama /backup / 2>/dev/null || df -h'"
|
||
estimated_downtime: "N/A"
|
||
risk: low
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "備份失敗屬基礎設施維運問題，先自動收集只讀證據，再交由緊急介入或後續 Playbook 修復"
|
||
secondary_teams: []
|
||
optimization: []
|
||
reasoning: "[規則匹配] 備份失敗先自動 SSH 只讀診斷，避免 LLM 誤判為 K8s deployment 重啟。"
|
||
|
||
# ── DevOps 工具層 ─────────────────────────────────────────
|
||
# 2026-04-14 Claude Sonnet 4.6: Task 2.2 ADR-076 — 新增 devops_tool / ssl_cert / external_site 三類規則
|
||
# 設計原則: CI/CD 工具與外部服務均為 NO_ACTION，不可自動修復（誤操作風險過高）
|
||
|
||
- id: gitea_down
|
||
priority: 125
|
||
description: Gitea CI/CD 服務下線（不自動修復）
|
||
match:
|
||
alertname:
|
||
- GiteaDown
|
||
- GiteaServiceDown
|
||
- GiteaUnhealthy
|
||
message:
|
||
- gitea
|
||
- git server
|
||
- ci/cd down
|
||
response:
|
||
action_title: "Gitea ({instance}) 下線 — 需人工確認"
|
||
description: "⚠️ 規則匹配: Gitea CI/CD 服務 ({instance}) 無法連線，影響所有部署流程。不自動重啟（誤觸 CD 風險過高）。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: ""
|
||
estimated_downtime: "N/A"
|
||
risk: critical
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "Gitea 是 CI/CD 核心，自動重啟有誤觸部署風險，需人工確認狀態後手動操作"
|
||
secondary_teams: []
|
||
optimization:
|
||
- type: HEALTH_CHECK
|
||
description: "確認 Gitea 服務狀態"
|
||
command: "ssh {host} 'cd /data/gitea && docker compose ps && docker compose logs --tail=20 gitea'"
|
||
reasoning: "[規則匹配] Gitea 下線不自動修復，通知後由人工確認狀態再操作，避免 CD pipeline 誤觸發。"
|
||
|
||
- id: ssl_cert_expiring
|
||
priority: 126
|
||
description: SSL/TLS 憑證即將到期或已到期
|
||
match:
|
||
alertname:
|
||
- SSLCertExpiringSoon
|
||
- SSLCertExpired
|
||
- CertificateExpirationWarning
|
||
- TLSCertExpiring
|
||
message:
|
||
- ssl cert
|
||
- certificate expir
|
||
- tls cert
|
||
- cert will expire
|
||
response:
|
||
action_title: "SSL 憑證 ({instance}) 即將到期 — 需人工更新"
|
||
description: "⚠️ 規則匹配: SSL/TLS 憑證 ({instance}) 即將到期或已到期。無自動修復，需人工確認 cert-manager 或執行 certbot 更新。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: ""
|
||
estimated_downtime: "N/A"
|
||
risk: medium
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "SSL 憑證更新需域名驗證，屬基礎設施團隊責任"
|
||
secondary_teams: []
|
||
optimization:
|
||
- type: CERT_RENEWAL
|
||
description: "確認 cert-manager 自動更新狀態"
|
||
command: "kubectl get certificate,certificaterequest -A && kubectl get secret -n awoooi-prod | grep tls"
|
||
reasoning: "[規則匹配] SSL 憑證到期無法自動修復，需人工操作 certbot 或確認 cert-manager 自動更新是否正常。"
|
||
|
||
- id: external_site_down
|
||
priority: 127
|
||
description: 外部網站或服務下線（MoWooo 系列 / HTTP probe 失敗）
|
||
match:
|
||
alertname:
|
||
- MoWoooWorkDown
|
||
- MoWoooDevDown
|
||
- ExternalSiteDown
|
||
- WebsiteDown
|
||
- BlackboxProbeFailed
|
||
message:
|
||
- external site
|
||
- website down
|
||
- mowooo
|
||
- http probe failed
|
||
- probe failed
|
||
response:
|
||
action_title: "外部網站 {instance} 下線 — 僅通知"
|
||
description: "⚠️ 規則匹配: 外部網站 ({instance}) HTTP probe 失敗。此為外部服務，無自動修復動作，等待服務恢復。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: ""
|
||
estimated_downtime: "N/A"
|
||
risk: medium
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "外部網站超出系統控制範圍，無法自動修復，通知後人工跟進"
|
||
secondary_teams: []
|
||
optimization:
|
||
- type: STATUS_CHECK
|
||
description: "手動確認外部網站狀態"
|
||
command: "curl -sv {instance} --max-time 10 2>&1 | grep -E '(HTTP|Connected|Failed)'"
|
||
reasoning: "[規則匹配] 外部網站下線屬外部依賴，通知統帥後等待服務恢復，必要時切換備援路徑。"
|
||
|
||
# 2026-04-24 ogt + Claude Sonnet 4.6: Sentry / ClickHouse 監控告警 — 外部服務，禁止 kubectl 操作
|
||
- id: sentry_clickhouse_alert
|
||
priority: 60
|
||
description: Sentry 或 ClickHouse 監控告警（外部服務，不是 K8s workload）
|
||
match:
|
||
alertname:
|
||
- SentryClickHouseMemoryPressure
|
||
- SentryClickHouseCpuHigh
|
||
- SentryClickHouseDiskUsageHigh
|
||
- ClickHouseMemoryHigh
|
||
- ClickHouseMemoryPressure
|
||
- ClickHouseCpuHigh
|
||
- ClickHouseReplicationLag
|
||
- ClickHouseQuerySlow
|
||
- SentryWorkerQueueHigh
|
||
- SentryKafkaLag
|
||
- SentryBacklogHigh
|
||
response:
|
||
action_title: "⚠️ Sentry/ClickHouse 告警 — 需 SSH 人工排查"
|
||
description: "⚠️ Sentry/ClickHouse 屬外部監控服務，無法透過 kubectl 自動修復。請 SSH 登入服務主機排查根因：clickhouse-client / docker stats / journalctl -xe。若記憶體壓力持續，考慮調整 ClickHouse max_memory_usage 設定或清理舊資料。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: ""
|
||
estimated_downtime: "N/A"
|
||
risk: high
|
||
responsibility: INFRA
|
||
responsibility_reasoning: "Sentry/ClickHouse 基礎設施由 INFRA 團隊管理"
|
||
secondary_teams: []
|
||
optimization: []
|
||
reasoning: "[規則匹配] Sentry/ClickHouse 非 K8s 服務，kubectl 操作無效。需 SSH 進入服務主機，確認記憶體/CPU/磁碟狀況後手動介入。"
|
||
|
||
# ── 通用兜底 ────────────────────────────────────────────────
|
||
|
||
- id: generic_fallback
|
||
priority: 999
|
||
description: 通用兜底規則 (無法匹配的告警)
|
||
match:
|
||
alertname:
|
||
- "*"
|
||
response:
|
||
action_title: "重新啟動 {target} 服務"
|
||
description: "⚙️ 規則匹配: {target} 發生異常，需進一步診斷確認根因。"
|
||
suggested_action: NO_ACTION
|
||
kubectl_command: ""
|
||
estimated_downtime: "N/A"
|
||
risk: medium
|
||
responsibility: COLLAB
|
||
responsibility_reasoning: "告警資訊不足以判定單一責任團隊，建議多團隊協同排查"
|
||
secondary_teams: [BE, INFRA]
|
||
optimization: []
|
||
reasoning: "[規則匹配] 未知告警類型，無法安全判斷修復動作，由人工或 LLM 診斷後決策。"
|