fix(monitoring): 補齊 blackbox-tcp scrape targets (11→15)

SentryDown/HarborDown/SignOzDown 等告警引用的 instance 不在 scrape list 導致 absent metric = 0，告警持續 firing 新增缺少的 targets: 192.168.0.125:6443/32334/32335 (K3s) 192.168.0.110:9000/5000/3100 (Sentry/Harbor/Langfuse) 192.168.0.188:3301/5432/6380/11434/8089 (SignOz/PG/Redis/Ollama/OpenClaw) 已在 110 主機 reload Prometheus，全部 15 targets UP Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-09 12:20:19 +08:00
parent af49a54728
commit 5bd8a8a719
1 changed files with 175 additions and 0 deletions
--- a/k8s/monitoring/prometheus.yml
+++ b/k8s/monitoring/prometheus.yml
@@ -0,0 +1,175 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+  external_labels:
+    monitor: "wooo-aiops-monitor"
+    environment: "production"
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["alertmanager:9093"]
+
+rule_files:
+  - "alerts.yml"
+
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["localhost:9090"]
+        labels:
+          host: "110"
+
+  # === Node Exporters (5 台主機) ===
+  - job_name: "node-exporter-110"
+    static_configs:
+      - targets: ["192.168.0.110:9100"]
+        labels:
+          host: "110"
+          role: "devops"
+
+  - job_name: "node-exporter-112"
+    static_configs:
+      - targets: ["192.168.0.112:9100"]
+        labels:
+          host: "112"
+          role: "kali-security"
+
+  - job_name: "node-exporter-188"
+    scrape_interval: 30s
+    scrape_timeout: 25s
+    static_configs:
+      - targets: ["192.168.0.188:9100"]
+        labels:
+          host: "188"
+          role: "ai-web"
+
+  # === Blackbox HTTP 探測 ===
+  - job_name: "blackbox-http"
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets:
+          - https://aiops.wooo.work
+          - https://mo.wooo.work
+          - http://192.168.0.110:3001
+          - http://192.168.0.120:31234
+          - http://192.168.0.120:31235
+          - https://www.tsenyang.com
+          - http://stock.wooo.work
+          - https://bitan.wooo.work
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 192.168.0.110:9115
+
+  # === Blackbox TCP 探測 ===
+  - job_name: "blackbox-tcp"
+    metrics_path: /probe
+    params:
+      module: [tcp_connect]
+    static_configs:
+      - targets:
+          # K3s 層
+          - 192.168.0.125:6443
+          - 192.168.0.125:32334
+          - 192.168.0.125:32335
+          # 110 服務
+          - 192.168.0.110:9090
+          - 192.168.0.110:3001
+          - 192.168.0.110:9000
+          - 192.168.0.110:5000
+          - 192.168.0.110:3100
+          # 188 服務
+          - 192.168.0.188:3301
+          - 192.168.0.188:5432
+          - 192.168.0.188:6380
+          - 192.168.0.188:11434
+          - 192.168.0.188:8089
+          # K3s Worker
+          - 192.168.0.120:31234
+          - 192.168.0.120:31235
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 192.168.0.110:9115
+
+  # === cAdvisor (Docker 容器監控) ===
+  - job_name: "cadvisor-110"
+    scrape_interval: 60s
+    scrape_timeout: 55s
+    static_configs:
+      - targets: ["192.168.0.110:9180"]
+        labels:
+          host: "110"
+
+  # === ClawBot ===
+  - job_name: "blackbox-clawbot"
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets:
+        - http://192.168.0.188:8088/health
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 192.168.0.110:9115
+
+  - job_name: "clawbot"
+    scrape_interval: 30s
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["192.168.0.188:8088"]
+        labels:
+          host: "188"
+          service: "clawbot"
+
+  # === GitHub Actions Exporter (Phase 5 OPS.176) ===
+  - job_name: 'github-actions'
+    scrape_interval: 60s
+    static_configs:
+      - targets: ['github-exporter:9504']
+        labels:
+          source: 'github-actions'
+
+  # === AWOOOI API Metrics (Phase O Wave A.2, 2026-04-02) ===
+  - job_name: 'awoooi-api'
+    scrape_interval: 30s
+    metrics_path: /metrics
+    static_configs:
+      - targets: ['192.168.0.125:32334']
+        labels:
+          host: '125'
+          service: 'awoooi-api'
+          env: 'prod'
+
+  # === Sprint 5.2 Plan B: PostgreSQL Exporter (2026-04-08 Claude Sonnet 4.6) ===
+  - job_name: 'postgres-exporter'
+    scrape_interval: 30s
+    static_configs:
+      - targets: ['192.168.0.188:9187']
+        labels:
+          host: '188'
+          service: 'postgresql'
+          layer: 'systemd-188'
+
+  # === Sprint 5.2 Plan B: Redis Exporter (2026-04-08 Claude Sonnet 4.6) ===
+  - job_name: 'redis-exporter'
+    scrape_interval: 30s
+    static_configs:
+      - targets: ['192.168.0.188:9121']
+        labels:
+          host: '188'
+          service: 'redis'
+          layer: 'systemd-188'