# =============================================================================
# K3s Supplemental Alerts - 補充告警規則
# =============================================================================
# 首席架構師審查 2026-03-28: 補齊缺失的監控告警
# 部署位置: 192.168.0.188 Prometheus/Alertmanager
# =============================================================================

groups:
  # ===== 基礎設施服務告警 =====
  - name: infrastructure_services
    rules:
      # Redis 連線失敗
      - alert: RedisDown
        expr: probe_success{job="blackbox-tcp", instance="192.168.0.188:6380"} == 0
        for: 1m
        labels:
          severity: critical
          team: ops
          component: redis
        annotations:
          summary: "🔴 Redis 服務離線"
          description: "Redis Stack (192.168.0.188:6380) 已離線超過 1 分鐘"

      # PostgreSQL 連線失敗
      - alert: PostgreSQLDown
        expr: probe_success{job="blackbox-tcp", instance="192.168.0.188:5432"} == 0
        for: 1m
        labels:
          severity: critical
          team: ops
          component: postgresql
        annotations:
          summary: "🔴 PostgreSQL 服務離線"
          description: "PostgreSQL (192.168.0.188:5432) 已離線超過 1 分鐘"

      # OpenClaw 服務離線
      - alert: OpenClawDown
        expr: probe_success{job="blackbox-tcp", instance="192.168.0.188:8089"} == 0
        for: 2m
        labels:
          severity: warning
          team: ops
          component: openclaw
        annotations:
          summary: "⚠️ OpenClaw 服務離線"
          description: "OpenClaw (192.168.0.188:8089) 已離線超過 2 分鐘"

      # Harbor Registry 離線
      - alert: HarborDown
        expr: probe_success{job="blackbox-tcp", instance="192.168.0.110:5000"} == 0
        for: 2m
        labels:
          severity: warning
          team: ops
          component: harbor
        annotations:
          summary: "⚠️ Harbor Registry 離線"
          description: "Harbor (192.168.0.110:5000) 已離線超過 2 分鐘，影響 CI/CD"

      # Langfuse LLMOps 離線
      - alert: LangfuseDown
        expr: probe_success{job="blackbox-tcp", instance="192.168.0.110:3100"} == 0
        for: 2m
        labels:
          severity: warning
          team: ops
          component: langfuse
        annotations:
          summary: "⚠️ Langfuse LLMOps 離線"
          description: "Langfuse (192.168.0.110:3100) 已離線超過 2 分鐘"

  # ===== HPA 告警 =====
  - name: hpa_alerts
    rules:
      # HPA 達到最大副本數
      - alert: HPAMaxedOut
        expr: kube_horizontalpodautoscaler_status_current_replicas{namespace="awoooi-prod"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace="awoooi-prod"}
        for: 10m
        labels:
          severity: warning
          team: ops
          component: hpa
        annotations:
          summary: "⚠️ HPA 達到最大副本數"
          description: "{{ $labels.horizontalpodautoscaler }} 已達到最大 {{ $value }} 副本"

      # HPA 無法獲取指標
      - alert: HPAScalingDisabled
        expr: kube_horizontalpodautoscaler_status_condition{condition="ScalingActive", status="false", namespace="awoooi-prod"} == 1
        for: 5m
        labels:
          severity: warning
          team: ops
          component: hpa
        annotations:
          summary: "⚠️ HPA 擴展停用"
          description: "{{ $labels.horizontalpodautoscaler }} 無法擴展 (可能指標不可用)"

  # ===== Worker 告警 =====
  - name: worker_alerts
    rules:
      # Worker Pod 不可用
      - alert: WorkerUnavailable
        expr: kube_deployment_status_replicas_available{deployment="awoooi-worker", namespace="awoooi-prod"} == 0
        for: 2m
        labels:
          severity: critical
          team: ops
          component: worker
        annotations:
          summary: "🔴 Signal Worker 不可用"
          description: "awoooi-worker 沒有可用副本，Redis Streams 訊號無法處理"

  # ===== ArgoCD 告警 =====
  - name: argocd_alerts
    rules:
      # ArgoCD Application 同步失敗
      - alert: ArgoCDSyncFailed
        expr: argocd_app_info{sync_status!="Synced", namespace="argocd"} == 1
        for: 10m
        labels:
          severity: warning
          team: ops
          component: argocd
        annotations:
          summary: "⚠️ ArgoCD 同步失敗"
          description: "Application {{ $labels.name }} 同步狀態: {{ $labels.sync_status }}"

  # ===== 資源使用告警 =====
  - name: resource_alerts
    rules:
      # 節點 CPU 使用率過高
      - alert: NodeHighCPU
        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
        for: 10m
        labels:
          severity: warning
          team: ops
          component: node
        annotations:
          summary: "⚠️ 節點 CPU 使用率過高"
          description: "{{ $labels.instance }} CPU 使用率 {{ $value | printf \"%.1f\" }}%"

      # 節點記憶體使用率過高
      - alert: NodeHighMemory
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
        for: 10m
        labels:
          severity: warning
          team: ops
          component: node
        annotations:
          summary: "⚠️ 節點記憶體使用率過高"
          description: "{{ $labels.instance }} 記憶體使用率 {{ $value | printf \"%.1f\" }}%"

      # 容器 OOM Killed
      - alert: ContainerOOMKilled
        expr: increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled", namespace="awoooi-prod"}[1h]) > 0
        labels:
          severity: warning
          team: ops
          component: container
        annotations:
          summary: "⚠️ 容器 OOM Killed"
          description: "{{ $labels.pod }}/{{ $labels.container }} 過去 1 小時因 OOM 被終止"

  # ===== TLS 證書告警 =====
  - name: tls_certificate_alerts
    rules:
      # TLS 證書即將過期 (30 天內)
      - alert: TLSCertExpiringIn30Days
        expr: probe_ssl_earliest_cert_expiry{job="blackbox-https"} - time() < 86400 * 30
        for: 1h
        labels:
          severity: warning
          team: ops
          component: tls
        annotations:
          summary: "⚠️ TLS 證書即將過期"
          description: "{{ $labels.instance }} 證書將在 {{ $value | humanizeDuration }} 後過期"

      # TLS 證書即將過期 (7 天內)
      - alert: TLSCertExpiringIn7Days
        expr: probe_ssl_earliest_cert_expiry{job="blackbox-https"} - time() < 86400 * 7
        for: 1h
        labels:
          severity: critical
          team: ops
          component: tls
        annotations:
          summary: "🔴 TLS 證書即將過期 (7 天內)"
          description: "{{ $labels.instance }} 證書將在 {{ $value | humanizeDuration }} 後過期，請立即更新！"

      # TLS 證書已過期
      - alert: TLSCertExpired
        expr: probe_ssl_earliest_cert_expiry{job="blackbox-https"} - time() < 0
        labels:
          severity: critical
          team: ops
          component: tls
        annotations:
          summary: "🔴🔴 TLS 證書已過期"
          description: "{{ $labels.instance }} 證書已過期！HTTPS 服務可能無法正常運作"

      # TLS 探測失敗
      - alert: TLSProbeFailure
        expr: probe_success{job="blackbox-https"} == 0
        for: 5m
        labels:
          severity: warning
          team: ops
          component: tls
        annotations:
          summary: "⚠️ TLS 探測失敗"
          description: "無法連線到 {{ $labels.instance }} 進行 TLS 檢查"

  # ===== kube-state-metrics 擴充告警 (P3 2026-03-29) =====
  - name: kube_state_extended
    rules:
      # CronJob 上次執行失敗
      - alert: CronJobLastRunFailed
        expr: kube_job_status_failed{namespace="awoooi-prod"} > 0
        for: 1m
        labels:
          severity: warning
          team: ops
          component: cronjob
        annotations:
          summary: "⚠️ CronJob 執行失敗"
          description: "Job {{ $labels.job_name }} 執行失敗"

      # DaemonSet 缺少 Pod
      - alert: DaemonSetMissingPods
        expr: kube_daemonset_status_number_unavailable{namespace=~"awoooi-prod|kube-system|velero"} > 0
        for: 5m
        labels:
          severity: warning
          team: ops
          component: daemonset
        annotations:
          summary: "⚠️ DaemonSet 缺少 Pod"
          description: "{{ $labels.daemonset }} 缺少 {{ $value }} 個 Pod"

      # StatefulSet 副本不足
      - alert: StatefulSetReplicasMismatch
        expr: kube_statefulset_status_replicas_ready{namespace="awoooi-prod"} != kube_statefulset_replicas{namespace="awoooi-prod"}
        for: 5m
        labels:
          severity: warning
          team: ops
          component: statefulset
        annotations:
          summary: "⚠️ StatefulSet 副本不足"
          description: "{{ $labels.statefulset }} 預期副本數與就緒數不符"

      # 容器長時間等待 (ImagePullBackOff/CrashLoopBackOff)
      - alert: ContainerWaiting
        expr: kube_pod_container_status_waiting_reason{namespace="awoooi-prod", reason=~"ImagePullBackOff|CrashLoopBackOff|ErrImagePull"} == 1
        for: 10m
        labels:
          severity: warning
          team: ops
          component: container
        annotations:
          summary: "⚠️ 容器等待中"
          description: "{{ $labels.pod }}/{{ $labels.container }} 處於 {{ $labels.reason }} 狀態"

      # PDB 違規 (可用 Pod 數低於 minAvailable)
      - alert: PDBViolation
        expr: kube_poddisruptionbudget_status_current_healthy{namespace="awoooi-prod"} < kube_poddisruptionbudget_status_desired_healthy{namespace="awoooi-prod"}
        for: 5m
        labels:
          severity: warning
          team: ops
          component: pdb
        annotations:
          summary: "⚠️ PDB 違規"
          description: "{{ $labels.poddisruptionbudget }} 健康 Pod 數低於期望值"

      # 節點 taint 未被容忍 (排程問題偵測)
      - alert: NodeUnschedulable
        expr: kube_node_spec_unschedulable == 1
        for: 30m
        labels:
          severity: info
          team: ops
          component: node
        annotations:
          summary: "ℹ️ 節點標記為不可排程"
          description: "節點 {{ $labels.node }} 已被標記為 cordon/unschedulable 超過 30 分鐘"
