diff --git a/.claude/settings.json b/.claude/settings.json index 063101fb..f84a825b 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -407,7 +407,24 @@ "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"echo ''0936223270'' | sudo -S cat /etc/nginx/sites-available/awoooi.wooo.work.conf 2>/dev/null\")", "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"echo ''0936223270'' | sudo -S rm /etc/nginx/conf.d/awoooi-prod.conf && echo ''0936223270'' | sudo -S nginx -t 2>&1\")", "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"echo ''0936223270'' | sudo -S nginx -s reload 2>&1\")", - "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"echo ''0936223270'' | sudo -S systemctl reload nginx 2>&1\")" + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"echo ''0936223270'' | sudo -S systemctl reload nginx 2>&1\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"docker logs openclaw 2>&1 | tail -30\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"docker ps -a --format ''table {{.Names}}\\\\t{{.Status}}\\\\t{{.Image}}'' 2>&1 | head -15\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"docker logs clawbot 2>&1 | grep -i telegram | tail -20\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"docker logs clawbot 2>&1 | tail -30\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"docker exec alertmanager cat /etc/alertmanager/alertmanager.yml 2>&1 | head -30\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"curl -sf ''http://localhost:9093/api/v1/alerts'' | jq ''.data | length'' 2>/dev/null || curl -sf ''http://localhost:9093/api/v2/alerts'' | jq ''length'' 2>/dev/null\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"docker exec alertmanager wget -qO- ''http://localhost:9093/api/v2/alerts'' 2>&1 | head -100\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n awoooi-prod logs -l app=awoooi-worker --tail=50 2>&1\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"cat /home/ollama/alertmanager/alertmanager.yml 2>/dev/null || docker exec alertmanager cat /etc/alertmanager/alertmanager.yml\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"docker cp /tmp/alertmanager.yml alertmanager:/etc/alertmanager/alertmanager.yml && docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml && docker kill -s SIGHUP alertmanager\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"docker inspect alertmanager --format ''{{range .Mounts}}{{.Source}} -> {{.Destination}}{{println}}{{end}}''\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"docker exec alertmanager cat /etc/alertmanager/alertmanager.yml\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"docker restart alertmanager && sleep 3 && docker exec alertmanager cat /etc/alertmanager/alertmanager.yml\")", + "Bash(sshpass -p '0936223270' ssh ollama@192.168.0.188 \"docker logs clawbot 2>&1 | grep -i ''telegram\\\\|webhook\\\\|alert'' | tail -10\")", + "Bash(ssh wooo@192.168.0.120 \"kubectl logs deployment/awoooi-api -n awoooi-prod --tail=30 2>/dev/null | grep -E ''''POST|webhook|alertmanager|ManualTest''''\")", + "Bash(ssh wooo@192.168.0.120 \"kubectl logs deployment/awoooi-api -n awoooi-prod --tail=30 2>/dev/null | grep -iE ''''POST|webhook''''\")", + "Bash(ssh wooo@192.168.0.120 \"kubectl logs deployment/awoooi-api -n awoooi-prod --tail=50 2>/dev/null | grep -iE ''''POST.*webhook|alertmanager_webhook|NewFingerprint''''\")" ], "deny": [ "Bash(rm -rf *)", diff --git a/apps/api/src/api/v1/audit_logs.py b/apps/api/src/api/v1/audit_logs.py index 32daeb6c..a1caca91 100644 --- a/apps/api/src/api/v1/audit_logs.py +++ b/apps/api/src/api/v1/audit_logs.py @@ -14,8 +14,6 @@ Endpoints: from datetime import timedelta from typing import Any -from src.utils.timezone import now_taipei - from fastapi import APIRouter, HTTPException, Query, status from pydantic import BaseModel from sqlalchemy import func, select @@ -23,6 +21,7 @@ from sqlalchemy import func, select from src.core.logging import get_logger from src.db.base import get_db_context from src.db.models import AuditLog +from src.utils.timezone import now_taipei router = APIRouter(prefix="/audit-logs", tags=["Audit Logs"]) logger = get_logger("awoooi.audit") @@ -191,7 +190,6 @@ async def get_audit_stats() -> AuditStatsResponse: Returns: AuditStatsResponse: 統計資訊 """ - from datetime import timedelta async with get_db_context() as db: # Total count diff --git a/apps/api/src/api/v1/incidents.py b/apps/api/src/api/v1/incidents.py index 80004175..9f51f4a5 100644 --- a/apps/api/src/api/v1/incidents.py +++ b/apps/api/src/api/v1/incidents.py @@ -19,8 +19,6 @@ Phase 6.4 核心功能: from typing import Any -from src.utils.timezone import now_taipei - from fastapi import APIRouter, HTTPException, status from pydantic import BaseModel, Field @@ -30,6 +28,7 @@ from src.models.approval import ApprovalRequestResponse from src.models.incident import Incident, IncidentStatus, Severity from src.services.decision_manager import get_decision_manager from src.services.proposal_service import get_proposal_service +from src.utils.timezone import now_taipei router = APIRouter(prefix="/incidents", tags=["Incidents"]) logger = get_logger("awoooi.incidents") @@ -417,7 +416,6 @@ async def submit_feedback( Raises: HTTPException: 404 事件不存在 """ - from datetime import datetime from sqlalchemy import select @@ -531,7 +529,6 @@ async def debug_resolve_incident(incident_id: str) -> dict[str, Any]: DEBUG: 直接更新 Incident 狀態為 RESOLVED 用於測試 resolve_incident_after_approval 邏輯 """ - from datetime import datetime from sqlalchemy import select @@ -653,7 +650,6 @@ async def sync_incidents_from_approvals() -> SyncResult: """ 掃描所有 pending Approvals，為缺少 Incident 的補建 """ - from datetime import datetime from uuid import UUID from src.models.incident import Signal diff --git a/apps/api/src/api/v1/sentry_webhook.py b/apps/api/src/api/v1/sentry_webhook.py index ed4c5891..e189c17d 100644 --- a/apps/api/src/api/v1/sentry_webhook.py +++ b/apps/api/src/api/v1/sentry_webhook.py @@ -18,20 +18,19 @@ import uuid import httpx import structlog - from fastapi import APIRouter, BackgroundTasks, HTTPException, Request from pydantic import BaseModel -from src.utils.timezone import now_taipei_iso -from src.services.telegram_gateway import get_telegram_gateway -from src.services.approval_db import get_approval_service from src.models.approval import ( ApprovalRequestCreate, - RiskLevel, BlastRadius, DataImpact, DryRunCheck, + RiskLevel, ) +from src.services.approval_db import get_approval_service +from src.services.telegram_gateway import get_telegram_gateway +from src.utils.timezone import now_taipei_iso logger = structlog.get_logger(__name__) diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index e6dd2c19..86314dc6 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -26,8 +26,6 @@ import hashlib import hmac from typing import Literal -from src.utils.timezone import now_taipei - from fastapi import APIRouter, BackgroundTasks, Header, HTTPException, Request, status from pydantic import BaseModel, Field @@ -51,6 +49,7 @@ from src.services.openclaw import get_openclaw # Phase 5: Telegram Gateway (行動戰情室) from src.services.telegram_gateway import TelegramGatewayError, get_telegram_gateway +from src.utils.timezone import now_taipei router = APIRouter(prefix="/webhooks", tags=["Webhooks"]) logger = get_logger("awoooi.webhooks") diff --git a/apps/api/src/plugins/finops/cost_analyzer.py b/apps/api/src/plugins/finops/cost_analyzer.py index 3d207f56..cb5dd2bc 100644 --- a/apps/api/src/plugins/finops/cost_analyzer.py +++ b/apps/api/src/plugins/finops/cost_analyzer.py @@ -14,12 +14,12 @@ Phase 3.3: 商業變現能力 - Day-1 ROI import logging from dataclasses import dataclass, field -from datetime import timedelta - -from src.utils.timezone import now_taipei +from datetime import datetime, timedelta from enum import Enum from typing import Literal +from src.utils.timezone import now_taipei + logger = logging.getLogger(__name__) diff --git a/apps/api/src/plugins/mcp/mcp_bridge.py b/apps/api/src/plugins/mcp/mcp_bridge.py index 21380cf7..d83fdbf1 100644 --- a/apps/api/src/plugins/mcp/mcp_bridge.py +++ b/apps/api/src/plugins/mcp/mcp_bridge.py @@ -18,13 +18,13 @@ import re import uuid from dataclasses import dataclass, field from datetime import datetime - -from src.utils.timezone import now_taipei from enum import Enum from typing import Any import httpx +from src.utils.timezone import now_taipei + logger = logging.getLogger(__name__) diff --git a/apps/api/src/services/executor.py b/apps/api/src/services/executor.py index 70231907..c41b9cc6 100644 --- a/apps/api/src/services/executor.py +++ b/apps/api/src/services/executor.py @@ -21,7 +21,6 @@ Supported Operations: import asyncio import time from dataclasses import dataclass -from src.utils.timezone import now_taipei, now_taipei_iso from enum import Enum from pathlib import Path from typing import Any @@ -32,6 +31,7 @@ from src.core.config import settings from src.db.base import get_db_context from src.db.models import AuditLog from src.models.approval import ApprovalRequest +from src.utils.timezone import now_taipei logger = structlog.get_logger(__name__) diff --git a/apps/api/src/services/notifications/base.py b/apps/api/src/services/notifications/base.py index 70d9290d..40424d50 100644 --- a/apps/api/src/services/notifications/base.py +++ b/apps/api/src/services/notifications/base.py @@ -12,11 +12,11 @@ Phase 6: leWOOOgo Output Plugins from abc import ABC, abstractmethod from dataclasses import dataclass, field from datetime import datetime - -from src.utils.timezone import now_taipei from enum import Enum from typing import Any +from src.utils.timezone import now_taipei + class NotificationStatus(str, Enum): """通知狀態""" diff --git a/apps/api/src/services/openclaw.py b/apps/api/src/services/openclaw.py index 1731cae5..045f3bed 100644 --- a/apps/api/src/services/openclaw.py +++ b/apps/api/src/services/openclaw.py @@ -23,9 +23,9 @@ import json import random import re import time -import httpx +from datetime import datetime -from src.utils.timezone import now_taipei_iso +import httpx import structlog from src.core.config import settings @@ -34,6 +34,7 @@ from src.models.ai import ( OpenClawDecision, ) from src.services.signoz_client import GoldMetrics, get_signoz_client +from src.utils.timezone import now_taipei_iso logger = structlog.get_logger(__name__) diff --git a/apps/api/src/utils/timezone.py b/apps/api/src/utils/timezone.py index 971815ce..f4bdade4 100644 --- a/apps/api/src/utils/timezone.py +++ b/apps/api/src/utils/timezone.py @@ -6,7 +6,7 @@ AWOOOI - 時區工具 🔴 HARD RULE: 全系統使用台北時區，禁止 UTC """ -from datetime import datetime, timezone, timedelta +from datetime import UTC, datetime, timedelta, timezone # 台北時區 (UTC+8) TAIPEI_TZ = timezone(timedelta(hours=8)) @@ -38,7 +38,7 @@ def to_taipei(dt: datetime) -> datetime: """ if dt.tzinfo is None: # naive datetime，假設是 UTC - dt = dt.replace(tzinfo=timezone.utc) + dt = dt.replace(tzinfo=UTC) return dt.astimezone(TAIPEI_TZ) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 644dfaf1..b4c73d40 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -5,15 +5,37 @@ --- -## 📍 當前狀態 (2026-03-25 00:20) +## 📍 當前狀態 (2026-03-25 01:20) | 項目 | 狀態 | |------|------| -| **當前 Phase** | **Phase 10 完善 + Incident 同步** | +| **當前 Phase** | **Phase 10 完善** | | **Day** | Day 7 | -| **下一步** | CD 5d03a82 部署 → 驗證 Incidents 同步 | -| **重大修復** | ✅ Sentry Tunnel Nginx 修復 + Incidents 同步端點 + Ollama 0.18.2 | -| **CI/CD** | ⏳ 5d03a82 排隊 (Runner 忙碌中) | +| **下一步** | 測試簽核流程 (驗證原始內容保留) | +| **重大修復** | ✅ **簽核保留原始內容** - OpenClaw 1859893 | +| **CI/CD** | ✅ 23501633819 完成 | + +### 🔴 2026-03-25 01:20 簽核內容保留修復 + +**問題**: Telegram 簽核後只顯示「✅ 動作 xxx 已批准」，原始告警內容被覆蓋 + +**根因**: OpenClaw `_handle_approval_callback()` 使用 `edit_message_text()` 完全覆蓋原始訊息 + +**修復** (commit 1859893): +- 取得 `query.message.text` 原始內容 +- 組合: 原始內容 + 分隔線 + 簽核鋼印 +- 更新 `feedback_approval_preserve_content.md` + +### 🔴 2026-03-25 Alertmanager 災難修復 + +**問題**: Claude 錯誤將 Alertmanager 指向 OpenClaw (8088)，導致 Telegram 發送舊 AIOPS 格式 + +**修復**: +- Alertmanager 改指向 `http://192.168.0.120:32334/api/v1/webhooks/alertmanager` +- 刪除錯誤 Memory `feedback_alertmanager_openclaw_flow.md` +- 新增正確 Memory `feedback_alertmanager_awoooi_flow.md` +- 更新 DevOps Skill 04 +- 停用 runner-healthcheck.yml 中 ubuntu-latest jobs (GitHub Billing) ### 🧠 認知覺醒計畫 Phase 6 施工順序 (C-Suite 2026-03-23 統帥方案) @@ -41,6 +63,11 @@ | 時間 | 事件 | 負責人 | |------|------|--------| +| 2026-03-25 01:10 | **✅ CD 23501633819 部署完成**: API/Web/Worker 全部更新，Alertmanager webhook 路徑修復生效 | Claude Code | +| 2026-03-25 01:05 | **🔧 NetworkPolicy DNS 修復**: CoreDNS podSelector 修正，Telegram 發送恢復 | Claude Code | +| 2026-03-25 01:00 | **📝 feedback_approval_preserve_content.md**: 簽核後保留原始內容鐵律 | Claude Code | +| 2026-03-25 00:55 | **🔧 CI ubuntu-latest Jobs 停用 (ad00eda)**: external-sentinel + telegram-connectivity | Claude Code | +| 2026-03-25 00:50 | **🔴🔴 Alertmanager 路由修復**: Alertmanager 改指向 AWOOOI API (K3s 32334)，修復舊 AIOPS 格式災難 + Memory/Skill 更新 | Claude Code | | 2026-03-25 00:20 | **🔧 sync-from-approvals 端點 (5d03a82)**: 為舊 Approvals 補建 Incidents，修復活躍事件顯示 0 問題 | Claude Code | | 2026-03-25 00:10 | **🔧 Sentry Tunnel Nginx 修復 (41bd213)**: 將 /api/sentry-tunnel 路由到前端 (不是後端 FastAPI) | Claude Code | | 2026-03-25 00:05 | **✅ CD 23498719881 部署成功**: Incident-Approval 同步 + ApprovalCard UX 修復生效 | Claude Code |