wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

feat: add all application source code

Browse Source 
- apps/api: FastAPI backend with Dockerfile
- apps/web: Next.js frontend with Dockerfile
- apps/sensor: Signal collection agent
- packages: shared packages

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
OG T
2026-03-22 18:57:44 +08:00
parent a840bf975b
commit 196d269b92
245 changed files with 42207 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
apps/api/src/plugins/__init__.py Normal file
View File

28
apps/api/src/plugins/finops/__init__.py Normal file
View File

@@ -0,0 +1,28 @@
"""
FinOps Plugin - 成本優化引擎
Phase 3.3: 閒置資源掃描與成本換算
"""
from .cost_analyzer import (
IdleResourceScanner,
idle_scanner,
CostReport,
WastedResource,
RecommendedAction,
ResourceType,
PricingConfig,
SavingsType,
WasteReason,
)
__all__ = [
"IdleResourceScanner",
"idle_scanner",
"CostReport",
"WastedResource",
"RecommendedAction",
"ResourceType",
"PricingConfig",
"SavingsType",
"WasteReason",
]

625
apps/api/src/plugins/finops/cost_analyzer.py Normal file
View File

@@ -0,0 +1,625 @@
"""
FinOps Cost Analyzer - 閒置資源掃描與成本換算
Phase 3.3: 商業變現能力 - Day-1 ROI
核心功能:
1. Orphaned PVCs (孤兒儲存卷) - 沒有被任何 Pod 掛載
2. Zombie Pods (殭屍容器) - CPU 使用率連續 7 天 < 1%
3. Over-provisioned Nodes (過度配置節點) - Request 高但 Usage 低
輸出格式:
- total_wasted_usd: 每月浪費金額
- recommended_actions: ClawBot 可執行的建議清單
"""
import logging
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from enum import Enum
from typing import Literal
logger = logging.getLogger(__name__)
# ==================== Types ====================
class ResourceType(str, Enum):
"""資源類型"""
PVC = "pvc" # PersistentVolumeClaim
POD = "pod" # Pod
NODE = "node" # Node
DEPLOYMENT = "deployment" # Deployment
SERVICE = "service" # Service
class WasteReason(str, Enum):
"""浪費原因"""
ORPHANED = "orphaned" # 孤兒資源 (無連結)
ZOMBIE = "zombie" # 殭屍 (幾乎無活動)
OVER_PROVISIONED = "over_provisioned" # 過度配置
IDLE = "idle" # 閒置
@dataclass
class WastedResource:
"""浪費的資源"""
resource_type: ResourceType
name: str
namespace: str
reason: WasteReason
details: str
monthly_cost_usd: float
created_at: datetime
last_used_at: datetime | None = None
# 資源規格
spec: dict = field(default_factory=dict)
def to_dict(self) -> dict:
return {
"resourceType": self.resource_type.value,
"name": self.name,
"namespace": self.namespace,
"reason": self.reason.value,
"details": self.details,
"monthlyCostUsd": round(self.monthly_cost_usd, 2),
"createdAt": self.created_at.isoformat(),
"lastUsedAt": self.last_used_at.isoformat() if self.last_used_at else None,
"spec": self.spec,
}
class SavingsType(str, Enum):
"""節省類型 - 區分真實省錢 vs 釋放資源"""
REALIZABLE = "realizable" # 真實省錢 (例如刪除 PVC → AWS 帳單立刻減少)
FREED = "freed" # 釋放資源 (例如刪除 Pod → 除非 Node 縮容否則不省錢)
@dataclass
class RecommendedAction:
"""建議的優化動作 (ClawBot 可執行)"""
action_id: str
action_type: Literal["delete", "scale_down", "resize", "migrate"]
resource_type: ResourceType
resource_name: str
namespace: str
description: str
estimated_savings_usd: float
risk_level: Literal["low", "medium", "high", "critical"]
command_hint: str # 給 ClawBot 的執行提示
savings_type: SavingsType = SavingsType.REALIZABLE # 節省類型
def to_dict(self) -> dict:
return {
"actionId": self.action_id,
"actionType": self.action_type,
"resourceType": self.resource_type.value,
"resourceName": self.resource_name,
"namespace": self.namespace,
"description": self.description,
"estimatedSavingsUsd": round(self.estimated_savings_usd, 2),
"riskLevel": self.risk_level,
"commandHint": self.command_hint,
"savingsType": self.savings_type.value,
}
@dataclass
class CostReport:
"""成本報告 (ClawBot 整合用)"""
scan_id: str
scanned_at: datetime
cluster_name: str
# 核心指標
total_wasted_usd: float
total_resources_scanned: int
wasted_resources_count: int
# 詳細資料
wasted_resources: list[WastedResource]
recommended_actions: list[RecommendedAction]
# 分類統計
waste_by_type: dict[str, float]
waste_by_namespace: dict[str, float]
def to_dict(self) -> dict:
"""輸出 ClawBot 可讀取的 JSON 格式"""
return {
"scanId": self.scan_id,
"scannedAt": self.scanned_at.isoformat(),
"clusterName": self.cluster_name,
# ClawBot 核心關注
"totalWastedUsd": round(self.total_wasted_usd, 2),
"totalResourcesScanned": self.total_resources_scanned,
"wastedResourcesCount": self.wasted_resources_count,
# 詳細資料
"wastedResources": [r.to_dict() for r in self.wasted_resources],
"recommendedActions": [a.to_dict() for a in self.recommended_actions],
# 統計
"wasteByType": {k: round(v, 2) for k, v in self.waste_by_type.items()},
"wasteByNamespace": {k: round(v, 2) for k, v in self.waste_by_namespace.items()},
# 摘要 (給 AI 的自然語言描述)
"summary": self._generate_summary(),
}
def _generate_summary(self) -> str:
"""產生 AI 可讀的摘要"""
if self.total_wasted_usd < 10:
return f"Cluster {self.cluster_name} is well-optimized. Only ${self.total_wasted_usd:.2f}/month potential savings."
top_waste = max(self.waste_by_type.items(), key=lambda x: x[1]) if self.waste_by_type else ("none", 0)
return (
f"Cluster {self.cluster_name} has ${self.total_wasted_usd:.2f}/month in wasted resources. "
f"Found {self.wasted_resources_count} idle resources. "
f"Biggest waste: {top_waste[0]} (${top_waste[1]:.2f}/month). "
f"{len(self.recommended_actions)} optimization actions available."
)
# ==================== Pricing Configuration ====================
@dataclass
class PricingConfig:
"""
費率配置 (可依雲端供應商調整)
預設值基於 AWS 美東區域 (us-east-1)
"""
# 儲存 (per GB/month)
storage_gp3_per_gb: float = 0.08 # EBS gp3
storage_gp2_per_gb: float = 0.10 # EBS gp2
storage_io1_per_gb: float = 0.125 # EBS io1
storage_standard_per_gb: float = 0.05 # Standard HDD
# 運算 (per vCPU/month, 假設 on-demand)
compute_per_vcpu: float = 30.0 # ~$0.04/hr * 720hr
compute_per_gb_ram: float = 4.0 # ~$0.005/hr/GB * 720hr
# 網路
load_balancer_per_month: float = 18.0 # ALB/NLB 固定費
nat_gateway_per_month: float = 32.0 # NAT Gateway
# ╔════════════════════════════════════════════════════════════════╗
# ║ SAFETY_BUFFER: 縮容安全係數 ║
# ║ 避免建議縮到剛好 actual usage，造成 OOM/CPU throttling ║
# ║ 公式: wasted = requested - (actual × 1.2) ║
# ╚════════════════════════════════════════════════════════════════╝
safety_buffer: float = 1.2
def get_storage_price(self, storage_class: str) -> float:
"""依 StorageClass 取得費率"""
mapping = {
"gp3": self.storage_gp3_per_gb,
"gp2": self.storage_gp2_per_gb,
"io1": self.storage_io1_per_gb,
"standard": self.storage_standard_per_gb,
}
return mapping.get(storage_class.lower(), self.storage_gp3_per_gb)
# 預設費率
DEFAULT_PRICING = PricingConfig()
# ==================== Idle Resource Scanner ====================
class IdleResourceScanner:
"""
閒置資源掃描器
偵測並量化 K8s 叢集中的浪費資源，
轉換為美金金額，供 ClawBot 決策
"""
def __init__(self, pricing: PricingConfig | None = None):
self.pricing = pricing or DEFAULT_PRICING
self._scan_counter = 0
async def full_scan(self, cluster_name: str = "default") -> CostReport:
"""
執行完整掃描
Returns:
CostReport 包含所有浪費資源與建議動作
"""
self._scan_counter += 1
scan_id = f"scan-{self._scan_counter:04d}-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}"
logger.info(f"[FinOps] Starting full scan: {scan_id}")
# 執行各類掃描
orphaned_pvcs = await self._scan_orphaned_pvcs()
zombie_pods = await self._scan_zombie_pods()
over_provisioned = await self._scan_over_provisioned_nodes()
# 合併所有浪費資源
all_wasted = orphaned_pvcs + zombie_pods + over_provisioned
# 產生建議動作
actions = self._generate_recommendations(all_wasted)
# 計算統計
total_wasted = sum(r.monthly_cost_usd for r in all_wasted)
waste_by_type = self._group_by_type(all_wasted)
waste_by_ns = self._group_by_namespace(all_wasted)
report = CostReport(
scan_id=scan_id,
scanned_at=datetime.utcnow(),
cluster_name=cluster_name,
total_wasted_usd=total_wasted,
total_resources_scanned=self._get_mock_total_resources(),
wasted_resources_count=len(all_wasted),
wasted_resources=all_wasted,
recommended_actions=actions,
waste_by_type=waste_by_type,
waste_by_namespace=waste_by_ns,
)
logger.info(
f"[FinOps] Scan complete: {scan_id} - "
f"${total_wasted:.2f}/month wasted, {len(actions)} actions"
)
return report
# ==================== Orphaned PVCs ====================
async def _scan_orphaned_pvcs(self) -> list[WastedResource]:
"""
掃描孤兒 PVC
孤兒 PVC = 已建立但沒有被任何 Pod 掛載的 PersistentVolumeClaim
常見原因: Pod 刪除後忘記清理 PVC
"""
# Phase 3: Mock 資料 (實際連接 K8s API 待 Phase 4)
mock_orphans = [
{
"name": "data-postgres-backup-old",
"namespace": "database",
"size_gb": 500,
"storage_class": "gp3",
"created": datetime.utcnow() - timedelta(days=90),
"last_used": datetime.utcnow() - timedelta(days=60),
},
{
"name": "logs-elasticsearch-2023",
"namespace": "logging",
"size_gb": 200,
"storage_class": "gp2",
"created": datetime.utcnow() - timedelta(days=180),
"last_used": datetime.utcnow() - timedelta(days=120),
},
{
"name": "cache-redis-temp",
"namespace": "default",
"size_gb": 50,
"storage_class": "gp3",
"created": datetime.utcnow() - timedelta(days=30),
"last_used": None,
},
]
results = []
for pvc in mock_orphans:
price_per_gb = self.pricing.get_storage_price(pvc["storage_class"])
monthly_cost = pvc["size_gb"] * price_per_gb
results.append(WastedResource(
resource_type=ResourceType.PVC,
name=pvc["name"],
namespace=pvc["namespace"],
reason=WasteReason.ORPHANED,
details=f"PVC not mounted by any Pod. Size: {pvc['size_gb']}GB ({pvc['storage_class']})",
monthly_cost_usd=monthly_cost,
created_at=pvc["created"],
last_used_at=pvc["last_used"],
spec={
"sizeGb": pvc["size_gb"],
"storageClass": pvc["storage_class"],
},
))
logger.info(f"[FinOps] Found {len(results)} orphaned PVCs")
return results
# ==================== Zombie Pods ====================
async def _scan_zombie_pods(self) -> list[WastedResource]:
"""
掃描殭屍 Pod
殭屍 Pod = CPU 使用率連續 7 天 < 1% 的 Pod
常見原因: 被遺忘的測試 Pod、已下線但未刪除的服務
"""
mock_zombies = [
{
"name": "legacy-api-5d7b8c9f6-abc12",
"namespace": "legacy",
"cpu_request": 2.0, # vCPU
"mem_request_gb": 4.0,
"avg_cpu_percent": 0.3,
"created": datetime.utcnow() - timedelta(days=120),
"last_active": datetime.utcnow() - timedelta(days=45),
},
{
"name": "test-worker-batch-xyz99",
"namespace": "testing",
"cpu_request": 1.0,
"mem_request_gb": 2.0,
"avg_cpu_percent": 0.1,
"created": datetime.utcnow() - timedelta(days=60),
"last_active": datetime.utcnow() - timedelta(days=30),
},
{
"name": "debug-shell-admin",
"namespace": "default",
"cpu_request": 0.5,
"mem_request_gb": 1.0,
"avg_cpu_percent": 0.0,
"created": datetime.utcnow() - timedelta(days=14),
"last_active": datetime.utcnow() - timedelta(days=10),
},
]
results = []
for pod in mock_zombies:
# 計算成本: CPU + Memory
cpu_cost = pod["cpu_request"] * self.pricing.compute_per_vcpu
mem_cost = pod["mem_request_gb"] * self.pricing.compute_per_gb_ram
monthly_cost = cpu_cost + mem_cost
results.append(WastedResource(
resource_type=ResourceType.POD,
name=pod["name"],
namespace=pod["namespace"],
reason=WasteReason.ZOMBIE,
details=(
f"CPU usage < 1% for 7+ days. "
f"Avg: {pod['avg_cpu_percent']:.1f}%. "
f"Resources: {pod['cpu_request']} vCPU, {pod['mem_request_gb']}GB RAM"
),
monthly_cost_usd=monthly_cost,
created_at=pod["created"],
last_used_at=pod["last_active"],
spec={
"cpuRequest": pod["cpu_request"],
"memoryGb": pod["mem_request_gb"],
"avgCpuPercent": pod["avg_cpu_percent"],
},
))
logger.info(f"[FinOps] Found {len(results)} zombie Pods")
return results
# ==================== Over-provisioned Nodes ====================
async def _scan_over_provisioned_nodes(self) -> list[WastedResource]:
"""
掃描過度配置節點
過度配置 = Request 很高但實際 Usage 很低
例如: Request 8 vCPU 但只用 1 vCPU
"""
mock_nodes = [
{
"name": "worker-large-01",
"namespace": "kube-system",
"total_cpu": 16.0,
"total_mem_gb": 64.0,
"requested_cpu": 12.0,
"requested_mem_gb": 48.0,
"actual_cpu": 2.0,
"actual_mem_gb": 8.0,
"created": datetime.utcnow() - timedelta(days=200),
},
{
"name": "worker-gpu-unused",
"namespace": "kube-system",
"total_cpu": 8.0,
"total_mem_gb": 32.0,
"requested_cpu": 4.0,
"requested_mem_gb": 16.0,
"actual_cpu": 0.5,
"actual_mem_gb": 2.0,
"created": datetime.utcnow() - timedelta(days=90),
},
]
results = []
for node in mock_nodes:
# ╔════════════════════════════════════════════════════════════════╗
# ║ 安全緩衝計算: wasted = requested - (actual × SAFETY_BUFFER) ║
# ║ 避免縮容建議導致 OOM / CPU throttling ║
# ╚════════════════════════════════════════════════════════════════╝
buffered_cpu = node["actual_cpu"] * self.pricing.safety_buffer
buffered_mem = node["actual_mem_gb"] * self.pricing.safety_buffer
wasted_cpu = node["requested_cpu"] - buffered_cpu
wasted_mem = node["requested_mem_gb"] - buffered_mem
if wasted_cpu < 1 and wasted_mem < 4:
continue # 浪費不夠顯著 (含安全緩衝後)
cpu_waste_cost = wasted_cpu * self.pricing.compute_per_vcpu
mem_waste_cost = wasted_mem * self.pricing.compute_per_gb_ram
monthly_cost = cpu_waste_cost + mem_waste_cost
utilization = node["actual_cpu"] / node["requested_cpu"] * 100
results.append(WastedResource(
resource_type=ResourceType.NODE,
name=node["name"],
namespace=node["namespace"],
reason=WasteReason.OVER_PROVISIONED,
details=(
f"Utilization: {utilization:.0f}%. "
f"Requested: {node['requested_cpu']} vCPU, {node['requested_mem_gb']}GB. "
f"Actual: {node['actual_cpu']} vCPU, {node['actual_mem_gb']}GB"
),
monthly_cost_usd=monthly_cost,
created_at=node["created"],
last_used_at=datetime.utcnow(),
spec={
"totalCpu": node["total_cpu"],
"totalMemoryGb": node["total_mem_gb"],
"requestedCpu": node["requested_cpu"],
"requestedMemoryGb": node["requested_mem_gb"],
"actualCpu": node["actual_cpu"],
"actualMemoryGb": node["actual_mem_gb"],
"utilizationPercent": utilization,
},
))
logger.info(f"[FinOps] Found {len(results)} over-provisioned resources")
return results
# ==================== Recommendations ====================
def _generate_recommendations(
self,
wasted: list[WastedResource],
) -> list[RecommendedAction]:
"""
產生優化建議 (ClawBot 可執行)
"""
actions = []
action_counter = 0
for resource in wasted:
action_counter += 1
action_id = f"action-{action_counter:03d}"
if resource.resource_type == ResourceType.PVC:
# ✅ REALIZABLE: 刪除 PVC → AWS 帳單立刻減少
actions.append(RecommendedAction(
action_id=action_id,
action_type="delete",
resource_type=resource.resource_type,
resource_name=resource.name,
namespace=resource.namespace,
description=f"Delete orphaned PVC '{resource.name}' - not mounted by any Pod",
estimated_savings_usd=resource.monthly_cost_usd,
risk_level="low",
command_hint=f"kubectl delete pvc {resource.name} -n {resource.namespace}",
savings_type=SavingsType.REALIZABLE,
))
elif resource.resource_type == ResourceType.POD:
# ⚠️ FREED: 刪除 Pod 只是釋放資源，除非 Node 縮容否則不省錢
risk = "medium" if resource.monthly_cost_usd > 50 else "low"
actions.append(RecommendedAction(
action_id=action_id,
action_type="delete",
resource_type=resource.resource_type,
resource_name=resource.name,
namespace=resource.namespace,
description=f"Delete zombie Pod '{resource.name}' - CPU < 1% for 7+ days",
estimated_savings_usd=resource.monthly_cost_usd,
risk_level=risk,
command_hint=f"kubectl delete pod {resource.name} -n {resource.namespace}",
savings_type=SavingsType.FREED,
))
elif resource.resource_type == ResourceType.NODE:
# ✅ REALIZABLE: Node 縮容/刪除 → AWS 帳單減少
actions.append(RecommendedAction(
action_id=action_id,
action_type="resize",
resource_type=resource.resource_type,
resource_name=resource.name,
namespace=resource.namespace,
description=(
f"Resize node '{resource.name}' - "
f"utilization only {resource.spec.get('utilizationPercent', 0):.0f}%"
),
estimated_savings_usd=resource.monthly_cost_usd,
risk_level="high",
command_hint=f"# Consider migrating workloads and downsizing {resource.name}",
savings_type=SavingsType.REALIZABLE,
))
# 按節省金額排序 (最大節省優先)
actions.sort(key=lambda a: a.estimated_savings_usd, reverse=True)
return actions
# ==================== Utilities ====================
def _group_by_type(self, resources: list[WastedResource]) -> dict[str, float]:
"""依類型分組統計"""
result: dict[str, float] = {}
for r in resources:
key = r.resource_type.value
result[key] = result.get(key, 0) + r.monthly_cost_usd
return result
def _group_by_namespace(self, resources: list[WastedResource]) -> dict[str, float]:
"""依 Namespace 分組統計"""
result: dict[str, float] = {}
for r in resources:
result[r.namespace] = result.get(r.namespace, 0) + r.monthly_cost_usd
return result
def _get_mock_total_resources(self) -> int:
"""Mock: 總掃描資源數"""
return 150 # 假設叢集有 150 個資源
def calculate_monthly_savings(self, report: CostReport) -> dict:
"""
計算月度節省摘要
╔════════════════════════════════════════════════════════════════╗
║ 嚴格區分真實省錢 vs 釋放資源 ║
║ - realizableSavingsUsd: 刪除後 AWS 帳單立刻減少 ║
║ - freedResourcesUsd: 釋放 Pod/Container，需要 Node 縮容才省錢 ║
╚════════════════════════════════════════════════════════════════╝
Returns:
ClawBot 可直接使用的 JSON 格式
"""
realizable = sum(
a.estimated_savings_usd
for a in report.recommended_actions
if a.savings_type == SavingsType.REALIZABLE
)
freed = sum(
a.estimated_savings_usd
for a in report.recommended_actions
if a.savings_type == SavingsType.FREED
)
return {
"totalWastedUsd": round(report.total_wasted_usd, 2),
# ⚠️ 嚴格區分
"realizableSavingsUsd": round(realizable, 2), # 真實省錢
"freedResourcesUsd": round(freed, 2), # 釋放資源 (需縮容才省錢)
"potentialSavingsUsd": round(realizable + freed, 2), # 總計 (參考用)
"actionCount": len(report.recommended_actions),
"topActions": [
{
"action": a.description,
"savings": round(a.estimated_savings_usd, 2),
"risk": a.risk_level,
"savingsType": a.savings_type.value,
}
for a in report.recommended_actions[:5] # Top 5
],
"annualProjection": round(realizable * 12, 2), # 年度預估僅計真實省錢
"annualProjectionWithFreed": round((realizable + freed) * 12, 2),
}
# 全域實例
idle_scanner = IdleResourceScanner()

20
apps/api/src/plugins/mcp/__init__.py Normal file
View File

@@ -0,0 +1,20 @@
"""
MCP (Model Context Protocol) Integration
Phase 3: 企業功能 - AI 與外部工具橋樑
"""
from .mcp_bridge import (
MCPBridge,
mcp_bridge,
MCPTool,
MCPToolResult,
MCPServer,
)
__all__ = [
"MCPBridge",
"mcp_bridge",
"MCPTool",
"MCPToolResult",
"MCPServer",
]

543
apps/api/src/plugins/mcp/mcp_bridge.py Normal file
View File

@@ -0,0 +1,543 @@
"""
MCP Bridge - AI 與外部工具橋樑
Phase 3: 企業功能 - ADR-001 MCP 協議採用
核心功能:
1. list_tools(server_name) - 動態獲取 MCP Server 工具清單
2. call_tool(server_name, tool_name, parameters) - 執行工具
資安機制:
- Rehydration: 執行前將 [IP_1] 還原為真實值
- 符合 leWOOOgo ActionExecutor 介面
MCP Protocol Spec: https://modelcontextprotocol.io/
"""
import logging
import re
import uuid
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
from typing import Any
import httpx
logger = logging.getLogger(__name__)
# ==================== Types ====================
class MCPTransport(str, Enum):
"""MCP 傳輸方式"""
STDIO = "stdio" # 標準輸入輸出 (本地程式)
HTTP = "http" # HTTP/SSE (遠端服務)
WEBSOCKET = "ws" # WebSocket (即時雙向)
@dataclass
class MCPTool:
"""MCP 工具定義"""
name: str
description: str
input_schema: dict[str, Any]
server_name: str
@dataclass
class MCPToolResult:
"""工具執行結果 (符合 ActionResult 介面)"""
success: bool
execution_id: str
output: Any | None = None
error: str | None = None
duration: float = 0.0
timestamp: datetime = field(default_factory=datetime.utcnow)
def to_dict(self) -> dict:
return {
"success": self.success,
"executionId": self.execution_id,
"output": self.output,
"error": self.error,
"duration": self.duration,
"timestamp": self.timestamp.isoformat(),
}
@dataclass
class MCPServer:
"""MCP Server 配置"""
name: str
transport: MCPTransport
endpoint: str # 執行檔路徑 (stdio) 或 URL (http/ws)
args: list[str] = field(default_factory=list)
env: dict[str, str] = field(default_factory=dict)
enabled: bool = True
# ==================== Rehydration Engine ====================
class RehydrationEngine:
"""
資安標籤還原器
將 Privacy Shield 產生的 [IP_1], [EMAIL_1], [SECRET_1] 等標籤
還原為真實值，以便 MCP Tool 執行
"""
# 標籤格式: [TYPE_N]
LABEL_PATTERN = re.compile(r'\[(IP|EMAIL|SECRET|CC|PHONE|ID)_(\d+)\]')
def unredact(
self,
data: Any,
mapping: dict[str, str],
) -> Any:
"""
還原脫敏資料
Args:
data: 可能包含脫敏標籤的資料 (str, dict, list)
mapping: 原始值 → 標籤 的映射表 (來自 Privacy Shield)
Returns:
還原後的資料
"""
# 反轉映射: 標籤 → 原始值
reverse_mapping = {v: k for k, v in mapping.items()}
return self._recursive_unredact(data, reverse_mapping)
def _recursive_unredact(
self,
data: Any,
reverse_mapping: dict[str, str],
) -> Any:
"""遞迴還原各種資料結構"""
if isinstance(data, str):
return self._unredact_string(data, reverse_mapping)
elif isinstance(data, dict):
return {
k: self._recursive_unredact(v, reverse_mapping)
for k, v in data.items()
}
elif isinstance(data, list):
return [
self._recursive_unredact(item, reverse_mapping)
for item in data
]
else:
return data
def _unredact_string(
self,
text: str,
reverse_mapping: dict[str, str],
) -> str:
"""
還原字串中的標籤
⚠️ 重要: 按標籤長度從長到短排序替換
避免 [IP_1] 被先替換而污染 [IP_10] → 結果變成 "192.168.1.1000"
"""
result = text
# 按標籤長度降序排序，確保 [IP_10] 先於 [IP_1] 處理
sorted_labels = sorted(
reverse_mapping.items(),
key=lambda x: len(x[0]),
reverse=True,
)
for label, original in sorted_labels:
# 使用精準邊界匹配，避免部分替換
result = result.replace(label, original)
return result
def validate_no_labels(self, data: Any) -> tuple[bool, list[str]]:
"""
驗證資料中是否還有未還原的標籤
Returns:
(is_clean, remaining_labels)
"""
remaining = []
self._find_labels(data, remaining)
return len(remaining) == 0, remaining
def _find_labels(self, data: Any, found: list[str]) -> None:
"""遞迴搜尋標籤"""
if isinstance(data, str):
matches = self.LABEL_PATTERN.findall(data)
for match in matches:
label = f"[{match[0]}_{match[1]}]"
if label not in found:
found.append(label)
elif isinstance(data, dict):
for v in data.values():
self._find_labels(v, found)
elif isinstance(data, list):
for item in data:
self._find_labels(item, found)
# ==================== MCP Bridge ====================
class MCPBridge:
"""
MCP 協議橋樑
連接 AI 與外部 MCP Server，實現動態工具調用
符合 leWOOOgo ActionExecutor 介面設計
"""
def __init__(self):
self.rehydrator = RehydrationEngine()
self._servers: dict[str, MCPServer] = {}
self._tool_cache: dict[str, list[MCPTool]] = {}
self._http_client = httpx.AsyncClient(timeout=30.0)
# 註冊 Mock Servers (Phase 3: 先驗證介面)
self._register_mock_servers()
def _register_mock_servers(self) -> None:
"""註冊 Mock MCP Servers (開發測試用)"""
self._servers["kubernetes"] = MCPServer(
name="kubernetes",
transport=MCPTransport.HTTP,
endpoint="http://localhost:8081/mcp",
)
self._servers["filesystem"] = MCPServer(
name="filesystem",
transport=MCPTransport.STDIO,
endpoint="/usr/local/bin/mcp-filesystem",
args=["--root", "/tmp"],
)
self._servers["database"] = MCPServer(
name="database",
transport=MCPTransport.HTTP,
endpoint="http://localhost:8082/mcp",
)
def register_server(self, server: MCPServer) -> None:
"""註冊 MCP Server"""
self._servers[server.name] = server
logger.info(f"MCP Server registered: {server.name} ({server.transport.value})")
async def list_tools(self, server_name: str) -> list[MCPTool]:
"""
動態獲取 MCP Server 工具清單
Args:
server_name: MCP Server 名稱
Returns:
可用工具列表
"""
if server_name not in self._servers:
raise ValueError(f"Unknown MCP Server: {server_name}")
# 快取檢查
if server_name in self._tool_cache:
return self._tool_cache[server_name]
server = self._servers[server_name]
tools = await self._fetch_tools(server)
self._tool_cache[server_name] = tools
return tools
async def _fetch_tools(self, server: MCPServer) -> list[MCPTool]:
"""從 MCP Server 獲取工具清單"""
if server.transport == MCPTransport.HTTP:
return await self._fetch_tools_http(server)
elif server.transport == MCPTransport.STDIO:
return await self._fetch_tools_stdio(server)
else:
raise NotImplementedError(f"Transport not supported: {server.transport}")
async def _fetch_tools_http(self, server: MCPServer) -> list[MCPTool]:
"""HTTP 方式獲取工具 (Mock 實作)"""
# Phase 3: Mock 回傳，實際連接待 MCP Server 部署
mock_tools = {
"kubernetes": [
MCPTool(
name="kubectl_get",
description="Get Kubernetes resources",
input_schema={
"type": "object",
"properties": {
"resource": {"type": "string"},
"namespace": {"type": "string"},
"name": {"type": "string"},
},
"required": ["resource"],
},
server_name=server.name,
),
MCPTool(
name="kubectl_delete",
description="Delete Kubernetes resources",
input_schema={
"type": "object",
"properties": {
"resource": {"type": "string"},
"namespace": {"type": "string"},
"name": {"type": "string"},
},
"required": ["resource", "name"],
},
server_name=server.name,
),
MCPTool(
name="kubectl_scale",
description="Scale Kubernetes deployment",
input_schema={
"type": "object",
"properties": {
"deployment": {"type": "string"},
"namespace": {"type": "string"},
"replicas": {"type": "integer"},
},
"required": ["deployment", "replicas"],
},
server_name=server.name,
),
],
"database": [
MCPTool(
name="query",
description="Execute SQL query",
input_schema={
"type": "object",
"properties": {
"sql": {"type": "string"},
"params": {"type": "array"},
},
"required": ["sql"],
},
server_name=server.name,
),
],
}
return mock_tools.get(server.name, [])
async def _fetch_tools_stdio(self, server: MCPServer) -> list[MCPTool]:
"""STDIO 方式獲取工具 (Mock 實作)"""
# Phase 3: Mock 回傳
return [
MCPTool(
name="read_file",
description="Read file contents",
input_schema={
"type": "object",
"properties": {"path": {"type": "string"}},
"required": ["path"],
},
server_name=server.name,
),
MCPTool(
name="write_file",
description="Write file contents",
input_schema={
"type": "object",
"properties": {
"path": {"type": "string"},
"content": {"type": "string"},
},
"required": ["path", "content"],
},
server_name=server.name,
),
]
# ╔════════════════════════════════════════════════════════════════╗
# ║ ⚠️ SECURITY CRITICAL - DO NOT LOG REHYDRATED PARAMETERS ⚠️ ║
# ║ ║
# ║ After rehydration, `parameters` contains REAL sensitive ║
# ║ data (IPs, emails, secrets). Logging them defeats the ║
# ║ entire purpose of Privacy Shield. ║
# ║ ║
# ║ ALLOWED: logger.info(f"Calling {tool_name}") ║
# ║ FORBIDDEN: logger.info(f"Params: {parameters}") ║
# ╚════════════════════════════════════════════════════════════════╝
async def call_tool(
self,
server_name: str,
tool_name: str,
parameters: dict[str, Any],
redaction_mapping: dict[str, str] | None = None,
) -> MCPToolResult:
"""
執行 MCP 工具
⚠️ 資安關鍵路徑：
1. Rehydration - 還原脫敏標籤為真實值
2. 驗證 - 確保無殘留標籤
3. 執行 - 調用 MCP Server
4. 結果 - 返回 ActionResult 格式
⛔ 禁止 logging 任何已 rehydrate 的 parameters！
Args:
server_name: MCP Server 名稱
tool_name: 工具名稱
parameters: 工具參數 (可能包含脫敏標籤)
redaction_mapping: Privacy Shield 映射表 (原始值 → 標籤)
Returns:
MCPToolResult (符合 ActionResult 介面)
"""
execution_id = str(uuid.uuid4())
start_time = datetime.utcnow()
try:
# ========================================
# 1. Rehydration: 還原脫敏標籤
# ========================================
if redaction_mapping:
logger.info(f"[{execution_id}] Rehydrating {len(redaction_mapping)} labels")
parameters = self.rehydrator.unredact(parameters, redaction_mapping)
# ========================================
# 2. 驗證: 確保無殘留標籤
# ========================================
is_clean, remaining = self.rehydrator.validate_no_labels(parameters)
if not is_clean:
logger.error(f"[{execution_id}] Unrehydrated labels found: {remaining}")
return MCPToolResult(
success=False,
execution_id=execution_id,
error=f"Security violation: Unrehydrated labels found: {remaining}",
duration=self._calc_duration(start_time),
)
# ========================================
# 3. 執行: 調用 MCP Server
# ========================================
logger.info(f"[{execution_id}] Calling {server_name}.{tool_name}")
if server_name not in self._servers:
raise ValueError(f"Unknown MCP Server: {server_name}")
server = self._servers[server_name]
result = await self._execute_tool(server, tool_name, parameters)
# ========================================
# 4. 結果: 返回 ActionResult 格式
# ========================================
return MCPToolResult(
success=True,
execution_id=execution_id,
output=result,
duration=self._calc_duration(start_time),
)
except Exception as e:
logger.error(f"[{execution_id}] Tool execution failed: {e}")
return MCPToolResult(
success=False,
execution_id=execution_id,
error=str(e),
duration=self._calc_duration(start_time),
)
async def _execute_tool(
self,
server: MCPServer,
tool_name: str,
parameters: dict[str, Any],
) -> Any:
"""執行 MCP 工具 (實際調用)"""
if server.transport == MCPTransport.HTTP:
return await self._execute_http(server, tool_name, parameters)
elif server.transport == MCPTransport.STDIO:
return await self._execute_stdio(server, tool_name, parameters)
else:
raise NotImplementedError(f"Transport not supported: {server.transport}")
async def _execute_http(
self,
server: MCPServer,
tool_name: str,
parameters: dict[str, Any],
) -> Any:
"""HTTP 方式執行工具 (Mock 實作)"""
# Phase 3: Mock 執行，實際連接待 MCP Server 部署
logger.info(f"[MOCK] HTTP call to {server.endpoint}: {tool_name}({parameters})")
# 模擬不同工具的回傳
mock_responses = {
"kubectl_get": {"items": [{"name": "pod-1"}, {"name": "pod-2"}]},
"kubectl_delete": {"deleted": True, "resource": parameters.get("name")},
"kubectl_scale": {"scaled": True, "replicas": parameters.get("replicas")},
"query": {"rows": [], "affected": 0},
}
return mock_responses.get(tool_name, {"status": "ok"})
async def _execute_stdio(
self,
server: MCPServer,
tool_name: str,
parameters: dict[str, Any],
) -> Any:
"""STDIO 方式執行工具 (Mock 實作)"""
# Phase 3: Mock 執行
logger.info(f"[MOCK] STDIO call to {server.endpoint}: {tool_name}({parameters})")
mock_responses = {
"read_file": f"[Mock] Contents of {parameters.get('path')}",
"write_file": {"written": True, "path": parameters.get("path")},
}
return mock_responses.get(tool_name, {"status": "ok"})
def _calc_duration(self, start_time: datetime) -> float:
"""計算執行時間 (毫秒)"""
return (datetime.utcnow() - start_time).total_seconds() * 1000
# ==================== ActionExecutor 介面對齊 ====================
def get_supported_operations(self) -> list[str]:
"""取得支援的操作列表 (符合 ActionExecutor 介面)"""
operations = []
for server_name, tools in self._tool_cache.items():
for tool in tools:
operations.append(f"{server_name}.{tool.name}")
return operations
async def execute(
self,
operation: str,
parameters: dict[str, Any],
redaction_mapping: dict[str, str] | None = None,
) -> MCPToolResult:
"""
執行操作 (符合 ActionExecutor.execute 介面)
Args:
operation: 格式為 "server_name.tool_name"
parameters: 工具參數
redaction_mapping: Privacy Shield 映射表
Returns:
MCPToolResult
"""
parts = operation.split(".", 1)
if len(parts) != 2:
return MCPToolResult(
success=False,
execution_id=str(uuid.uuid4()),
error=f"Invalid operation format: {operation}. Expected: server.tool",
)
server_name, tool_name = parts
return await self.call_tool(server_name, tool_name, parameters, redaction_mapping)
async def close(self) -> None:
"""關閉連線"""
await self._http_client.aclose()
# 全域實例
mcp_bridge = MCPBridge()

17
apps/api/src/plugins/security/__init__.py Normal file
View File

@@ -0,0 +1,17 @@
"""
AWOOOI Security Plugins
"""
from .privacy_shield import (
PrivacyShield,
privacy_shield,
SensitiveDataType,
RedactionResult,
)
__all__ = [
"PrivacyShield",
"privacy_shield",
"SensitiveDataType",
"RedactionResult",
]

341
apps/api/src/plugins/security/privacy_shield.py Normal file
View File

@@ -0,0 +1,341 @@
"""
Privacy Shield - BFF 脫敏攔截器
Phase 2.4: 資料清理引擎
在送給 LLM 之前，自動脫敏機敏資料：
- IPv4/IPv6 地址 → [IP_1], [IP_2], ...
- Email 信箱 → [EMAIL_1], [EMAIL_2], ...
- UUIDs/Tokens → [SECRET_1], [SECRET_2], ...
- API Keys (sk-*) → [SECRET_1], [SECRET_2], ...
特色：一致性雜湊 (Consistent Hashing)
- 同一段 Log 裡的同一個 IP，會被替換成同一個標籤
- AI 仍能辨識「這兩個 IP 是同一個」
"""
import re
from dataclasses import dataclass, field
from enum import Enum
from typing import Callable
# ==================== Types ====================
class SensitiveDataType(str, Enum):
"""機敏資料類型"""
IP_ADDRESS = "IP"
EMAIL = "EMAIL"
SECRET = "SECRET" # UUID, Token, API Key
CREDIT_CARD = "CC" # 未來擴充
PHONE = "PHONE" # 未來擴充
ID_NUMBER = "ID" # 未來擴充
@dataclass
class RedactionMatch:
"""單次脫敏匹配"""
original: str
redacted: str
data_type: SensitiveDataType
start: int
end: int
@dataclass
class RedactionResult:
"""脫敏結果"""
original_text: str
redacted_text: str
matches: list[RedactionMatch]
mapping: dict[str, str] # 原始值 → 脫敏標籤 (可逆映射)
@property
def has_sensitive_data(self) -> bool:
return len(self.matches) > 0
@property
def stats(self) -> dict[str, int]:
"""各類型脫敏統計"""
stats: dict[str, int] = {}
for match in self.matches:
key = match.data_type.value
stats[key] = stats.get(key, 0) + 1
return stats
# ==================== Regex Patterns ====================
# IPv4: 192.168.1.1
PATTERN_IPV4 = re.compile(
r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}'
r'(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b'
)
# IPv6: 2001:0db8:85a3::8a2e:0370:7334 (簡化版)
PATTERN_IPV6 = re.compile(
r'\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b|' # 完整格式
r'\b(?:[0-9a-fA-F]{1,4}:){1,7}:\b|' # 壓縮格式
r'\b(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}\b|'
r'\b(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}\b|'
r'\b(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}\b|'
r'\b(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}\b|'
r'\b(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}\b|'
r'\b[0-9a-fA-F]{1,4}:(?::[0-9a-fA-F]{1,4}){1,6}\b|'
r'\b::(?:[0-9a-fA-F]{1,4}:){0,5}[0-9a-fA-F]{1,4}\b|'
r'\b::1\b' # localhost
)
# Email: user@example.com
PATTERN_EMAIL = re.compile(
r'\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b'
)
# UUID: 550e8400-e29b-41d4-a716-446655440000
PATTERN_UUID = re.compile(
r'\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-'
r'[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b'
)
# API Keys: sk-xxx, pk-xxx, key-xxx, token-xxx
PATTERN_API_KEY = re.compile(
r'\b(?:sk|pk|api|key|token|bearer|secret|password|pwd|auth)[-_]?'
r'[a-zA-Z0-9]{16,}\b',
re.IGNORECASE
)
# Generic long tokens (32+ hex/alphanumeric)
PATTERN_LONG_TOKEN = re.compile(
r'\b[a-zA-Z0-9]{32,}\b'
)
# JWT-like tokens (xxx.xxx.xxx)
PATTERN_JWT = re.compile(
r'\beyJ[a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*\b'
)
# ==================== Privacy Shield Engine ====================
@dataclass
class ConsistentMapper:
"""
一致性映射器
確保同一個值在同一個上下文中被映射到同一個標籤
例如：192.168.1.1 總是映射到 [IP_1]
"""
prefix: str
_counter: int = 0
_mapping: dict[str, str] = field(default_factory=dict)
_reverse: dict[str, str] = field(default_factory=dict)
def get_label(self, value: str) -> str:
"""取得或建立標籤"""
if value not in self._mapping:
self._counter += 1
label = f"[{self.prefix}_{self._counter}]"
self._mapping[value] = label
self._reverse[label] = value
return self._mapping[value]
def get_original(self, label: str) -> str | None:
"""反查原始值 (用於還原)"""
return self._reverse.get(label)
@property
def mapping(self) -> dict[str, str]:
return self._mapping.copy()
class PrivacyShield:
"""
Privacy Shield 脫敏引擎
BFF 層攔截器，在送給 LLM 前自動脫敏機敏資料
使用一致性雜湊確保同值同標籤，AI 仍能辨識上下文關係
"""
def __init__(self):
# 預設啟用的規則 (可動態配置)
self.rules: list[tuple[re.Pattern, SensitiveDataType]] = [
(PATTERN_API_KEY, SensitiveDataType.SECRET), # API Key 優先
(PATTERN_JWT, SensitiveDataType.SECRET), # JWT Token
(PATTERN_UUID, SensitiveDataType.SECRET), # UUID
(PATTERN_EMAIL, SensitiveDataType.EMAIL), # Email
(PATTERN_IPV6, SensitiveDataType.IP_ADDRESS), # IPv6 先於 IPv4
(PATTERN_IPV4, SensitiveDataType.IP_ADDRESS), # IPv4
(PATTERN_LONG_TOKEN, SensitiveDataType.SECRET), # 長 Token (最後)
]
def redact(self, text: str) -> RedactionResult:
"""
執行脫敏
Args:
text: 原始文字 (Log、錯誤訊息、使用者輸入等)
Returns:
RedactionResult 包含脫敏後文字、匹配列表、映射表
"""
# 每次 redact 使用獨立的 mapper，確保同一批文字內一致
mappers: dict[SensitiveDataType, ConsistentMapper] = {
SensitiveDataType.IP_ADDRESS: ConsistentMapper(prefix="IP"),
SensitiveDataType.EMAIL: ConsistentMapper(prefix="EMAIL"),
SensitiveDataType.SECRET: ConsistentMapper(prefix="SECRET"),
}
matches: list[RedactionMatch] = []
redacted_positions: set[tuple[int, int]] = set()
# 1. 收集所有匹配 (避免重疊)
all_matches: list[tuple[re.Match, SensitiveDataType]] = []
for pattern, data_type in self.rules:
for match in pattern.finditer(text):
# 檢查是否與已匹配區域重疊
start, end = match.start(), match.end()
overlaps = any(
not (end <= s or start >= e)
for s, e in redacted_positions
)
if not overlaps:
all_matches.append((match, data_type))
redacted_positions.add((start, end))
# 2. 按位置排序 (從後往前替換，避免位移)
all_matches.sort(key=lambda x: x[0].start(), reverse=True)
# 3. 執行替換
result_text = text
for match, data_type in all_matches:
original = match.group()
mapper = mappers[data_type]
label = mapper.get_label(original)
# 記錄匹配
matches.append(RedactionMatch(
original=original,
redacted=label,
data_type=data_type,
start=match.start(),
end=match.end(),
))
# 替換文字
result_text = (
result_text[:match.start()] +
label +
result_text[match.end():]
)
# 反轉 matches 順序 (恢復正序)
matches.reverse()
# 合併所有映射
combined_mapping: dict[str, str] = {}
for mapper in mappers.values():
combined_mapping.update(mapper.mapping)
return RedactionResult(
original_text=text,
redacted_text=result_text,
matches=matches,
mapping=combined_mapping,
)
def redact_batch(self, texts: list[str]) -> list[RedactionResult]:
"""批次脫敏 (每個文字獨立映射)"""
return [self.redact(text) for text in texts]
def redact_with_shared_context(self, texts: list[str]) -> tuple[list[str], dict[str, str]]:
"""
共享上下文批次脫敏
多段文字共用同一個映射器，確保跨文字的同值同標籤
適用於：多行 Log、對話歷史等
"""
mappers: dict[SensitiveDataType, ConsistentMapper] = {
SensitiveDataType.IP_ADDRESS: ConsistentMapper(prefix="IP"),
SensitiveDataType.EMAIL: ConsistentMapper(prefix="EMAIL"),
SensitiveDataType.SECRET: ConsistentMapper(prefix="SECRET"),
}
results: list[str] = []
for text in texts:
result_text = text
redacted_positions: set[tuple[int, int]] = set()
all_matches: list[tuple[re.Match, SensitiveDataType]] = []
for pattern, data_type in self.rules:
for match in pattern.finditer(text):
start, end = match.start(), match.end()
overlaps = any(
not (end <= s or start >= e)
for s, e in redacted_positions
)
if not overlaps:
all_matches.append((match, data_type))
redacted_positions.add((start, end))
all_matches.sort(key=lambda x: x[0].start(), reverse=True)
for match, data_type in all_matches:
original = match.group()
label = mappers[data_type].get_label(original)
result_text = (
result_text[:match.start()] +
label +
result_text[match.end():]
)
results.append(result_text)
# 合併映射
combined_mapping: dict[str, str] = {}
for mapper in mappers.values():
combined_mapping.update(mapper.mapping)
return results, combined_mapping
def restore(self, text: str, mapping: dict[str, str]) -> str:
"""
還原脫敏文字 (用於除錯或日誌記錄)
⚠️ 警告：只應在 BFF 內部使用，絕不可還原後送給外部系統
"""
result = text
# 反轉映射
reverse_mapping = {v: k for k, v in mapping.items()}
for label, original in reverse_mapping.items():
result = result.replace(label, original)
return result
# ==================== FastAPI Middleware Integration ====================
def create_privacy_middleware(shield: "PrivacyShield"):
"""
建立 FastAPI 中間件
用於自動脫敏請求/回應中的機敏資料
"""
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from starlette.responses import Response
import json
class PrivacyShieldMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next: Callable) -> Response:
# TODO: 實作請求/回應脫敏
# 目前僅作為範例骨架
response = await call_next(request)
return response
return PrivacyShieldMiddleware
# 全域引擎實例
privacy_shield = PrivacyShield()