""" AwoooP Operator Console — Platform Operator Service ==================================================== leWOOOgo 積木化：DB 存取集中在 Service 層，Router 不直接引用 get_db。 ADR-106（AwoooP Agent Platform） 2026-05-05 ogt + Claude Sonnet 4.6 """ from __future__ import annotations import asyncio import json import os import re import time import uuid from collections import defaultdict from collections.abc import Mapping from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, get_args from urllib.parse import urlencode from uuid import UUID import httpx import structlog from fastapi import HTTPException, status from sqlalchemy import func, select, text, update from sqlalchemy import or_ as sa_or from src.core.config import get_settings from src.db.awooop_models import ( AwoooPContractRevision, AwoooPConversationEvent, AwoooPMcpGatewayAudit, AwoooPOutboundMessage, AwoooPRunState, AwoooPRunStepJournal, ) from src.db.base import get_db_context from src.db.models import IncidentRecord, MCPAuditLog from src.services.audit_sink import write_audit from src.services.awooop_ansible_audit_service import summarize_ansible_execution from src.services.awooop_approval_token import issue_approval_token, record_approval from src.services.awooop_truth_chain_service import ( _summarize_gateway_mcp, _summarize_mcp, fetch_truth_chain, ) from src.services.governance_km_stale_review_service import ( query_km_stale_owner_review_completion_queue, ) from src.services.ollama_endpoint_resolver import ( OllamaEndpointSelection, OllamaWorkloadType, resolve_ollama_order, ) from src.services.ollama_failover_manager import ( OllamaEndpoint, OllamaRoutingResult, get_ollama_failover_manager, ) from src.services.ollama_health_monitor import HealthReport, HealthStatus from src.services.operator_outcome import build_operator_outcome from src.services.operator_summary_cache import ( get_cached_operator_summary_async, store_operator_summary_async, ) from src.services.run_state_machine import transition from src.services.snapshot_paths import resolve_repo_root logger = structlog.get_logger(__name__) _MAX_CONTRACTS = 200 _DEFAULT_PER_PAGE = 50 _MAX_PER_PAGE = 200 _MAX_EVENTS = 100 _MAX_TIMELINE_ITEMS = 100 _MAX_LIST_CONTEXT_ROWS = 500 _RUN_CONTEXT_QUERY_CHUNK_SIZE = 500 _MAX_STEP_SUMMARY_CHARS = 128 _AI_ROUTE_STATUS_SELECT_TIMEOUT_SECONDS = 12.0 _AI_ROUTE_STATUS_CONNECTIVITY_TIMEOUT_SECONDS = 2.5 _REMEDIATION_HISTORY_LIMIT = 20 _ADR100_GATE5_PROJECTION_TRIGGER = "adr100_runtime_replay_gate5" _CALLBACK_REPLY_CACHE_TTL_SECONDS = int( os.getenv("AWOOOP_CALLBACK_REPLY_CACHE_TTL_SECONDS", "20") ) _INCIDENT_ID_RE = re.compile(r"\bINC-\d{8}-[A-Z0-9]{4,}\b") _REMEDIATION_STATUS_FILTERS = { "mcp_observed", "no_evidence", "read_only_dry_run", "write_observed", "blocked", "observed", } _CALLBACK_REPLY_STATUS_FILTERS = { "no_callback", "sent", "fallback_sent", "rescue_sent", "failed", "observed", } _CALLBACK_REPLY_RAW_STATUS_BY_FILTER = { "sent": "callback_reply_sent", "fallback_sent": "callback_reply_fallback_sent", "rescue_sent": "callback_reply_rescue_sent", "failed": "callback_reply_failed", } _CALLBACK_REPLY_ACTION_RE = re.compile(r"^[a-z0-9_:-]{1,64}$", re.IGNORECASE) _CICD_STATUS_FILTERS = {"running", "success", "failed", "pending"} _CICD_STAGE_RE = re.compile(r"^[a-z0-9_:-]{1,64}$", re.IGNORECASE) _AI_ROUTE_STATUS_SCHEMA_VERSION = "awooop_ai_route_status_v1" _AI_ROUTE_WORKLOADS = set(get_args(OllamaWorkloadType)) _AI_ROUTE_REPAIR_EVIDENCE_PROVIDER = "ai_route_repair" _AI_ROUTE_REPAIR_EVIDENCE_STAGE = "repair_diagnosis" _SOURCE_CORRELATION_SCHEMA_VERSION = "source_provider_correlation_v1" _SOURCE_CORRELATION_PROVIDERS = ("sentry", "signoz") _SOURCE_CORRELATION_EVENT_LIMIT = 200 _SOURCE_CORRELATION_LOOKBACK_DAYS = 7 _SOURCE_CORRELATION_PRE_WINDOW_HOURS = 2 _KM_STALE_COMPLETION_CALLBACK_SCHEMA_VERSION = ( "km_stale_owner_review_completion_callback_summary_v1" ) _CALLBACK_EVIDENCE_CAPTURE_STATUS_SCHEMA_VERSION = "callback_evidence_capture_status_v1" _CALLBACK_REPLY_AUDIT_SUMMARY_SCHEMA_VERSION = ( "telegram_callback_reply_audit_summary_v1" ) # ============================================================================= # Tenants # ============================================================================= _PUBLIC_GATEWAY_INVENTORY_SNAPSHOT = ( "docs/security/public-gateway-preflight-inventory.snapshot.json" ) _SOURCE_CONTROL_READINESS_SNAPSHOT = ( "docs/security/source-control-primary-readiness-gate.snapshot.json" ) _DOMAIN_PRODUCT_OVERRIDES: dict[str, dict[str, str]] = { "aiops.wooo.work": { "product_id": "awoooi", "product_name": "AWOOOI / AwoooP / IwoooS", "category": "core_platform", }, "awoooi.wooo.work": { "product_id": "awoooi", "product_name": "AWOOOI / AwoooP / IwoooS", "category": "core_platform", }, "api.awoooi.wooo.work": { "product_id": "awoooi", "product_name": "AWOOOI / AwoooP / IwoooS", "category": "core_platform", }, "app.awoooi.wooo.work": { "product_id": "awoooi", "product_name": "AWOOOI / AwoooP / IwoooS", "category": "core_platform", }, "api.aiops.wooo.work": { "product_id": "awoooi", "product_name": "AWOOOI / AwoooP / IwoooS", "category": "core_platform", }, "clawbot.aiops.wooo.work": { "product_id": "awoooi", "product_name": "AWOOOI / AwoooP / IwoooS", "category": "core_platform", }, "command.aiops.wooo.work": { "product_id": "awoooi", "product_name": "AWOOOI / AwoooP / IwoooS", "category": "core_platform", }, "security.wooo.work": { "product_id": "awoooi", "product_name": "AWOOOI / AwoooP / IwoooS", "category": "core_platform", }, "mo.wooo.work": { "product_id": "ewoooc", "product_name": "EwoooC / Mo", "category": "business_product", }, "2026fifa.wooo.work": { "product_id": "2026-fifa-world-cup", "product_name": "2026 FIFA World Cup", "category": "business_product", }, "vibework.wooo.work": { "product_id": "vibework", "product_name": "VibeWork", "category": "business_product", }, "agent.wooo.work": { "product_id": "agent-bounty-protocol", "product_name": "Agent Bounty Protocol", "category": "business_product", }, "stock.wooo.work": { "product_id": "stockplatform", "product_name": "StockPlatform", "category": "business_product", }, "bitan.wooo.work": { "product_id": "bitan-pharmacy", "product_name": "Bitan Pharmacy", "category": "business_product", }, "tsenyang.com": { "product_id": "tsenyang-website", "product_name": "TsenYang Website", "category": "public_site", }, "www.tsenyang.com": { "product_id": "tsenyang-website", "product_name": "TsenYang Website", "category": "public_site", }, "tsenyang.wooo.work": { "product_id": "tsenyang-website", "product_name": "TsenYang Website", "category": "public_site", }, "vtuber.wooo.work": { "product_id": "vtuber", "product_name": "VTuber", "category": "public_site", }, "design.wooo.work": { "product_id": "wooo-open-design", "product_name": "WOOO Open Design", "category": "platform_tool", }, "grist.wooo.work": { "product_id": "data-workspace", "product_name": "Data Workspace / Grist", "category": "platform_tool", }, "n8n.wooo.work": { "product_id": "workflow-automation", "product_name": "Workflow Automation / n8n", "category": "platform_tool", }, "vault.wooo.work": { "product_id": "security-secrets-platform", "product_name": "Security / Vault", "category": "platform_tool", }, "ollama.wooo.work": { "product_id": "ai-model-gateway", "product_name": "AI Model Gateway / Ollama", "category": "platform_tool", }, "gitea.wooo.work": { "product_id": "source-control", "product_name": "Source Control / DevOps", "category": "platform_tool", }, "gitlab.wooo.work": { "product_id": "source-control", "product_name": "Source Control / DevOps", "category": "platform_tool", }, "harbor.wooo.work": { "product_id": "source-control", "product_name": "Source Control / DevOps", "category": "platform_tool", }, "registry.wooo.work": { "product_id": "source-control", "product_name": "Source Control / DevOps", "category": "platform_tool", }, "sentry.wooo.work": { "product_id": "observability-tooling", "product_name": "Observability / LLMOps", "category": "platform_tool", }, "signoz.wooo.work": { "product_id": "observability-tooling", "product_name": "Observability / LLMOps", "category": "platform_tool", }, "langfuse.wooo.work": { "product_id": "observability-tooling", "product_name": "Observability / LLMOps", "category": "platform_tool", }, "monitor.wooo.work": { "product_id": "observability-tooling", "product_name": "Observability / LLMOps", "category": "platform_tool", }, } _CANONICAL_PRODUCT_SURFACES: tuple[dict[str, Any], ...] = ( { "product_id": "awoooi", "product_name": "AWOOOI / AwoooP / IwoooS", "project_id": "awoooi", "category": "core_platform", "surface_kind": "platform_product", "owner_lane": "S4.9-S4.13", "coverage_status": "read_only_visible", "public_routes": [ "awoooi.wooo.work", "aiops.wooo.work", "api.awoooi.wooo.work", "app.awoooi.wooo.work", "api.aiops.wooo.work", "clawbot.aiops.wooo.work", "command.aiops.wooo.work", "security.wooo.work", ], "source_keys": ["wooo/awoooi"], }, { "product_id": "ewoooc", "product_name": "EwoooC / Mo", "project_id": "ewoooc", "category": "business_product", "surface_kind": "product_console", "owner_lane": "S4.10", "coverage_status": "owner_response_required", "public_routes": ["mo.wooo.work"], "source_keys": ["wooo/ewoooc / root/momo-pro-system / momo working trees"], }, { "product_id": "2026-fifa-world-cup", "product_name": "2026 FIFA World Cup", "project_id": "2026FIFAWorldCup", "category": "business_product", "surface_kind": "product_site", "owner_lane": "product_onboarding", "coverage_status": "owner_response_required", "public_routes": ["2026fifa.wooo.work"], "source_keys": ["2026FIFAWorldCup"], }, { "product_id": "vibework", "product_name": "VibeWork", "project_id": "vibework", "category": "business_product", "surface_kind": "product_site", "owner_lane": "S4.10", "coverage_status": "owner_response_required", "public_routes": ["vibework.wooo.work"], "source_keys": ["vibework"], }, { "product_id": "agent-bounty-protocol", "product_name": "Agent Bounty Protocol", "project_id": "agent-bounty-protocol", "category": "business_product", "surface_kind": "agent_protocol", "owner_lane": "S4.10", "coverage_status": "owner_response_required", "public_routes": ["agent.wooo.work"], "source_keys": ["agent-bounty-protocol"], }, { "product_id": "stockplatform", "product_name": "StockPlatform", "project_id": "stockplatform", "category": "business_product", "surface_kind": "product_site", "owner_lane": "P0-CRON-001", "coverage_status": "read_only_visible", "public_routes": ["stock.wooo.work"], "source_keys": ["stockplatform"], }, { "product_id": "bitan-pharmacy", "product_name": "Bitan Pharmacy", "project_id": "bitan-pharmacy", "category": "business_product", "surface_kind": "product_site", "owner_lane": "P0-CRON-002", "coverage_status": "read_only_visible", "public_routes": ["bitan.wooo.work"], "source_keys": ["bitan-pharmacy"], }, { "product_id": "tsenyang-website", "product_name": "TsenYang Website", "project_id": "tsenyang-website", "category": "public_site", "surface_kind": "public_site", "owner_lane": "S4.10", "coverage_status": "owner_response_required", "public_routes": ["tsenyang.com", "www.tsenyang.com", "tsenyang.wooo.work"], "source_keys": ["tsenyang-website"], }, { "product_id": "vtuber", "product_name": "VTuber", "project_id": "vtuber", "category": "public_site", "surface_kind": "public_site", "owner_lane": "public_gateway", "coverage_status": "read_only_visible", "public_routes": ["vtuber.wooo.work"], "source_keys": [], }, { "product_id": "wooo-open-design", "product_name": "WOOO Open Design", "project_id": "open-design", "category": "platform_tool", "surface_kind": "design_system", "owner_lane": "S4.10", "coverage_status": "read_only_visible", "public_routes": ["design.wooo.work"], "source_keys": ["open-design"], }, { "product_id": "workflow-automation", "product_name": "Workflow Automation / n8n", "project_id": "__platform__", "category": "platform_tool", "surface_kind": "automation_tool", "owner_lane": "observability_tooling", "coverage_status": "read_only_candidate", "public_routes": ["n8n.wooo.work"], "source_keys": [], }, { "product_id": "data-workspace", "product_name": "Data Workspace / Grist", "project_id": "__platform__", "category": "platform_tool", "surface_kind": "data_tool", "owner_lane": "observability_tooling", "coverage_status": "read_only_candidate", "public_routes": ["grist.wooo.work"], "source_keys": [], }, { "product_id": "security-secrets-platform", "product_name": "Security / Vault", "project_id": "__platform__", "category": "platform_tool", "surface_kind": "security_tool", "owner_lane": "S4.12", "coverage_status": "read_only_candidate", "public_routes": ["vault.wooo.work"], "source_keys": [], }, { "product_id": "ai-model-gateway", "product_name": "AI Model Gateway / Ollama", "project_id": "__platform__", "category": "platform_tool", "surface_kind": "ai_provider_tool", "owner_lane": "P1-004", "coverage_status": "read_only_candidate", "public_routes": ["ollama.wooo.work"], "source_keys": [], }, { "product_id": "source-control", "product_name": "Source Control / DevOps", "project_id": "__platform__", "category": "platform_tool", "surface_kind": "tooling", "owner_lane": "S4.9-S4.13", "coverage_status": "read_only_visible", "public_routes": [ "gitea.wooo.work", "gitlab.wooo.work", "harbor.wooo.work", "registry.wooo.work", ], "source_keys": ["wooo/wooo-infra-config"], }, { "product_id": "observability-tooling", "product_name": "Observability / LLMOps", "project_id": "__platform__", "category": "platform_tool", "surface_kind": "tooling", "owner_lane": "P1-003", "coverage_status": "read_only_visible", "public_routes": [ "sentry.wooo.work", "signoz.wooo.work", "langfuse.wooo.work", "monitor.wooo.work", ], "source_keys": [], }, ) _ADDITIONAL_PUBLIC_ROUTES: tuple[dict[str, Any], ...] = ( { "domain": "awoooi.wooo.work", "coverage_status": "read_only_visible", "source": "awoooi_canonical_route", }, { "domain": "api.awoooi.wooo.work", "coverage_status": "read_only_candidate", "source": "repo_domain_scan_candidate", }, { "domain": "app.awoooi.wooo.work", "coverage_status": "read_only_candidate", "source": "repo_domain_scan_candidate", }, { "domain": "api.aiops.wooo.work", "coverage_status": "read_only_candidate", "source": "repo_domain_scan_candidate", }, { "domain": "clawbot.aiops.wooo.work", "coverage_status": "read_only_candidate", "source": "repo_domain_scan_candidate", }, { "domain": "command.aiops.wooo.work", "coverage_status": "read_only_candidate", "source": "repo_domain_scan_candidate", }, { "domain": "security.wooo.work", "coverage_status": "read_only_candidate", "source": "repo_domain_scan_candidate", }, { "domain": "2026fifa.wooo.work", "coverage_status": "owner_response_required", "source": "cross_workspace_product_candidate", }, { "domain": "vibework.wooo.work", "coverage_status": "owner_response_required", "source": "vibework_onboarding_handoff", }, { "domain": "agent.wooo.work", "coverage_status": "owner_response_required", "source": "agent_bounty_onboarding_handoff", }, { "domain": "tsenyang.wooo.work", "coverage_status": "owner_response_required", "source": "repo_domain_scan_candidate", }, { "domain": "design.wooo.work", "coverage_status": "read_only_visible", "source": "open_design_route_candidate", }, { "domain": "grist.wooo.work", "coverage_status": "read_only_candidate", "source": "repo_domain_scan_candidate", }, { "domain": "n8n.wooo.work", "coverage_status": "read_only_candidate", "source": "repo_domain_scan_candidate", }, { "domain": "vault.wooo.work", "coverage_status": "read_only_candidate", "source": "repo_domain_scan_candidate", }, { "domain": "ollama.wooo.work", "coverage_status": "read_only_candidate", "source": "repo_domain_scan_candidate", }, { "domain": "monitor.wooo.work", "coverage_status": "read_only_candidate", "source": "repo_domain_scan_candidate", }, ) def _load_committed_snapshot(relative_path: str) -> dict[str, Any]: repo_root = resolve_repo_root(Path(__file__)) snapshot_path = repo_root / relative_path try: return json.loads(snapshot_path.read_text(encoding="utf-8")) except FileNotFoundError: logger.warning("tenant_asset_snapshot_missing", path=relative_path) except json.JSONDecodeError as exc: logger.warning( "tenant_asset_snapshot_invalid_json", path=relative_path, error=str(exc), ) return {} def _domain_product(domain: str) -> dict[str, str]: return _DOMAIN_PRODUCT_OVERRIDES.get( domain, { "product_id": domain.replace(".", "-"), "product_name": domain, "category": "public_route", }, ) def _route_asset_from_gateway_row(row: Mapping[str, Any]) -> dict[str, Any]: domain = str(row.get("domain") or "") product = _domain_product(domain) route_smoke_accepted = bool(row.get("route_smoke_accepted")) owner_response_accepted = bool(row.get("owner_response_accepted")) coverage_status = ( "verified" if route_smoke_accepted and owner_response_accepted else "read_only_pending_smoke" ) return { "domain": domain, "product_id": product["product_id"], "product_name": product["product_name"], "category": product["category"], "coverage_status": coverage_status, "control_tier": row.get("control_tier") or "unknown", "upstream_count": int(row.get("upstream_count") or 0), "admin_route_count": int(row.get("admin_route_count") or 0), "websocket_route_count": int(row.get("websocket_route_count") or 0), "public_route_smoke_required": bool(row.get("public_route_smoke_required")), "route_smoke_accepted": route_smoke_accepted, "owner_response_accepted": owner_response_accepted, "runtime_gate_count": 0, "action_button_count": 0, "source": "public_gateway_preflight_inventory", } def _build_route_assets(public_gateway_snapshot: Mapping[str, Any]) -> list[dict[str, Any]]: seen_domains: set[str] = set() route_assets: list[dict[str, Any]] = [] for row in public_gateway_snapshot.get("route_impacts") or []: if not isinstance(row, Mapping): continue asset = _route_asset_from_gateway_row(row) if not asset["domain"] or asset["domain"] in seen_domains: continue seen_domains.add(asset["domain"]) route_assets.append(asset) for row in _ADDITIONAL_PUBLIC_ROUTES: domain = str(row["domain"]) if domain in seen_domains: continue product = _domain_product(domain) route_assets.append( { "domain": domain, "product_id": product["product_id"], "product_name": product["product_name"], "category": product["category"], "coverage_status": row["coverage_status"], "control_tier": "candidate", "upstream_count": 0, "admin_route_count": 0, "websocket_route_count": 0, "public_route_smoke_required": True, "route_smoke_accepted": False, "owner_response_accepted": False, "runtime_gate_count": 0, "action_button_count": 0, "source": row["source"], } ) return route_assets def _build_source_repo_assets( source_control_snapshot: Mapping[str, Any], ) -> list[dict[str, Any]]: source_assets: list[dict[str, Any]] = [] for row in source_control_snapshot.get("repo_readiness") or []: if not isinstance(row, Mapping): continue source_key = str(row.get("source_key") or "") github_repo = str(row.get("github_repo") or "") product = { "product_id": (source_key or github_repo).replace("/", "-") or "unknown-source", "product_name": source_key or github_repo or "unknown-source", "category": "source_repo", } for surface in _CANONICAL_PRODUCT_SURFACES: if source_key in surface.get("source_keys", []) or github_repo in surface.get( "source_keys", [] ): product = { "product_id": str(surface["product_id"]), "product_name": str(surface["product_name"]), "category": str(surface["category"]), } break source_assets.append( { "github_repo": github_repo, "source_key": source_key, "product_id": product["product_id"], "product_name": product["product_name"], "category": product["category"], "scope_status": row.get("scope_status") or "unknown", "readiness_state": row.get("readiness_state") or "unknown", "risk": row.get("risk") or "UNKNOWN", "primary_ready": bool(row.get("primary_ready")), "blocker_count": len(row.get("blockers") or []), "runtime_gate_count": 0, "action_button_count": 0, } ) return source_assets def build_tenant_asset_inventory(tenants: list[Mapping[str, Any]]) -> dict[str, Any]: """Build a read-only product / route / source-control asset view for tenants.""" public_gateway = _load_committed_snapshot(_PUBLIC_GATEWAY_INVENTORY_SNAPSHOT) source_control = _load_committed_snapshot(_SOURCE_CONTROL_READINESS_SNAPSHOT) public_routes = _build_route_assets(public_gateway) source_repos = _build_source_repo_assets(source_control) public_route_domains = {asset["domain"] for asset in public_routes} source_keys = {asset["source_key"] for asset in source_repos} products: list[dict[str, Any]] = [] for surface in _CANONICAL_PRODUCT_SURFACES: public_route_count = len( [domain for domain in surface["public_routes"] if domain in public_route_domains] ) source_repo_count = len( [source for source in surface["source_keys"] if source in source_keys] ) missing_public_routes = [ domain for domain in surface["public_routes"] if domain not in public_route_domains ] products.append( { **surface, "public_route_count": public_route_count, "source_repo_count": source_repo_count, "missing_public_routes": missing_public_routes, "owner_response_received_count": 0, "owner_response_accepted_count": 0, "runtime_gate_count": 0, "action_button_count": 0, } ) source_summary = source_control.get("summary") or {} gateway_summary = public_gateway.get("summary") or {} return { "schema_version": "awooop_tenant_asset_inventory_v1", "mode": "read_only_global_asset_inventory", "evidence_refs": [ _PUBLIC_GATEWAY_INVENTORY_SNAPSHOT, _SOURCE_CONTROL_READINESS_SNAPSHOT, "docs/security/SECURITY-MIRROR-STATUS-ROLLUP.md", ], "summary": { "tenant_table_count": len(tenants), "product_surface_count": len(products), "public_route_count": len(public_routes), "public_gateway_snapshot_route_count": int( gateway_summary.get("route_impact_count") or 0 ), "source_candidate_repo_count": int( source_summary.get("candidate_repo_count") or len(source_repos) ), "source_in_scope_repo_count": int(source_summary.get("in_scope_repo_count") or 0), "source_primary_ready_count": int(source_summary.get("primary_ready_count") or 0), "owner_response_received_count": 0, "owner_response_accepted_count": 0, "runtime_gate_count": 0, "action_button_count": 0, }, "products": products, "public_routes": public_routes, "source_repos": source_repos, "boundaries": [ "read_only_inventory_only=true", "owner_response_received_count=0", "owner_response_accepted_count=0", "runtime_execution_authorized=false", "active_runtime_gate_count=0", "action_buttons_allowed=false", "repo_creation_authorized=false", "refs_sync_authorized=false", "workflow_modification_authorized=false", "public_route_change_authorized=false", ], } async def list_tenants() -> dict[str, Any]: """列出所有 AwoooP 租戶（Operator Console，不依 RLS 過濾）。""" async with get_db_context("awoooi") as db: result = await db.execute( text(""" SELECT project_id, display_name, migration_mode, budget_limit_usd, is_active, created_at FROM awooop_operator_list_projects() """) ) rows = list(result.mappings().all()) tenants = [ { "project_id": r["project_id"], "display_name": r["display_name"], "migration_mode": r["migration_mode"], "budget_limit_usd": r["budget_limit_usd"], "is_active": r["is_active"], "created_at": r["created_at"], } for r in rows ] return { "tenants": tenants, "total": len(tenants), "asset_inventory": build_tenant_asset_inventory(tenants), } # ============================================================================= # Contracts # ============================================================================= async def list_contracts( project_id: str | None, lifecycle_status: str | None, ) -> dict[str, Any]: """列出合約 revisions（可 filter by project_id / lifecycle_status）。""" async with get_db_context("awoooi") as db: stmt = select(AwoooPContractRevision).order_by( AwoooPContractRevision.created_at.desc() ) if project_id is not None: stmt = stmt.where(AwoooPContractRevision.project_id == project_id) if lifecycle_status is not None: stmt = stmt.where( AwoooPContractRevision.lifecycle_status == lifecycle_status ) count_stmt = select(func.count()).select_from(stmt.subquery()) total_result = await db.execute(count_stmt) total = total_result.scalar_one() stmt = stmt.limit(_MAX_CONTRACTS) result = await db.execute(stmt) rows = list(result.scalars().all()) contracts = [ { "revision_id": r.revision_id, "contract_id": r.contract_id, "contract_family": r.contract_family, "lifecycle_status": r.lifecycle_status, "body_hash": r.body_hash, "version_major": r.version_major, "version_minor": r.version_minor, "created_at": r.created_at, "project_id": r.project_id, } for r in rows ] return {"contracts": contracts, "total": total} # ============================================================================= # Runs # ============================================================================= async def list_runs( project_id: str | None, state: str | None, remediation_status: str | None, callback_reply_status: str | None, incident_id: str | None, page: int, per_page: int, ) -> dict[str, Any]: """列出 runs，支援 project/state/evidence/callback/incident filter 與分頁。""" _validate_remediation_status_filter(remediation_status) _validate_callback_reply_status_filter(callback_reply_status) _validate_incident_id_filter(incident_id) async with get_db_context("awoooi") as db: stmt = select(AwoooPRunState).order_by(AwoooPRunState.created_at.desc()) if project_id is not None: stmt = stmt.where(AwoooPRunState.project_id == project_id) if state is not None: stmt = stmt.where(AwoooPRunState.state == state) offset = (page - 1) * per_page if remediation_status or incident_id or callback_reply_status: result = await db.execute(stmt) candidate_rows = list(result.scalars().all()) context_limit = _list_filter_context_limit(len(candidate_rows)) inbound_by_run, outbound_by_run = await _load_run_message_context( db, candidate_rows, limit=context_limit, ) remediation_summaries = await _build_run_remediation_summaries( runs=candidate_rows, inbound_by_run=inbound_by_run, outbound_by_run=outbound_by_run, ) callback_reply_summaries = { row.run_id: _run_callback_reply_summary(outbound_by_run.get(row.run_id, [])) for row in candidate_rows } filtered_rows = [ row for row in candidate_rows if _remediation_summary_matches_status( remediation_summaries.get(row.run_id), remediation_status, ) and _remediation_summary_matches_incident_id( remediation_summaries.get(row.run_id), incident_id, ) and _callback_reply_summary_matches_status( callback_reply_summaries.get(row.run_id), callback_reply_status, ) ] total = len(filtered_rows) rows = filtered_rows[offset : offset + per_page] else: count_stmt = select(func.count()).select_from(stmt.subquery()) total_result = await db.execute(count_stmt) total = total_result.scalar_one() stmt = stmt.offset(offset).limit(per_page) result = await db.execute(stmt) rows = list(result.scalars().all()) inbound_by_run, outbound_by_run = await _load_run_message_context(db, rows) remediation_summaries = await _build_run_remediation_summaries( runs=rows, inbound_by_run=inbound_by_run, outbound_by_run=outbound_by_run, ) callback_reply_summaries = { row.run_id: _run_callback_reply_summary(outbound_by_run.get(row.run_id, [])) for row in rows } runs = [ { "run_id": r.run_id, "project_id": r.project_id, "agent_id": r.agent_id, "state": r.state, "is_shadow": r.is_shadow, "cost_usd": r.cost_usd, "step_count": r.step_count, "created_at": r.created_at, "timeout_at": r.timeout_at, "remediation_summary": remediation_summaries.get(r.run_id), "callback_reply_summary": callback_reply_summaries.get(r.run_id), } for r in rows ] return {"runs": runs, "total": total, "page": page, "per_page": per_page} async def list_callback_replies( project_id: str | None, callback_reply_status: str | None, action: str | None, incident_id: str | None, page: int, per_page: int, refresh: bool = False, ) -> dict[str, Any]: """列出 Telegram detail/history callback reply evidence，不改 runtime 狀態。""" _validate_callback_reply_status_filter(callback_reply_status) callback_action = _validate_callback_reply_action_filter(action) _validate_incident_id_filter(incident_id) normalized_project_id = project_id or "awoooi" if callback_reply_status == "no_callback": return { "items": [], "total": 0, "page": page, "per_page": per_page, } callback_summary_cache_key = { "project_id": project_id or "__all__", "callback_reply_status": callback_reply_status or "", "action": callback_action or "", "incident_id": incident_id or "", "page": page, "per_page": per_page, } if not refresh: cached_response = await get_cached_operator_summary_async( "callback_replies", callback_summary_cache_key, ttl_seconds=_CALLBACK_REPLY_CACHE_TTL_SECONDS, ) if cached_response is not None: logger.info( "operator_callback_replies_cache_hit", project_id=normalized_project_id, callback_reply_status=callback_reply_status, action=callback_action, incident_id=incident_id, page=page, per_page=per_page, ttl_seconds=_CALLBACK_REPLY_CACHE_TTL_SECONDS, ) return cached_response where_clauses = [ "m.source_envelope ? 'callback_reply'", ] params: dict[str, Any] = { "limit": per_page, "offset": (page - 1) * per_page, } if project_id: where_clauses.append("m.project_id = :project_id") params["project_id"] = project_id raw_status = _CALLBACK_REPLY_RAW_STATUS_BY_FILTER.get( str(callback_reply_status or "") ) if raw_status: where_clauses.append( "m.source_envelope #>> '{callback_reply,status}' = :raw_status" ) params["raw_status"] = raw_status if callback_action: where_clauses.append( "LOWER(m.source_envelope #>> '{callback_reply,action}') = :callback_action" ) params["callback_action"] = callback_action if incident_id: where_clauses.append( "m.source_envelope #>> '{callback_reply,incident_id}' = :incident_id" ) params["incident_id"] = incident_id where_sql = " AND ".join(where_clauses) count_sql = text(f""" SELECT COUNT(*) AS total FROM awooop_outbound_message m WHERE {where_sql} """) list_sql = text(f""" SELECT m.message_id, m.project_id, m.run_id, m.channel_type, m.message_type, m.content_preview, m.provider_message_id, m.send_status, m.send_error, m.queued_at, m.sent_at, m.triggered_by_state, m.source_envelope -> 'callback_reply' AS callback_reply, m.source_envelope -> 'awooop_status_chain' AS persisted_awooop_status_chain, m.source_envelope -> 'km_stale_completion_summary' AS persisted_km_stale_completion_summary, r.agent_id, r.state AS run_state, r.created_at AS run_created_at FROM awooop_outbound_message m LEFT JOIN awooop_run_state r ON r.project_id = m.project_id AND r.run_id = m.run_id WHERE {where_sql} ORDER BY COALESCE(m.sent_at, m.queued_at) DESC, m.message_id DESC LIMIT :limit OFFSET :offset """) async with get_db_context(normalized_project_id) as db: count_result = await db.execute(count_sql, params) total = count_result.scalar_one() rows_result = await db.execute(list_sql, params) rows = list(rows_result.mappings().all()) audit_summary = await _fetch_callback_reply_audit_summary( db, project_id=normalized_project_id, ) items = [_callback_reply_event_item(row) for row in rows] status_chain_cache: dict[tuple[str, str], dict[str, Any]] = {} km_completion_queue_cache: dict[str, Any] = {} km_completion_summary_cache: dict[tuple[str, str | None], dict[str, Any]] = {} for item in items: incident = item.get("incident_id") item_project_id = str(item.get("project_id") or project_id or "awoooi") if not incident: item["awooop_status_chain"] = _build_awooop_status_chain( incident_ids=[], source_id=None, ) item["km_stale_completion_summary"] = ( _empty_km_stale_completion_summary( project_id=item_project_id, incident_id=None, status_value="no_incident", reason="callback_reply_missing_incident_id", ) ) continue incident_id = str(incident) status_chain_cache_key = (item_project_id, incident_id) cached = status_chain_cache.get(status_chain_cache_key) if cached is not None: item["awooop_status_chain"] = cached else: remediation_history = await _fetch_run_remediation_history( [incident_id], limit=5, ) chain = await _fetch_awooop_status_chain( incident_ids=[incident_id], project_id=item_project_id, remediation_history=remediation_history, ) status_chain_cache[status_chain_cache_key] = chain item["awooop_status_chain"] = chain km_summary_cache_key = (item_project_id, incident_id) km_summary = km_completion_summary_cache.get(km_summary_cache_key) if km_summary is None: km_summary = await _fetch_km_stale_completion_summary_for_incident( project_id=item_project_id, incident_id=incident_id, queue_cache=km_completion_queue_cache, ) km_completion_summary_cache[km_summary_cache_key] = km_summary item["km_stale_completion_summary"] = km_summary response = { "items": items, "total": total, "page": page, "per_page": per_page, "summary": audit_summary, } logger.info( "operator_callback_replies_fetched", project_id=normalized_project_id, callback_reply_status=callback_reply_status, action=callback_action, incident_id=incident_id, page=page, per_page=per_page, total=total, cache_status="miss", cache_ttl_seconds=_CALLBACK_REPLY_CACHE_TTL_SECONDS, ) return await store_operator_summary_async( "callback_replies", callback_summary_cache_key, response, ttl_seconds=_CALLBACK_REPLY_CACHE_TTL_SECONDS, ) async def _fetch_callback_reply_audit_summary( db: Any, *, project_id: str, ) -> dict[str, Any]: """Summarize Telegram outbound mirror and callback evidence capture coverage.""" result = await db.execute( text(""" WITH outbound AS ( SELECT m.*, EXISTS ( SELECT 1 FROM jsonb_each( CASE WHEN jsonb_typeof( COALESCE( m.source_envelope -> 'source_refs', '{}'::jsonb ) ) = 'object' THEN COALESCE( m.source_envelope -> 'source_refs', '{}'::jsonb ) ELSE '{}'::jsonb END ) AS refs(key, value) WHERE jsonb_typeof(refs.value) = 'array' AND refs.value <> '[]'::jsonb ) AS has_trace_ref FROM awooop_outbound_message m WHERE m.project_id = :project_id AND m.channel_type = 'telegram' ), trace_gap_cutoff AS ( SELECT MAX(COALESCE(sent_at, queued_at)) AS latest_missing_trace_ref_at FROM outbound WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND NOT has_trace_ref ), inbound_callbacks AS ( SELECT COUNT(*) AS inbound_callback_total, COUNT(*) FILTER ( WHERE received_at >= NOW() - INTERVAL '24 hours' ) AS inbound_callback_recent_24h_total, MAX(received_at) AS inbound_callback_latest_at FROM awooop_conversation_event WHERE project_id = :project_id AND channel_type = 'telegram' AND content_type = 'callback_query' ) SELECT COUNT(*) AS outbound_total, COUNT(*) FILTER ( WHERE source_envelope <> '{}'::jsonb ) AS outbound_source_envelope_total, COUNT(*) FILTER ( WHERE source_envelope ? 'source_refs' ) AS outbound_source_refs_total, COUNT(*) FILTER ( WHERE has_trace_ref ) AS outbound_trace_ref_total, COUNT(*) FILTER ( WHERE COALESCE( source_envelope #> '{source_refs,incident_ids}', '[]'::jsonb ) <> '[]'::jsonb ) AS outbound_incident_ref_total, COUNT(*) FILTER ( WHERE source_envelope #>> '{reply_markup,present}' = 'true' ) AS outbound_reply_markup_total, COUNT(*) FILTER ( WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND COALESCE( source_envelope #> '{source_refs,incident_ids}', '[]'::jsonb ) = '[]'::jsonb ) AS outbound_reply_markup_missing_incident_ref_total, COUNT(*) FILTER ( WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND COALESCE( source_envelope #> '{source_refs,incident_ids}', '[]'::jsonb ) = '[]'::jsonb AND COALESCE(sent_at, queued_at) >= NOW() - INTERVAL '1 hour' ) AS outbound_reply_markup_missing_incident_ref_recent_1h_total, COUNT(*) FILTER ( WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND COALESCE( source_envelope #> '{source_refs,incident_ids}', '[]'::jsonb ) = '[]'::jsonb AND COALESCE(sent_at, queued_at) >= NOW() - INTERVAL '24 hours' ) AS outbound_reply_markup_missing_incident_ref_recent_24h_total, MAX(COALESCE(sent_at, queued_at)) FILTER ( WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND COALESCE( source_envelope #> '{source_refs,incident_ids}', '[]'::jsonb ) = '[]'::jsonb ) AS outbound_reply_markup_missing_incident_ref_latest_sent_at, COUNT(*) FILTER ( WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND NOT has_trace_ref ) AS outbound_reply_markup_missing_trace_ref_total, COUNT(*) FILTER ( WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND NOT has_trace_ref AND COALESCE(sent_at, queued_at) >= NOW() - INTERVAL '1 hour' ) AS outbound_reply_markup_missing_trace_ref_recent_1h_total, COUNT(*) FILTER ( WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND NOT has_trace_ref AND COALESCE(sent_at, queued_at) >= NOW() - INTERVAL '24 hours' ) AS outbound_reply_markup_missing_trace_ref_recent_24h_total, MAX(COALESCE(sent_at, queued_at)) FILTER ( WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND NOT has_trace_ref ) AS outbound_reply_markup_missing_trace_ref_latest_sent_at, COUNT(*) FILTER ( WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND has_trace_ref AND trace_gap_cutoff.latest_missing_trace_ref_at IS NOT NULL AND COALESCE(sent_at, queued_at) > trace_gap_cutoff.latest_missing_trace_ref_at ) AS outbound_reply_markup_trace_ref_after_gap_total, MIN(COALESCE(sent_at, queued_at)) FILTER ( WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND has_trace_ref AND trace_gap_cutoff.latest_missing_trace_ref_at IS NOT NULL AND COALESCE(sent_at, queued_at) > trace_gap_cutoff.latest_missing_trace_ref_at ) AS outbound_reply_markup_trace_ref_after_gap_first_sent_at, MAX(COALESCE(sent_at, queued_at)) FILTER ( WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND has_trace_ref AND trace_gap_cutoff.latest_missing_trace_ref_at IS NOT NULL AND COALESCE(sent_at, queued_at) > trace_gap_cutoff.latest_missing_trace_ref_at ) AS outbound_reply_markup_trace_ref_after_gap_latest_sent_at, COALESCE(( SELECT jsonb_agg( jsonb_build_object( 'prefix', prefix, 'total', total, 'recent_24h_total', recent_24h_total, 'first_sent_at', first_sent_at, 'last_sent_at', last_sent_at ) ORDER BY total DESC, prefix ASC ) FROM ( SELECT COALESCE( NULLIF( source_envelope #>> '{reply_markup,buttons,0,callback_prefix}', '' ), 'unknown' ) AS prefix, COUNT(*) AS total, COUNT(*) FILTER ( WHERE COALESCE(sent_at, queued_at) >= NOW() - INTERVAL '24 hours' ) AS recent_24h_total, MIN(COALESCE(sent_at, queued_at)) AS first_sent_at, MAX(COALESCE(sent_at, queued_at)) AS last_sent_at FROM outbound WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND COALESCE( source_envelope #> '{source_refs,incident_ids}', '[]'::jsonb ) = '[]'::jsonb GROUP BY 1 ORDER BY total DESC, prefix ASC LIMIT 5 ) missing_prefixes ), '[]'::jsonb) AS outbound_reply_markup_missing_incident_ref_top_prefixes, COALESCE(( SELECT jsonb_agg( jsonb_build_object( 'prefix', prefix, 'total', total, 'recent_24h_total', recent_24h_total, 'first_sent_at', first_sent_at, 'last_sent_at', last_sent_at ) ORDER BY total DESC, prefix ASC ) FROM ( SELECT COALESCE( NULLIF( source_envelope #>> '{reply_markup,buttons,0,callback_prefix}', '' ), 'unknown' ) AS prefix, COUNT(*) AS total, COUNT(*) FILTER ( WHERE COALESCE(sent_at, queued_at) >= NOW() - INTERVAL '24 hours' ) AS recent_24h_total, MIN(COALESCE(sent_at, queued_at)) AS first_sent_at, MAX(COALESCE(sent_at, queued_at)) AS last_sent_at FROM outbound WHERE source_envelope #>> '{reply_markup,present}' = 'true' AND NOT has_trace_ref GROUP BY 1 ORDER BY total DESC, prefix ASC LIMIT 5 ) missing_trace_prefixes ), '[]'::jsonb) AS outbound_reply_markup_missing_trace_ref_top_prefixes, COUNT(*) FILTER ( WHERE send_status = 'failed' ) AS outbound_failed_total, COUNT(*) FILTER ( WHERE source_envelope ? 'callback_reply' ) AS callback_total, COUNT(*) FILTER ( WHERE source_envelope #>> '{callback_reply,status}' = 'callback_reply_sent' ) AS callback_sent_total, COUNT(*) FILTER ( WHERE source_envelope #>> '{callback_reply,status}' = 'callback_reply_fallback_sent' ) AS callback_fallback_total, COUNT(*) FILTER ( WHERE source_envelope #>> '{callback_reply,status}' = 'callback_reply_rescue_sent' ) AS callback_rescue_total, COUNT(*) FILTER ( WHERE source_envelope #>> '{callback_reply,status}' = 'callback_reply_failed' ) AS callback_failed_total, COUNT(*) FILTER ( WHERE LOWER(source_envelope #>> '{callback_reply,action}') = 'detail' ) AS callback_detail_total, COUNT(*) FILTER ( WHERE LOWER(source_envelope #>> '{callback_reply,action}') = 'history' ) AS callback_history_total, COUNT(*) FILTER ( WHERE source_envelope ? 'callback_reply' AND source_envelope ? 'awooop_status_chain' AND source_envelope ? 'km_stale_completion_summary' ) AS callback_snapshot_captured_total, COUNT(*) FILTER ( WHERE source_envelope ? 'callback_reply' AND ( source_envelope ? 'awooop_status_chain' OR source_envelope ? 'km_stale_completion_summary' ) AND NOT ( source_envelope ? 'awooop_status_chain' AND source_envelope ? 'km_stale_completion_summary' ) ) AS callback_snapshot_partial_total, COUNT(*) FILTER ( WHERE source_envelope ? 'callback_reply' AND NOT ( source_envelope ? 'awooop_status_chain' OR source_envelope ? 'km_stale_completion_summary' ) ) AS callback_snapshot_missing_total, COUNT(DISTINCT source_envelope #>> '{callback_reply,incident_id}') FILTER ( WHERE source_envelope ? 'callback_reply' AND COALESCE( source_envelope #>> '{callback_reply,incident_id}', '' ) <> '' ) AS callback_incident_total, MAX(inbound_callbacks.inbound_callback_total) AS inbound_callback_total, MAX(inbound_callbacks.inbound_callback_recent_24h_total) AS inbound_callback_recent_24h_total, MAX(inbound_callbacks.inbound_callback_latest_at) AS inbound_callback_latest_at, MAX(COALESCE(sent_at, queued_at)) AS latest_outbound_at, MAX(COALESCE(sent_at, queued_at)) FILTER ( WHERE source_envelope ? 'callback_reply' ) AS latest_callback_at FROM outbound CROSS JOIN trace_gap_cutoff CROSS JOIN inbound_callbacks """), {"project_id": project_id}, ) return _callback_reply_audit_summary_from_row( result.mappings().one(), project_id=project_id, ) def _callback_reply_audit_summary_from_row( row: Mapping[str, Any], *, project_id: str, ) -> dict[str, Any]: """Convert aggregate SQL row into the public callback evidence audit summary.""" outbound_total = _safe_int(row.get("outbound_total")) callback_total = _safe_int(row.get("callback_total")) captured = _safe_int(row.get("callback_snapshot_captured_total")) partial = _safe_int(row.get("callback_snapshot_partial_total")) missing = _safe_int(row.get("callback_snapshot_missing_total")) outbound_incident_refs = _safe_int(row.get("outbound_incident_ref_total")) top_missing_prefixes = _reply_markup_gap_prefixes_from_value( row.get("outbound_reply_markup_missing_incident_ref_top_prefixes") ) top_missing_trace_prefixes = _reply_markup_gap_prefixes_from_value( row.get("outbound_reply_markup_missing_trace_ref_top_prefixes") ) missing_trace_total = _safe_int( row.get("outbound_reply_markup_missing_trace_ref_total") ) missing_trace_recent_1h = _safe_int( row.get("outbound_reply_markup_missing_trace_ref_recent_1h_total") ) missing_trace_recent_24h = _safe_int( row.get("outbound_reply_markup_missing_trace_ref_recent_24h_total") ) trace_gap_status, trace_gap_next_action = _trace_ref_gap_decision( total=missing_trace_total, recent_1h=missing_trace_recent_1h, recent_24h=missing_trace_recent_24h, ) trace_ref_after_gap_total = _safe_int( row.get("outbound_reply_markup_trace_ref_after_gap_total") ) trace_gap_recovery_status = _trace_ref_gap_recovery_status( missing_total=missing_trace_total, after_gap_total=trace_ref_after_gap_total, ) inbound_callback_total = _safe_int(row.get("inbound_callback_total")) inbound_callback_recent_24h_total = _safe_int( row.get("inbound_callback_recent_24h_total") ) if inbound_callback_total > 0: inbound_callback_mirror_status = "capturing" inbound_callback_next_action = "none" elif callback_total > 0: inbound_callback_mirror_status = "reply_only_gap" inbound_callback_next_action = "press_any_telegram_callback_after_rollout" else: inbound_callback_mirror_status = "no_callback_observed" inbound_callback_next_action = "press_any_telegram_callback_after_rollout" if callback_total <= 0: snapshot_status = "no_callback" next_action = "press_telegram_detail_or_history" elif captured > 0 and (missing > 0 or partial > 0): snapshot_status = "partial" next_action = "review_legacy_callback_snapshot_gap" elif partial > 0: snapshot_status = "partial" next_action = "press_telegram_detail_or_history_after_rollout" elif missing > 0: snapshot_status = "not_captured" next_action = "press_telegram_detail_or_history_after_rollout" elif outbound_total > 0 and outbound_incident_refs == 0: snapshot_status = "captured" next_action = "review_outbound_source_refs" else: snapshot_status = "captured" next_action = "none" return { "schema_version": _CALLBACK_REPLY_AUDIT_SUMMARY_SCHEMA_VERSION, "project_id": project_id, "outbound_total": outbound_total, "outbound_source_envelope_total": _safe_int( row.get("outbound_source_envelope_total") ), "outbound_source_refs_total": _safe_int( row.get("outbound_source_refs_total") ), "outbound_trace_ref_total": _safe_int(row.get("outbound_trace_ref_total")), "outbound_incident_ref_total": outbound_incident_refs, "outbound_reply_markup_total": _safe_int( row.get("outbound_reply_markup_total") ), "outbound_reply_markup_missing_incident_ref_total": _safe_int( row.get("outbound_reply_markup_missing_incident_ref_total") ), "outbound_reply_markup_missing_incident_ref_recent_1h_total": _safe_int( row.get("outbound_reply_markup_missing_incident_ref_recent_1h_total") ), "outbound_reply_markup_missing_incident_ref_recent_24h_total": _safe_int( row.get("outbound_reply_markup_missing_incident_ref_recent_24h_total") ), "outbound_reply_markup_missing_incident_ref_latest_sent_at": row.get( "outbound_reply_markup_missing_incident_ref_latest_sent_at" ), "outbound_reply_markup_missing_trace_ref_total": missing_trace_total, "outbound_reply_markup_missing_trace_ref_recent_1h_total": ( missing_trace_recent_1h ), "outbound_reply_markup_missing_trace_ref_recent_24h_total": ( missing_trace_recent_24h ), "outbound_reply_markup_missing_trace_ref_latest_sent_at": row.get( "outbound_reply_markup_missing_trace_ref_latest_sent_at" ), "outbound_reply_markup_trace_ref_gap_status": trace_gap_status, "outbound_reply_markup_trace_ref_gap_next_action": trace_gap_next_action, "outbound_reply_markup_trace_ref_after_gap_total": ( trace_ref_after_gap_total ), "outbound_reply_markup_trace_ref_after_gap_first_sent_at": row.get( "outbound_reply_markup_trace_ref_after_gap_first_sent_at" ), "outbound_reply_markup_trace_ref_after_gap_latest_sent_at": row.get( "outbound_reply_markup_trace_ref_after_gap_latest_sent_at" ), "outbound_reply_markup_trace_ref_gap_recovery_status": ( trace_gap_recovery_status ), "outbound_reply_markup_missing_incident_ref_top_prefixes": ( top_missing_prefixes ), "outbound_reply_markup_missing_trace_ref_top_prefixes": ( top_missing_trace_prefixes ), "outbound_failed_total": _safe_int(row.get("outbound_failed_total")), "callback_total": callback_total, "callback_sent_total": _safe_int(row.get("callback_sent_total")), "callback_fallback_total": _safe_int(row.get("callback_fallback_total")), "callback_rescue_total": _safe_int(row.get("callback_rescue_total")), "callback_failed_total": _safe_int(row.get("callback_failed_total")), "callback_detail_total": _safe_int(row.get("callback_detail_total")), "callback_history_total": _safe_int(row.get("callback_history_total")), "callback_snapshot_captured_total": captured, "callback_snapshot_partial_total": partial, "callback_snapshot_missing_total": missing, "callback_incident_total": _safe_int(row.get("callback_incident_total")), "inbound_callback_total": inbound_callback_total, "inbound_callback_recent_24h_total": inbound_callback_recent_24h_total, "inbound_callback_latest_at": row.get("inbound_callback_latest_at"), "inbound_callback_mirror_status": inbound_callback_mirror_status, "inbound_callback_next_action": inbound_callback_next_action, "snapshot_status": snapshot_status, "next_action": next_action, "latest_outbound_at": row.get("latest_outbound_at"), "latest_callback_at": row.get("latest_callback_at"), } def _trace_ref_gap_decision( *, total: int, recent_1h: int, recent_24h: int, ) -> tuple[str, str]: """Classify reply_markup messages without any source_refs into operator actions.""" if total <= 0: return "clean", "none" if recent_1h > 0: return "active_gap", "inspect_recent_outbound_source_refs" if recent_24h > 0: return "recent_backlog", "watch_24h_decay" return "legacy_backlog", "backfill_or_archive_legacy_callbacks" def _trace_ref_gap_recovery_status( *, missing_total: int, after_gap_total: int, ) -> str: """Describe whether traced reply_markup messages resumed after the last gap.""" if missing_total <= 0: return "not_needed" if after_gap_total > 0: return "recovered_after_gap" return "no_recovery_signal" def _reply_markup_gap_prefixes_from_value(value: Any) -> list[dict[str, Any]]: if not isinstance(value, list): return [] prefixes: list[dict[str, Any]] = [] for item in value: if not isinstance(item, Mapping): continue prefix = str(item.get("prefix") or "unknown").strip() or "unknown" prefixes.append({ "prefix": prefix[:80], "total": _safe_int(item.get("total")), "recent_24h_total": _safe_int(item.get("recent_24h_total")), "first_sent_at": item.get("first_sent_at"), "last_sent_at": item.get("last_sent_at"), }) if len(prefixes) >= 5: break return prefixes async def _fetch_km_stale_completion_summary_for_incident( *, project_id: str, incident_id: str | None, queue_cache: dict[str, Any] | None = None, ) -> dict[str, Any]: """Fetch read-only KM owner-review completion context for callback evidence.""" normalized_project_id = project_id or "awoooi" normalized_incident_id = str(incident_id or "").strip() or None if not normalized_incident_id: return _empty_km_stale_completion_summary( project_id=normalized_project_id, incident_id=None, status_value="no_incident", reason="callback_reply_missing_incident_id", ) cache = queue_cache if queue_cache is not None else {} queue = cache.get(normalized_project_id) if queue is None: try: queue = await query_km_stale_owner_review_completion_queue( project_id=normalized_project_id, status_bucket="all", limit=100, ) except Exception as exc: logger.warning( "operator_km_stale_completion_summary_fetch_failed", project_id=normalized_project_id, incident_id=normalized_incident_id, error=str(exc), ) return _empty_km_stale_completion_summary( project_id=normalized_project_id, incident_id=normalized_incident_id, status_value="fetch_failed", reason="km_stale_completion_queue_fetch_failed", ) cache[normalized_project_id] = queue return _build_km_stale_completion_summary( queue=queue, project_id=normalized_project_id, incident_id=normalized_incident_id, ) async def list_cicd_events( *, project_id: str | None, stage: str | None, status_filter: str | None, limit: int, ) -> dict[str, Any]: """列出 CI/CD notification evidence，來源是 alert_operation_log。""" safe_limit = max(1, min(limit, 50)) normalized_stage = _validate_cicd_stage_filter(stage) normalized_status = _validate_cicd_status_filter(status_filter) # alert_operation_log 目前是 legacy/global evidence table，CI/CD notification # 只屬於 AWOOOI production；非 awoooi project filter 回空集合，避免誤導多租戶 UI。 if project_id and project_id != "awoooi": return {"items": [], "total": 0, "limit": safe_limit} where_clauses = [ "event_type = 'ALERT_RECEIVED'", "actor = 'alertmanager'", """ COALESCE( context #>> '{labels,alertname}', context ->> 'alertname', '' ) LIKE 'CI_%' """, ] params: dict[str, Any] = {"limit": safe_limit} if normalized_stage: where_clauses.append( "LOWER(COALESCE(context #>> '{labels,stage}', '')) = :stage" ) params["stage"] = normalized_stage if normalized_status: where_clauses.append( "LOWER(COALESCE(context #>> '{labels,status}', '')) = :status" ) params["status"] = normalized_status where_sql = " AND ".join(where_clauses) sql = text(f""" SELECT id, action_detail, success, created_at, context, COALESCE( context #>> '{{labels,alertname}}', context ->> 'alertname', '' ) AS alertname, context #>> '{{labels,stage}}' AS stage, context #>> '{{labels,status}}' AS status, context #>> '{{labels,severity}}' AS severity, context #>> '{{labels,commit}}' AS commit_sha, context #>> '{{labels,triggered_by}}' AS triggered_by, context #>> '{{labels,duration_seconds}}' AS duration_seconds, context #>> '{{annotations,summary}}' AS summary, context #>> '{{annotations,description}}' AS description, context #>> '{{annotations,workflow_url}}' AS workflow_url, context ->> 'alert_id' AS alert_id, context ->> 'source' AS source FROM alert_operation_log WHERE {where_sql} ORDER BY created_at DESC, id DESC LIMIT :limit """) async with get_db_context("awoooi") as db: result = await db.execute(sql, params) rows = list(result.mappings().all()) items = [_cicd_event_item_from_row(row, project_id=project_id or "awoooi") for row in rows] return {"items": items, "total": len(items), "limit": safe_limit} async def get_ai_route_status( workload_type: str | None = None, ) -> dict[str, Any]: """回傳目前 AI/Ollama provider routing 的只讀狀態，供 Operator Console 顯示。""" workload = _validate_ai_route_workload(workload_type) policy_order = _ai_route_policy_order(workload) checked_at = _utc_now_naive() try: route = await asyncio.wait_for( get_ollama_failover_manager().select_provider(task_type=workload), timeout=_AI_ROUTE_STATUS_SELECT_TIMEOUT_SECONDS, ) except TimeoutError: logger.warning( "ai_route_status_check_timeout", workload_type=workload, timeout_seconds=_AI_ROUTE_STATUS_SELECT_TIMEOUT_SECONDS, ) return await _ai_route_lightweight_status_from_policy( workload=workload, policy_order=policy_order, checked_at=checked_at, route_reason="route_check_timeout", route_error=( f"route status timed out after " f"{_AI_ROUTE_STATUS_SELECT_TIMEOUT_SECONDS:g}s" ), ) except Exception as exc: return await _ai_route_lightweight_status_from_policy( workload=workload, policy_order=policy_order, checked_at=checked_at, route_reason="route_check_failed", route_error=str(exc), ) health = _ai_route_health_map(route) response = { "schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION, "workload_type": workload, "policy_order": policy_order, "selected_provider": route.primary.provider_name, "selected_url": route.primary.url or None, "selected_model": route.primary.model, "fallback_chain": [ _ai_route_runtime_endpoint_item(endpoint, priority=index + 2) for index, endpoint in enumerate(route.fallback_chain) ], "route_reason": route.routing_reason, "route_source": "ollama_failover_manager", "route_error": None, "health": health, "checked_at": checked_at, } response.update(_ai_route_lane_state( policy_order=policy_order, selected_provider=route.primary.provider_name, health=health, )) return await _ai_route_response_with_repair_evidence(response) def _validate_ai_route_workload(workload_type: str | None) -> OllamaWorkloadType: """Normalize and validate workload filter for the public route status endpoint.""" workload = str(workload_type or "deep_rca").strip() or "deep_rca" if workload not in _AI_ROUTE_WORKLOADS: allowed = ", ".join(sorted(_AI_ROUTE_WORKLOADS)) raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=f"Unsupported workload_type: {workload}. Allowed: {allowed}", ) return workload # type: ignore[return-value] def _ai_route_policy_order(workload: OllamaWorkloadType) -> list[dict[str, Any]]: """Expose configured policy order: GCP-A -> GCP-B -> 111 -> Gemini.""" items = [ _ai_route_policy_endpoint_item(endpoint, priority=index + 1) for index, endpoint in enumerate(resolve_ollama_order(workload)) ] items.append({ "priority": len(items) + 1, "provider_name": "gemini", "url": None, "workload_type": workload, "reason": "final_cloud_fallback_after_all_ollama_endpoints", "role": "final_fallback", "runtime": "cloud", }) return items async def _ai_route_lightweight_status_from_policy( *, workload: OllamaWorkloadType, policy_order: list[dict[str, Any]], checked_at: datetime, route_reason: str, route_error: str, ) -> dict[str, Any]: """Fallback read model for route status; never changes the execution router.""" endpoints = list(resolve_ollama_order(workload)) try: reports = await asyncio.gather( *[_ai_route_probe_connectivity(endpoint) for endpoint in endpoints], ) except Exception as exc: logger.warning( "ai_route_status_lightweight_probe_failed", workload_type=workload, route_reason=route_reason, error=str(exc), ) response = _ai_route_unavailable_status( workload=workload, policy_order=policy_order, checked_at=checked_at, route_reason=route_reason, route_error=route_error, route_source="ollama_failover_manager", ) return await _ai_route_response_with_repair_evidence(response) health_by_provider = { endpoint.provider_name: _ai_route_health_item(report) for endpoint, report in zip(endpoints, reports, strict=False) } selected_index = next( ( index for index, report in enumerate(reports) if report.status != HealthStatus.OFFLINE ), None, ) if selected_index is None: response = { "schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION, "workload_type": workload, "policy_order": policy_order, "selected_provider": "gemini", "selected_url": None, "selected_model": None, "fallback_chain": [], "route_reason": ( f"{route_reason}; lightweight connectivity found all Ollama " "endpoints offline; final fallback policy is Gemini" ), "route_source": "lightweight_connectivity_fallback", "route_error": None, "health": health_by_provider, "checked_at": checked_at, } response.update(_ai_route_lane_state( policy_order=policy_order, selected_provider="gemini", health=health_by_provider, )) return await _ai_route_response_with_repair_evidence(response) selected = endpoints[selected_index] model = get_settings().OLLAMA_HEALTH_CHECK_MODEL fallback_chain = [ _ai_route_runtime_policy_endpoint_item( endpoint, priority=index + 1, model=model, ) for index, endpoint in enumerate(endpoints[selected_index + 1 :], start=selected_index + 1) ] fallback_chain.append({ "priority": len(endpoints) + 1, "provider_name": "gemini", "url": None, "model": None, "runtime": "cloud", }) response = { "schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION, "workload_type": workload, "policy_order": policy_order, "selected_provider": selected.provider_name, "selected_url": selected.url, "selected_model": model, "fallback_chain": fallback_chain, "route_reason": ( f"{route_reason}; lightweight connectivity selected " f"{selected.provider_name}" ), "route_source": "lightweight_connectivity_fallback", "route_error": None, "health": health_by_provider, "checked_at": checked_at, } response.update(_ai_route_lane_state( policy_order=policy_order, selected_provider=selected.provider_name, health=health_by_provider, )) return await _ai_route_response_with_repair_evidence(response) async def _ai_route_probe_connectivity( endpoint: OllamaEndpointSelection, ) -> HealthReport: """Cheap read-only /api/tags probe for Operator Console status fallback.""" if not endpoint.url: return HealthReport( status=HealthStatus.OFFLINE, host=endpoint.url, reason="no_ollama_endpoint_url", ) start = time.perf_counter() try: async with httpx.AsyncClient( timeout=httpx.Timeout(_AI_ROUTE_STATUS_CONNECTIVITY_TIMEOUT_SECONDS), ) as client: response = await client.get(f"{endpoint.url.rstrip('/')}/api/tags") latency_ms = (time.perf_counter() - start) * 1000 if response.status_code == 200: return HealthReport( status=HealthStatus.HEALTHY, host=endpoint.url, latency_ms=latency_ms, reason="status_only_connectivity_ok", ) return HealthReport( status=HealthStatus.OFFLINE, host=endpoint.url, latency_ms=latency_ms, reason=f"status_only_connectivity_http_{response.status_code}", ) except Exception as exc: latency_ms = (time.perf_counter() - start) * 1000 return HealthReport( status=HealthStatus.OFFLINE, host=endpoint.url, latency_ms=latency_ms, reason=f"status_only_connectivity_error:{type(exc).__name__}", ) def _ai_route_runtime_policy_endpoint_item( endpoint: OllamaEndpointSelection, *, priority: int, model: str, ) -> dict[str, Any]: return { "priority": priority, "provider_name": endpoint.provider_name, "url": endpoint.url or None, "model": model, "runtime": "ollama", } def _ai_route_unavailable_status( *, workload: OllamaWorkloadType, policy_order: list[dict[str, Any]], checked_at: datetime, route_reason: str, route_error: str, route_source: str, ) -> dict[str, Any]: response = { "schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION, "workload_type": workload, "policy_order": policy_order, "selected_provider": None, "selected_url": None, "selected_model": None, "fallback_chain": [], "route_reason": route_reason, "route_source": route_source, "route_error": route_error, "health": {}, "checked_at": checked_at, } response.update(_ai_route_lane_state( policy_order=policy_order, selected_provider=None, health={}, )) return response async def _ai_route_response_with_repair_evidence( response: dict[str, Any], ) -> dict[str, Any]: """Attach latest read-only repair dossier evidence when a lane is degraded.""" response["repair_evidence"] = None if response.get("lane_mode") not in { "degraded_failover", "cloud_fallback", "unavailable", }: return response target_provider = _ai_route_repair_evidence_target(response) response["repair_evidence"] = await _latest_ai_route_repair_evidence( target_provider=target_provider, ) return response def _ai_route_repair_evidence_target(response: Mapping[str, Any]) -> str | None: skipped_lanes = response.get("skipped_lanes") if isinstance(skipped_lanes, list): for lane in skipped_lanes: if not isinstance(lane, dict): continue provider_name = str(lane.get("provider_name") or "").strip() if provider_name and lane.get("action_required") is True: return provider_name for lane in skipped_lanes: if isinstance(lane, dict): provider_name = str(lane.get("provider_name") or "").strip() if provider_name: return provider_name policy_order = response.get("policy_order") if isinstance(policy_order, list): for item in policy_order: if not isinstance(item, dict): continue if item.get("runtime") == "ollama": provider_name = str(item.get("provider_name") or "").strip() if provider_name: return provider_name return None async def _latest_ai_route_repair_evidence( *, project_id: str = "awoooi", target_provider: str | None = None, ) -> dict[str, Any] | None: """Fetch the newest AI route repair diagnosis stored in AwoooP event DB.""" params: dict[str, Any] = { "project_id": project_id, "provider": _AI_ROUTE_REPAIR_EVIDENCE_PROVIDER, "stage": _AI_ROUTE_REPAIR_EVIDENCE_STAGE, } target_clause = "" if target_provider: target_clause = """ AND COALESCE( NULLIF(source_envelope #>> '{log_correlation,target_resource}', ''), NULLIF(source_envelope #>> '{extra,payload,target_resource}', '') ) = :target_provider """ params["target_provider"] = target_provider try: item = await _fetch_latest_ai_route_repair_evidence( params=params, target_clause=target_clause, ) if item is None and target_provider: params.pop("target_provider", None) item = await _fetch_latest_ai_route_repair_evidence( params=params, target_clause="", ) return item except Exception as exc: logger.warning( "ai_route_repair_evidence_fetch_failed", project_id=project_id, target_provider=target_provider, error=str(exc), ) return None async def _fetch_latest_ai_route_repair_evidence( *, params: dict[str, Any], target_clause: str, ) -> dict[str, Any] | None: sql = text(f""" SELECT event_id, run_id, provider_event_id, source_envelope, provider_ts, received_at FROM awooop_conversation_event WHERE project_id = :project_id AND LOWER(COALESCE( NULLIF(source_envelope->>'provider', ''), NULLIF(split_part(provider_event_id, ':', 1), ''), channel_type )) = :provider AND LOWER(COALESCE(NULLIF(source_envelope->>'stage', ''), 'received')) = :stage {target_clause} ORDER BY received_at DESC, event_id DESC LIMIT 1 """) async with get_db_context("awoooi") as db: result = await db.execute(sql, params) row = result.mappings().first() return _ai_route_repair_evidence_item(row) if row else None def _ai_route_repair_evidence_item( row: Mapping[str, Any], ) -> dict[str, Any]: """Project route-repair source envelopes into a compact operator-safe view.""" envelope = _as_dict(row.get("source_envelope")) extra = _as_dict(envelope.get("extra")) payload = _as_dict(extra.get("payload")) log_correlation = _as_dict(envelope.get("log_correlation")) live_probe = _as_dict(payload.get("live_probe")) observed_state = _as_dict(payload.get("observed_state")) side_effects = _ai_route_repair_side_effects(payload.get("side_effects")) evidence = { "schema_version": ( payload.get("schema_version") or envelope.get("schema_version") or "ai_route_repair_evidence_projection_v1" ), "provider": ( envelope.get("provider") or str(row.get("provider_event_id") or "").split(":", 1)[0] ), "stage": envelope.get("stage") or _AI_ROUTE_REPAIR_EVIDENCE_STAGE, "provider_event_id": row.get("provider_event_id"), "conversation_event_id": _string_or_none(row.get("event_id")), "run_id": _string_or_none(row.get("run_id")), "alertname": log_correlation.get("alertname"), "severity": log_correlation.get("severity"), "fingerprint": log_correlation.get("fingerprint"), "target_resource": ( log_correlation.get("target_resource") or payload.get("target_resource") or observed_state.get("target_resource") ), "observed_state": observed_state, "live_probe": live_probe, "access_blockers": _as_string_list(payload.get("access_blockers")), "side_effects": side_effects, "source_ref_count": _source_ref_count(envelope), "provider_ts": row.get("provider_ts"), "received_at": row.get("received_at"), } evidence["work_item"] = _ai_route_repair_work_item(evidence) evidence["playbook_recommendation"] = _ai_route_repair_playbook_recommendation( evidence ) evidence["owner_action"] = _ai_route_repair_owner_action(evidence) return evidence def _ai_route_repair_side_effects(value: Any) -> dict[str, bool | None]: raw = _as_dict(value) return { "incident_created": _bool_or_none(raw.get("incident_created")), "telegram_sent": _bool_or_none(raw.get("telegram_sent")), "approval_created": _bool_or_none(raw.get("approval_created")), "runtime_route_changed": _bool_or_none(raw.get("runtime_route_changed")), } def _as_string_list(value: Any) -> list[str]: if isinstance(value, list): return [str(item) for item in value if str(item or "").strip()] if value not in (None, ""): return [str(value)] return [] def _string_or_none(value: Any) -> str | None: if value in (None, ""): return None return str(value) def _bool_or_none(value: Any) -> bool | None: return value if isinstance(value, bool) else None def _source_ref_count(envelope: Any) -> int: source_refs = _as_dict(_as_dict(envelope).get("source_refs")) total = 0 for value in source_refs.values(): if isinstance(value, list): total += len([item for item in value if str(item or "").strip()]) elif value not in (None, ""): total += 1 return total def _recent_event_source_summary(row: AwoooPConversationEvent) -> dict[str, Any]: """Return redaction-safe source context for recent channel events.""" envelope = _as_dict(row.source_envelope) extra = _as_dict(envelope.get("extra")) telegram_callback = _as_dict( extra.get("telegram_callback_query") or envelope.get("telegram_callback_query") ) log_correlation = _as_dict(envelope.get("log_correlation")) provider = ( envelope.get("provider") or str(row.provider_event_id or "").split(":", 1)[0] or row.channel_type ) summary: dict[str, Any] = { "schema_version": "awooop_recent_event_source_summary_v1", "provider": provider, "stage": envelope.get("stage"), "provider_event_id": row.provider_event_id, "source_ref_count": _source_ref_count(envelope), "redaction_version": envelope.get("redaction_version"), } if telegram_callback: summary["telegram_callback_query"] = { "action": _string_or_none( telegram_callback.get("callback_action") ), "callback_ref": _string_or_none( telegram_callback.get("callback_ref") ), "incident_id": _string_or_none( telegram_callback.get("incident_id") ), "approval_id": _string_or_none( telegram_callback.get("approval_id") ), "message_id": _string_or_none( telegram_callback.get("message_id") ), "username_present": _bool_or_none( telegram_callback.get("username_present") ), } if log_correlation: summary["log_correlation"] = { "alertname": _string_or_none(log_correlation.get("alertname")), "severity": _string_or_none(log_correlation.get("severity")), "namespace": _string_or_none(log_correlation.get("namespace")), "target_resource": _string_or_none( log_correlation.get("target_resource") ), "fingerprint": _string_or_none( log_correlation.get("fingerprint") ), } return summary def _recent_channel_event_item(row: AwoooPConversationEvent) -> dict[str, Any]: """Project one recent channel event into the Operator Console DTO.""" return { "event_id": row.event_id, "project_id": row.project_id, "channel_type": row.channel_type, "provider_event_id": row.provider_event_id, "channel_chat_id": row.channel_chat_id, "run_id": row.run_id, "content_type": row.content_type, "content_preview": row.content_preview, "is_duplicate": row.is_duplicate, "received_at": row.received_at, "source_summary": _recent_event_source_summary(row), } def _ai_route_repair_work_item(evidence: Mapping[str, Any]) -> dict[str, Any]: target = str(evidence.get("target_resource") or "unknown").strip() blockers = _as_string_list(evidence.get("access_blockers")) open_item = bool(blockers) work_item_id = f"ai-route-repair:{target or 'unknown'}" return { "schema_version": "awooop_ai_route_repair_work_item_v1", "work_item_id": work_item_id, "status": "open" if open_item else "watching", "kind": "ai_route_primary_lane_repair", "next_step": ( "restore_primary_ollama_lane_access" if open_item else "continue_route_monitoring" ), "reason": "primary_lane_unavailable" if open_item else "primary_lane_observed", "needs_human": open_item, "owner": "cloud_sre_operator", "target_resource": target or None, "target_href": "/awooop/runs", "decision_effect": "none", "safety_level": "read_only_work_item_projection", "writes_incident_state": False, "writes_auto_repair_result": False, "writes_runtime_route": False, } def _ai_route_repair_playbook_recommendation( evidence: Mapping[str, Any], ) -> dict[str, Any]: blockers = set(_as_string_list(evidence.get("access_blockers"))) live_probe = _as_dict(evidence.get("live_probe")) steps: list[dict[str, Any]] = [] if any(blocker.startswith("gcloud_") for blocker in blockers): steps.append({ "step": "verify_cloud_control_plane_access", "scope": "gcp_compute_read", "mode": "manual_or_approved", }) if "gcp_a_ssh_refused" in blockers or ( live_probe.get("gcp_a_direct_22") == "connection_refused" ): steps.append({ "step": "restore_gcp_a_os_access", "scope": "gcp_serial_console_or_os_login", "mode": "manual_or_approved", }) if "gcp_a_ollama_11434_refused" in blockers or ( live_probe.get("gcp_a_direct_11434") == "connection_refused" ): steps.append({ "step": "restore_ollama_service_on_gcp_a", "scope": "systemd_ollama", "mode": "manual_or_approved", }) if live_probe.get("proxy_110_11435") == "http_502": steps.append({ "step": "verify_110_proxy_after_gcp_a_recovery", "scope": "nginx_proxy_readback", "mode": "read_only_verification", }) steps.append({ "step": "verify_ai_route_status_returns_primary", "scope": "awooop_ai_route_status", "mode": "read_only_verification", }) return { "schema_version": "awooop_ai_route_playbook_recommendation_v1", "playbook_id": "ai_route_primary_lane_recovery", "status": "candidate_from_live_evidence", "safe_to_auto_execute": False, "requires_approval": True, "decision_effect": "none", "steps": steps, } def _ai_route_repair_owner_action(evidence: Mapping[str, Any]) -> dict[str, Any]: work_item = _as_dict(evidence.get("work_item")) playbook = _as_dict(evidence.get("playbook_recommendation")) return { "schema_version": "awooop_ai_route_owner_action_v1", "lead_agent": "Hermes", "supporting_agents": ["OpenClaw", "ElephantAlpha"], "human_owner": "Cloud/SRE owner", "automation_state": "blocked_by_external_cloud_or_os_access", "next_step": work_item.get("next_step") or "continue_route_monitoring", "playbook_id": playbook.get("playbook_id"), "safe_to_auto_repair": False, "blocking_reason": work_item.get("reason") or "unknown", } def _ai_route_lane_state( *, policy_order: list[dict[str, Any]], selected_provider: str | None, health: dict[str, dict[str, Any]], ) -> dict[str, Any]: """Expose failover lane state separately from policy labels.""" selected_index = next( ( index for index, item in enumerate(policy_order) if item.get("provider_name") == selected_provider ), None, ) active_item = ( policy_order[selected_index] if selected_index is not None else None ) skipped_items = policy_order[:selected_index] if selected_index is not None else [] skipped_lanes = [ _ai_route_lane_item(item, health.get(str(item.get("provider_name")))) for item in skipped_items if item.get("runtime") == "ollama" ] if not selected_provider or active_item is None: lane_mode = "unavailable" operator_action = { "human_required": True, "action": "inspect_ai_router", "reason": "no_active_provider", } elif active_item.get("runtime") == "cloud": lane_mode = "cloud_fallback" operator_action = { "human_required": True, "action": "restore_ollama_lanes", "reason": "all_ollama_lanes_unavailable", } elif skipped_lanes: lane_mode = "degraded_failover" operator_action = { "human_required": True, "action": "repair_skipped_primary_lane", "reason": "fallback_lane_active", } else: lane_mode = "primary" operator_action = { "human_required": False, "action": "monitor", "reason": "primary_lane_active", } return { "lane_mode": lane_mode, "active_lane": ( _ai_route_lane_item(active_item, health.get(str(active_item.get("provider_name")))) if active_item else None ), "skipped_lanes": skipped_lanes, "operator_action": operator_action, } def _ai_route_lane_item( item: dict[str, Any], health_item: dict[str, Any] | None, ) -> dict[str, Any]: return { "priority": item.get("priority"), "provider_name": item.get("provider_name"), "role": item.get("role"), "runtime": item.get("runtime"), "url": item.get("url"), "health_status": (health_item or {}).get("status", "not_checked"), "reason": (health_item or {}).get("reason") or item.get("reason"), "action_required": (health_item or {}).get("status") not in { "healthy", "not_checked", None, }, } def _ai_route_policy_endpoint_item( endpoint: OllamaEndpointSelection, *, priority: int, ) -> dict[str, Any]: role = { "ollama_gcp_a": "primary", "ollama_gcp_b": "secondary", "ollama_local": "local_fallback", }.get(endpoint.provider_name, "ollama") return { "priority": priority, "provider_name": endpoint.provider_name, "url": endpoint.url, "workload_type": endpoint.workload_type, "reason": endpoint.reason, "role": role, "runtime": "ollama", } def _ai_route_runtime_endpoint_item( endpoint: OllamaEndpoint, *, priority: int, ) -> dict[str, Any]: return { "priority": priority, "provider_name": endpoint.provider_name, "url": endpoint.url or None, "model": endpoint.model, "runtime": "ollama" if endpoint.provider_name.startswith("ollama") else "cloud", } def _ai_route_health_map(route: OllamaRoutingResult) -> dict[str, dict[str, Any]]: """Convert failover health reports into provider keyed status for the UI.""" health: dict[str, dict[str, Any]] = { "ollama_gcp_a": _ai_route_health_item(route.health_gcp_a), } if route.health_gcp_b: health["ollama_gcp_b"] = _ai_route_health_item(route.health_gcp_b) else: health["ollama_gcp_b"] = _ai_route_not_checked_health_item() if route.health_local: health["ollama_local"] = _ai_route_health_item(route.health_local) else: health["ollama_local"] = _ai_route_not_checked_health_item() return health def _ai_route_health_item(report: HealthReport) -> dict[str, Any]: payload = report.to_dict() payload["checked"] = True return payload def _ai_route_not_checked_health_item() -> dict[str, Any]: return { "status": "not_checked", "host": "", "latency_ms": None, "reason": "standby_not_checked_primary_healthy", "checked_at": None, "from_cache": False, "checked": False, } def _timeline_item( *, ts: Any, kind: str, title: str, status: str, summary: str | None = None, metadata: dict[str, Any] | None = None, ) -> dict[str, Any]: """Build one Operator Console timeline item.""" return { "ts": ts, "kind": kind, "title": title, "status": status, "summary": summary, "metadata": metadata or {}, } def _utc_now_naive() -> datetime: """回傳與 AwoooP timestamp-without-timezone 欄位相容的 UTC 時間。""" return datetime.now(UTC).replace(tzinfo=None) def _truncate_step_summary(value: str | None) -> str | None: """壓縮 Step summary，避免超過 DB 欄位與前端 timeline 需要的短摘要。""" if not value: return None compact = " ".join(str(value).split()) if len(compact) <= _MAX_STEP_SUMMARY_CHARS: return compact return f"{compact[: _MAX_STEP_SUMMARY_CHARS - 1]}…" def _approval_step_title(tool_name: str, step_seq: int) -> str: """將 operator_console.* step 轉成人能一眼理解的 timeline 標題。""" if tool_name == "operator_console.approve": return f"人工審批 {step_seq}: 核准" if tool_name == "operator_console.reject": return f"人工審批 {step_seq}: 拒絕" return f"Step {step_seq}: {tool_name}" def _outbound_timeline_title( channel_type: str, message_type: str, content_preview: str | None, callback_reply: dict[str, Any] | None = None, ) -> str: """將 legacy Telegram outbound 分類成 Operator 看得懂的語義標題。""" channel = channel_type.upper() preview = content_preview or "" if callback_reply: action = str(callback_reply.get("action") or "").strip() status = str(callback_reply.get("status") or "").strip() action_label = { "detail": "詳情", "history": "歷史", }.get(action, action or "Callback") status_label = { "callback_reply_sent": "已送出", "callback_reply_fallback_sent": " fallback 已送出", "callback_reply_rescue_sent": "救援已送出", "callback_reply_failed": "送出失敗", }.get(status, status or "已記錄") return f"{channel}：{action_label}回覆{status_label}" if "RUNBOOK REVIEW" in preview: return f"{channel}：Runbook 待人工審核" if "[AWOOOI CI/CD]" in preview or "AWOOOI CI/CD" in preview: return f"{channel}：CI/CD 狀態通知" if "AI 治理警報" in preview: return f"{channel}：AI 治理警報" if "HANDOFF REQUIRED" in preview or "AI 自動修復失敗" in preview: return f"{channel}：AI 自動修復失敗，已轉人工" if "AUTO RESOLVED" in preview or "AI 自動修復完成" in preview: return f"{channel}：AI 自動修復完成" if "ESCALATION" in preview or "事故升級" in preview: return f"{channel}：事故升級通知" if "ACTION REQUIRED" in preview: return f"{channel}：告警審批卡" fallback = { "approval_request": "人工審批請求", "error": "錯誤回覆", "final": "處置結果", "interim": "漸進式狀態回饋", }.get(message_type, message_type) return f"{channel}：{fallback}" def _outbound_callback_reply(source_envelope: Any) -> dict[str, Any] | None: """Extract Telegram callback reply evidence from outbound source envelope.""" if not isinstance(source_envelope, dict): return None callback_reply = source_envelope.get("callback_reply") return callback_reply if isinstance(callback_reply, dict) else None def _callback_reply_public_status(callback_reply: dict[str, Any]) -> str: """Map raw Telegram callback reply result into the Operator Console filter value.""" raw_status = str(callback_reply.get("status") or "") return { "callback_reply_sent": "sent", "callback_reply_fallback_sent": "fallback_sent", "callback_reply_rescue_sent": "rescue_sent", "callback_reply_failed": "failed", }.get(raw_status, "observed") def _callback_reply_evidence_capture_status( *, callback_reply: Mapping[str, Any], persisted_awooop_status_chain: dict[str, Any] | None, persisted_km_stale_completion_summary: dict[str, Any] | None, event_at: Any, ) -> dict[str, Any]: """Explain whether callback-time evidence snapshots were persisted.""" captured: list[str] = [] missing: list[str] = [] if persisted_awooop_status_chain: captured.append("awooop_status_chain") else: missing.append("awooop_status_chain") if persisted_km_stale_completion_summary: captured.append("km_stale_completion_summary") else: missing.append("km_stale_completion_summary") if not missing: status_value = "captured" reason = "ok" next_action = "none" elif captured: status_value = "partial" reason = "partial_snapshot_rollout_transition" next_action = "press_telegram_detail_or_history_after_rollout" else: status_value = "not_captured" raw_status = str(callback_reply.get("status") or "") reason = ( "callback_reply_delivery_failed_snapshot_missing" if raw_status == "callback_reply_failed" else "legacy_callback_before_snapshot_rollout" ) next_action = "press_telegram_detail_or_history_after_rollout" return { "schema_version": _CALLBACK_EVIDENCE_CAPTURE_STATUS_SCHEMA_VERSION, "status": status_value, "reason": reason, "action": str(callback_reply.get("action") or "").strip() or None, "captured": captured, "missing": missing, "snapshot_rollout": "t167_t169", "next_action": next_action, "event_at": event_at, } def _callback_reply_capture_status_from_outbound( row: AwoooPOutboundMessage, callback_reply: Mapping[str, Any], ) -> dict[str, Any]: """Build capture status directly from one outbound source envelope.""" source_envelope = _as_dict(row.source_envelope) return _callback_reply_evidence_capture_status( callback_reply=callback_reply, persisted_awooop_status_chain=( _as_dict(source_envelope.get("awooop_status_chain")) or None ), persisted_km_stale_completion_summary=( _as_dict(source_envelope.get("km_stale_completion_summary")) or None ), event_at=row.sent_at or row.queued_at, ) def _callback_reply_event_item(row: Mapping[str, Any]) -> dict[str, Any]: """Convert one callback reply outbound row into a read-only evidence item.""" callback_reply = _as_dict(row.get("callback_reply")) action = str(callback_reply.get("action") or "").strip() or None incident_id = str(callback_reply.get("incident_id") or "").strip() or None project_id = str(row.get("project_id") or "") run_id = row.get("run_id") status_value = _callback_reply_public_status(callback_reply) event_at = row.get("sent_at") or row.get("queued_at") persisted_awooop_status_chain = _as_dict( row.get("persisted_awooop_status_chain"), ) or None persisted_km_stale_completion_summary = _as_dict( row.get("persisted_km_stale_completion_summary"), ) or None return { "message_id": row.get("message_id"), "run_id": run_id, "project_id": project_id, "status": status_value, "needs_human": status_value == "failed", "action": action, "incident_id": incident_id, "event_at": event_at, "channel_type": row.get("channel_type"), "message_type": row.get("message_type"), "send_status": row.get("send_status"), "send_error": row.get("send_error"), "provider_message_id": row.get("provider_message_id"), "triggered_by_state": row.get("triggered_by_state"), "content_preview": row.get("content_preview"), "run_state": row.get("run_state"), "agent_id": row.get("agent_id"), "run_created_at": row.get("run_created_at"), "callback_reply": callback_reply, "persisted_awooop_status_chain": persisted_awooop_status_chain, "persisted_km_stale_completion_summary": ( persisted_km_stale_completion_summary ), "evidence_capture_status": _callback_reply_evidence_capture_status( callback_reply=callback_reply, persisted_awooop_status_chain=persisted_awooop_status_chain, persisted_km_stale_completion_summary=( persisted_km_stale_completion_summary ), event_at=event_at, ), "run_detail_href": ( f"/awooop/runs/{run_id}?project_id={project_id}" if run_id and project_id else None ), } def _empty_km_stale_completion_summary( *, project_id: str, incident_id: str | None, status_value: str, reason: str | None = None, ) -> dict[str, Any]: """Build the nullable KM owner-review summary shape for callback evidence.""" return { "schema_version": _KM_STALE_COMPLETION_CALLBACK_SCHEMA_VERSION, "project_id": project_id, "incident_id": incident_id, "status": status_value, "missing_reason": reason, "total": 0, "returned": 0, "pending_count": 0, "ready_count": 0, "blocked_count": 0, "completed_count": 0, "failed_count": 0, "writes_on_read": False, "manual_review_required": True, "batch_writes_allowed": False, "items_truncated": False, "related_total": 0, "related_items": [], "work_item": _km_stale_callback_owner_review_work_item( project_id=project_id, incident_id=incident_id, status_value=status_value, reason=reason, ), } def _object_field(payload: Any, name: str, default: Any = None) -> Any: if isinstance(payload, Mapping): return payload.get(name, default) return getattr(payload, name, default) def _object_int_field(payload: Any, name: str) -> int: try: return int(_object_field(payload, name, 0) or 0) except (TypeError, ValueError): return 0 def _build_km_stale_completion_summary( *, queue: Any, project_id: str, incident_id: str, ) -> dict[str, Any]: """Summarize KM owner-review completion queue state for one incident.""" related_items: list[dict[str, Any]] = [] for item in list(_object_field(queue, "items", []) or []): if str(_object_field(item, "related_incident_id") or "").strip() != incident_id: continue related_items.append({ "entry_id": _object_field(item, "entry_id"), "title": _object_field(item, "title"), "dispatch_id": _object_field(item, "dispatch_id"), "governance_event_id": _object_field(item, "governance_event_id"), "readiness": _object_field(item, "readiness"), "workflow_stage": _object_field(item, "workflow_stage"), "next_action": _object_field(item, "next_action"), "priority_tier": _object_field(item, "priority_tier"), "recommended_completion_outcome": _object_field( item, "recommended_completion_outcome", ), "can_preview": bool(_object_field(item, "can_preview", False)), }) total = _object_int_field(queue, "total") returned = _object_int_field(queue, "returned") return { "schema_version": _KM_STALE_COMPLETION_CALLBACK_SCHEMA_VERSION, "project_id": project_id, "incident_id": incident_id, "status": "matched_owner_review" if related_items else "no_related_owner_review", "missing_reason": None if related_items else "no_matching_completion_item", "total": total, "returned": returned, "pending_count": _object_int_field(queue, "pending_count"), "ready_count": _object_int_field(queue, "ready_count"), "blocked_count": _object_int_field(queue, "blocked_count"), "completed_count": _object_int_field(queue, "completed_count"), "failed_count": _object_int_field(queue, "failed_count"), "writes_on_read": bool(_object_field(queue, "writes_on_read", False)), "manual_review_required": bool( _object_field(queue, "manual_review_required", True) ), "batch_writes_allowed": bool( _object_field(queue, "batch_writes_allowed", False) ), "items_truncated": total > returned, "related_total": len(related_items), "related_items": related_items[:3], "work_item": _km_stale_callback_owner_review_work_item( project_id=project_id, incident_id=incident_id, status_value=( "matched_owner_review" if related_items else "no_related_owner_review" ), reason=None if related_items else "no_matching_completion_item", ), } def _km_stale_callback_owner_review_work_item( *, project_id: str, incident_id: str | None, status_value: str, reason: str | None, ) -> dict[str, Any] | None: """Generate a read-only Work Items link for callback evidence gaps.""" if not incident_id or status_value != "no_related_owner_review": return None work_item_id = f"km-callback-owner-review:{project_id}:{incident_id}" target_query = urlencode( { "project_id": project_id, "incident_id": incident_id, "callback_reply_status": "sent", } ) work_item_query = urlencode( { "project_id": project_id, "work_item_id": work_item_id, "incident_id": incident_id, } ) return { "schema_version": "km_stale_callback_owner_review_work_item_v1", "work_item_id": work_item_id, "kind": "km_stale_callback_owner_review", "status": "open", "project_id": project_id, "incident_id": incident_id, "reason": reason or "no_matching_completion_item", "title": "Telegram callback incident has no matching KM owner-review item", "next_step": "review_or_queue_km_owner_review", "target_surface": "awooop_runs_callback_evidence", "target_href": f"/awooop/runs?{target_query}", "work_item_href": f"/awooop/work-items?{work_item_query}", "triage": { "schema_version": "km_stale_callback_owner_review_triage_v1", "flow_stage": "callback_observed_owner_review_link_missing", "ai_lead_agent": "Hermes", "supporting_agents": ["OpenClaw", "ElephantAlpha"], "automation_state": "manual_owner_review_required", "safe_to_auto_repair": False, "blocking_reason": reason or "no_matching_completion_item", "matching_strategy": "related_incident_id_exact_match", "already_done": [ "callback_reply_persisted", "completion_queue_checked", "generated_read_only_work_item", ], "next_actions": [ "review_runs_callback_evidence", "queue_matching_km_stale_candidate", "complete_owner_review_after_owner_approval", ], }, "writes_on_read": False, "manual_review_required": True, "batch_writes_allowed": False, } def _outbound_timeline_status( send_status: str, callback_reply: dict[str, Any] | None, ) -> str: """Prefer callback delivery status when the outbound row records one.""" if callback_reply: status = callback_reply.get("status") if isinstance(status, str) and status: return status return send_status def _outbound_timeline_summary( *, content_preview: str | None, send_error: str | None, callback_reply: dict[str, Any] | None, ) -> str | None: """Summarize callback reply state without forcing operators to inspect raw JSON.""" if not callback_reply: return content_preview or send_error parts = [ f"callback={callback_reply.get('action') or '--'}", f"incident={callback_reply.get('incident_id') or '--'}", f"status={callback_reply.get('status') or '--'}", ] parse_mode = callback_reply.get("parse_mode") if parse_mode: parts.append(f"parse_mode={parse_mode}") error = callback_reply.get("error") if error: parts.append(f"error={error}") if content_preview: parts.append(str(content_preview)) return " · ".join(parts) def _outbound_timeline_metadata( row: AwoooPOutboundMessage, callback_reply: dict[str, Any] | None, ) -> dict[str, Any]: """Build compact outbound metadata with callback fields first when present.""" metadata: dict[str, Any] = {} if callback_reply: metadata.update({ "callback_status": callback_reply.get("status"), "callback_action": callback_reply.get("action"), "callback_incident_id": callback_reply.get("incident_id"), "callback_parse_mode": callback_reply.get("parse_mode"), }) metadata.update({ "message_type": row.message_type, "provider_message_id": row.provider_message_id, "triggered_by_state": row.triggered_by_state, }) return metadata def _validate_cicd_stage_filter(value: str | None) -> str | None: """Normalize a CI/CD stage filter without allowing arbitrary SQL fragments.""" if value is None: return None stage = value.strip().lower() if not stage: return None if not _CICD_STAGE_RE.fullmatch(stage): raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="stage 格式錯誤，僅允許 a-z、0-9、底線、冒號與短橫線", ) return stage def _validate_cicd_status_filter(value: str | None) -> str | None: """Normalize and validate CI/CD status filter.""" if value is None: return None status_value = value.strip().lower() if not status_value: return None if status_value not in _CICD_STATUS_FILTERS: allowed = ", ".join(sorted(_CICD_STATUS_FILTERS)) raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=f"status 必須是: {allowed}", ) return status_value def _cicd_duration_seconds(value: Any) -> int: """Coerce Alertmanager duration_seconds label into a non-negative integer.""" try: duration = int(str(value or "0")) except (TypeError, ValueError): return 0 return max(duration, 0) def _cicd_event_needs_attention(status_value: str | None, severity: str | None) -> bool: """Return whether a CI/CD evidence row should be highlighted for operators.""" normalized_status = str(status_value or "").lower() normalized_severity = str(severity or "").lower() return normalized_status in {"failed", "pending"} or normalized_severity in { "critical", "warning", } def _cicd_event_item_from_row(row: Mapping[str, Any], *, project_id: str) -> dict[str, Any]: """Convert one alert_operation_log CI/CD row into an operator-facing item.""" context = _as_dict(row.get("context")) labels = _as_dict(context.get("labels")) annotations = _as_dict(context.get("annotations")) status_value = str(row.get("status") or labels.get("status") or "").lower() or None severity = str(row.get("severity") or labels.get("severity") or "").lower() or None summary = row.get("summary") or annotations.get("summary") description = row.get("description") or annotations.get("description") workflow_url = row.get("workflow_url") or annotations.get("workflow_url") return { "id": str(row.get("id") or ""), "project_id": project_id, "alertname": str(row.get("alertname") or labels.get("alertname") or ""), "stage": row.get("stage") or labels.get("stage"), "status": status_value, "severity": severity, "commit_sha": row.get("commit_sha") or labels.get("commit"), "triggered_by": row.get("triggered_by") or labels.get("triggered_by"), "duration_seconds": _cicd_duration_seconds( row.get("duration_seconds") or labels.get("duration_seconds") ), "summary": str(summary).strip() if summary else None, "description": str(description).strip() if description else None, "workflow_url": str(workflow_url).strip() if workflow_url else None, "alert_id": row.get("alert_id") or context.get("alert_id"), "source": row.get("source") or context.get("source"), "action_detail": row.get("action_detail"), "needs_attention": _cicd_event_needs_attention(status_value, severity), "created_at": row.get("created_at"), } def _run_callback_reply_summary( outbound_messages: list[AwoooPOutboundMessage], ) -> dict[str, Any]: """Summarize Telegram detail/history callback reply delivery for Run List.""" callback_rows: list[tuple[AwoooPOutboundMessage, dict[str, Any]]] = [] for row in outbound_messages: callback_reply = _outbound_callback_reply(row.source_envelope) if callback_reply: callback_rows.append((row, callback_reply)) if not callback_rows: return { "schema_version": "awooop_run_callback_reply_summary_v1", "status": "no_callback", "total": 0, "sent": 0, "fallback_sent": 0, "rescue_sent": 0, "failed": 0, "needs_human": False, "latest_status": None, "latest_action": None, "latest_incident_id": None, "latest_at": None, "latest_provider_message_id": None, "capture_status": "no_callback", "capture_captured": 0, "capture_partial": 0, "capture_not_captured": 0, "latest_capture_status": None, "latest_capture_missing": [], "latest_capture_next_action": None, } sorted_rows = sorted( callback_rows, key=lambda item: str(item[0].sent_at or item[0].queued_at or ""), reverse=True, ) latest_row, latest_callback = sorted_rows[0] statuses = [ str(callback.get("status") or "") for _, callback in sorted_rows ] failed = statuses.count("callback_reply_failed") latest_status = str(latest_callback.get("status") or "") summary_status = _callback_reply_public_status(latest_callback) capture_rows = [ _callback_reply_capture_status_from_outbound(row, callback) for row, callback in sorted_rows ] capture_statuses = [ str(capture.get("status") or "observed") for capture in capture_rows ] capture_not_captured = capture_statuses.count("not_captured") capture_partial = capture_statuses.count("partial") capture_captured = capture_statuses.count("captured") latest_capture = capture_rows[0] if capture_rows else {} if capture_not_captured > 0: capture_status = "not_captured" elif capture_partial > 0: capture_status = "partial" elif capture_captured > 0 and capture_captured == len(capture_rows): capture_status = "captured" else: capture_status = "observed" return { "schema_version": "awooop_run_callback_reply_summary_v1", "status": summary_status, "total": len(sorted_rows), "sent": statuses.count("callback_reply_sent"), "fallback_sent": statuses.count("callback_reply_fallback_sent"), "rescue_sent": statuses.count("callback_reply_rescue_sent"), "failed": failed, "needs_human": failed > 0 or latest_status == "callback_reply_failed", "latest_status": latest_status or None, "latest_action": latest_callback.get("action"), "latest_incident_id": latest_callback.get("incident_id"), "latest_at": latest_row.sent_at or latest_row.queued_at, "latest_provider_message_id": latest_row.provider_message_id, "capture_status": capture_status, "capture_captured": capture_captured, "capture_partial": capture_partial, "capture_not_captured": capture_not_captured, "latest_capture_status": latest_capture.get("status"), "latest_capture_missing": latest_capture.get("missing") or [], "latest_capture_next_action": latest_capture.get("next_action"), } def _mcp_gateway_summary_row(row: AwoooPMcpGatewayAudit) -> dict[str, Any]: """Convert SQLAlchemy audit rows into the truth-chain summary shape.""" return { "agent_id": row.agent_id, "tool_name": row.tool_name, "result_status": row.result_status, "block_gate": row.block_gate, "gate_result": row.gate_result or {}, } def _as_dict(value: Any) -> dict[str, Any]: """Return dict payloads defensively; DB JSON fields may be null or stale.""" return value if isinstance(value, dict) else {} def _append_unique(values: list[str], candidate: Any) -> None: """Append non-empty string once while preserving discovery order.""" text_value = str(candidate or "").strip() if text_value and text_value not in values: values.append(text_value) def _append_incident_ids_from_text(values: list[str], text_value: Any) -> None: """Extract incident ids from legacy text payloads.""" if not text_value: return for incident_id in _INCIDENT_ID_RE.findall(str(text_value)): _append_unique(values, incident_id) def _append_incident_ids_from_source_envelope(values: list[str], envelope: Any) -> None: """Extract incident ids from AwoooP channel event source_refs.""" source_refs = _as_dict(_as_dict(envelope).get("source_refs")) incident_ids = source_refs.get("incident_ids") if isinstance(incident_ids, list): for incident_id in incident_ids: _append_unique(values, incident_id) else: _append_unique(values, incident_ids) def _collect_run_incident_ids( *, run: AwoooPRunState, inbound_events: list[AwoooPConversationEvent], outbound_messages: list[AwoooPOutboundMessage], ) -> list[str]: """Collect incident ids that tie a Run back to legacy incident evidence.""" incident_ids: list[str] = [] _append_incident_ids_from_text(incident_ids, run.trigger_ref) _append_incident_ids_from_text(incident_ids, run.error_detail) for event in inbound_events: _append_incident_ids_from_source_envelope(incident_ids, event.source_envelope) _append_incident_ids_from_text(incident_ids, event.content_preview) _append_incident_ids_from_text(incident_ids, event.content_redacted) for message in outbound_messages: _append_incident_ids_from_source_envelope(incident_ids, message.source_envelope) _append_incident_ids_from_text(incident_ids, message.content_preview) _append_incident_ids_from_text(incident_ids, message.send_error) return incident_ids async def _load_run_message_context( db: Any, runs: list[AwoooPRunState], *, limit: int = _MAX_LIST_CONTEXT_ROWS, ) -> tuple[ dict[UUID, list[AwoooPConversationEvent]], dict[UUID, list[AwoooPOutboundMessage]], ]: """Load list-page sidecar events needed to link runs back to incidents.""" if not runs: return {}, {} inbound_by_run: dict[UUID, list[AwoooPConversationEvent]] = defaultdict(list) outbound_by_run: dict[UUID, list[AwoooPOutboundMessage]] = defaultdict(list) remaining_inbound = max(int(limit), 0) remaining_outbound = max(int(limit), 0) for batch in _iter_run_context_batches(runs): if remaining_inbound > 0: inbound_filters = [AwoooPConversationEvent.run_id.in_(batch["run_ids"])] if batch["trigger_refs"]: inbound_filters.append(AwoooPConversationEvent.provider_event_id.in_( batch["trigger_refs"] )) if batch["trigger_event_ids"]: inbound_filters.append(AwoooPConversationEvent.event_id.in_( batch["trigger_event_ids"] )) inbound_result = await db.execute( select(AwoooPConversationEvent) .where(sa_or(*inbound_filters)) .order_by(AwoooPConversationEvent.received_at.desc()) .limit(remaining_inbound) ) inbound_events = list(inbound_result.scalars().all()) remaining_inbound = max(remaining_inbound - len(inbound_events), 0) for event in inbound_events: target_run_id = ( event.run_id if event.run_id in batch["run_ids_set"] else None ) if target_run_id is None: target_run_id = batch["trigger_ref_to_run"].get( str(event.provider_event_id) ) if target_run_id is None: target_run_id = batch["trigger_ref_to_run"].get(str(event.event_id)) if target_run_id is not None: inbound_by_run[target_run_id].append(event) if remaining_outbound > 0: outbound_result = await db.execute( select(AwoooPOutboundMessage) .where(AwoooPOutboundMessage.run_id.in_(batch["run_ids"])) .order_by(AwoooPOutboundMessage.queued_at.desc()) .limit(remaining_outbound) ) outbound_messages = list(outbound_result.scalars().all()) remaining_outbound = max(remaining_outbound - len(outbound_messages), 0) for message in outbound_messages: outbound_by_run[message.run_id].append(message) return dict(inbound_by_run), dict(outbound_by_run) def _list_filter_context_limit(candidate_count: int) -> int: return min(max(candidate_count * 4, _MAX_LIST_CONTEXT_ROWS), 20_000) def _iter_run_context_batches( runs: list[AwoooPRunState], ) -> list[dict[str, Any]]: """Split run context lookups below asyncpg's bind-parameter ceiling.""" batches: list[dict[str, Any]] = [] for start in range(0, len(runs), _RUN_CONTEXT_QUERY_CHUNK_SIZE): batch_runs = runs[start : start + _RUN_CONTEXT_QUERY_CHUNK_SIZE] run_ids = [run.run_id for run in batch_runs] trigger_refs = [str(run.trigger_ref) for run in batch_runs if run.trigger_ref] trigger_ref_to_run = { str(run.trigger_ref): run.run_id for run in batch_runs if run.trigger_ref } trigger_event_ids: list[UUID] = [] for trigger_ref in trigger_refs: try: trigger_event_ids.append(uuid.UUID(trigger_ref)) except ValueError: continue batches.append({ "run_ids": run_ids, "run_ids_set": set(run_ids), "trigger_refs": trigger_refs, "trigger_ref_to_run": trigger_ref_to_run, "trigger_event_ids": trigger_event_ids, }) return batches def _route_label_from_remediation(item: dict[str, Any]) -> str: """Render remediation MCP route consistently with Telegram / Work Items.""" return "/".join( str(part) for part in ( item.get("agent_id"), item.get("tool_name"), item.get("required_scope"), ) if part ) or "--" def _route_label_from_legacy_mcp(record: dict[str, Any]) -> str: """Render self-built/legacy MCP evidence as agent/tool/scope for list UX.""" tool = record.get("tool_name") server = record.get("mcp_server") tool_label = ".".join(str(part) for part in (server, tool) if part) or tool return "/".join( str(part) for part in ( record.get("agent_role"), tool_label, "read", ) if part ) or "--" def _remediation_timeline_status(item: dict[str, Any]) -> str: if item.get("success") is False or item.get("allowed") is False: return "failed" if item.get("verification_result_preview") == "success": return "success" return "warning" def _remediation_timeline_summary(item: dict[str, Any]) -> str: return ( f"incident={item.get('incident_id') or '--'} " f"mode={item.get('mode') or '--'} " f"preview={item.get('verification_result_preview') or '--'} " f"route={_route_label_from_remediation(item)} " f"writes_incident={item.get('writes_incident_state')} " f"writes_auto_repair={item.get('writes_auto_repair_result')}" )[:500] def _legacy_mcp_timeline_status(record: dict[str, Any]) -> str: if record.get("success") is True: return "success" if record.get("success") is False: return "failed" return "warning" def _legacy_mcp_timeline_summary(record: dict[str, Any]) -> str: return ( f"incident={record.get('incident_id') or '--'} " f"agent={record.get('agent_role') or '--'} " f"node={record.get('flywheel_node') or '--'} " f"duration_ms={record.get('duration_ms') if record.get('duration_ms') is not None else '--'} " f"error={record.get('error_message') or '--'}" )[:500] def _run_remediation_list_summary( *, run: AwoooPRunState, incident_ids: list[str], items: list[dict[str, Any]], legacy_mcp_records: list[dict[str, Any]] | None = None, errors: list[dict[str, str]] | None = None, ) -> dict[str, Any]: """Summarize durable ADR-100 dry-run and MCP investigation evidence for list UX.""" sorted_items = sorted( (item for item in items if isinstance(item, dict)), key=lambda item: str(item.get("created_at") or ""), reverse=True, ) sorted_mcp_records = sorted( (record for record in (legacy_mcp_records or []) if isinstance(record, dict)), key=lambda record: str(record.get("created_at") or ""), reverse=True, ) latest = sorted_items[0] if sorted_items else {} latest_mcp = sorted_mcp_records[0] if sorted_mcp_records else {} writes_incident = latest.get("writes_incident_state") writes_auto_repair = latest.get("writes_auto_repair_result") route = ( _route_label_from_remediation(latest) if latest else _route_label_from_legacy_mcp(latest_mcp) if latest_mcp else "--" ) write_observed = writes_incident is True or writes_auto_repair is True is_read_only = ( bool(latest) and latest.get("required_scope") == "read" and writes_incident is False and writes_auto_repair is False ) mcp_total = len(sorted_mcp_records) mcp_success = sum(1 for record in sorted_mcp_records if record.get("success") is True) mcp_failed = sum(1 for record in sorted_mcp_records if record.get("success") is False) if not sorted_items: status_value = "mcp_observed" if mcp_total > 0 else "no_evidence" elif latest.get("success") is False or latest.get("allowed") is False: status_value = "blocked" elif write_observed: status_value = "write_observed" elif is_read_only: status_value = "read_only_dry_run" else: status_value = "observed" return { "schema_version": "awooop_run_remediation_summary_v1", "source": "alert_operation_log" if sorted_items else "mcp_audit_log" if mcp_total > 0 else "none", "incident_ids": incident_ids, "total": len(sorted_items), "evidence_total": len(sorted_items) + mcp_total, "status": status_value, "has_dry_run": bool(sorted_items), "has_mcp_investigation": mcp_total > 0, "is_read_only": is_read_only, "human_gate_open": run.state == "waiting_approval", "latest_at": latest.get("created_at"), "latest_preview": latest.get("verification_result_preview"), "latest_mode": latest.get("mode"), "latest_route": route, "latest_agent_id": latest.get("agent_id") or latest_mcp.get("agent_role"), "latest_tool_name": latest.get("tool_name") or latest_mcp.get("tool_name"), "latest_required_scope": latest.get("required_scope") or ("read" if latest_mcp else None), "writes_incident_state": writes_incident, "writes_auto_repair_result": writes_auto_repair, "mcp_observation_total": mcp_total, "mcp_observation_success": mcp_success, "mcp_observation_failed": mcp_failed, "latest_mcp_server": latest_mcp.get("mcp_server"), "errors": errors or [], } def _safe_int(value: Any) -> int: try: return int(value or 0) except (TypeError, ValueError): return 0 def _has_repair_execution_evidence(facts: dict[str, Any]) -> bool: return ( _safe_int(facts.get("auto_repair_execution_records")) > 0 or _safe_int(facts.get("effective_execution_records")) > 0 ) def _has_nonrepair_operation_evidence(facts: dict[str, Any]) -> bool: return ( _safe_int(facts.get("automation_operation_records")) > 0 and not _has_repair_execution_evidence(facts) ) def _latest_remediation_history_item( history: dict[str, Any] | None, ) -> dict[str, Any]: if not isinstance(history, dict): return {} items = history.get("items") if isinstance(history.get("items"), list) else [] latest = items[0] if items and isinstance(items[0], dict) else {} return latest def _remediation_evidence_state(history: dict[str, Any] | None) -> str: """Classify ADR-100 evidence with the same operator semantics as Telegram.""" if not isinstance(history, dict): return "missing" total = _safe_int(history.get("total")) if total <= 0: if history.get("status") == "fetch_failed": return "fetch_failed" return "missing" latest = _latest_remediation_history_item(history) if latest.get("writes_incident_state") or latest.get("writes_auto_repair_result"): return "write_observed" if latest.get("allowed") is False or latest.get("success") is False: return "blocked" if ( str(latest.get("safety_level") or "").lower() == "read_only" or str(latest.get("required_scope") or "").lower() == "read" ): return "read_only" return "observed" def _select_status_chain_source_id( incident_ids: list[str], remediation_history: dict[str, Any] | None, ) -> str | None: latest_incident_id = str( _latest_remediation_history_item(remediation_history).get("incident_id") or "" ).strip() if latest_incident_id and latest_incident_id in incident_ids: return latest_incident_id return incident_ids[0] if incident_ids else latest_incident_id or None def _status_chain_mcp_section(truth_chain: dict[str, Any] | None) -> dict[str, Any]: mcp = truth_chain.get("mcp") if isinstance(truth_chain, dict) else {} if not isinstance(mcp, dict): mcp = {} gateway = mcp.get("awooop_gateway") if isinstance(mcp.get("awooop_gateway"), dict) else {} legacy = mcp.get("legacy") if isinstance(mcp.get("legacy"), dict) else {} top_tools: list[dict[str, Any]] = [] seen_tools: set[str] = set() for source, summary in (("gateway", gateway), ("legacy", legacy)): by_tool = summary.get("by_tool") if isinstance(summary, dict) else [] if not isinstance(by_tool, list): continue for item in by_tool: if not isinstance(item, dict): continue tool_name = str(item.get("tool_name") or "unknown").strip() or "unknown" key = f"{source}:{tool_name}" if key in seen_tools: continue seen_tools.add(key) top_tools.append({ "source": source, "tool_name": tool_name, "total": ( _safe_int(item.get("total")) or _safe_int(item.get("success")) + _safe_int(item.get("failed")) + _safe_int(item.get("blocked")) ), "success": _safe_int(item.get("success")), "failed": _safe_int(item.get("failed")), "blocked": _safe_int(item.get("blocked")), "last_error": item.get("last_error"), }) if len(top_tools) >= 5: break if len(top_tools) >= 5: break return { "gateway": { "total": _safe_int(gateway.get("total")), "success": _safe_int(gateway.get("success")), "failed": _safe_int(gateway.get("failed")), "blocked": _safe_int(gateway.get("blocked")), "first_class_total": _safe_int(gateway.get("first_class_total")), "legacy_bridge_total": _safe_int(gateway.get("legacy_bridge_total")), "policy_enforced_total": _safe_int(gateway.get("policy_enforced_total")), "stage": gateway.get("stage"), "stage_status": gateway.get("stage_status"), }, "legacy": { "total": _safe_int(legacy.get("total")), "success": _safe_int(legacy.get("success")), "failed": _safe_int(legacy.get("failed")), }, "top_tools": top_tools, } def _first_non_empty(row: Mapping[str, Any], keys: tuple[str, ...]) -> Any: for key in keys: value = row.get(key) if value not in (None, ""): return value return None def _status_chain_execution_section(truth_chain: dict[str, Any] | None) -> dict[str, Any]: execution = truth_chain.get("execution") if isinstance(truth_chain, dict) else {} if not isinstance(execution, dict): execution = {} ops = execution.get("automation_operation_log") if not isinstance(ops, list): ops = [] latest_op = ops[0] if ops and isinstance(ops[0], dict) else {} playbook_ids: list[str] = [] playbook_paths: list[str] = [] for row in ops: if not isinstance(row, dict): continue _append_unique(playbook_ids, row.get("matched_playbook_id")) _append_unique(playbook_ids, row.get("input_playbook_id")) _append_unique(playbook_ids, row.get("output_playbook_id")) _append_unique(playbook_paths, row.get("input_playbook_path")) _append_unique(playbook_paths, row.get("output_playbook_path")) _append_unique(playbook_paths, row.get("input_ansible_playbook_path")) _append_unique(playbook_paths, row.get("output_ansible_playbook_path")) ansible = execution.get("ansible") if isinstance(execution.get("ansible"), dict) else {} ansible_records = ansible.get("records") if isinstance(ansible.get("records"), list) else [] latest_ansible = ( ansible_records[0] if ansible_records and isinstance(ansible_records[0], dict) else {} ) candidate_catalog = ( ansible.get("candidate_catalog") if isinstance(ansible.get("candidate_catalog"), dict) else {} ) candidates = ( candidate_catalog.get("candidates") if isinstance(candidate_catalog.get("candidates"), list) else [] ) ansible_summary = ( ansible.get("summary") if isinstance(ansible.get("summary"), dict) else summarize_ansible_execution([row for row in ansible_records if isinstance(row, dict)]) ) return { "operation_total": len(ops), "latest_operation_type": latest_op.get("operation_type"), "latest_status": latest_op.get("status"), "latest_actor": latest_op.get("actor"), "latest_action": _first_non_empty(latest_op, ("input_action", "output_action")), "latest_executor": _first_non_empty( latest_op, ( "input_executor", "output_executor", "input_execution_backend", "output_execution_backend", ), ), "playbook_ids": playbook_ids[:5], "playbook_paths": playbook_paths[:5], "ansible": { "considered": bool(ansible.get("considered")), "record_total": len(ansible_records), "candidate_count": len(candidates), "not_used_reason": ansible.get("not_used_reason"), "check_mode_total": ansible_summary.get("check_mode_total"), "apply_total": ansible_summary.get("apply_total"), "rollback_total": ansible_summary.get("rollback_total"), "pending_check_mode_total": ansible_summary.get("pending_check_mode_total"), "applied_success_total": ansible_summary.get("applied_success_total"), "applied": ansible_summary.get("applied"), "controlled_apply": ansible_summary.get("controlled_apply"), "latest_operation_type": ansible_summary.get("latest_operation_type") or latest_ansible.get("operation_type"), "latest_status": ansible_summary.get("latest_status") or latest_ansible.get("status"), "latest_catalog_id": ansible_summary.get("latest_catalog_id") or latest_ansible.get("catalog_id"), "latest_playbook_path": ansible_summary.get("latest_playbook_path") or latest_ansible.get("playbook_path"), "latest_execution_mode": ansible_summary.get("latest_execution_mode") or latest_ansible.get("execution_mode"), "latest_check_mode": ( ansible_summary.get("latest_check_mode") if ansible_summary.get("latest_check_mode") is not None else latest_ansible.get("check_mode") ), "latest_returncode": ansible_summary.get("latest_returncode"), "latest_apply_executed": ansible_summary.get("latest_apply_executed"), "approval_source": ansible_summary.get("approval_source"), "candidate_playbooks": [ { "catalog_id": item.get("catalog_id"), "playbook_path": item.get("playbook_path"), "risk_level": item.get("risk_level"), "match_score": item.get("match_score"), } for item in candidates[:3] if isinstance(item, dict) ], }, } def _source_ref_values(envelope: Any, key: str) -> list[str]: if not isinstance(envelope, dict): return [] source_refs = envelope.get("source_refs") if not isinstance(source_refs, dict): return [] raw_values = source_refs.get(key) if isinstance(raw_values, list): return [str(item) for item in raw_values if str(item or "").strip()] if raw_values not in (None, ""): return [str(raw_values)] return [] def _source_correlation_empty( incident_ids: list[str], *, status_value: str, missing_reason: str, ) -> dict[str, Any]: return { "schema_version": _SOURCE_CORRELATION_SCHEMA_VERSION, "status": status_value, "missing_reason": missing_reason, "incident_ids": incident_ids, "direct_ref_total": 0, "candidate_total": 0, "applied_link_total": 0, "provider_event_total": 0, "latest_applied_link_at": None, "verification_status": status_value, "providers": { provider: { "direct_ref_total": 0, "candidate_total": 0, "applied_link_total": 0, "latest_event_at": None, "latest_heartbeat_at": None, "latest_applied_link_at": None, } for provider in _SOURCE_CORRELATION_PROVIDERS }, "top_candidates": [], "matching_criteria": [ "source_correlation_linked_stage", "direct_source_ref", "fingerprint_overlap", "alertname_overlap", "service_or_namespace_overlap", "severity_overlap", ], } def _normalize_correlation_value(value: Any) -> str: if hasattr(value, "value"): value = value.value return str(value or "").strip().lower() def _append_correlation_term(values: list[str], value: Any) -> None: term = _normalize_correlation_value(value) if term in {"", "--", "n/a", "none", "null", "unknown"}: return if len(term) < 2: return if term not in values: values.append(term) def _intersection(left: list[str], right: list[str]) -> list[str]: right_set = set(right) return [item for item in left if item in right_set] def _as_utc_naive(value: Any) -> datetime | None: if not isinstance(value, datetime): return None if value.tzinfo is not None: return value.astimezone(UTC).replace(tzinfo=None) return value def _iso_or_none(value: Any) -> str | None: if hasattr(value, "isoformat"): return value.isoformat() if value in (None, ""): return None return str(value) def _incident_correlation_context(record: IncidentRecord) -> dict[str, list[str]]: """Build compact incident terms used only for read-only source matching.""" alertnames: list[str] = [] severities: list[str] = [] fingerprints: list[str] = [] namespaces: list[str] = [] targets: list[str] = [] _append_correlation_term(alertnames, record.alertname) _append_correlation_term(severities, record.severity) for service in record.affected_services or []: _append_correlation_term(targets, service) for signal in record.signals or []: if not isinstance(signal, dict): continue _append_correlation_term(alertnames, signal.get("alert_name")) _append_correlation_term(severities, signal.get("severity")) _append_correlation_term(fingerprints, signal.get("fingerprint")) labels = _as_dict(signal.get("labels")) annotations = _as_dict(signal.get("annotations")) _append_correlation_term(alertnames, labels.get("alertname")) _append_correlation_term(fingerprints, labels.get("fingerprint")) for key in ( "namespace", "kubernetes_namespace", ): _append_correlation_term(namespaces, labels.get(key)) for key in ( "service", "service_name", "pod", "pod_name", "deployment", "deployment_name", "container", "job", "instance", "target", "target_resource", "workload", "app", "app.kubernetes.io/name", ): _append_correlation_term(targets, labels.get(key)) for key in ("summary", "description"): _append_correlation_term(alertnames, annotations.get(key)) return { "incident_ids": [record.incident_id], "alertnames": alertnames, "severities": severities, "fingerprints": fingerprints, "namespaces": namespaces, "targets": targets, } def _source_event_correlation_context(row: Mapping[str, Any]) -> dict[str, Any]: envelope = _as_dict(row.get("source_envelope")) source_refs = _as_dict(envelope.get("source_refs")) log_correlation = _as_dict(envelope.get("log_correlation")) labels = _as_dict(envelope.get("labels")) annotations = _as_dict(envelope.get("annotations")) alertnames: list[str] = [] severities: list[str] = [] fingerprints: list[str] = [] namespaces: list[str] = [] targets: list[str] = [] _append_correlation_term(alertnames, log_correlation.get("alertname")) _append_correlation_term(alertnames, labels.get("alertname")) for value in _source_ref_values(envelope, "signoz_alerts"): _append_correlation_term(alertnames, value) _append_correlation_term(severities, log_correlation.get("severity")) _append_correlation_term(severities, labels.get("severity")) _append_correlation_term(fingerprints, log_correlation.get("fingerprint")) _append_correlation_term(fingerprints, labels.get("fingerprint")) for value in _source_ref_values(envelope, "fingerprints"): _append_correlation_term(fingerprints, value) for key in ("namespace", "kubernetes_namespace"): _append_correlation_term(namespaces, log_correlation.get(key)) _append_correlation_term(namespaces, labels.get(key)) for key in ( "target_resource", "service", "service_name", "pod", "pod_name", "deployment", "deployment_name", "container", "job", "instance", "target", "workload", "app", "app.kubernetes.io/name", ): _append_correlation_term(targets, log_correlation.get(key)) _append_correlation_term(targets, labels.get(key)) for key in ("summary", "description"): _append_correlation_term(alertnames, annotations.get(key)) return { "provider": str(row.get("provider") or envelope.get("provider") or "").lower(), "stage": str(row.get("stage") or envelope.get("stage") or ""), "provider_event_id": row.get("provider_event_id") or envelope.get("provider_event_id"), "received_at": row.get("received_at"), "source_refs": source_refs, "incident_ids": _source_ref_values(envelope, "incident_ids"), "alertnames": alertnames, "severities": severities, "fingerprints": fingerprints, "namespaces": namespaces, "targets": targets, } def _score_source_correlation_event( incident_context: dict[str, list[str]], event_context: dict[str, Any], ) -> dict[str, Any]: """Return a deterministic, read-only source-match score for UI evidence.""" reasons: list[str] = [] score = 0 is_direct = False if _intersection(incident_context["incident_ids"], event_context["incident_ids"]): is_direct = True score += 100 reasons.append("direct_incident_ref") fingerprint_hits = _intersection( incident_context["fingerprints"], event_context["fingerprints"], ) if fingerprint_hits: is_direct = True score += 80 reasons.append("fingerprint_overlap") if _intersection(incident_context["alertnames"], event_context["alertnames"]): score += 35 reasons.append("alertname_overlap") if _intersection(incident_context["targets"], event_context["targets"]): score += 25 reasons.append("target_overlap") if _intersection(incident_context["namespaces"], event_context["namespaces"]): score += 10 reasons.append("namespace_overlap") if _intersection(incident_context["severities"], event_context["severities"]): score += 5 reasons.append("severity_overlap") return { "is_direct": is_direct, "is_candidate": bool(is_direct or score >= 35), "score": min(score, 100), "reasons": reasons[:5], } def _is_source_correlation_applied_link( event_context: dict[str, Any], scored: dict[str, Any], ) -> bool: """Applied source links must be append-only events that still match directly.""" return ( str(event_context.get("stage") or "").lower() == "source_correlation_linked" and bool(scored.get("is_direct")) ) async def _fetch_source_correlation_summary( *, project_id: str, incident_ids: list[str], ) -> dict[str, Any]: """Fetch read-only Sentry/SigNoz evidence candidates for incident status-chain.""" if not incident_ids: return _source_correlation_empty( incident_ids, status_value="no_incident_context", missing_reason="no_incident_ids", ) safe_project_id = project_id or "awoooi" async with get_db_context(safe_project_id) as db: incident_result = await db.execute( select(IncidentRecord) .where(IncidentRecord.project_id == safe_project_id) .where(IncidentRecord.incident_id.in_(incident_ids)) ) incident_rows = list(incident_result.scalars().all()) if not incident_rows: heartbeat_rows = [] source_rows = [] else: now = _utc_now_naive() created_candidates = [ value for value in (_as_utc_naive(row.created_at) for row in incident_rows) if value is not None ] earliest_created = min(created_candidates) if created_candidates else now window_start = max( earliest_created - timedelta(hours=_SOURCE_CORRELATION_PRE_WINDOW_HOURS), now - timedelta(days=_SOURCE_CORRELATION_LOOKBACK_DAYS), ) provider_sql = ( "LOWER(COALESCE(NULLIF(source_envelope->>'provider', ''), " "NULLIF(split_part(provider_event_id, ':', 1), ''), channel_type))" ) source_result = await db.execute( text(f""" SELECT event_id::text AS event_id, project_id, channel_type, provider_event_id, content_preview, source_envelope, received_at, {provider_sql} AS provider, LOWER(COALESCE(source_envelope->>'stage', '')) AS stage FROM awooop_conversation_event WHERE project_id = :project_id AND {provider_sql} IN ('sentry', 'signoz') AND LOWER(COALESCE(source_envelope->>'stage', '')) <> 'heartbeat' AND received_at >= :window_start ORDER BY received_at DESC LIMIT :limit """), { "project_id": safe_project_id, "window_start": window_start, "limit": _SOURCE_CORRELATION_EVENT_LIMIT, }, ) source_rows = list(source_result.mappings().all()) heartbeat_result = await db.execute( text(f""" SELECT {provider_sql} AS provider, MAX(received_at) AS latest_heartbeat_at FROM awooop_conversation_event WHERE project_id = :project_id AND {provider_sql} IN ('sentry', 'signoz') AND LOWER(COALESCE(source_envelope->>'stage', '')) = 'heartbeat' AND received_at >= :window_start GROUP BY {provider_sql} """), { "project_id": safe_project_id, "window_start": window_start, }, ) heartbeat_rows = list(heartbeat_result.mappings().all()) if not incident_rows: summary = _source_correlation_empty( incident_ids, status_value="no_incident_context", missing_reason="incident_not_found", ) return summary contexts = [_incident_correlation_context(row) for row in incident_rows] summary = _source_correlation_empty( incident_ids, status_value="missing", missing_reason="no_matching_provider_source_event", ) providers = summary["providers"] for heartbeat in heartbeat_rows: provider = str(heartbeat.get("provider") or "").lower() if provider in providers: providers[provider]["latest_heartbeat_at"] = _iso_or_none( heartbeat.get("latest_heartbeat_at") ) top_candidates: list[dict[str, Any]] = [] for row in source_rows: event_context = _source_event_correlation_context(row) provider = str(event_context.get("provider") or "").lower() if provider not in providers: continue provider_item = providers[provider] if provider_item.get("latest_event_at") is None: provider_item["latest_event_at"] = _iso_or_none(row.get("received_at")) best_match: dict[str, Any] | None = None for context in contexts: scored = _score_source_correlation_event(context, event_context) if best_match is None or scored["score"] > best_match["score"]: best_match = scored if not best_match or not best_match["is_candidate"]: continue summary["provider_event_total"] += 1 if best_match["is_direct"]: summary["direct_ref_total"] += 1 provider_item["direct_ref_total"] += 1 else: summary["candidate_total"] += 1 provider_item["candidate_total"] += 1 is_applied_link = _is_source_correlation_applied_link( event_context, best_match, ) if is_applied_link: applied_at = _iso_or_none(row.get("received_at")) summary["applied_link_total"] += 1 provider_item["applied_link_total"] += 1 if summary.get("latest_applied_link_at") is None: summary["latest_applied_link_at"] = applied_at if provider_item.get("latest_applied_link_at") is None: provider_item["latest_applied_link_at"] = applied_at top_candidates.append( { "provider": provider, "provider_event_id": str(event_context.get("provider_event_id") or ""), "stage": str(event_context.get("stage") or ""), "score": best_match["score"], "match_type": "direct" if best_match["is_direct"] else "candidate", "link_state": ( "applied" if is_applied_link else "direct_ref" if best_match["is_direct"] else "candidate" ), "verification_status": ( "applied_link_verified" if is_applied_link else "direct_ref_verified" if best_match["is_direct"] else "candidate_only" ), "reasons": best_match["reasons"], "received_at": _iso_or_none(row.get("received_at")), } ) if summary["applied_link_total"] > 0: summary["status"] = "linked" summary["verification_status"] = "applied_link_verified" summary["missing_reason"] = None elif summary["direct_ref_total"] > 0: summary["status"] = "linked" summary["verification_status"] = "direct_ref_verified" summary["missing_reason"] = None elif summary["candidate_total"] > 0: summary["status"] = "candidate_found" summary["verification_status"] = "candidate_only" summary["missing_reason"] = None elif any(item.get("latest_heartbeat_at") for item in providers.values()): summary["status"] = "provider_fresh_no_match" summary["verification_status"] = "provider_fresh_no_match" summary["missing_reason"] = "provider_heartbeat_present_but_no_incident_match" summary["top_candidates"] = sorted( top_candidates, key=lambda item: (item.get("score") or 0, item.get("received_at") or ""), reverse=True, )[:5] return summary def _status_chain_source_section(truth_chain: dict[str, Any] | None) -> dict[str, Any]: channel = truth_chain.get("channel") if isinstance(truth_chain, dict) else {} if not isinstance(channel, dict): channel = {} inbound_events = channel.get("inbound_events") outbound_messages = channel.get("outbound_messages") if not isinstance(inbound_events, list): inbound_events = [] if not isinstance(outbound_messages, list): outbound_messages = [] source_refs: dict[str, list[str]] = { "alert_ids": [], "sentry_issue_ids": [], "signoz_alerts": [], "fingerprints": [], "incident_ids": [], } inbound_channels: list[str] = [] for row in inbound_events: if not isinstance(row, dict): continue _append_unique(inbound_channels, row.get("channel_type")) envelope = row.get("source_envelope") for key in source_refs: for value in _source_ref_values(envelope, key): _append_unique(source_refs[key], value) latest_inbound = inbound_events[0] if inbound_events and isinstance(inbound_events[0], dict) else {} latest_outbound = ( outbound_messages[0] if outbound_messages and isinstance(outbound_messages[0], dict) else {} ) return { "inbound_total": len(inbound_events), "outbound_total": len(outbound_messages), "inbound_channels": inbound_channels[:5], "refs": {key: values[:5] for key, values in source_refs.items()}, "latest_inbound": { "channel_type": latest_inbound.get("channel_type"), "provider_event_id": latest_inbound.get("provider_event_id"), "content_type": latest_inbound.get("content_type"), "is_duplicate": latest_inbound.get("is_duplicate"), "received_at": latest_inbound.get("received_at"), }, "latest_outbound": { "channel_type": latest_outbound.get("channel_type"), "message_type": latest_outbound.get("message_type"), "send_status": latest_outbound.get("send_status"), "sent_at": latest_outbound.get("sent_at"), }, } def _build_awooop_status_chain( *, incident_ids: list[str], truth_chain: dict[str, Any] | None = None, remediation_history: dict[str, Any] | None = None, source_id: str | None = None, fetch_error: str | None = None, source_correlation: dict[str, Any] | None = None, ) -> dict[str, Any]: """Build the shared read-only status chain used by Telegram and Operator UI.""" truth_status = ( truth_chain.get("truth_status") if isinstance(truth_chain, dict) and isinstance(truth_chain.get("truth_status"), dict) else {} ) quality = ( truth_chain.get("automation_quality") if isinstance(truth_chain, dict) and isinstance(truth_chain.get("automation_quality"), dict) else {} ) facts = quality.get("facts") if isinstance(quality.get("facts"), dict) else {} latest = _latest_remediation_history_item(remediation_history) remediation_state = _remediation_evidence_state(remediation_history) remediation_total = ( _safe_int(remediation_history.get("total")) if isinstance(remediation_history, dict) else 0 ) latest_route = _route_label_from_remediation(latest) if latest else "--" current_stage = str(truth_status.get("current_stage") or "unknown") stage_status = str(truth_status.get("stage_status") or "unknown") verdict = str(quality.get("verdict") or "unknown") verification = ( facts.get("verification_result") or latest.get("verification_result_preview") or "missing" ) auto_repair_records = _safe_int(facts.get("auto_repair_execution_records")) operation_records = _safe_int(facts.get("automation_operation_records")) has_repair_execution = _has_repair_execution_evidence(facts) has_nonrepair_operation = _has_nonrepair_operation_evidence(facts) gateway_total = _safe_int(facts.get("mcp_gateway_total")) km_entries = _safe_int(facts.get("knowledge_entries")) needs_human = bool(truth_status.get("needs_human")) if verdict == "auto_repaired_verified": repair_state = "auto_repaired_verified" next_step = "monitor_for_regression" elif verdict == "approval_rejected_no_execution": repair_state = "approval_rejected_no_execution" next_step = "monitor_or_reopen_if_alert_recurs" elif verdict == "approval_expired_manual_review": repair_state = "approval_expired_manual_review" next_step = "reopen_close_or_escalate_expired_approval" elif has_repair_execution: repair_state = ( "executed_pending_verification" if str(verification) == "missing" else "executed" ) next_step = "verify_execution_result" elif has_nonrepair_operation: repair_state = "diagnostic_or_audit_recorded" next_step = "manual_review_or_collect_repair_evidence" elif remediation_state == "read_only": repair_state = "read_only_dry_run" next_step = "approve_or_escalate_from_awooop" elif remediation_state == "write_observed": repair_state = "write_observed_manual_review" next_step = "review_write_evidence" elif remediation_state == "blocked": repair_state = "blocked_manual_required" next_step = "manual_investigation" elif needs_human: repair_state = "manual_required" next_step = "manual_investigation" else: repair_state = "no_execution_evidence" next_step = "collect_evidence_or_wait" if remediation_state in {"blocked", "fetch_failed"}: needs_human = True if ( remediation_state == "write_observed" and repair_state != "auto_repaired_verified" ): needs_human = True mcp_section = _status_chain_mcp_section(truth_chain) execution_section = _status_chain_execution_section(truth_chain) source_section = _status_chain_source_section(truth_chain) if source_correlation is not None: source_section["correlation"] = source_correlation blockers = [ str(item) for item in [ *(truth_status.get("blockers") if isinstance(truth_status.get("blockers"), list) else []), *(quality.get("blockers") if isinstance(quality.get("blockers"), list) else []), ] if item ] if fetch_error: blockers.append("truth_chain_fetch_failed") outcome = {} if isinstance(quality.get("operator_outcome"), dict): outcome = dict(quality["operator_outcome"]) else: outcome = build_operator_outcome( truth_status=truth_status, automation_quality=quality, remediation_state=remediation_state, fetch_error=fetch_error, source_id=source_id, ) if outcome: needs_human = bool(needs_human or outcome.get("needs_human")) next_step = str(outcome.get("next_action") or next_step) return { "schema_version": "awooop_status_chain_v1", "source": "truth_chain+adr100_history", "source_id": source_id, "incident_ids": incident_ids, "current_stage": current_stage, "stage_status": stage_status, "verdict": verdict, "repair_state": repair_state, "verification": str(verification), "needs_human": needs_human, "next_step": next_step, "operator_outcome": outcome, "blockers": blockers[:8], "fetch_error": fetch_error, "evidence": { "auto_repair_records": auto_repair_records, "operation_records": operation_records, "mcp_gateway_total": gateway_total, "knowledge_entries": km_entries, "remediation_total": remediation_total, "remediation_state": remediation_state, "latest_route": latest_route, "latest_mode": latest.get("mode"), "latest_at": latest.get("created_at"), "latest_preview": latest.get("verification_result_preview"), }, "writes": { "incident": latest.get("writes_incident_state"), "auto_repair": latest.get("writes_auto_repair_result"), }, "mcp": mcp_section, "execution": execution_section, "source_refs": source_section, } async def _fetch_awooop_status_chain( *, incident_ids: list[str], project_id: str, remediation_history: dict[str, Any] | None, ) -> dict[str, Any]: """Fetch read-only truth-chain state and merge it with ADR-100 evidence.""" source_id = _select_status_chain_source_id(incident_ids, remediation_history) truth_chain: dict[str, Any] | None = None fetch_error: str | None = None if source_id: try: truth_chain = await fetch_truth_chain( source_id=source_id, project_id=project_id or "awoooi", ) except Exception as exc: fetch_error = str(exc) logger.warning( "operator_awooop_status_chain_fetch_failed", source_id=source_id, project_id=project_id, error=fetch_error, ) try: source_correlation = await _fetch_source_correlation_summary( incident_ids=incident_ids, project_id=project_id or "awoooi", ) except Exception as exc: logger.warning( "operator_source_correlation_fetch_failed", incident_ids=incident_ids, project_id=project_id, error=str(exc), ) source_correlation = _source_correlation_empty( incident_ids, status_value="fetch_failed", missing_reason="source_correlation_fetch_failed", ) return _build_awooop_status_chain( incident_ids=incident_ids, truth_chain=truth_chain, remediation_history=remediation_history, source_id=source_id, fetch_error=fetch_error, source_correlation=source_correlation, ) async def get_awooop_status_chain( *, project_id: str | None, incident_ids: list[str], ) -> dict[str, Any]: """Return the shared AwoooP status chain for UI surfaces without writing state.""" normalized_incident_ids: list[str] = [] for incident_id in incident_ids: safe_incident_id = str(incident_id or "").strip() if not safe_incident_id: continue _validate_incident_id_filter(safe_incident_id) _append_unique(normalized_incident_ids, safe_incident_id) if not normalized_incident_ids: return _build_awooop_status_chain(incident_ids=[], source_id=None) remediation_history = await _fetch_run_remediation_history( normalized_incident_ids, limit=5, ) return await _fetch_awooop_status_chain( incident_ids=normalized_incident_ids, project_id=project_id or "awoooi", remediation_history=remediation_history, ) def _validate_remediation_status_filter(value: str | None) -> None: if value is None: return if value not in _REMEDIATION_STATUS_FILTERS: allowed = ", ".join(sorted(_REMEDIATION_STATUS_FILTERS)) raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"remediation_status 必須是: {allowed}", ) def _validate_callback_reply_status_filter(value: str | None) -> None: if value is None: return if value not in _CALLBACK_REPLY_STATUS_FILTERS: allowed = ", ".join(sorted(_CALLBACK_REPLY_STATUS_FILTERS)) raise HTTPException( status_code=422, detail=f"callback_reply_status 必須是: {allowed}", ) def _validate_callback_reply_action_filter(value: str | None) -> str | None: if value is None: return None normalized = value.strip().lower() if not normalized: return None if not _CALLBACK_REPLY_ACTION_RE.fullmatch(normalized): raise HTTPException( status_code=422, detail="callback action 格式錯誤，僅允許 a-z、0-9、底線、冒號與短橫線", ) return normalized def _validate_incident_id_filter(value: str | None) -> None: if value is None: return if not _INCIDENT_ID_RE.fullmatch(value): raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="incident_id 格式錯誤，必須是 INC-YYYYMMDD-XXXX", ) def _remediation_summary_matches_status( summary: dict[str, Any] | None, remediation_status: str | None, ) -> bool: if remediation_status is None: return True status_value = str((summary or {}).get("status") or "no_evidence") return status_value == remediation_status def _callback_reply_summary_matches_status( summary: dict[str, Any] | None, callback_reply_status: str | None, ) -> bool: if callback_reply_status is None: return True status_value = str((summary or {}).get("status") or "no_callback") if callback_reply_status == "observed": return status_value != "no_callback" return status_value == callback_reply_status def _remediation_summary_matches_incident_id( summary: dict[str, Any] | None, incident_id: str | None, ) -> bool: if incident_id is None: return True incident_ids = (summary or {}).get("incident_ids") return isinstance(incident_ids, list) and incident_id in incident_ids async def _build_run_remediation_summaries( *, runs: list[AwoooPRunState], inbound_by_run: dict[UUID, list[AwoooPConversationEvent]], outbound_by_run: dict[UUID, list[AwoooPOutboundMessage]], ) -> dict[UUID, dict[str, Any]]: """Build remediation summaries for list endpoints without writing state.""" if not runs: return {} incident_ids_by_run: dict[UUID, list[str]] = {} all_incident_ids: list[str] = [] for run in runs: incident_ids = _collect_run_incident_ids( run=run, inbound_events=inbound_by_run.get(run.run_id, []), outbound_messages=outbound_by_run.get(run.run_id, []), ) incident_ids_by_run[run.run_id] = incident_ids for incident_id in incident_ids: _append_unique(all_incident_ids, incident_id) histories_by_incident: dict[str, list[dict[str, Any]]] = {} legacy_mcp_by_incident: dict[str, list[dict[str, Any]]] = {} errors_by_incident: dict[str, dict[str, str]] = {} if all_incident_ids: from src.services.adr100_remediation_service import Adr100RemediationService service = Adr100RemediationService(record_history=False) for incident_id in all_incident_ids: try: history = await service.history( limit=_REMEDIATION_HISTORY_LIMIT, incident_id=incident_id, ) histories_by_incident[incident_id] = [ item for item in history.get("items", []) if isinstance(item, dict) ] except Exception as exc: logger.warning( "run_list_remediation_history_fetch_failed", incident_id=incident_id, error=str(exc), ) errors_by_incident[incident_id] = { "incident_id": incident_id, "error": str(exc), } legacy_mcp_by_incident = await _fetch_legacy_mcp_by_incident_ids( all_incident_ids, limit=min(max(len(all_incident_ids) * _REMEDIATION_HISTORY_LIMIT, 100), 5_000), ) summaries: dict[UUID, dict[str, Any]] = {} for run in runs: incident_ids = incident_ids_by_run.get(run.run_id, []) items: list[dict[str, Any]] = [] legacy_mcp_records: list[dict[str, Any]] = [] errors: list[dict[str, str]] = [] for incident_id in incident_ids: items.extend(histories_by_incident.get(incident_id, [])) legacy_mcp_records.extend(legacy_mcp_by_incident.get(incident_id, [])) if incident_id in errors_by_incident: errors.append(errors_by_incident[incident_id]) summaries[run.run_id] = _run_remediation_list_summary( run=run, incident_ids=incident_ids, items=items, legacy_mcp_records=legacy_mcp_records, errors=errors, ) return summaries def _timeline_sort_key(item: dict[str, Any], fallback_ts: Any) -> str: """Normalize mixed DB datetime / ISO string timestamps for timeline sorting.""" value = item.get("ts") or fallback_ts if hasattr(value, "isoformat"): return value.isoformat() return str(value or "") def _summarize_run_remediation_by_work_item( items: list[dict[str, Any]], ) -> list[dict[str, Any]]: summary: dict[str, dict[str, Any]] = {} for item in items: key = str(item.get("work_item_id") or item.get("incident_id") or item.get("id")) if key not in summary: summary[key] = { "work_item_id": item.get("work_item_id"), "incident_id": item.get("incident_id"), "count": 0, "latest_at": item.get("created_at"), "latest_preview": item.get("verification_result_preview"), "latest_mode": item.get("mode"), "latest_route": _route_label_from_remediation(item), } summary[key]["count"] += 1 return list(summary.values()) async def _fetch_run_remediation_history( incident_ids: list[str], *, limit: int = _REMEDIATION_HISTORY_LIMIT, ) -> dict[str, Any]: """Fetch durable ADR-100 remediation dry-run evidence linked to run incidents.""" if not incident_ids: return { "schema_version": "awooop_run_remediation_evidence_v1", "source": "alert_operation_log", "incident_ids": [], "total": 0, "limit": limit, "items": [], "by_work_item": [], "errors": [], } from src.services.adr100_remediation_service import Adr100RemediationService service = Adr100RemediationService(record_history=False) items: list[dict[str, Any]] = [] errors: list[dict[str, str]] = [] for incident_id in incident_ids: try: history = await service.history(limit=limit, incident_id=incident_id) items.extend( item for item in history.get("items", []) if isinstance(item, dict) ) except Exception as exc: logger.warning( "run_remediation_history_fetch_failed", incident_id=incident_id, error=str(exc), ) errors.append({"incident_id": incident_id, "error": str(exc)}) items.sort(key=lambda item: str(item.get("created_at") or ""), reverse=True) visible_items = items[:limit] return { "schema_version": "awooop_run_remediation_evidence_v1", "source": "alert_operation_log", "incident_ids": incident_ids, "total": len(items), "limit": limit, "items": visible_items, "by_work_item": _summarize_run_remediation_by_work_item(visible_items), "errors": errors, } def _legacy_mcp_record(row: MCPAuditLog) -> dict[str, Any]: return { "id": row.id, "session_id": row.session_id, "flywheel_node": row.flywheel_node, "mcp_server": row.mcp_server, "tool_name": row.tool_name, "duration_ms": row.duration_ms, "success": row.success, "error_message": row.error_message, "incident_id": row.incident_id, "agent_role": row.agent_role, "created_at": row.created_at, } async def _fetch_legacy_mcp_by_incident_ids( incident_ids: list[str], *, limit: int, ) -> dict[str, list[dict[str, Any]]]: """Fetch legacy/self-built MCP rows for list evidence summaries.""" if not incident_ids: return {} async with get_db_context("awoooi") as db: result = await db.execute( select(MCPAuditLog) .where(MCPAuditLog.incident_id.in_(incident_ids)) .order_by(MCPAuditLog.created_at.desc()) .limit(limit) ) rows = list(result.scalars().all()) by_incident: dict[str, list[dict[str, Any]]] = defaultdict(list) for row in rows: if row.incident_id: by_incident[row.incident_id].append(_legacy_mcp_record(row)) return dict(by_incident) async def _fetch_run_legacy_mcp_history( incident_ids: list[str], *, limit: int = _MAX_TIMELINE_ITEMS, ) -> dict[str, Any]: """Fetch legacy/self-built MCP audit rows linked through incident ids.""" if not incident_ids: return { "schema_version": "awooop_run_legacy_mcp_evidence_v1", "source": "mcp_audit_log", "incident_ids": [], "total": 0, "limit": limit, "records": [], "summary": _summarize_mcp([]), } async with get_db_context("awoooi") as db: result = await db.execute( select(MCPAuditLog) .where(MCPAuditLog.incident_id.in_(incident_ids)) .order_by(MCPAuditLog.created_at.desc()) .limit(limit) ) rows = list(result.scalars().all()) records = [_legacy_mcp_record(row) for row in rows] return { "schema_version": "awooop_run_legacy_mcp_evidence_v1", "source": "mcp_audit_log", "incident_ids": incident_ids, "total": len(records), "limit": limit, "records": records, "summary": _summarize_mcp(records), } async def get_run_detail( run_id: str, project_id: str | None = None, ) -> dict[str, Any]: """取得單一 Run 的處置脈絡，供 AwoooP Run detail / Timeline 顯示。""" try: run_uuid = uuid.UUID(run_id) except ValueError as exc: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"run_id 格式錯誤: {exc}", ) from exc async with get_db_context(project_id or "awoooi") as db: run_stmt = select(AwoooPRunState).where(AwoooPRunState.run_id == run_uuid) if project_id is not None: run_stmt = run_stmt.where(AwoooPRunState.project_id == project_id) run_result = await db.execute(run_stmt) run = run_result.scalar_one_or_none() if run is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"run {run_id!r} 不存在", ) steps_result = await db.execute( select(AwoooPRunStepJournal) .where(AwoooPRunStepJournal.run_id == run_uuid) .order_by(AwoooPRunStepJournal.step_seq.asc()) .limit(_MAX_TIMELINE_ITEMS) ) steps = list(steps_result.scalars().all()) inbound_where = [AwoooPConversationEvent.run_id == run_uuid] if run.trigger_ref: try: trigger_event_uuid = uuid.UUID(run.trigger_ref) inbound_where.append(AwoooPConversationEvent.event_id == trigger_event_uuid) except ValueError: inbound_where.append( AwoooPConversationEvent.provider_event_id == run.trigger_ref ) inbound_result = await db.execute( select(AwoooPConversationEvent) .where(sa_or(*inbound_where)) .order_by(AwoooPConversationEvent.received_at.asc()) .limit(_MAX_TIMELINE_ITEMS) ) inbound_events = list(inbound_result.scalars().all()) outbound_result = await db.execute( select(AwoooPOutboundMessage) .where(AwoooPOutboundMessage.run_id == run_uuid) .order_by(AwoooPOutboundMessage.queued_at.asc()) .limit(_MAX_TIMELINE_ITEMS) ) outbound_messages = list(outbound_result.scalars().all()) mcp_result = await db.execute( select(AwoooPMcpGatewayAudit) .where(AwoooPMcpGatewayAudit.run_id == run_uuid) .order_by(AwoooPMcpGatewayAudit.created_at.asc()) .limit(_MAX_TIMELINE_ITEMS) ) mcp_calls = list(mcp_result.scalars().all()) run_payload = { "run_id": run.run_id, "project_id": run.project_id, "agent_id": run.agent_id, "state": run.state, "is_shadow": run.is_shadow, "trace_id": run.trace_id, "trigger_type": run.trigger_type, "trigger_ref": run.trigger_ref, "cost_usd": run.cost_usd, "step_count": run.step_count, "attempt_count": run.attempt_count, "max_attempts": run.max_attempts, "error_code": run.error_code, "error_detail": run.error_detail, "created_at": run.created_at, "started_at": run.started_at, "completed_at": run.completed_at, "timeout_at": run.timeout_at, "heartbeat_at": run.heartbeat_at, } step_items = [ { "step_id": row.step_id, "step_seq": row.step_seq, "tool_name": row.tool_name, "result_status": row.result_status, "was_blocked": row.was_blocked, "block_reason": row.block_reason, "error_code": row.error_code, "latency_ms": row.latency_ms, "created_at": row.created_at, "completed_at": row.completed_at, } for row in steps ] inbound_items = [ { "event_id": row.event_id, "channel_type": row.channel_type, "provider_event_id": row.provider_event_id, "content_preview": row.content_preview, "is_duplicate": row.is_duplicate, "received_at": row.received_at, } for row in inbound_events ] outbound_items = [] for row in outbound_messages: callback_reply = _outbound_callback_reply(row.source_envelope) outbound_items.append({ "message_id": row.message_id, "channel_type": row.channel_type, "message_type": row.message_type, "content_preview": row.content_preview, "send_status": row.send_status, "send_error": row.send_error, "provider_message_id": row.provider_message_id, "queued_at": row.queued_at, "sent_at": row.sent_at, "triggered_by_state": row.triggered_by_state, "callback_reply": callback_reply, }) def _mcp_item(row: AwoooPMcpGatewayAudit) -> dict[str, Any]: gate_result = row.gate_result if isinstance(row.gate_result, dict) else {} return { "call_id": row.call_id, "agent_id": row.agent_id, "tool_name": row.tool_name, "result_status": row.result_status, "block_gate": row.block_gate, "block_reason": row.block_reason, "latency_ms": row.latency_ms, "created_at": row.created_at, "required_scope": gate_result.get("required_scope"), "policy_enforced": gate_result.get("policy_enforced"), "is_shadow": gate_result.get("is_shadow"), "gate_result": gate_result, } mcp_items = [_mcp_item(row) for row in mcp_calls] mcp_gateway_summary = _summarize_gateway_mcp([ _mcp_gateway_summary_row(row) for row in mcp_calls ]) incident_ids = _collect_run_incident_ids( run=run, inbound_events=inbound_events, outbound_messages=outbound_messages, ) legacy_mcp_history = await _fetch_run_legacy_mcp_history(incident_ids) remediation_history = await _fetch_run_remediation_history(incident_ids) awooop_status_chain = await _fetch_awooop_status_chain( incident_ids=incident_ids, project_id=run.project_id, remediation_history=remediation_history, ) timeline: list[dict[str, Any]] = [ _timeline_item( ts=run.created_at, kind="run", title="Run 建立", status=run.state, summary=f"{run.trigger_type or 'unknown'} → {run.agent_id}", metadata={"trace_id": run.trace_id, "trigger_ref": run.trigger_ref}, ) ] if run.started_at: timeline.append( _timeline_item( ts=run.started_at, kind="run", title="Run 開始執行", status="running", summary=run.worker_id, ) ) for row in inbound_events: timeline.append( _timeline_item( ts=row.received_at, kind="inbound", title=f"{row.channel_type} 入站事件", status="duplicate" if row.is_duplicate else "received", summary=row.content_preview, metadata={"provider_event_id": row.provider_event_id}, ) ) for row in steps: is_approval_step = row.tool_name.startswith("operator_console.") timeline.append( _timeline_item( ts=row.completed_at or row.created_at, kind="approval" if is_approval_step else "step", title=_approval_step_title(row.tool_name, row.step_seq), status=row.result_status, summary=row.block_reason or row.error_code, metadata={ "was_blocked": row.was_blocked, "latency_ms": row.latency_ms, }, ) ) for row in mcp_calls: gate_result = row.gate_result if isinstance(row.gate_result, dict) else {} scope = gate_result.get("required_scope") policy_enforced = gate_result.get("policy_enforced") summary = row.block_reason if summary is None: summary = ( f"agent={row.agent_id or 'unknown'}" f" scope={scope or 'unknown'}" f" policy_enforced={policy_enforced}" ) timeline.append( _timeline_item( ts=row.created_at, kind="mcp", title=f"MCP: {row.tool_name}", status=row.result_status, summary=summary, metadata={ "agent_id": row.agent_id, "block_gate": row.block_gate, "required_scope": scope, "policy_enforced": policy_enforced, "latency_ms": row.latency_ms, }, ) ) for record in legacy_mcp_history.get("records", []): if not isinstance(record, dict): continue tool_route = "/".join( part for part in ( str(record.get("mcp_server") or ""), str(record.get("tool_name") or ""), ) if part ) or "unknown" timeline.append( _timeline_item( ts=record.get("created_at"), kind="mcp", title=f"Legacy MCP: {tool_route}", status=_legacy_mcp_timeline_status(record), summary=_legacy_mcp_timeline_summary(record), metadata={ "incident_id": record.get("incident_id"), "agent_role": record.get("agent_role"), "flywheel_node": record.get("flywheel_node"), "history_source": "mcp_audit_log", }, ) ) for item in remediation_history.get("items", []): if not isinstance(item, dict): continue timeline.append( _timeline_item( ts=item.get("created_at"), kind="remediation", title="ADR-100 補救試跑", status=_remediation_timeline_status(item), summary=_remediation_timeline_summary(item), metadata={ "incident_id": item.get("incident_id"), "work_item_id": item.get("work_item_id"), "mcp_route": _route_label_from_remediation(item), "writes_incident_state": item.get("writes_incident_state"), "writes_auto_repair_result": item.get("writes_auto_repair_result"), "history_source": "alert_operation_log", }, ) ) for row in outbound_messages: callback_reply = _outbound_callback_reply(row.source_envelope) timeline.append( _timeline_item( ts=row.sent_at or row.queued_at, kind="outbound", title=_outbound_timeline_title( row.channel_type, row.message_type, row.content_preview, callback_reply, ), status=_outbound_timeline_status(row.send_status, callback_reply), summary=_outbound_timeline_summary( content_preview=row.content_preview, send_error=row.send_error, callback_reply=callback_reply, ), metadata=_outbound_timeline_metadata(row, callback_reply), ) ) if run.completed_at: timeline.append( _timeline_item( ts=run.completed_at, kind="run", title="Run 結束", status=run.state, summary=run.error_detail or run.error_code, ) ) timeline = sorted( timeline, key=lambda item: _timeline_sort_key(item, run.created_at), )[:_MAX_TIMELINE_ITEMS] return { "run": run_payload, "steps": step_items, "inbound_events": inbound_items, "outbound_messages": outbound_items, "mcp_calls": mcp_items, "mcp_gateway": mcp_gateway_summary, "mcp_legacy": legacy_mcp_history, "remediation_history": remediation_history, "awooop_status_chain": awooop_status_chain, "timeline": timeline, "counts": { "steps": len(step_items), "inbound_events": len(inbound_items), "outbound_messages": len(outbound_items), "mcp_calls": len(mcp_items), "legacy_mcp_calls": legacy_mcp_history.get("total", 0), "remediation_history": remediation_history.get("total", 0), "timeline": len(timeline), }, } # ============================================================================= # Channel Events # ============================================================================= async def list_recent_channel_events( *, project_id: str | None, channel_type: str | None, provider_prefix: str | None, limit: int, ) -> dict[str, Any]: """列出最近 channel events，供 Operator Console 顯示收斂/鏡像脈絡。""" safe_limit = max(1, min(limit, _MAX_EVENTS)) async with get_db_context("awoooi") as db: stmt = select(AwoooPConversationEvent).order_by( AwoooPConversationEvent.received_at.desc() ) if project_id is not None: stmt = stmt.where(AwoooPConversationEvent.project_id == project_id) if channel_type is not None: stmt = stmt.where(AwoooPConversationEvent.channel_type == channel_type) if provider_prefix is not None: stmt = stmt.where( AwoooPConversationEvent.provider_event_id.like( f"{provider_prefix}%" ) ) result = await db.execute(stmt.limit(safe_limit)) rows = list(result.scalars().all()) events = [_recent_channel_event_item(r) for r in rows] return {"events": events, "total": len(events), "limit": safe_limit} # ============================================================================= # Approvals # ============================================================================= async def list_approvals( project_id: str | None, run_id: str | None = None, remediation_status: str | None = None, ) -> dict[str, Any]: """列出 waiting_approval runs，可依 project_id / run_id / remediation_status 篩選。""" _validate_remediation_status_filter(remediation_status) run_uuid: UUID | None = None if run_id: try: run_uuid = uuid.UUID(run_id) except ValueError as exc: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"run_id 格式錯誤: {exc}", ) from exc async with get_db_context("awoooi") as db: stmt = ( select(AwoooPRunState) .where(AwoooPRunState.state == "waiting_approval") .order_by(AwoooPRunState.created_at.asc()) ) if project_id is not None: stmt = stmt.where(AwoooPRunState.project_id == project_id) if run_uuid is not None: stmt = stmt.where(AwoooPRunState.run_id == run_uuid) count_stmt = select(func.count()).select_from(stmt.subquery()) total_result = await db.execute(count_stmt) total = total_result.scalar_one() result = await db.execute(stmt) rows = list(result.scalars().all()) inbound_by_run, outbound_by_run = await _load_run_message_context(db, rows) remediation_summaries = await _build_run_remediation_summaries( runs=rows, inbound_by_run=inbound_by_run, outbound_by_run=outbound_by_run, ) if remediation_status: rows = [ row for row in rows if _remediation_summary_matches_status( remediation_summaries.get(row.run_id), remediation_status, ) ] total = len(rows) status_chain_cache: dict[tuple[str, tuple[str, ...]], dict[str, Any]] = {} items = [] for r in rows: summary = remediation_summaries.get(r.run_id) summary_incident_ids = summary.get("incident_ids") if isinstance(summary, dict) else [] incident_ids = [ str(incident_id) for incident_id in summary_incident_ids if isinstance(incident_id, str) and incident_id ] cache_key = (r.project_id, tuple(incident_ids)) status_chain = status_chain_cache.get(cache_key) if status_chain is None: status_chain = await get_awooop_status_chain( project_id=r.project_id, incident_ids=incident_ids, ) status_chain_cache[cache_key] = status_chain items.append({ "run_id": r.run_id, "project_id": r.project_id, "agent_id": r.agent_id, "trigger_type": r.trigger_type, "trigger_ref": r.trigger_ref, "is_shadow": r.is_shadow, "created_at": r.created_at, "timeout_at": r.timeout_at, "remediation_summary": summary, "awooop_status_chain": status_chain, }) return {"approvals": items, "total": total, "items": items} async def decide_approval( run_id: str, project_id: str, decision: str, approver_id: str, reason: str | None, ) -> dict[str, Any]: """核准或拒絕一個待審核的 run（ADR-116 Gate 5）。""" try: run_uuid = uuid.UUID(run_id) except ValueError as exc: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"run_id 格式錯誤: {exc}", ) from exc async with get_db_context(project_id) as db: result = await db.execute( select(AwoooPRunState).where( AwoooPRunState.run_id == run_uuid, AwoooPRunState.project_id == project_id, ) ) run = result.scalar_one_or_none() if run is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"run {run_id!r} 不存在或非此 project 所有", ) if run.state != "waiting_approval": raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail=f"run {run_id!r} 目前狀態為 {run.state!r}，無法審核（需為 waiting_approval）", ) is_projection_only_gate5 = run.trigger_type == _ADR100_GATE5_PROJECTION_TRIGGER approval_token_jti: str | None = None new_state: str if is_projection_only_gate5: await _record_approval_projection_guard_step( run_id=run_uuid, project_id=project_id, decision=decision, approver_id=approver_id, reason=reason, ) try: await write_audit( project_id=project_id, action=f"run.approval.{decision}.blocked", resource_type="run", resource_id=run_id, details={ "approver_id": approver_id, "decision": decision, "reason": reason, "new_state": "waiting_approval", "trigger_type": _ADR100_GATE5_PROJECTION_TRIGGER, "block_reason": "adr100_runtime_replay_gate5_projection_only", "execution_authorized": False, "repair_executed": False, }, run_id=run_id, ) except Exception as exc: logger.warning( "approval_projection_guard_audit_write_failed", run_id=run_id, error=str(exc), ) raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail=( "adr100_runtime_replay_gate5_projection_only: " "此 AwoooP 簽核列只投影 legacy Gate 5 approval 與狀態鏈，" "尚未接上 auto_repair_executor 執行 handoff，不能直接由平台按鈕轉成 running。" ), ) if decision == "approve": token = issue_approval_token( project_id=project_id, run_id=run_id, tool_name="operator_console_approve", approver_id=approver_id, ) try: await record_approval( project_id=project_id, run_id=run_id, tool_name="operator_console_approve", approver_id=approver_id, token=token, ) except Exception as exc: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"核准記錄失敗: {exc}", ) from exc await transition(run_uuid, project_id, "running") new_state = "running" await _record_approval_decision_step( run_id=run_uuid, project_id=project_id, decision=decision, approver_id=approver_id, reason=reason, ) import base64 import json as _json try: p_b64 = token.split(".")[1] padding = 4 - len(p_b64) % 4 if padding != 4: p_b64 += "=" * padding payload = _json.loads(base64.urlsafe_b64decode(p_b64)) approval_token_jti = payload.get("jti") except Exception: approval_token_jti = None else: await transition( run_uuid, project_id, "cancelled", error_code="E-APPR-REJECTED", error_detail=f"operator 拒絕: approver={approver_id!r}, reason={reason!r}", ) new_state = "cancelled" await _record_approval_decision_step( run_id=run_uuid, project_id=project_id, decision=decision, approver_id=approver_id, reason=reason, ) try: await write_audit( project_id=project_id, action=f"run.approval.{decision}", resource_type="run", resource_id=run_id, details={ "approver_id": approver_id, "decision": decision, "reason": reason, "new_state": new_state, }, run_id=run_id, ) except Exception as exc: logger.warning("approval_audit_write_failed", run_id=run_id, error=str(exc)) return { "run_id": run_id, "decision": decision, "new_state": new_state, "approval_token_jti": approval_token_jti, } async def _record_approval_projection_guard_step( *, run_id: UUID, project_id: str, decision: str, approver_id: str, reason: str | None, ) -> None: summary = _truncate_step_summary( "projection_only_gate5; " f"approver={approver_id}; decision={decision}; reason={reason or '-'}" ) try: async with get_db_context(project_id) as db: max_result = await db.execute( select(func.coalesce(func.max(AwoooPRunStepJournal.step_seq), 0)).where( AwoooPRunStepJournal.run_id == run_id, AwoooPRunStepJournal.project_id == project_id, ) ) step_seq = int(max_result.scalar_one()) + 1 db.add( AwoooPRunStepJournal( run_id=run_id, project_id=project_id, step_seq=step_seq, tool_name="operator_console.approval_projection_guard", result_status="failed", error_code="E-ADR100-GATE5-PROJECTION", was_blocked=True, block_reason=summary, completed_at=_utc_now_naive(), ) ) await db.execute( update(AwoooPRunState) .where( AwoooPRunState.run_id == run_id, AwoooPRunState.project_id == project_id, ) .values(step_count=AwoooPRunState.step_count + 1) ) logger.info( "approval_projection_guard_step_recorded", run_id=str(run_id), project_id=project_id, decision=decision, approver_id=approver_id, ) except Exception as exc: logger.warning( "approval_projection_guard_step_record_failed", run_id=str(run_id), project_id=project_id, decision=decision, error=str(exc), ) async def _record_approval_decision_step( *, run_id: UUID, project_id: str, decision: str, approver_id: str, reason: str | None, ) -> None: """把 Operator Console 的人工審批決策寫進 Run Step Journal。 這是治理與可觀測節點，不是執行閘門本身；寫入失敗不可反向阻擋 已完成的 approve / reject，否則會讓人工決策狀態機產生二次故障。 """ tool_name = ( "operator_console.approve" if decision == "approve" else "operator_console.reject" ) summary = _truncate_step_summary( f"approver={approver_id}; decision={decision}; reason={reason or '-'}" ) try: async with get_db_context(project_id) as db: max_result = await db.execute( select(func.coalesce(func.max(AwoooPRunStepJournal.step_seq), 0)).where( AwoooPRunStepJournal.run_id == run_id, AwoooPRunStepJournal.project_id == project_id, ) ) step_seq = int(max_result.scalar_one()) + 1 db.add( AwoooPRunStepJournal( run_id=run_id, project_id=project_id, step_seq=step_seq, tool_name=tool_name, result_status="success", block_reason=summary, completed_at=_utc_now_naive(), ) ) await db.execute( update(AwoooPRunState) .where( AwoooPRunState.run_id == run_id, AwoooPRunState.project_id == project_id, ) .values(step_count=AwoooPRunState.step_count + 1) ) logger.info( "approval_decision_step_recorded", run_id=str(run_id), project_id=project_id, decision=decision, approver_id=approver_id, ) except Exception as exc: logger.warning( "approval_decision_step_record_failed", run_id=str(run_id), project_id=project_id, decision=decision, error=str(exc), )