diff --git a/docs/guides/observability_ui_governance.md b/docs/guides/observability_ui_governance.md index db63c54..a82e010 100644 --- a/docs/guides/observability_ui_governance.md +++ b/docs/guides/observability_ui_governance.md @@ -64,6 +64,7 @@ python3 scripts/sync_observability_css.py Guard 會檢查: +- 觀測台頁面契約集中在 `scripts/observability_contract.py`,新增/改名頁面先改這裡。 - Times / Georgia 等非規範標題字型。 - SQL/Jinja exception 文案外露。 - 圖表 `style="height:..."` 與靜態 `style="width:..."`。 diff --git a/scripts/check_observability_pages.py b/scripts/check_observability_pages.py index 50d9b93..b8f1161 100644 --- a/scripts/check_observability_pages.py +++ b/scripts/check_observability_pages.py @@ -13,32 +13,7 @@ import sys import urllib.error import urllib.request - -PAGES = [ - ("/observability/overview", "總覽"), - ("/observability/agent_orchestration", "Agent"), - ("/observability/business_intel", "商業"), - ("/observability/host_health", "主機"), - ("/observability/ai_calls", "AI 呼叫"), - ("/observability/budget", "預算"), - ("/observability/promotion_review", "晉升"), - ("/observability/rag_queries", "RAG"), - ("/observability/quality_trend", "品質"), - ("/observability/ppt_audit_history", "PPT"), -] - -EXPECTED_MARKERS = { - "/observability/overview": ["觀測台總覽", "主機健康", "AI 呼叫"], - "/observability/agent_orchestration": ["Agent 編排矩陣", "LLM", "MCP", "RAG"], - "/observability/business_intel": ["商業面 × AI", "AI", "競品"], - "/observability/host_health": ["主機健康", "Ollama", "AutoHeal"], - "/observability/ai_calls": ["AI 呼叫", "Provider", "RAG"], - "/observability/budget": ["預算控管", "force", "throttle"], - "/observability/promotion_review": ["RAG 晉升審核", "Promotion", "ai_insights"], - "/observability/rag_queries": ["RAG 召回詳情", "最近 50", "hits"], - "/observability/quality_trend": ["反饋趨勢", "Caller", "蒸餾"], - "/observability/ppt_audit_history": ["PPT 視覺審核", "AiderHeal", "audit"], -} +from observability_contract import CSS_ASSET_CHECKS, OBSERVABILITY_PAGES MIN_HTML_BYTES = 2500 @@ -49,20 +24,6 @@ GLOBAL_REQUIRED_MARKERS = [ "momo-obs-link", ] -ASSET_CHECKS = [ - ( - "/static/css/observability-system.css", - "觀測台 CSS", - [ - "--obs-title-size", - "--obs-value-size", - ".momo-observability-mode", - ".obs-chart-frame", - ".obs-modal-preview", - ], - ), -] - ERROR_NEEDLES = [ "Traceback", "UndefinedError", @@ -93,21 +54,21 @@ def main() -> int: failed = False print(f"Observability page smoke: {args.base_url.rstrip('/')}") - for path, label in PAGES: + for page in OBSERVABILITY_PAGES: try: - status, html = fetch_page(args.base_url, path, args.timeout) + status, html = fetch_page(args.base_url, page.url, args.timeout) except urllib.error.HTTPError as exc: - print(f"- {label}: HTTP {exc.code}, FAIL") + print(f"- {page.short_label}: HTTP {exc.code}, FAIL") failed = True continue except Exception as exc: - print(f"- {label}: {type(exc).__name__}: {exc}, FAIL") + print(f"- {page.short_label}: {type(exc).__name__}: {exc}, FAIL") failed = True continue found = [needle for needle in ERROR_NEEDLES if needle in html] missing_markers = [ - marker for marker in EXPECTED_MARKERS.get(path, []) + marker for marker in page.markers if marker not in html ] missing_global_markers = [ @@ -127,34 +88,34 @@ def main() -> int: issues.append(f"missing_global_markers={missing_global_markers}") if too_small: issues.append(f"html_too_small={len(html.encode('utf-8'))}B") - print(f"- {label}: HTTP {status}, issues={issues}, FAIL") + print(f"- {page.short_label}: HTTP {status}, issues={issues}, FAIL") failed = True else: - print(f"- {label}: HTTP {status}, issues=none") + print(f"- {page.short_label}: HTTP {status}, issues=none") - for path, label, markers in ASSET_CHECKS: + for asset in CSS_ASSET_CHECKS: try: - status, body = fetch_page(args.base_url, path, args.timeout) + status, body = fetch_page(args.base_url, asset.url, args.timeout) except urllib.error.HTTPError as exc: - print(f"- {label}: HTTP {exc.code}, FAIL") + print(f"- {asset.label}: HTTP {exc.code}, FAIL") failed = True continue except Exception as exc: - print(f"- {label}: {type(exc).__name__}: {exc}, FAIL") + print(f"- {asset.label}: {type(exc).__name__}: {exc}, FAIL") failed = True continue - missing_markers = [marker for marker in markers if marker not in body] + missing_markers = [marker for marker in asset.markers if marker not in body] if status != 200 or missing_markers: issues = [] if status != 200: issues.append("bad_status") if missing_markers: issues.append(f"missing_asset_markers={missing_markers}") - print(f"- {label}: HTTP {status}, issues={issues}, FAIL") + print(f"- {asset.label}: HTTP {status}, issues={issues}, FAIL") failed = True else: - print(f"- {label}: HTTP {status}, markers=ok") + print(f"- {asset.label}: HTTP {status}, markers=ok") if failed: print("Observability page smoke: FAIL") diff --git a/scripts/check_observability_ui.py b/scripts/check_observability_ui.py index 8736342..18149bb 100644 --- a/scripts/check_observability_ui.py +++ b/scripts/check_observability_ui.py @@ -14,34 +14,11 @@ import sys from dataclasses import dataclass from pathlib import Path +from observability_contract import OBSERVABILITY_PAGES ROOT = Path(__file__).resolve().parents[1] -TEMPLATE_PATHS = [ - "templates/admin/observability_overview.html", - "templates/admin/agent_orchestration.html", - "templates/admin/business_intel.html", - "templates/admin/host_health.html", - "templates/admin/ai_calls_dashboard.html", - "templates/admin/budget.html", - "templates/admin/promotion_review.html", - "templates/admin/rag_queries.html", - "templates/admin/quality_trend.html", - "templates/admin/ppt_audit_history.html", -] - -OBSERVABILITY_NAV_ITEMS = [ - ("templates/admin/observability_overview.html", "obs_overview", "/observability/overview", "觀測台總覽"), - ("templates/admin/agent_orchestration.html", "obs_agent_orchestration", "/observability/agent_orchestration", "Agent 編排矩陣"), - ("templates/admin/business_intel.html", "obs_business_intel", "/observability/business_intel", "商業面 × AI"), - ("templates/admin/host_health.html", "obs_host_health", "/observability/host_health", "主機健康"), - ("templates/admin/ai_calls_dashboard.html", "obs_ai_calls", "/observability/ai_calls", "AI 呼叫"), - ("templates/admin/budget.html", "obs_budget", "/observability/budget", "預算控管"), - ("templates/admin/promotion_review.html", "obs_promotion_review", "/observability/promotion_review", "RAG 晉升審核"), - ("templates/admin/rag_queries.html", "obs_rag_queries", "/observability/rag_queries", "RAG 召回詳情"), - ("templates/admin/quality_trend.html", "obs_quality_trend", "/observability/quality_trend", "反饋趨勢"), - ("templates/admin/ppt_audit_history.html", "obs_ppt_audit", "/observability/ppt_audit_history", "PPT 視覺審核"), -] +TEMPLATE_PATHS = [page.template for page in OBSERVABILITY_PAGES] CSS_PATH = Path("static/css/observability-system.css") WEB_CSS_PATH = Path("web/static/css/observability-system.css") @@ -267,42 +244,44 @@ def scan_nav_contract() -> list[str]: route_text = route_path.read_text(encoding="utf-8") if route_path.exists() else "" findings: list[str] = [] - for template_path, active_page, url, label in OBSERVABILITY_NAV_ITEMS: - template = ROOT / template_path + for page in OBSERVABILITY_PAGES: + template = ROOT / page.template if not template.exists(): - findings.append(f"{template_path}: missing observability template for `{label}`") + findings.append(f"{page.template}: missing observability template for `{page.label}`") continue template_text = template.read_text(encoding="utf-8") active_pattern = re.compile( - r"set\s+active_page\s*=\s*['\"]" + re.escape(active_page) + r"['\"]" + r"set\s+active_page\s*=\s*['\"]" + re.escape(page.active_page) + r"['\"]" + ) + route_has_contract = ( + page.template.replace("templates/", "") in route_text + and page.active_page in route_text ) - route_has_contract = template_path.replace("templates/", "") in route_text and active_page in route_text if not active_pattern.search(template_text) and not route_has_contract: findings.append( - f"{template_path}: active_page must be `{active_page}` in template or route for `{label}`" + f"{page.template}: active_page must be `{page.active_page}` in template or route for `{page.label}`" ) - if active_page not in shell_text: - findings.append(f"{SHELL_PATH}: sidebar missing active_page `{active_page}`") - if url not in shell_text: - findings.append(f"{SHELL_PATH}: sidebar missing URL `{url}` for `{label}`") - if label not in shell_text: - findings.append(f"{SHELL_PATH}: sidebar missing label `{label}`") - if active_page not in base_text: - findings.append(f"{BASE_PATH}: momo-observability-mode list missing `{active_page}`") + if page.active_page not in shell_text: + findings.append(f"{SHELL_PATH}: sidebar missing active_page `{page.active_page}`") + if page.url not in shell_text: + findings.append(f"{SHELL_PATH}: sidebar missing URL `{page.url}` for `{page.label}`") + if page.label not in shell_text: + findings.append(f"{SHELL_PATH}: sidebar missing label `{page.label}`") + if page.active_page not in base_text: + findings.append(f"{BASE_PATH}: momo-observability-mode list missing `{page.active_page}`") - route_suffix = url.replace("/observability", "", 1) or "/" route_pattern = re.compile( r"@admin_observability_bp\.route\(\s*['\"]" - + re.escape(route_suffix) + + re.escape(page.route_suffix) + r"['\"]" ) if route_text and not route_pattern.search(route_text): - findings.append(f"{ROUTE_PATH}: missing route `{route_suffix}` for `{label}`") + findings.append(f"{ROUTE_PATH}: missing route `{page.route_suffix}` for `{page.label}`") sidebar_observability_links = set(re.findall(r'href="(/observability/[^"]+)"', shell_text)) - expected_links = {url for _, _, url, _ in OBSERVABILITY_NAV_ITEMS} + expected_links = {page.url for page in OBSERVABILITY_PAGES} extra_links = sorted(sidebar_observability_links - expected_links) missing_links = sorted(expected_links - sidebar_observability_links) for url in extra_links: @@ -333,7 +312,7 @@ def main() -> int: print(f"- css guardrails checked: {len(REQUIRED_CSS_SNIPPETS)}") print(f"- sidebar/nav guardrails checked: {len(REQUIRED_SHELL_SNIPPETS)}") print(f"- base/topbar guardrails checked: {len(REQUIRED_BASE_SNIPPETS) + len(FORBIDDEN_BASE_PATTERNS)}") - print(f"- nav contract checked: {len(OBSERVABILITY_NAV_ITEMS)} pages") + print(f"- nav contract checked: {len(OBSERVABILITY_PAGES)} pages") return 0 diff --git a/scripts/observability_contract.py b/scripts/observability_contract.py new file mode 100644 index 0000000..9099f90 --- /dev/null +++ b/scripts/observability_contract.py @@ -0,0 +1,125 @@ +"""Single source of truth for AI observability QA contracts.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ObservabilityPage: + template: str + active_page: str + url: str + label: str + short_label: str + markers: tuple[str, ...] + + @property + def route_suffix(self) -> str: + return self.url.replace("/observability", "", 1) or "/" + + +@dataclass(frozen=True) +class AssetCheck: + url: str + label: str + markers: tuple[str, ...] + + +OBSERVABILITY_PAGES = ( + ObservabilityPage( + "templates/admin/observability_overview.html", + "obs_overview", + "/observability/overview", + "觀測台總覽", + "總覽", + ("觀測台總覽", "主機健康", "AI 呼叫"), + ), + ObservabilityPage( + "templates/admin/agent_orchestration.html", + "obs_agent_orchestration", + "/observability/agent_orchestration", + "Agent 編排矩陣", + "Agent", + ("Agent 編排矩陣", "LLM", "MCP", "RAG"), + ), + ObservabilityPage( + "templates/admin/business_intel.html", + "obs_business_intel", + "/observability/business_intel", + "商業面 × AI", + "商業", + ("商業面 × AI", "AI", "競品"), + ), + ObservabilityPage( + "templates/admin/host_health.html", + "obs_host_health", + "/observability/host_health", + "主機健康", + "主機", + ("主機健康", "Ollama", "AutoHeal"), + ), + ObservabilityPage( + "templates/admin/ai_calls_dashboard.html", + "obs_ai_calls", + "/observability/ai_calls", + "AI 呼叫", + "AI 呼叫", + ("AI 呼叫", "Provider", "RAG"), + ), + ObservabilityPage( + "templates/admin/budget.html", + "obs_budget", + "/observability/budget", + "預算控管", + "預算", + ("預算控管", "force", "throttle"), + ), + ObservabilityPage( + "templates/admin/promotion_review.html", + "obs_promotion_review", + "/observability/promotion_review", + "RAG 晉升審核", + "晉升", + ("RAG 晉升審核", "Promotion", "ai_insights"), + ), + ObservabilityPage( + "templates/admin/rag_queries.html", + "obs_rag_queries", + "/observability/rag_queries", + "RAG 召回詳情", + "RAG", + ("RAG 召回詳情", "最近 50", "hits"), + ), + ObservabilityPage( + "templates/admin/quality_trend.html", + "obs_quality_trend", + "/observability/quality_trend", + "反饋趨勢", + "品質", + ("反饋趨勢", "Caller", "蒸餾"), + ), + ObservabilityPage( + "templates/admin/ppt_audit_history.html", + "obs_ppt_audit", + "/observability/ppt_audit_history", + "PPT 視覺審核", + "PPT", + ("PPT 視覺審核", "AiderHeal", "audit"), + ), +) + + +CSS_ASSET_CHECKS = ( + AssetCheck( + "/static/css/observability-system.css", + "觀測台 CSS", + ( + "--obs-title-size", + "--obs-value-size", + ".momo-observability-mode", + ".obs-chart-frame", + ".obs-modal-preview", + ), + ), +)