431 lines
14 KiB
Python
431 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""Guardrail checks for AI observability UI templates.
|
||
|
||
This script is intentionally lightweight: it scans the observability templates
|
||
and shared CSS for regression signals that previously hurt the war-room UI,
|
||
including unreadable typography, raw SQL errors, oversized titles, and inline
|
||
chart sizing.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
import sys
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
|
||
from observability_contract import OBSERVABILITY_PAGES
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
|
||
TEMPLATE_PATHS = [page.template for page in OBSERVABILITY_PAGES]
|
||
|
||
CSS_PATH = Path("static/css/observability-system.css")
|
||
WEB_CSS_PATH = Path("web/static/css/observability-system.css")
|
||
SHELL_PATH = Path("templates/components/_ewoooc_shell.html")
|
||
BASE_PATH = Path("templates/ewoooc_base.html")
|
||
BASE_JS_PATH = Path("web/static/js/ewoooc-base.js")
|
||
ROUTE_PATH = Path("routes/admin_observability_routes.py")
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Rule:
|
||
code: str
|
||
pattern: re.Pattern[str]
|
||
message: str
|
||
|
||
|
||
TEMPLATE_RULES = [
|
||
Rule(
|
||
"legacy_serif_font",
|
||
re.compile(r"\b(Georgia|Times New Roman)\b", re.IGNORECASE),
|
||
"觀測台標題不得回退到 Times/Georgia；使用 Noto Sans TC / Inter 與 CSS token。",
|
||
),
|
||
Rule(
|
||
"raw_error_copy",
|
||
re.compile(
|
||
r"(查詢失敗:|ProgrammingError|OperationalError|SQLAlchemyError|sqlite3\.|UndefinedError|Traceback|Internal Server Error|relation\s+"|relation\s+\"|alert\(['\"]Error[:：]|載入錯誤：\$\{e\}|unknown)"
|
||
),
|
||
"不得把 SQL/Jinja exception 或 raw failure 文案直接顯示給使用者。",
|
||
),
|
||
Rule(
|
||
"inline_height",
|
||
re.compile(r"style=\"[^\"]*height\s*:", re.IGNORECASE),
|
||
"圖表/區塊高度請使用 .obs-chart-frame 系列或 CSS class，不要 inline height。",
|
||
),
|
||
Rule(
|
||
"static_inline_width",
|
||
re.compile(r"style=\"(?=[^\"]*width\s*:)(?![^\"]*\{\{)[^\"]*\"", re.IGNORECASE),
|
||
"靜態寬度請移到 CSS class；僅允許 progress 類動態 width={{ ... }}。",
|
||
),
|
||
Rule(
|
||
"oversized_title",
|
||
re.compile(r"font-size:\s*clamp\([^;]*(3\.8rem|4\.45rem)", re.IGNORECASE),
|
||
"頁面大標不得使用過大 clamp；使用 --obs-title-size token。",
|
||
),
|
||
Rule(
|
||
"pure_white_surface",
|
||
re.compile(r"background\s*:\s*#fff\s*;", re.IGNORECASE),
|
||
"觀測台 hover/card surface 不用純白；使用暖色 paper/surface token。",
|
||
),
|
||
Rule(
|
||
"wrong_business_active_page",
|
||
re.compile(r"active_page\s*=\s*['\"]obs_business['\"]"),
|
||
"商業戰情頁 active_page 必須是 obs_business_intel，否則觀測台 CSS 不會載入。",
|
||
),
|
||
]
|
||
|
||
REQUIRED_CSS_SNIPPETS = [
|
||
"--obs-title-size",
|
||
"--obs-value-size",
|
||
"--obs-matrix-dot",
|
||
"v3.11 V2 workbench normalization",
|
||
"v3.10 terminal dot-matrix layer",
|
||
".momo-observability-mode",
|
||
".obs-chart-frame",
|
||
".obs-chart-frame-tall",
|
||
".obs-chart-frame-slim",
|
||
".obs-modal-preview",
|
||
".obs-progress-xs",
|
||
".obs-table-shell",
|
||
]
|
||
|
||
REQUIRED_SHELL_SNIPPETS = [
|
||
"AI 中樞",
|
||
"AI 觀測台",
|
||
"戰情室",
|
||
"系統與成本",
|
||
"RAG 與品質",
|
||
"momo-nav-tree",
|
||
"momo-nav-subtree",
|
||
"momo-nav-subtitle",
|
||
"momo-nav-sublink",
|
||
]
|
||
|
||
FORBIDDEN_SHELL_PATTERNS = [
|
||
Rule(
|
||
"pure_black_sidebar",
|
||
re.compile(r"\.momo-sidebar\s*\{[^}]*background\s*:\s*(#000|black)\b", re.IGNORECASE | re.DOTALL),
|
||
"側欄不得回退成純黑背景；必須維持暖色 token 化背景。",
|
||
),
|
||
Rule(
|
||
"low_contrast_sublink",
|
||
re.compile(r"\.momo-nav-sublink\s*\{[^}]*color\s*:\s*rgba\([^)]*,\s*0\.(?:[0-5][0-9]|60)\)", re.IGNORECASE | re.DOTALL),
|
||
"第二/三層導覽文字透明度過低，會看不清楚；需維持足夠對比。",
|
||
),
|
||
]
|
||
|
||
REQUIRED_BASE_SNIPPETS = [
|
||
"observability-system.css",
|
||
"momo-observability-mode",
|
||
"/observability/overview",
|
||
"/observability/api/health_indicator",
|
||
"momo-obs-link",
|
||
"momo-obs-badge",
|
||
"classList.add('is-alert')",
|
||
]
|
||
|
||
FORBIDDEN_BASE_PATTERNS = [
|
||
Rule(
|
||
"inline_obs_badge_style",
|
||
re.compile(r"id=\"momo-obs-badge\"[^>]*style=", re.IGNORECASE | re.DOTALL),
|
||
"Topbar 觀測台 badge 不得使用 inline style；請使用 .momo-obs-badge。",
|
||
),
|
||
Rule(
|
||
"hardcoded_obs_alert_color",
|
||
re.compile(r"(#dc3545|link\.style\.color|badge\.style\.display)", re.IGNORECASE),
|
||
"Topbar 觀測台告警狀態不得用 JS/inline 硬改色；請切換 is-alert / hidden。",
|
||
),
|
||
]
|
||
|
||
FORBIDDEN_OBSERVABILITY_COPY = [
|
||
"AI Observability Command Room",
|
||
"Business Intelligence",
|
||
"Agent Command Matrix",
|
||
"AI Traffic Control",
|
||
"AI Cost Governance",
|
||
"RAG Recall Radar",
|
||
"Quality Diagnostics",
|
||
"PPT Visual QA Pipeline",
|
||
"RAG Promotion Gate",
|
||
"Infrastructure Lifeline",
|
||
"Risk Signals",
|
||
"AI Calls / 24h",
|
||
"Host Cascade",
|
||
"AI Runtime",
|
||
"Learning Loop",
|
||
"Total Calls",
|
||
"Ollama Share",
|
||
"Paid Cost",
|
||
"Provider Split",
|
||
"Caller Orchestration",
|
||
"Model Economics",
|
||
"Recent Calls",
|
||
"Budget Ratio",
|
||
"Budget Lines",
|
||
"Provider Mix",
|
||
"Burn Rate",
|
||
"Saved Call",
|
||
"Query Stream",
|
||
"Caller Quality",
|
||
"Worst Avg",
|
||
"RAG Scores",
|
||
"Caller Feedback",
|
||
"RAG Feedback",
|
||
"Learning Pool",
|
||
"Root Cause",
|
||
"Action Outcomes",
|
||
"Audit History",
|
||
"Generated Files",
|
||
"30d Audit Mix",
|
||
"Failure Hotspots",
|
||
"Awaiting Review",
|
||
"Review Queue",
|
||
"Ollama Down",
|
||
"AIOps Open",
|
||
"Heal Rate",
|
||
"Cost Throttle",
|
||
"MCP Workload",
|
||
"Operation Ollama-First",
|
||
"RAG hits",
|
||
"Cache hits",
|
||
"30d episodes",
|
||
"MCP calls",
|
||
"force-throttle",
|
||
]
|
||
|
||
|
||
def line_number(text: str, index: int) -> int:
|
||
return text.count("\n", 0, index) + 1
|
||
|
||
|
||
def scan_file(relative_path: Path) -> list[str]:
|
||
path = ROOT / relative_path
|
||
if not path.exists():
|
||
return [f"{relative_path}: missing required observability file"]
|
||
|
||
text = path.read_text(encoding="utf-8")
|
||
findings: list[str] = []
|
||
for rule in TEMPLATE_RULES:
|
||
for match in rule.pattern.finditer(text):
|
||
line = line_number(text, match.start())
|
||
snippet = match.group(0).replace("\n", " ")[:90]
|
||
findings.append(
|
||
f"{relative_path}:{line}: [{rule.code}] {rule.message} ({snippet})"
|
||
)
|
||
|
||
return findings
|
||
|
||
|
||
def scan_css() -> list[str]:
|
||
path = ROOT / CSS_PATH
|
||
if not path.exists():
|
||
return [f"{CSS_PATH}: missing required observability CSS"]
|
||
|
||
text = path.read_text(encoding="utf-8")
|
||
findings: list[str] = []
|
||
for snippet in REQUIRED_CSS_SNIPPETS:
|
||
if snippet not in text:
|
||
findings.append(f"{CSS_PATH}: missing required token/class `{snippet}`")
|
||
|
||
terminal_marker = (
|
||
"/* v3.10 terminal dot-matrix layer: this must stay at EOF to win the cascade. */"
|
||
)
|
||
marker_count = text.count(terminal_marker)
|
||
if marker_count != 1:
|
||
findings.append(
|
||
f"{CSS_PATH}: expected exactly one terminal dot-matrix layer, found {marker_count}"
|
||
)
|
||
else:
|
||
marker_pos = text.rfind(terminal_marker)
|
||
tail = text[marker_pos:]
|
||
if len(tail) > 4200:
|
||
findings.append(
|
||
f"{CSS_PATH}: terminal dot-matrix layer must remain near EOF so later rules cannot override it"
|
||
)
|
||
|
||
for legacy_marker in (
|
||
"v3.8 dot-matrix integration",
|
||
"v3.9 final dot-matrix layer",
|
||
):
|
||
if legacy_marker in text:
|
||
findings.append(
|
||
f"{CSS_PATH}: remove obsolete `{legacy_marker}`; keep only the v3.10 terminal layer"
|
||
)
|
||
|
||
web_path = ROOT / WEB_CSS_PATH
|
||
if not web_path.exists():
|
||
findings.append(
|
||
f"{WEB_CSS_PATH}: missing mirrored CSS served by Flask static route"
|
||
)
|
||
else:
|
||
web_text = web_path.read_text(encoding="utf-8")
|
||
if web_text != text:
|
||
findings.append(
|
||
f"{WEB_CSS_PATH}: must match {CSS_PATH} so production /static CSS is in sync; run `python3 scripts/sync_observability_css.py`"
|
||
)
|
||
|
||
return findings
|
||
|
||
|
||
def scan_required_snippets(relative_path: Path, snippets: list[str], label: str) -> list[str]:
|
||
path = ROOT / relative_path
|
||
if not path.exists():
|
||
return [f"{relative_path}: missing required {label} file"]
|
||
|
||
text = path.read_text(encoding="utf-8")
|
||
findings: list[str] = []
|
||
for snippet in snippets:
|
||
if snippet not in text:
|
||
findings.append(f"{relative_path}: missing required {label} snippet `{snippet}`")
|
||
return findings
|
||
|
||
|
||
def scan_base_topbar() -> list[str]:
|
||
path = ROOT / BASE_PATH
|
||
if not path.exists():
|
||
return [f"{BASE_PATH}: missing required base/topbar file"]
|
||
|
||
text = path.read_text(encoding="utf-8")
|
||
js_path = ROOT / BASE_JS_PATH
|
||
js_text = js_path.read_text(encoding="utf-8") if js_path.exists() else ""
|
||
combined_text = f"{text}\n{js_text}"
|
||
findings: list[str] = []
|
||
if not js_path.exists():
|
||
findings.append(f"{BASE_JS_PATH}: missing required base/topbar JS file")
|
||
for snippet in REQUIRED_BASE_SNIPPETS:
|
||
if snippet not in combined_text:
|
||
findings.append(f"{BASE_PATH}: missing required base/topbar snippet `{snippet}`")
|
||
for rule in FORBIDDEN_BASE_PATTERNS:
|
||
for match in rule.pattern.finditer(combined_text):
|
||
line = line_number(text, match.start())
|
||
snippet = match.group(0).replace("\n", " ")[:90]
|
||
findings.append(
|
||
f"{BASE_PATH}:{line}: [{rule.code}] {rule.message} ({snippet})"
|
||
)
|
||
return findings
|
||
|
||
|
||
def scan_observability_copy() -> list[str]:
|
||
findings: list[str] = []
|
||
for template_path in TEMPLATE_PATHS:
|
||
path = ROOT / Path(template_path)
|
||
if not path.exists():
|
||
findings.append(f"{template_path}: missing required observability page")
|
||
continue
|
||
|
||
text = path.read_text(encoding="utf-8")
|
||
for snippet in FORBIDDEN_OBSERVABILITY_COPY:
|
||
if snippet in text:
|
||
findings.append(
|
||
f"{template_path}: legacy English observability copy `{snippet}` must be localized to the V2 workbench language"
|
||
)
|
||
return findings
|
||
|
||
|
||
def scan_shell() -> list[str]:
|
||
path = ROOT / SHELL_PATH
|
||
if not path.exists():
|
||
return [f"{SHELL_PATH}: missing required shell file"]
|
||
|
||
text = path.read_text(encoding="utf-8")
|
||
findings = scan_required_snippets(SHELL_PATH, REQUIRED_SHELL_SNIPPETS, "sidebar/nav")
|
||
for rule in FORBIDDEN_SHELL_PATTERNS:
|
||
for match in rule.pattern.finditer(text):
|
||
line = line_number(text, match.start())
|
||
snippet = match.group(0).replace("\n", " ")[:90]
|
||
findings.append(
|
||
f"{SHELL_PATH}:{line}: [{rule.code}] {rule.message} ({snippet})"
|
||
)
|
||
return findings
|
||
|
||
|
||
def scan_nav_contract() -> list[str]:
|
||
shell_path = ROOT / SHELL_PATH
|
||
base_path = ROOT / BASE_PATH
|
||
route_path = ROOT / ROUTE_PATH
|
||
if not shell_path.exists() or not base_path.exists():
|
||
return []
|
||
|
||
shell_text = shell_path.read_text(encoding="utf-8")
|
||
base_text = base_path.read_text(encoding="utf-8")
|
||
route_text = route_path.read_text(encoding="utf-8") if route_path.exists() else ""
|
||
findings: list[str] = []
|
||
|
||
for page in OBSERVABILITY_PAGES:
|
||
template = ROOT / page.template
|
||
if not template.exists():
|
||
findings.append(f"{page.template}: missing observability template for `{page.label}`")
|
||
continue
|
||
|
||
template_text = template.read_text(encoding="utf-8")
|
||
active_pattern = re.compile(
|
||
r"set\s+active_page\s*=\s*['\"]" + re.escape(page.active_page) + r"['\"]"
|
||
)
|
||
route_has_contract = (
|
||
page.template.replace("templates/", "") in route_text
|
||
and page.active_page in route_text
|
||
)
|
||
if not active_pattern.search(template_text) and not route_has_contract:
|
||
findings.append(
|
||
f"{page.template}: active_page must be `{page.active_page}` in template or route for `{page.label}`"
|
||
)
|
||
|
||
if page.active_page not in shell_text:
|
||
findings.append(f"{SHELL_PATH}: sidebar missing active_page `{page.active_page}`")
|
||
if page.url not in shell_text:
|
||
findings.append(f"{SHELL_PATH}: sidebar missing URL `{page.url}` for `{page.label}`")
|
||
if page.label not in shell_text:
|
||
findings.append(f"{SHELL_PATH}: sidebar missing label `{page.label}`")
|
||
if page.active_page not in base_text:
|
||
findings.append(f"{BASE_PATH}: momo-observability-mode list missing `{page.active_page}`")
|
||
|
||
route_pattern = re.compile(
|
||
r"@admin_observability_bp\.route\(\s*['\"]"
|
||
+ re.escape(page.route_suffix)
|
||
+ r"['\"]"
|
||
)
|
||
if route_text and not route_pattern.search(route_text):
|
||
findings.append(f"{ROUTE_PATH}: missing route `{page.route_suffix}` for `{page.label}`")
|
||
|
||
sidebar_observability_links = set(re.findall(r'href="(/observability/[^"]+)"', shell_text))
|
||
expected_links = {page.url for page in OBSERVABILITY_PAGES}
|
||
extra_links = sorted(sidebar_observability_links - expected_links)
|
||
missing_links = sorted(expected_links - sidebar_observability_links)
|
||
for url in extra_links:
|
||
findings.append(f"{SHELL_PATH}: unexpected observability sidebar URL `{url}`")
|
||
for url in missing_links:
|
||
findings.append(f"{SHELL_PATH}: missing observability sidebar URL `{url}`")
|
||
|
||
return findings
|
||
|
||
|
||
def main() -> int:
|
||
findings: list[str] = []
|
||
for template_path in TEMPLATE_PATHS:
|
||
findings.extend(scan_file(Path(template_path)))
|
||
findings.extend(scan_css())
|
||
findings.extend(scan_shell())
|
||
findings.extend(scan_base_topbar())
|
||
findings.extend(scan_observability_copy())
|
||
findings.extend(scan_nav_contract())
|
||
|
||
if findings:
|
||
print("Observability UI guard: FAIL")
|
||
for finding in findings:
|
||
print(f"- {finding}")
|
||
return 1
|
||
|
||
print("Observability UI guard: PASS")
|
||
print(f"- templates checked: {len(TEMPLATE_PATHS)}")
|
||
print(f"- css guardrails checked: {len(REQUIRED_CSS_SNIPPETS)}")
|
||
print(f"- sidebar/nav guardrails checked: {len(REQUIRED_SHELL_SNIPPETS)}")
|
||
print(f"- base/topbar guardrails checked: {len(REQUIRED_BASE_SNIPPETS) + len(FORBIDDEN_BASE_PATTERNS)}")
|
||
print(f"- observability copy guardrails checked: {len(TEMPLATE_PATHS)} pages × {len(FORBIDDEN_OBSERVABILITY_COPY)} terms")
|
||
print(f"- nav contract checked: {len(OBSERVABILITY_PAGES)} pages")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|