Files
awoooi/scripts/security/ai-provider-owner-response-acceptance.py
Your Name 2b8654704a
All checks were successful
Code Review / ai-code-review (push) Successful in 14s
CD Pipeline / tests (push) Successful in 1m34s
CD Pipeline / build-and-deploy (push) Successful in 4m5s
CD Pipeline / post-deploy-checks (push) Successful in 1m36s
feat(iwooos): 新增 AI provider owner response 驗收 gate
2026-06-15 18:57:27 +08:00

554 lines
24 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""
IwoooS AI provider / model routing owner response acceptance 只讀帳本產生器。
本工具建立 AI provider、Ollama proxy、fallback order、cost、privacy、
benchmark、dry-run 與 agent replacement candidate 的 metadata-only
acceptance ledger。它不呼叫任何模型、不讀 live endpoint、不切 provider、
不改 Nginx、不送 prompt、不建立外部 API 呼叫、不收 secret value也不開
runtime gate。
"""
from __future__ import annotations
import argparse
import hashlib
import json
import subprocess
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
TAIPEI = timezone(timedelta(hours=8))
ACCEPTANCE_FIELDS = [
"acceptance_candidate_id",
"surface_id",
"label",
"expected_scope",
"config_kind",
"control_tier",
"source_refs",
"source_ref_sha256",
"write_capable_surface",
"paid_provider_related",
"data_egress_related",
"requires_live_evidence",
"owner_response_ref",
"owner_role_or_team",
"decision",
"decision_reason",
"affected_scope",
"provider_owner",
"fallback_order_ref",
"dry_run_result_ref",
"benchmark_result_ref",
"cost_review_ref",
"privacy_review_ref",
"data_classification_ref",
"prompt_redaction_ref",
"secret_handling_ref",
"quota_budget_ref",
"latency_slo_ref",
"quality_gate_ref",
"rollback_owner",
"rollback_plan_ref",
"maintenance_window",
"postcheck_evidence_refs",
"redacted_evidence_refs",
"reviewer_outcome",
"followup_owner",
"not_approval",
]
REQUIRED_OWNER_FIELDS = [
"owner_role_or_team",
"decision",
"decision_reason",
"affected_scope",
"provider_owner",
"fallback_order_ref",
"dry_run_result_ref",
"benchmark_result_ref",
"cost_review_ref",
"privacy_review_ref",
"data_classification_ref",
"prompt_redaction_ref",
"secret_handling_ref",
"quota_budget_ref",
"latency_slo_ref",
"quality_gate_ref",
"rollback_owner",
"rollback_plan_ref",
"maintenance_window",
"postcheck_evidence_refs",
"redacted_evidence_refs",
"followup_owner",
"not_approval",
"no_secret_value_attestation",
]
REVIEWER_CHECKS = [
{"check_id": "owner_identity_present", "instruction": "owner role / team 與 provider owner 必須可追溯。"},
{"check_id": "decision_reason_present", "instruction": "decision 與 decision reason 必須同時存在。"},
{"check_id": "affected_scope_matches_surface", "instruction": "affected scope 必須能對回既有 AI provider surface。"},
{"check_id": "redacted_refs_only", "instruction": "evidence 只能是脫敏 ref、hash、ticket、commit 或 artifact pointer。"},
{"check_id": "secret_value_absent", "instruction": "不得出現 API key、token、cookie、private key、env dump、完整 DSN 或 partial secret。"},
{"check_id": "live_endpoint_absent", "instruction": "不得保存 raw live endpoint、內網位址或可直接連線 URL。"},
{"check_id": "fallback_order_present", "instruction": "必須提供 fallback order 與 degraded mode ref不接受憑印象切 provider。"},
{"check_id": "dry_run_result_present", "instruction": "必須有 dry-run 結果或明確不適用理由dry-run 不得呼叫 production LLM。"},
{"check_id": "benchmark_result_present", "instruction": "必須有 benchmark / replay / fixture 結果與評分方法 ref。"},
{"check_id": "cost_review_present", "instruction": "涉及雲端或付費 provider 時必須有成本上限、quota、告警與停損 ref。"},
{"check_id": "privacy_review_present", "instruction": "涉及外部 provider、prompt、log 或 embedding 時必須有資料外送與脫敏審查 ref。"},
{"check_id": "data_classification_present", "instruction": "必須標示可送出、不可送出、需遮罩與只能本地處理的資料分類。"},
{"check_id": "prompt_redaction_present", "instruction": "prompt / log / incident context 需有 redaction proof不接受 raw prompt 或 raw log。"},
{"check_id": "secret_handling_metadata_only", "instruction": "secret handling 只能描述注入路徑與 owner不得貼 secret value / hash / partial token。"},
{"check_id": "quota_budget_present", "instruction": "付費 provider、雲端 provider 或外部 API 必須有 quota budget 與 stop condition。"},
{"check_id": "latency_slo_present", "instruction": "需要 latency / timeout / circuit breaker / degradation SLO ref。"},
{"check_id": "quality_gate_present", "instruction": "模型切換或 agent replacement 必須有品質門檻與 incumbent baseline。"},
{"check_id": "rollback_owner_present", "instruction": "rollback owner 與 rollback plan 必須存在。"},
{"check_id": "maintenance_window_present", "instruction": "provider route、proxy、env 或 model change 必須另有維護窗口。"},
{"check_id": "postcheck_evidence_present", "instruction": "post-check 必須覆蓋 provider route、fallback、cost、privacy、observability 與 rollback stop condition。"},
{"check_id": "no_runtime_request", "instruction": "夾帶 provider switch、Nginx reload、env change、external call、prompt send、benchmark live call 或 deploy 時拒收。"},
{"check_id": "no_false_green_provider_health", "instruction": "不得把 provider route 200、dashboard up、模型回覆一次成功當成 routing / privacy / cost 已驗收。"},
{"check_id": "cross_project_sync_present", "instruction": "影響 AwoooP、IwoooS、agent-bounty、code review、StockPlatform 或公開網站時需有跨專案同步 ref。"},
{"check_id": "counts_transition_safe", "instruction": "只有 reviewer record 可更新 received / accepted不得同時開 runtime gate。"},
]
OUTCOME_LANES = [
{"lane_id": "waiting_owner_response", "meaning": "尚未收到 AI provider owner response所有 accepted / runtime count 維持 0。"},
{"lane_id": "quarantine_secret_or_raw_payload", "meaning": "收到 secret、raw prompt、raw log、raw endpoint、未脫敏截圖或 env dump 時隔離。"},
{"lane_id": "reject_execution_request", "meaning": "夾帶 provider switch、外部呼叫、prompt send、Nginx reload、env change 或 deploy 要求時拒收。"},
{"lane_id": "request_fallback_supplement", "meaning": "缺 fallback order、degraded mode、circuit breaker 或 rollback 時要求補件。"},
{"lane_id": "request_cost_privacy_supplement", "meaning": "缺 cost / quota / privacy / data classification / prompt redaction 時要求補件。"},
{"lane_id": "request_benchmark_supplement", "meaning": "缺 dry-run、benchmark、fixture replay、quality gate 或 incumbent baseline 時要求補件。"},
{"lane_id": "legacy_model_card_redaction_required", "meaning": "舊模型卡、runbook 或報表含 raw endpoint / 主機識別時,先改成脫敏 alias。"},
{"lane_id": "ready_for_ai_provider_review", "meaning": "metadata 合格後,只能進 AI provider reviewer review。"},
{"lane_id": "owner_review_only_update", "meaning": "只允許更新只讀 owner review ledger不切 provider、不呼叫模型、不改 proxy。"},
{"lane_id": "waiting_runtime_gate", "meaning": "即使 owner response acceptedruntime gate 仍需獨立人工批准。"},
]
BLOCKED_ACTIONS = [
"provider_switch",
"model_route_change",
"fallback_order_change",
"ollama_proxy_change",
"nginx_reload",
"env_change",
"runtime_config_change",
"external_provider_call",
"paid_provider_call",
"prompt_send",
"raw_prompt_storage",
"raw_log_storage",
"raw_endpoint_storage",
"live_endpoint_probe",
"benchmark_live_call",
"dry_run_external_call",
"quota_increase",
"cost_limit_change",
"privacy_boundary_change",
"data_egress_change",
"secret_value_collection",
"secret_hash_collection",
"partial_token_collection",
"api_key_rotation",
"secret_store_read",
"model_download",
"sdk_install",
"agent_runtime_enable",
"shadow_or_canary_enable",
"production_deploy",
"workflow_dispatch",
"host_write",
"ssh_read",
"ssh_write",
"active_scan",
"mark_owner_response_accepted_without_reviewer_record",
"open_runtime_gate",
"add_action_button",
]
SURFACES = [
{
"surface_id": "ai_router_provider_policy",
"label": "AI router provider policy / purpose mapping",
"expected_scope": "purpose routing、provider priority、degraded mode",
"config_kind": "provider_policy",
"source_refs": [
"apps/api/src/services/ai_router.py",
"apps/api/src/services/ai_provider_route_matrix.py",
"apps/api/models.json",
"docs/ai/AI-MODEL-CARDS.md",
],
"write_capable_surface": True,
"paid_provider_related": True,
"data_egress_related": True,
"requires_live_evidence": True,
},
{
"surface_id": "ollama_proxy_gateway",
"label": "Ollama proxy gateway / local fallback boundary",
"expected_scope": "proxy route、local fallback、gateway health",
"config_kind": "proxy_gateway",
"source_refs": [
"infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2",
"apps/api/src/services/ollama_endpoint_resolver.py",
"apps/api/src/services/ollama_failover_manager.py",
"apps/api/src/services/provider_proxy.py",
],
"write_capable_surface": True,
"paid_provider_related": False,
"data_egress_related": False,
"requires_live_evidence": True,
},
{
"surface_id": "fallback_order_and_circuit_breaker",
"label": "Fallback order / circuit breaker",
"expected_scope": "GCP / local / cloud fallback order、timeout、circuit breaker",
"config_kind": "routing_guard",
"source_refs": [
"apps/api/src/services/ollama_endpoint_circuit_breaker.py",
"apps/api/src/services/ollama_health_monitor.py",
"apps/api/src/services/failover_alerter.py",
"apps/web/messages/zh-TW.json",
],
"write_capable_surface": True,
"paid_provider_related": True,
"data_egress_related": True,
"requires_live_evidence": True,
},
{
"surface_id": "cost_budget_and_quota",
"label": "Cost budget / quota / paid provider stop condition",
"expected_scope": "quota、token budget、cost alert、paid provider stop condition",
"config_kind": "cost_policy",
"source_refs": [
"docs/ai/AI-MODEL-CARDS.md",
"apps/api/src/services/ai_rate_limiter.py",
"apps/api/src/services/ai_slo_calculator.py",
"apps/web/messages/zh-TW.json",
],
"write_capable_surface": False,
"paid_provider_related": True,
"data_egress_related": True,
"requires_live_evidence": True,
},
{
"surface_id": "privacy_data_egress_boundary",
"label": "Privacy / data egress / prompt redaction boundary",
"expected_scope": "prompt redaction、資料分級、外部 provider 資料外送",
"config_kind": "privacy_policy",
"source_refs": [
"docs/HARD_RULES.md",
"docs/ai/AI-MODEL-CARDS.md",
"apps/api/src/services/ai_providers/permissions.py",
"apps/api/src/services/alertmanager_llm_guard.py",
],
"write_capable_surface": False,
"paid_provider_related": True,
"data_egress_related": True,
"requires_live_evidence": True,
},
{
"surface_id": "benchmark_dry_run_pack",
"label": "Dry-run / benchmark / replay pack",
"expected_scope": "dry-run、benchmark、fixture replay、quality gate",
"config_kind": "evaluation_pack",
"source_refs": [
"docs/runbooks/OPENCLAW-REPLACEMENT-EVALUATION.md",
"docs/ai/agent-replacement-candidates.v1.json",
"scripts/agents/run-agent-replacement-replay.py",
"scripts/agents/evaluate-agent-promotion-gate.py",
],
"write_capable_surface": False,
"paid_provider_related": False,
"data_egress_related": True,
"requires_live_evidence": False,
},
{
"surface_id": "model_card_and_version_inventory",
"label": "Model card / model version inventory",
"expected_scope": "model card、version freshness、source-of-truth、redacted endpoint",
"config_kind": "model_metadata",
"source_refs": [
"docs/ai/AI-MODEL-CARDS.md",
"apps/api/src/services/model_registry.py",
"apps/api/src/services/model_version_tracker.py",
"apps/api/src/services/model_version_probe.py",
],
"write_capable_surface": True,
"paid_provider_related": False,
"data_egress_related": False,
"requires_live_evidence": False,
},
{
"surface_id": "agent_replacement_candidate_runtime_boundary",
"label": "Agent replacement / NemoTron / Hermes / OpenClaw candidate boundary",
"expected_scope": "agent candidate、promotion gate、shadow/canary boundary",
"config_kind": "agent_runtime_boundary",
"source_refs": [
"docs/runbooks/OPENCLAW-REPLACEMENT-EVALUATION.md",
"docs/ai/agent-market-watch-sources.v1.json",
"docs/ai/agent-replacement-candidates.v1.json",
"apps/api/src/services/ai_agent_tool_adoption_approval_package.py",
],
"write_capable_surface": True,
"paid_provider_related": True,
"data_egress_related": True,
"requires_live_evidence": True,
},
]
def git_short_sha(root: Path) -> str:
try:
result = subprocess.run(
["git", "rev-parse", "--short", "HEAD"],
cwd=root,
check=True,
capture_output=True,
text=True,
)
return result.stdout.strip()
except Exception:
return "unknown"
def ref_digest(root: Path, refs: list[str]) -> str:
digest = hashlib.sha256()
for ref in refs:
path = root / ref
digest.update(ref.encode("utf-8"))
if path.is_file():
digest.update(path.read_bytes())
elif path.is_dir():
digest.update(b"<dir>")
else:
digest.update(b"<missing>")
return digest.hexdigest()
def build_candidate(root: Path, surface: dict[str, Any]) -> dict[str, Any]:
return {
"acceptance_candidate_id": f"ai_provider_owner_response_acceptance:{surface['surface_id']}",
"status": "waiting_owner_response",
"surface_id": surface["surface_id"],
"label": surface["label"],
"expected_scope": surface["expected_scope"],
"config_kind": surface["config_kind"],
"control_tier": "C1",
"source_refs": surface["source_refs"],
"source_ref_sha256": ref_digest(root, surface["source_refs"]),
"write_capable_surface": surface["write_capable_surface"],
"paid_provider_related": surface["paid_provider_related"],
"data_egress_related": surface["data_egress_related"],
"requires_live_evidence": surface["requires_live_evidence"],
"owner_response_ref": None,
"owner_role_or_team": "pending_owner_response",
"decision": "pending_owner_response",
"decision_reason": "pending_owner_response",
"affected_scope": "pending_owner_response",
"provider_owner": "pending_owner_response",
"fallback_order_ref": None,
"dry_run_result_ref": None,
"benchmark_result_ref": None,
"cost_review_ref": None,
"privacy_review_ref": None,
"data_classification_ref": None,
"prompt_redaction_ref": None,
"secret_handling_ref": None,
"quota_budget_ref": None,
"latency_slo_ref": None,
"quality_gate_ref": None,
"rollback_owner": "pending_owner_response",
"rollback_plan_ref": None,
"maintenance_window": "pending_owner_response",
"postcheck_evidence_refs": [],
"redacted_evidence_refs": [],
"reviewer_outcome": "waiting_owner_response",
"followup_owner": "pending_owner_response",
"acceptance_fields": ACCEPTANCE_FIELDS,
"required_owner_fields": REQUIRED_OWNER_FIELDS,
"reviewer_checks": [item["check_id"] for item in REVIEWER_CHECKS],
"outcome_lanes": [item["lane_id"] for item in OUTCOME_LANES],
"blocked_actions": BLOCKED_ACTIONS,
"not_approval": True,
"request_sent": False,
"recipient_confirmed": False,
"owner_response_received": False,
"owner_response_accepted": False,
"owner_response_rejected": False,
"owner_response_quarantined": False,
"supplement_requested": False,
"fallback_order_accepted": False,
"dry_run_result_accepted": False,
"benchmark_result_accepted": False,
"cost_review_accepted": False,
"privacy_review_accepted": False,
"data_classification_accepted": False,
"prompt_redaction_accepted": False,
"secret_handling_accepted": False,
"quota_budget_accepted": False,
"latency_slo_accepted": False,
"quality_gate_accepted": False,
"rollback_owner_accepted": False,
"maintenance_window_accepted": False,
"postcheck_evidence_accepted": False,
"provider_switch_authorized": False,
"model_route_change_authorized": False,
"fallback_order_change_authorized": False,
"ollama_proxy_change_authorized": False,
"nginx_reload_authorized": False,
"env_change_authorized": False,
"external_provider_call_authorized": False,
"paid_provider_call_authorized": False,
"prompt_send_authorized": False,
"live_endpoint_probe_authorized": False,
"benchmark_live_call_authorized": False,
"dry_run_external_call_authorized": False,
"quota_increase_authorized": False,
"cost_limit_change_authorized": False,
"privacy_boundary_change_authorized": False,
"data_egress_change_authorized": False,
"secret_value_collection_allowed": False,
"secret_hash_collection_allowed": False,
"partial_token_collection_allowed": False,
"api_key_rotation_authorized": False,
"secret_store_read_authorized": False,
"model_download_authorized": False,
"sdk_install_authorized": False,
"agent_runtime_enable_authorized": False,
"shadow_or_canary_enable_authorized": False,
"production_deploy_authorized": False,
"workflow_dispatch_authorized": False,
"host_write_authorized": False,
"ssh_read_authorized": False,
"ssh_write_authorized": False,
"active_scan_authorized": False,
"runtime_gate": False,
"action_buttons_allowed": False,
}
def build_snapshot(root: Path, generated_at: str | None = None) -> dict[str, Any]:
generated = generated_at or datetime.now(TAIPEI).isoformat(timespec="seconds")
candidates = [build_candidate(root, surface) for surface in SURFACES]
summary = {
"acceptance_candidate_count": len(candidates),
"write_capable_acceptance_candidate_count": sum(1 for item in candidates if item["write_capable_surface"]),
"paid_provider_related_candidate_count": sum(1 for item in candidates if item["paid_provider_related"]),
"data_egress_candidate_count": sum(1 for item in candidates if item["data_egress_related"]),
"live_evidence_required_candidate_count": sum(1 for item in candidates if item["requires_live_evidence"]),
"acceptance_field_count": len(ACCEPTANCE_FIELDS),
"required_owner_field_count": len(REQUIRED_OWNER_FIELDS),
"reviewer_check_count": len(REVIEWER_CHECKS),
"outcome_lane_count": len(OUTCOME_LANES),
"blocked_action_count": len(BLOCKED_ACTIONS),
"owner_response_received_count": 0,
"owner_response_accepted_count": 0,
"owner_response_rejected_count": 0,
"owner_response_quarantined_count": 0,
"supplement_requested_count": 0,
"fallback_order_accepted_count": 0,
"dry_run_result_accepted_count": 0,
"benchmark_result_accepted_count": 0,
"cost_review_accepted_count": 0,
"privacy_review_accepted_count": 0,
"data_classification_accepted_count": 0,
"prompt_redaction_accepted_count": 0,
"secret_handling_accepted_count": 0,
"quota_budget_accepted_count": 0,
"latency_slo_accepted_count": 0,
"quality_gate_accepted_count": 0,
"rollback_owner_accepted_count": 0,
"maintenance_window_accepted_count": 0,
"postcheck_evidence_accepted_count": 0,
"provider_switch_authorized_count": 0,
"external_provider_call_authorized_count": 0,
"paid_provider_call_authorized_count": 0,
"prompt_send_authorized_count": 0,
"live_endpoint_probe_authorized_count": 0,
"secret_value_collection_allowed_count": 0,
"runtime_gate_count": 0,
"action_button_count": 0,
"coverage_percent_before_acceptance": 60,
"coverage_percent_after_acceptance": 64,
}
return {
"schema_version": "ai_provider_owner_response_acceptance_v1",
"status": "owner_response_acceptance_ledger_ready_no_runtime_action",
"generated_at": generated,
"git_sha": git_short_sha(root),
"source_paths": [
"docs/HARD_RULES.md",
"docs/ai/AI-MODEL-CARDS.md",
"docs/runbooks/OPENCLAW-REPLACEMENT-EVALUATION.md",
"scripts/security/ai-provider-owner-response-acceptance.py",
"scripts/security/high-value-config-control-coverage.py",
],
"summary": summary,
"acceptance_fields": ACCEPTANCE_FIELDS,
"required_owner_fields": REQUIRED_OWNER_FIELDS,
"reviewer_checks": REVIEWER_CHECKS,
"outcome_lanes": OUTCOME_LANES,
"blocked_actions": BLOCKED_ACTIONS,
"acceptance_candidates": candidates,
"boundaries": {
"not_authorization": True,
"runtime_execution_authorized": False,
"provider_switch_authorized": False,
"model_route_change_authorized": False,
"fallback_order_change_authorized": False,
"ollama_proxy_change_authorized": False,
"nginx_reload_authorized": False,
"external_provider_call_authorized": False,
"paid_provider_call_authorized": False,
"prompt_send_authorized": False,
"live_endpoint_probe_authorized": False,
"secret_value_collection_allowed": False,
"raw_prompt_storage_allowed": False,
"raw_endpoint_storage_allowed": False,
"sdk_install_authorized": False,
"agent_runtime_enable_authorized": False,
"shadow_or_canary_enable_authorized": False,
"production_deploy_authorized": False,
"host_write_authorized": False,
"active_scan_authorized": False,
"runtime_gate": False,
"action_buttons_allowed": False,
},
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--root", default=".")
parser.add_argument(
"--output",
default="docs/security/ai-provider-owner-response-acceptance.snapshot.json",
)
parser.add_argument("--generated-at", default=None)
args = parser.parse_args()
root = Path(args.root).resolve()
output = root / args.output
snapshot = build_snapshot(root, args.generated_at)
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(snapshot, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(
"AI_PROVIDER_OWNER_RESPONSE_ACCEPTANCE_OK "
f"candidates={snapshot['summary']['acceptance_candidate_count']} "
f"checks={snapshot['summary']['reviewer_check_count']} "
f"lanes={snapshot['summary']['outcome_lane_count']} "
f"accepted={snapshot['summary']['owner_response_accepted_count']} "
f"runtime_gate={snapshot['summary']['runtime_gate_count']}"
)
return 0
if __name__ == "__main__":
sys.exit(main())