新增市場情報 DB catalog 判讀
All checks were successful
CD Pipeline / deploy (push) Successful in 1m4s

This commit is contained in:
OoO
2026-05-18 20:18:03 +08:00
parent 9a88b32e90
commit 36e20216ed
9 changed files with 684 additions and 25 deletions

View File

@@ -320,7 +320,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '')
# ==========================================
# 系統版本與路徑
# ==========================================
SYSTEM_VERSION = "V10.205"
SYSTEM_VERSION = "V10.206"
LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log')
public_url = PUBLIC_URL # 用於模板顯示

View File

@@ -169,6 +169,7 @@ EwoooC 目前已有 MOMO EDM / 節慶活動資料、`promo_products`、PChome
- 2026-05-18 追加 alert review queue contract`/api/market_intel/opportunity_alert_plan` 補上 `market_alert_review_queue` 的 preview contract、required / audit / forbidden fields、priority lanes 與索引規劃。此階段只定義資料契約,不建立 review table、不寫 queue contract、不執行審核、不派送 Telegram、不呼叫 LLM。
- 2026-05-18 追加 alert review queue migration blueprint`market_alert_review_queue` 納入 `database/market_intel_models.py``migrations/032_market_intel_core_schema.sql` 與 migration blueprint補齊 additive CREATE TABLE / index / grant / rollback draft。此階段仍不執行 migration、不連 DB、不建立 review queue、不寫入審核資料。
- 2026-05-18 追加 migration apply drill preview`services.market_intel.migration_drill``/api/market_intel/migration_apply_drill` 集中正式 migration 前的只讀 schema probe、platform seed diff、人工套用清單、post-apply smoke、回滾演練與風險清單。預設 `execute=false` 不連 DB人工 smoke 可用 `execute=true` 觸發只讀 catalog / seed diff probe但仍不執行 psql、不跑 rollback、不寫 DB、不重啟容器、不掛 scheduler。
- 2026-05-18 追加 migration catalog review preview`services.market_intel.migration_catalog_review``/api/market_intel/migration_catalog_review` 將 schema_db_probe 與 platform_seed_db_diff 的只讀結果歸納為 `planned_no_probe` / `not_applied` / `partial_schema` / `already_applied` / `probe_error`,並輸出 risk level、apply path、finding 與下一步。預設 `execute=false` 不連 DB`execute=true` 仍只查 catalog / seed diff不執行 psql、不寫 DB、不跑 rollback、不掛 scheduler。
### Phase 4Coupang / Shopee Adapter

View File

@@ -19,7 +19,7 @@
| `edm_routes.py` | EDM 與節慶儀表板 | `/edm`, `/festival` |
| `monthly_routes.py` | 月結分析 | `/monthly_summary_analysis`, `/api/monthly_summary_data` |
| `daily_sales_routes.py` | 當日業績 | `/daily_sales`, `/daily_sales/export*` |
| `market_intel_routes.py` | 市場情報 Phase 43 migration apply drill preview | `/market_intel`, `/market_intel/*`, `/api/market_intel/status`, `/api/market_intel/schema`, `/api/market_intel/schema_smoke`, `/api/market_intel/schema_db_probe`, `/api/market_intel/platform_seed_db_diff`, `/api/market_intel/legacy_source_bridge`, `/api/market_intel/mcp_readiness`, `/api/market_intel/mcp_tool_contract`, `/api/market_intel/mcp_deploy_preflight`, `/api/market_intel/mcp_activation_runbook`, `/api/market_intel/mcp_fetch_gate`, `/api/market_intel/scheduler_plan`, `/api/market_intel/match_review_plan`, `/api/market_intel/opportunity_plan`, `/api/market_intel/opportunity_scoring_plan`, `/api/market_intel/opportunity_evidence_plan`, `/api/market_intel/opportunity_alert_plan`, `/api/market_intel/adapters`, `/api/market_intel/dry_run_plan`, `/api/market_intel/discovery_plan`, `/api/market_intel/manual_discovery`, `/api/market_intel/candidate_preview`, `/api/market_intel/platform_seed_plan`, `/api/market_intel/platform_seed_write_guard`, `/api/market_intel/platform_seed_writer_plan`, `/api/market_intel/migration_blueprint`, `/api/market_intel/migration_apply_drill`, `/api/market_intel/seed_writer_cli_status`, `/api/market_intel/write_approval_runbook`, `/api/market_intel/deployment_readiness` |
| `market_intel_routes.py` | 市場情報 Phase 44 migration catalog review preview | `/market_intel`, `/market_intel/*`, `/api/market_intel/status`, `/api/market_intel/schema`, `/api/market_intel/schema_smoke`, `/api/market_intel/schema_db_probe`, `/api/market_intel/platform_seed_db_diff`, `/api/market_intel/legacy_source_bridge`, `/api/market_intel/mcp_readiness`, `/api/market_intel/mcp_tool_contract`, `/api/market_intel/mcp_deploy_preflight`, `/api/market_intel/mcp_activation_runbook`, `/api/market_intel/mcp_fetch_gate`, `/api/market_intel/scheduler_plan`, `/api/market_intel/match_review_plan`, `/api/market_intel/opportunity_plan`, `/api/market_intel/opportunity_scoring_plan`, `/api/market_intel/opportunity_evidence_plan`, `/api/market_intel/opportunity_alert_plan`, `/api/market_intel/adapters`, `/api/market_intel/dry_run_plan`, `/api/market_intel/discovery_plan`, `/api/market_intel/manual_discovery`, `/api/market_intel/candidate_preview`, `/api/market_intel/platform_seed_plan`, `/api/market_intel/platform_seed_write_guard`, `/api/market_intel/platform_seed_writer_plan`, `/api/market_intel/migration_blueprint`, `/api/market_intel/migration_apply_drill`, `/api/market_intel/migration_catalog_review`, `/api/market_intel/seed_writer_cli_status`, `/api/market_intel/write_approval_runbook`, `/api/market_intel/deployment_readiness` |
| `api_routes.py` | 通用任務與查詢 API | `/api/run_task`, `/api/history/*` |
| `export_routes.py` | 匯出功能 | `/api/export/*` |
| `import_routes.py` | 匯入功能 | `/api/import_excel`, `/api/import/monthly_summary` |

View File

@@ -271,6 +271,17 @@ def market_intel_migration_apply_drill():
)
@market_intel_bp.route("/api/market_intel/migration_catalog_review")
@login_required
def market_intel_migration_catalog_review():
execute_requested = request.args.get("execute", "false").lower() == "true"
return jsonify(
_service().build_migration_catalog_review(
execute_requested=execute_requested,
)
)
@market_intel_bp.route("/api/market_intel/seed_writer_cli_status")
@login_required
def market_intel_seed_writer_cli_status():

View File

@@ -24,6 +24,7 @@ def build_deployment_readiness_preview(
opportunity_evidence_plan = service.build_opportunity_evidence_plan()
opportunity_alert_plan = service.build_opportunity_alert_plan()
migration_apply_drill = service.build_migration_apply_drill()
migration_catalog_review = service.build_migration_catalog_review()
checks = {
"schema_smoke_passed": bool(schema_smoke["passed"]),
"feature_flags_default_safe": bool(
@@ -120,6 +121,15 @@ def build_deployment_readiness_preview(
and not migration_apply_drill["api_executes_migration"]
and not migration_apply_drill["api_executes_rollback"]
),
"migration_catalog_review_preview_safe": bool(
migration_catalog_review["mode"] == "migration_catalog_review_preview"
and not migration_catalog_review["migration_executed"]
and not migration_catalog_review["rollback_executed"]
and not migration_catalog_review["database_write_executed"]
and not migration_catalog_review["database_commit_executed"]
and not migration_catalog_review["api_executes_migration"]
and not migration_catalog_review["api_executes_rollback"]
),
}
ready_for_production_deploy = all(checks.values())
blocked_reasons = [
@@ -255,6 +265,7 @@ def build_deployment_readiness_preview(
"/api/market_intel/opportunity_evidence_plan",
"/api/market_intel/opportunity_alert_plan",
"/api/market_intel/migration_apply_drill",
"/api/market_intel/migration_catalog_review",
],
"status": status.to_dict(),
"schema_smoke": schema_smoke,
@@ -266,6 +277,7 @@ def build_deployment_readiness_preview(
"write_approval_runbook": service.build_write_approval_runbook(),
"migration_blueprint": service.build_migration_blueprint(),
"migration_apply_drill": migration_apply_drill,
"migration_catalog_review": migration_catalog_review,
"seed_writer_cli_status": service.build_seed_writer_cli_status(),
"schema_db_probe": service.build_schema_db_probe(),
"platform_seed_db_diff": service.build_platform_seed_db_diff(),

View File

@@ -0,0 +1,331 @@
"""市場情報正式 DB catalog 判讀。
本模組只整合 schema_db_probe 與 platform_seed_db_diff 的只讀結果,產生
operator 可讀的風險判定;不執行 migration、不寫 DB、不建立 ORM session。
"""
def _catalog_state(schema_db_probe):
mode = schema_db_probe.get("mode")
if mode == "schema_db_probe_planned":
return "planned_no_probe"
if mode == "schema_db_probe_error":
return "probe_error"
if not schema_db_probe.get("read_only_query_executed"):
return "planned_no_probe"
if schema_db_probe.get("schema_tables_exist"):
return "already_applied"
if schema_db_probe.get("existing_tables"):
return "partial_schema"
return "not_applied"
def _risk_for_state(catalog_state, seed_state):
if catalog_state == "probe_error":
return "high"
if catalog_state == "partial_schema":
return "high"
if seed_state == "differs":
return "medium"
if catalog_state == "not_applied":
return "low"
if catalog_state == "already_applied":
return "low"
return "info"
def _apply_path_for_state(catalog_state, *, execute_requested):
if catalog_state == "planned_no_probe":
return "run_execute_true_read_only_probe_first"
if catalog_state == "probe_error":
return "blocked_until_probe_error_reviewed"
if catalog_state == "partial_schema":
return "blocked_until_partial_schema_reviewed"
if catalog_state == "already_applied":
return "skip_apply_verify_post_apply_state"
if catalog_state == "not_applied" and execute_requested:
return "manual_apply_review_candidate"
return "run_execute_true_read_only_probe_first"
def _seed_state(platform_seed_db_diff):
mode = platform_seed_db_diff.get("mode")
if mode == "platform_seed_db_diff_planned":
return "planned_no_probe"
if mode == "platform_seed_db_diff_error":
return "probe_error"
if platform_seed_db_diff.get("changed_codes"):
return "differs"
if platform_seed_db_diff.get("missing_codes"):
return "missing"
if platform_seed_db_diff.get("seed_rows_ready"):
return "ready"
return "unknown"
def _finding(key, severity, label, status):
return {
"key": key,
"severity": severity,
"label": label,
"status": status,
}
def _build_findings(*, catalog_state, schema_db_probe, seed_state, platform_seed_db_diff):
findings = []
missing_tables = schema_db_probe.get("missing_tables") or []
existing_tables = schema_db_probe.get("existing_tables") or []
if catalog_state == "planned_no_probe":
findings.append(
_finding(
"catalog_probe_not_executed",
"info",
"尚未執行正式 DB catalog 只讀探測,維持 planned 狀態。",
"planned",
)
)
elif catalog_state == "not_applied":
findings.append(
_finding(
"market_schema_not_applied",
"low",
"正式 DB 尚未建立 market_* schema可進入人工 migration review。",
"review",
)
)
elif catalog_state == "partial_schema":
findings.append(
_finding(
"partial_market_schema_detected",
"high",
f"正式 DB 只存在 {len(existing_tables)} 張 market_* 表,缺少 {len(missing_tables)} 張。",
"blocked",
)
)
elif catalog_state == "already_applied":
findings.append(
_finding(
"market_schema_already_applied",
"low",
"正式 DB 已存在所有 market_* 表migration apply 可能不需要重跑。",
"verify",
)
)
elif catalog_state == "probe_error":
findings.append(
_finding(
"catalog_probe_error",
"high",
"正式 DB catalog 只讀探測失敗,需人工檢查連線與權限。",
"blocked",
)
)
if seed_state == "probe_error":
findings.append(
_finding(
"seed_diff_probe_error",
"medium",
"platform seed DB diff 只讀探測失敗;若 market_platforms 尚未建立,需等 migration 後重跑。",
"review",
)
)
elif seed_state == "planned_no_probe":
findings.append(
_finding(
"seed_diff_not_executed",
"info",
"尚未執行 platform seed DB diff只能停留在 preview 判讀。",
"planned",
)
)
elif seed_state == "missing":
findings.append(
_finding(
"platform_seed_missing",
"low",
f"platform seed 尚缺 {len(platform_seed_db_diff.get('missing_codes') or [])} 筆,需等 migration 後獨立批准 seed writer。",
"review",
)
)
elif seed_state == "differs":
findings.append(
_finding(
"platform_seed_differs",
"medium",
f"platform seed 有 {len(platform_seed_db_diff.get('changed_codes') or [])} 筆與預期不同,需人工比對。",
"review",
)
)
elif seed_state == "ready":
findings.append(
_finding(
"platform_seed_ready",
"low",
"platform seed 與目前 DB 內容一致。",
"verify",
)
)
return findings
def build_migration_catalog_review_preview(
*,
runtime_status,
migration_blueprint,
schema_db_probe,
platform_seed_db_diff,
):
"""建立正式 DB catalog 判讀 payload不執行 migration 或 DB write。"""
execute_requested = bool(
schema_db_probe.get("execute_requested")
or platform_seed_db_diff.get("execute_requested")
)
catalog_state = _catalog_state(schema_db_probe)
seed_state = _seed_state(platform_seed_db_diff)
risk_level = _risk_for_state(catalog_state, seed_state)
apply_path = _apply_path_for_state(catalog_state, execute_requested=execute_requested)
safe_checks = {
"schema_probe_read_only_or_planned": bool(
not schema_db_probe.get("database_write_executed")
and not schema_db_probe.get("database_commit_executed")
and not schema_db_probe.get("database_session_created")
and not schema_db_probe.get("migration_executed")
),
"seed_diff_read_only_or_planned": bool(
not platform_seed_db_diff.get("database_write_executed")
and not platform_seed_db_diff.get("database_commit_executed")
and not platform_seed_db_diff.get("database_session_created")
and not platform_seed_db_diff.get("seed_write_executed")
and not platform_seed_db_diff.get("migration_executed")
),
"migration_file_matches_blueprint": bool(
migration_blueprint.get("file_created")
and migration_blueprint.get("file_matches_blueprint")
),
"forward_sql_additive_only": bool(
migration_blueprint.get("safety_checks", {}).get("forward_sql_additive_only")
and not migration_blueprint.get("forward_has_destructive_sql")
),
"feature_flags_default_safe": bool(
not runtime_status.enabled
and not runtime_status.crawler_enabled
and not runtime_status.write_enabled
),
"database_write_blocked": bool(not runtime_status.database_write_allowed),
"scheduler_detached": bool(not runtime_status.scheduler_attached),
}
review_ready = all(safe_checks.values())
ready_for_manual_migration_review = bool(
review_ready
and execute_requested
and catalog_state == "not_applied"
)
blocked_reasons = [
"migration_not_executed_by_catalog_review",
"api_never_runs_migration",
"database_write_still_blocked",
]
blocked_reasons.extend(
key for key, passed in safe_checks.items()
if not passed
)
if not execute_requested:
blocked_reasons.append("execute_false_planned_only")
if catalog_state == "probe_error":
blocked_reasons.append("catalog_probe_error")
if catalog_state == "partial_schema":
blocked_reasons.append("partial_schema_requires_manual_reconciliation")
if catalog_state == "already_applied":
blocked_reasons.append("market_schema_already_present")
expected_tables = schema_db_probe.get("expected_tables") or []
existing_tables = schema_db_probe.get("existing_tables") or []
missing_tables = schema_db_probe.get("missing_tables") or expected_tables
table_statuses = schema_db_probe.get("table_statuses") or [
{"table": table_name, "exists": False}
for table_name in expected_tables
]
return {
"mode": "migration_catalog_review_preview",
"execute_requested": execute_requested,
"catalog_state": catalog_state,
"seed_state": seed_state,
"risk_level": risk_level,
"apply_path": apply_path,
"review_ready": review_ready,
"ready_for_manual_migration_review": ready_for_manual_migration_review,
"ready_to_apply_migration": False,
"migration_executed": False,
"rollback_executed": False,
"database_connection_opened": bool(
schema_db_probe.get("database_connection_opened")
or platform_seed_db_diff.get("database_connection_opened")
),
"read_only_query_executed": bool(
schema_db_probe.get("read_only_query_executed")
or platform_seed_db_diff.get("read_only_query_executed")
),
"database_session_created": False,
"explicit_transaction_opened": False,
"database_write_executed": False,
"database_commit_executed": False,
"external_network_executed": False,
"scheduler_attached": False,
"api_executes_migration": False,
"api_executes_rollback": False,
"safe_checks": safe_checks,
"blocked_reasons": blocked_reasons,
"findings": _build_findings(
catalog_state=catalog_state,
schema_db_probe=schema_db_probe,
seed_state=seed_state,
platform_seed_db_diff=platform_seed_db_diff,
),
"table_catalog": {
"expected_count": len(expected_tables),
"existing_count": len(existing_tables),
"missing_count": len(missing_tables),
"expected_tables": expected_tables,
"existing_tables": existing_tables,
"missing_tables": missing_tables,
"table_statuses": table_statuses,
},
"seed_catalog": {
"expected_seed_count": platform_seed_db_diff.get("expected_seed_count", 0),
"existing_seed_count": platform_seed_db_diff.get("existing_seed_count", 0),
"missing_codes": platform_seed_db_diff.get("missing_codes") or [],
"changed_codes": platform_seed_db_diff.get("changed_codes") or [],
"matching_codes": platform_seed_db_diff.get("matching_codes") or [],
"seed_rows_ready": bool(platform_seed_db_diff.get("seed_rows_ready")),
},
"operator_next_steps": [
{
"key": "run_catalog_review_execute_true",
"label": "人工 smoke 時呼叫 /api/market_intel/migration_catalog_review?execute=true只讀判讀正式 DB 狀態",
"status": "required" if not execute_requested else "completed",
},
{
"key": "review_catalog_state",
"label": "依 catalog_state 決定是否是未套用、部分套用、已套用或探測錯誤",
"status": "required",
},
{
"key": "manual_migration_apply_window",
"label": "只有 catalog_state=not_applied 且人工批准後,才可在維護窗口手動執行 psql migration",
"status": "blocked_by_operator_approval",
},
{
"key": "post_apply_read_only_probe",
"label": "套用後重跑 execute=true只確認所有 market_* 表存在,不自動寫 seed",
"status": "required_after_manual_apply",
},
],
"manual_probe_targets": [
"/api/market_intel/schema_db_probe?execute=true",
"/api/market_intel/platform_seed_db_diff?execute=true",
"/api/market_intel/migration_catalog_review?execute=true",
],
}

View File

@@ -30,6 +30,9 @@ from services.market_intel.mcp_deploy_preflight import build_mcp_deploy_prefligh
from services.market_intel.mcp_fetch_gate import build_mcp_fetch_gate_preview
from services.market_intel.mcp_readiness import build_mcp_readiness_plan
from services.market_intel.migration_blueprint import build_migration_blueprint
from services.market_intel.migration_catalog_review import (
build_migration_catalog_review_preview,
)
from services.market_intel.migration_drill import build_migration_apply_drill_preview
from services.market_intel.opportunity_alerts import (
build_opportunity_alert_plan_preview,
@@ -85,7 +88,7 @@ class MarketIntelRuntimeStatus:
class MarketIntelService:
"""市場情報入口服務,先集中 feature gate 與安全狀態。"""
phase = "phase_43_migration_apply_drill_preview"
phase = "phase_44_migration_catalog_review_preview"
def get_runtime_status(self) -> MarketIntelRuntimeStatus:
return MarketIntelRuntimeStatus(
@@ -548,6 +551,37 @@ class MarketIntelService:
drill["phase"] = self.phase
return drill
def build_migration_catalog_review(
self,
*,
execute_requested=False,
schema_engine=None,
seed_diff_engine=None,
database_url=None,
database_type=None,
):
"""建立正式 DB catalog 判讀;不執行 migration、不寫 DB。"""
schema_db_probe = self.build_schema_db_probe(
execute_requested=execute_requested,
engine=schema_engine,
database_url=database_url,
database_type=database_type,
)
platform_seed_db_diff = self.build_platform_seed_db_diff(
execute_requested=execute_requested,
engine=seed_diff_engine,
database_url=database_url,
database_type=database_type,
)
review = build_migration_catalog_review_preview(
runtime_status=self.get_runtime_status(),
migration_blueprint=self.build_migration_blueprint(),
schema_db_probe=schema_db_probe,
platform_seed_db_diff=platform_seed_db_diff,
)
review["phase"] = self.phase
return review
def build_seed_writer_cli_status(
self,
platform_code="all",

View File

@@ -630,6 +630,24 @@
</div>
</div>
<div class="market-intel-panel" data-market-intel-catalog-review>
<div class="market-intel-preview-head">
<div>
<p class="market-intel-muted momo-mono mb-1">MIGRATION / CATALOG REVIEW</p>
<h2 class="market-intel-preview-title">正式 DB catalog 判讀</h2>
</div>
<button class="market-intel-icon-button" type="button" title="重新整理正式 DB catalog 判讀" data-market-intel-catalog-review-refresh>
<i class="fas fa-rotate-right" aria-hidden="true"></i>
</button>
</div>
<div class="market-intel-preview-meta" data-market-intel-catalog-review-meta>
<span class="market-intel-pill">loading</span>
</div>
<div data-market-intel-catalog-review-body>
<div class="market-intel-empty">讀取正式 DB catalog 判讀中...</div>
</div>
</div>
<div class="market-intel-panel" data-market-intel-approval>
<div class="market-intel-preview-head">
<div>
@@ -689,9 +707,10 @@
const opportunityAlertRoot = document.querySelector('[data-market-intel-opportunity-alert]');
const migrationRoot = document.querySelector('[data-market-intel-migration]');
const migrationDrillRoot = document.querySelector('[data-market-intel-migration-drill]');
const catalogReviewRoot = document.querySelector('[data-market-intel-catalog-review]');
const approvalRoot = document.querySelector('[data-market-intel-approval]');
const deployRoot = document.querySelector('[data-market-intel-deploy]');
if (!root && !writerRoot && !cliRoot && !dbProbeRoot && !seedDiffRoot && !legacyBridgeRoot && !mcpReadinessRoot && !mcpPreflightRoot && !mcpActivationRoot && !mcpFetchGateRoot && !schedulerRoot && !matchReviewRoot && !opportunityRoot && !opportunityScoringRoot && !opportunityEvidenceRoot && !opportunityAlertRoot && !migrationRoot && !migrationDrillRoot && !approvalRoot && !deployRoot) return;
if (!root && !writerRoot && !cliRoot && !dbProbeRoot && !seedDiffRoot && !legacyBridgeRoot && !mcpReadinessRoot && !mcpPreflightRoot && !mcpActivationRoot && !mcpFetchGateRoot && !schedulerRoot && !matchReviewRoot && !opportunityRoot && !opportunityScoringRoot && !opportunityEvidenceRoot && !opportunityAlertRoot && !migrationRoot && !migrationDrillRoot && !catalogReviewRoot && !approvalRoot && !deployRoot) return;
const meta = root ? root.querySelector('[data-market-intel-preview-meta]') : null;
const body = root ? root.querySelector('[data-market-intel-preview-body]') : null;
@@ -765,6 +784,10 @@
const migrationDrillBody = migrationDrillRoot ? migrationDrillRoot.querySelector('[data-market-intel-migration-drill-body]') : null;
const migrationDrillRefresh = migrationDrillRoot ? migrationDrillRoot.querySelector('[data-market-intel-migration-drill-refresh]') : null;
const migrationDrillEndpoint = "{{ url_for('market_intel.market_intel_migration_apply_drill') }}?execute=false";
const catalogReviewMeta = catalogReviewRoot ? catalogReviewRoot.querySelector('[data-market-intel-catalog-review-meta]') : null;
const catalogReviewBody = catalogReviewRoot ? catalogReviewRoot.querySelector('[data-market-intel-catalog-review-body]') : null;
const catalogReviewRefresh = catalogReviewRoot ? catalogReviewRoot.querySelector('[data-market-intel-catalog-review-refresh]') : null;
const catalogReviewEndpoint = "{{ url_for('market_intel.market_intel_migration_catalog_review') }}?execute=false";
const approvalMeta = approvalRoot ? approvalRoot.querySelector('[data-market-intel-approval-meta]') : null;
const approvalBody = approvalRoot ? approvalRoot.querySelector('[data-market-intel-approval-body]') : null;
const approvalRefresh = approvalRoot ? approvalRoot.querySelector('[data-market-intel-approval-refresh]') : null;
@@ -2433,6 +2456,109 @@
}
};
const renderCatalogReviewMeta = data => {
catalogReviewMeta.innerHTML = [
`mode=${data.mode || 'unknown'}`,
`catalog=${data.catalog_state || 'unknown'}`,
`risk=${data.risk_level || 'info'}`,
`tables=${data.table_catalog ? `${data.table_catalog.existing_count || 0}/${data.table_catalog.expected_count || 0}` : '0/0'}`,
`apply=${data.apply_path || 'blocked'}`
].map(item => `<span class="market-intel-pill">${escapeHtml(item)}</span>`).join('');
};
const renderCatalogReviewBody = data => {
const blockers = (data.blocked_reasons || []).join(' / ');
const checks = Object.entries(data.safe_checks || {});
const tableCatalog = data.table_catalog || {};
const seedCatalog = data.seed_catalog || {};
const findings = data.findings || [];
const nextSteps = data.operator_next_steps || [];
const probeTargets = data.manual_probe_targets || [];
const renderCheck = ([name, passed]) => `
<div class="market-intel-check">
<div>
<strong>${escapeHtml(name)}</strong>
</div>
<span>${passed ? 'PASS' : 'BLOCK'}</span>
</div>
`;
const renderNamedItem = item => `
<div class="market-intel-check">
<div>
<strong>${escapeHtml(item.key || item.label || 'item')}</strong>
<small>${escapeHtml(item.label || item.key || '')}</small>
</div>
<span>${escapeHtml(item.status || item.severity || 'review').toUpperCase()}</span>
</div>
`;
catalogReviewBody.innerHTML = `
<div class="market-intel-empty mb-3">此卡只判讀正式 DB catalog 與 platform seed diff 的只讀結果API 不執行 migration、不寫 DB、不跑 rollback、不掛 scheduler。${blockers ? `阻擋:${escapeHtml(blockers)}` : ''}</div>
<div class="market-intel-empty mb-3">tables=${escapeHtml(tableCatalog.existing_count || 0)}/${escapeHtml(tableCatalog.expected_count || 0)} / missing=${escapeHtml(tableCatalog.missing_count || 0)} / seed_missing=${escapeHtml((seedCatalog.missing_codes || []).length)} / seed_changed=${escapeHtml((seedCatalog.changed_codes || []).length)}</div>
<div class="market-intel-deploy-grid">
<div data-market-intel-catalog-review-checks>
<p class="market-intel-deploy-section-title">SAFE CHECKS</p>
<div class="market-intel-check-list">${
checks.length
? checks.map(renderCheck).join('')
: '<div class="market-intel-empty">尚未提供 safety checks。</div>'
}</div>
</div>
<div data-market-intel-catalog-review-findings>
<p class="market-intel-deploy-section-title">CATALOG FINDINGS</p>
<div class="market-intel-check-list">${
findings.length
? findings.map(renderNamedItem).join('')
: '<div class="market-intel-empty">尚未提供 catalog finding。</div>'
}</div>
</div>
<div data-market-intel-catalog-review-tables>
<p class="market-intel-deploy-section-title">TABLE STATUS</p>
<div class="market-intel-check-list">${
(tableCatalog.table_statuses || []).slice(0, 8).map(item => `
<div class="market-intel-check">
<div>
<strong>${escapeHtml(item.table)}</strong>
</div>
<span>${item.exists ? 'EXISTS' : 'MISSING'}</span>
</div>
`).join('') || '<div class="market-intel-empty">尚未提供 table status。</div>'
}</div>
</div>
<div data-market-intel-catalog-review-next>
<p class="market-intel-deploy-section-title">NEXT STEPS</p>
<div class="market-intel-check-list">${
nextSteps.length
? nextSteps.map(renderNamedItem).join('')
: '<div class="market-intel-empty">尚未提供下一步。</div>'
}</div>
</div>
<div data-market-intel-catalog-review-probes>
<p class="market-intel-deploy-section-title">READ-ONLY PROBES</p>
<div class="market-intel-check-list">${
probeTargets.length
? probeTargets.map(item => renderNamedItem({ key: item, label: item, status: 'manual' })).join('')
: '<div class="market-intel-empty">尚未提供 probe targets。</div>'
}</div>
</div>
</div>
`;
};
const loadCatalogReview = async () => {
if (!catalogReviewMeta || !catalogReviewBody) return;
catalogReviewBody.innerHTML = '<div class="market-intel-empty">讀取正式 DB catalog 判讀中...</div>';
try {
const response = await fetch(catalogReviewEndpoint, { credentials: 'same-origin' });
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const data = await response.json();
renderCatalogReviewMeta(data);
renderCatalogReviewBody(data);
} catch (error) {
catalogReviewMeta.innerHTML = '<span class="market-intel-pill">error</span>';
catalogReviewBody.innerHTML = `<div class="market-intel-empty">正式 DB catalog 判讀讀取失敗:${escapeHtml(error.message)}</div>`;
}
};
const renderApprovalMeta = data => {
approvalMeta.innerHTML = [
`mode=${data.mode || 'unknown'}`,
@@ -2646,6 +2772,9 @@
if (migrationDrillRefresh) {
migrationDrillRefresh.addEventListener('click', loadMigrationDrill);
}
if (catalogReviewRefresh) {
catalogReviewRefresh.addEventListener('click', loadCatalogReview);
}
if (approvalRefresh) {
approvalRefresh.addEventListener('click', loadApproval);
}
@@ -2670,6 +2799,7 @@
loadOpportunityAlert();
loadMigration();
loadMigrationDrill();
loadCatalogReview();
loadApproval();
loadDeploy();
})();

View File

@@ -488,6 +488,10 @@ def test_market_intel_preview_template_uses_safe_fetch_false_endpoint():
assert "data-market-intel-migration-drill-checks" in template
assert "data-market-intel-migration-drill-preapply" in template
assert "data-market-intel-migration-drill-rollback" in template
assert "data-market-intel-catalog-review" in template
assert "data-market-intel-catalog-review-checks" in template
assert "data-market-intel-catalog-review-findings" in template
assert "data-market-intel-catalog-review-tables" in template
assert "data-market-intel-approval" in template
assert "data-market-intel-approval-gates" in template
assert "data-market-intel-deploy" in template
@@ -511,6 +515,7 @@ def test_market_intel_preview_template_uses_safe_fetch_false_endpoint():
assert "market_intel.market_intel_opportunity_alert_plan" in template
assert "market_intel.market_intel_migration_blueprint" in template
assert "market_intel.market_intel_migration_apply_drill" in template
assert "market_intel.market_intel_migration_catalog_review" in template
assert "market_intel.market_intel_write_approval_runbook" in template
assert "market_intel.market_intel_deployment_readiness" in template
assert "required_manual_steps" in template
@@ -535,7 +540,7 @@ def test_legacy_source_bridge_default_is_planned_only():
bridge = MarketIntelService().build_legacy_source_bridge()
assert bridge["mode"] == "legacy_source_bridge_planned"
assert bridge["phase"] == "phase_43_migration_apply_drill_preview"
assert bridge["phase"] == "phase_44_migration_catalog_review_preview"
assert bridge["execute_requested"] is False
assert bridge["read_only_query_executed"] is False
assert bridge["database_connection_opened"] is False
@@ -693,7 +698,7 @@ def test_mcp_tool_contract_preview_is_read_only_and_whitelisted():
contract = MarketIntelService().build_mcp_tool_contract()
assert contract["mode"] == "mcp_tool_contract_preview"
assert contract["phase"] == "phase_43_migration_apply_drill_preview"
assert contract["phase"] == "phase_44_migration_catalog_review_preview"
assert contract["caller"] == "market_intel"
assert contract["contract_ready"] is True
assert contract["blocked_reasons"] == []
@@ -826,7 +831,7 @@ def test_mcp_activation_runbook_route_is_preview_only():
assert response.status_code == 200
assert data["mode"] == "mcp_activation_runbook_preview"
assert data["phase"] == "phase_43_migration_apply_drill_preview"
assert data["phase"] == "phase_44_migration_catalog_review_preview"
assert data["deployment_actions_executed"] is False
assert data["docker_command_executed"] is False
assert data["ssh_command_executed"] is False
@@ -839,7 +844,7 @@ def test_mcp_fetch_gate_default_blocks_external_fetch():
gate = MarketIntelService().build_mcp_fetch_gate(fetch_requested=True)
assert gate["mode"] == "mcp_fetch_gate_planned"
assert gate["phase"] == "phase_43_migration_apply_drill_preview"
assert gate["phase"] == "phase_44_migration_catalog_review_preview"
assert gate["fetch_requested"] is True
assert gate["manual_fetch_gate_open"] is False
assert gate["network_request_allowed"] is False
@@ -909,7 +914,7 @@ def test_mcp_fetch_gate_route_is_preview_only():
assert response.status_code == 200
assert data["mode"] == "mcp_fetch_gate_planned"
assert data["phase"] == "phase_43_migration_apply_drill_preview"
assert data["phase"] == "phase_44_migration_catalog_review_preview"
assert data["fetch_requested"] is False
assert data["network_request_allowed"] is False
assert data["external_network_executed"] is False
@@ -921,7 +926,7 @@ def test_scheduler_plan_preview_blocks_job_attachment():
plan = MarketIntelService().build_scheduler_plan()
assert plan["mode"] == "scheduler_attach_plan_preview"
assert plan["phase"] == "phase_43_migration_apply_drill_preview"
assert plan["phase"] == "phase_44_migration_catalog_review_preview"
assert plan["ready_to_attach_scheduler"] is False
assert plan["scheduler_attached"] is False
assert plan["scheduler_registration_executed"] is False
@@ -959,7 +964,7 @@ def test_scheduler_plan_route_is_preview_only():
assert response.status_code == 200
assert data["mode"] == "scheduler_attach_plan_preview"
assert data["phase"] == "phase_43_migration_apply_drill_preview"
assert data["phase"] == "phase_44_migration_catalog_review_preview"
assert data["scheduler_registration_executed"] is False
assert data["crawler_job_started"] is False
assert data["external_network_executed"] is False
@@ -970,7 +975,7 @@ def test_match_review_plan_preview_blocks_auto_confirm():
plan = MarketIntelService().build_match_review_plan()
assert plan["mode"] == "match_review_plan_preview"
assert plan["phase"] == "phase_43_migration_apply_drill_preview"
assert plan["phase"] == "phase_44_migration_catalog_review_preview"
assert plan["ready_for_review_queue"] is False
assert plan["review_queue_created"] is False
assert plan["auto_match_executed"] is False
@@ -1006,7 +1011,7 @@ def test_match_review_plan_route_is_preview_only():
assert response.status_code == 200
assert data["mode"] == "match_review_plan_preview"
assert data["phase"] == "phase_43_migration_apply_drill_preview"
assert data["phase"] == "phase_44_migration_catalog_review_preview"
assert data["review_queue_created"] is False
assert data["auto_confirm_executed"] is False
assert data["external_network_executed"] is False
@@ -1017,7 +1022,7 @@ def test_opportunity_plan_preview_blocks_alerts_and_ai_summary():
plan = MarketIntelService().build_opportunity_plan()
assert plan["mode"] == "opportunity_plan_preview"
assert plan["phase"] == "phase_43_migration_apply_drill_preview"
assert plan["phase"] == "phase_44_migration_catalog_review_preview"
assert plan["ready_for_opportunity_queue"] is False
assert plan["opportunity_queue_created"] is False
assert plan["threat_alert_dispatched"] is False
@@ -1058,7 +1063,7 @@ def test_opportunity_plan_route_is_preview_only():
assert response.status_code == 200
assert data["mode"] == "opportunity_plan_preview"
assert data["phase"] == "phase_43_migration_apply_drill_preview"
assert data["phase"] == "phase_44_migration_catalog_review_preview"
assert data["opportunity_queue_created"] is False
assert data["threat_alert_dispatched"] is False
assert data["ai_summary_generated"] is False
@@ -1069,7 +1074,7 @@ def test_opportunity_scoring_plan_preview_blocks_scoring_and_alerts():
plan = MarketIntelService().build_opportunity_scoring_plan()
assert plan["mode"] == "opportunity_scoring_plan_preview"
assert plan["phase"] == "phase_43_migration_apply_drill_preview"
assert plan["phase"] == "phase_44_migration_catalog_review_preview"
assert plan["ready_for_scoring_job"] is False
assert plan["scoring_job_created"] is False
assert plan["score_calculation_executed"] is False
@@ -1117,7 +1122,7 @@ def test_opportunity_scoring_plan_route_is_preview_only():
assert response.status_code == 200
assert data["mode"] == "opportunity_scoring_plan_preview"
assert data["phase"] == "phase_43_migration_apply_drill_preview"
assert data["phase"] == "phase_44_migration_catalog_review_preview"
assert data["scoring_job_created"] is False
assert data["score_calculation_executed"] is False
assert data["sample_scores_generated"] is False
@@ -1129,7 +1134,7 @@ def test_opportunity_evidence_plan_preview_blocks_queries_and_alerts():
plan = MarketIntelService().build_opportunity_evidence_plan()
assert plan["mode"] == "opportunity_evidence_plan_preview"
assert plan["phase"] == "phase_43_migration_apply_drill_preview"
assert plan["phase"] == "phase_44_migration_catalog_review_preview"
assert plan["ready_for_evidence_bundle"] is False
assert plan["evidence_bundle_created"] is False
assert plan["evidence_query_executed"] is False
@@ -1175,7 +1180,7 @@ def test_opportunity_evidence_plan_route_is_preview_only():
assert response.status_code == 200
assert data["mode"] == "opportunity_evidence_plan_preview"
assert data["phase"] == "phase_43_migration_apply_drill_preview"
assert data["phase"] == "phase_44_migration_catalog_review_preview"
assert data["evidence_bundle_created"] is False
assert data["evidence_query_executed"] is False
assert data["sample_evidence_generated"] is False
@@ -1188,7 +1193,7 @@ def test_opportunity_alert_plan_preview_blocks_dispatch_and_llm_calls():
plan = MarketIntelService().build_opportunity_alert_plan()
assert plan["mode"] == "opportunity_alert_plan_preview"
assert plan["phase"] == "phase_43_migration_apply_drill_preview"
assert plan["phase"] == "phase_44_migration_catalog_review_preview"
assert plan["ready_for_alert_candidates"] is False
assert plan["alert_candidate_created"] is False
assert plan["alert_queue_created"] is False
@@ -1273,7 +1278,7 @@ def test_opportunity_alert_plan_route_is_preview_only():
assert response.status_code == 200
assert data["mode"] == "opportunity_alert_plan_preview"
assert data["phase"] == "phase_43_migration_apply_drill_preview"
assert data["phase"] == "phase_44_migration_catalog_review_preview"
assert data["alert_candidate_created"] is False
assert data["alert_queue_created"] is False
assert data["review_queue_created"] is False
@@ -1351,7 +1356,7 @@ def test_mcp_deploy_preflight_route_is_preview_only():
assert response.status_code == 200
assert data["mode"] == "mcp_external_deploy_preflight_preview"
assert data["phase"] == "phase_43_migration_apply_drill_preview"
assert data["phase"] == "phase_44_migration_catalog_review_preview"
assert data["deployment_actions_executed"] is False
assert data["docker_command_executed"] is False
assert data["ssh_command_executed"] is False
@@ -1366,7 +1371,7 @@ def test_mcp_readiness_default_is_planned_only(monkeypatch):
readiness = MarketIntelService().build_mcp_readiness()
assert readiness["mode"] == "mcp_readiness_planned"
assert readiness["phase"] == "phase_43_migration_apply_drill_preview"
assert readiness["phase"] == "phase_44_migration_catalog_review_preview"
assert readiness["execute_requested"] is False
assert readiness["router_enabled"] is False
assert readiness["external_mcp_complete"] is False
@@ -1760,6 +1765,7 @@ def test_deployment_readiness_reports_app_only_release_gate():
assert readiness["checks"]["opportunity_evidence_plan_preview_safe"] is True
assert readiness["checks"]["opportunity_alert_plan_preview_safe"] is True
assert readiness["checks"]["migration_apply_drill_preview_safe"] is True
assert readiness["checks"]["migration_catalog_review_preview_safe"] is True
assert readiness["checks"]["writer_plan_dry_run_only"] is True
assert readiness["writer_plan_summary"]["writes_executed"] is False
assert "readiness_checks_not_all_passed" not in readiness["blocked_reasons"]
@@ -1787,6 +1793,7 @@ def test_deployment_readiness_reports_app_only_release_gate():
assert "/api/market_intel/opportunity_evidence_plan" in readiness["production_smoke_targets"]
assert "/api/market_intel/opportunity_alert_plan" in readiness["production_smoke_targets"]
assert "/api/market_intel/migration_apply_drill" in readiness["production_smoke_targets"]
assert "/api/market_intel/migration_catalog_review" in readiness["production_smoke_targets"]
assert readiness["write_approval_runbook"]["ready_for_real_write"] is False
assert readiness["write_approval_runbook"]["writes_executed"] is False
assert readiness["migration_blueprint"]["migration_executed"] is False
@@ -1797,6 +1804,11 @@ def test_deployment_readiness_reports_app_only_release_gate():
assert readiness["migration_apply_drill"]["rollback_executed"] is False
assert readiness["migration_apply_drill"]["database_write_executed"] is False
assert readiness["migration_apply_drill"]["api_executes_migration"] is False
assert readiness["migration_catalog_review"]["mode"] == "migration_catalog_review_preview"
assert readiness["migration_catalog_review"]["catalog_state"] == "planned_no_probe"
assert readiness["migration_catalog_review"]["migration_executed"] is False
assert readiness["migration_catalog_review"]["database_write_executed"] is False
assert readiness["migration_catalog_review"]["api_executes_migration"] is False
assert readiness["schema_db_probe"]["read_only_query_executed"] is False
assert readiness["platform_seed_db_diff"]["read_only_query_executed"] is False
assert readiness["legacy_source_bridge"]["read_only_query_executed"] is False
@@ -1914,7 +1926,7 @@ def test_migration_apply_drill_planned_is_safe_and_manual_only():
drill = MarketIntelService().build_migration_apply_drill()
assert drill["mode"] == "migration_apply_drill_preview"
assert drill["phase"] == "phase_43_migration_apply_drill_preview"
assert drill["phase"] == "phase_44_migration_catalog_review_preview"
assert drill["execute_requested"] is False
assert drill["schema_state"] == "planned_no_db_probe"
assert drill["drill_ready_for_operator_review"] is True
@@ -2029,7 +2041,7 @@ def test_migration_apply_drill_route_is_preview_only():
assert response.status_code == 200
assert data["mode"] == "migration_apply_drill_preview"
assert data["phase"] == "phase_43_migration_apply_drill_preview"
assert data["phase"] == "phase_44_migration_catalog_review_preview"
assert data["execute_requested"] is False
assert data["migration_executed"] is False
assert data["rollback_executed"] is False
@@ -2037,6 +2049,134 @@ def test_migration_apply_drill_route_is_preview_only():
assert data["api_executes_migration"] is False
def test_migration_catalog_review_planned_is_safe_and_diagnostic():
review = MarketIntelService().build_migration_catalog_review()
assert review["mode"] == "migration_catalog_review_preview"
assert review["phase"] == "phase_44_migration_catalog_review_preview"
assert review["execute_requested"] is False
assert review["catalog_state"] == "planned_no_probe"
assert review["seed_state"] == "planned_no_probe"
assert review["risk_level"] == "info"
assert review["apply_path"] == "run_execute_true_read_only_probe_first"
assert review["review_ready"] is True
assert review["ready_for_manual_migration_review"] is False
assert review["ready_to_apply_migration"] is False
assert review["read_only_query_executed"] is False
assert review["database_connection_opened"] is False
assert review["database_session_created"] is False
assert review["database_write_executed"] is False
assert review["database_commit_executed"] is False
assert review["migration_executed"] is False
assert review["rollback_executed"] is False
assert review["api_executes_migration"] is False
assert review["table_catalog"]["expected_count"] == 8
assert review["table_catalog"]["existing_count"] == 0
assert review["table_catalog"]["missing_count"] == 8
assert "execute_false_planned_only" in review["blocked_reasons"]
assert "migration_not_executed_by_catalog_review" in review["blocked_reasons"]
assert {item["key"] for item in review["findings"]} >= {
"catalog_probe_not_executed",
"seed_diff_not_executed",
}
def test_migration_catalog_review_sqlite_read_only_detects_partial_schema():
service = MarketIntelService()
seed_rows = service.build_platform_seed_plan()["seeds"]
engine = create_engine("sqlite:///:memory:")
with engine.begin() as conn:
conn.execute(
text(
"""
CREATE TABLE market_platforms (
code TEXT PRIMARY KEY,
name TEXT,
base_url TEXT,
enabled BOOLEAN,
crawl_policy_json TEXT
)
"""
)
)
for seed in seed_rows:
conn.execute(
text(
"""
INSERT INTO market_platforms
(code, name, base_url, enabled, crawl_policy_json)
VALUES
(:code, :name, :base_url, :enabled, :crawl_policy_json)
"""
),
{
"code": seed["code"],
"name": seed["name"],
"base_url": seed["base_url"],
"enabled": seed["enabled"],
"crawl_policy_json": json.dumps(
seed["crawl_policy_json"],
ensure_ascii=False,
sort_keys=True,
),
},
)
review = service.build_migration_catalog_review(
execute_requested=True,
schema_engine=engine,
seed_diff_engine=engine,
database_type="sqlite",
)
assert review["mode"] == "migration_catalog_review_preview"
assert review["execute_requested"] is True
assert review["read_only_query_executed"] is True
assert review["database_connection_opened"] is True
assert review["catalog_state"] == "partial_schema"
assert review["seed_state"] == "ready"
assert review["risk_level"] == "high"
assert review["apply_path"] == "blocked_until_partial_schema_reviewed"
assert review["ready_for_manual_migration_review"] is False
assert review["table_catalog"]["expected_count"] == 8
assert review["table_catalog"]["existing_count"] == 1
assert review["table_catalog"]["missing_count"] == 7
assert review["seed_catalog"]["seed_rows_ready"] is True
assert "partial_schema_requires_manual_reconciliation" in review["blocked_reasons"]
assert review["database_session_created"] is False
assert review["database_write_executed"] is False
assert review["database_commit_executed"] is False
assert review["migration_executed"] is False
assert {item["key"] for item in review["findings"]} >= {
"partial_market_schema_detected",
"platform_seed_ready",
}
def test_migration_catalog_review_route_is_preview_only():
from routes.market_intel_routes import market_intel_bp
app = Flask(__name__)
app.secret_key = "test-secret"
app.register_blueprint(market_intel_bp)
client = app.test_client()
with client.session_transaction() as session:
session["logged_in"] = True
response = client.get("/api/market_intel/migration_catalog_review?execute=false")
data = response.get_json()
assert response.status_code == 200
assert data["mode"] == "migration_catalog_review_preview"
assert data["phase"] == "phase_44_migration_catalog_review_preview"
assert data["execute_requested"] is False
assert data["catalog_state"] == "planned_no_probe"
assert data["migration_executed"] is False
assert data["rollback_executed"] is False
assert data["database_write_executed"] is False
assert data["api_executes_migration"] is False
def test_seed_writer_cli_status_blocks_real_write():
status = MarketIntelService().build_seed_writer_cli_status(
platform_code="all",