diff --git a/config.py b/config.py index 0fa53e0..4145faa 100644 --- a/config.py +++ b/config.py @@ -320,7 +320,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.205" +SYSTEM_VERSION = "V10.206" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/adr/ADR-035-cross-platform-market-campaign-intelligence.md b/docs/adr/ADR-035-cross-platform-market-campaign-intelligence.md index 3e41cdb..a324017 100644 --- a/docs/adr/ADR-035-cross-platform-market-campaign-intelligence.md +++ b/docs/adr/ADR-035-cross-platform-market-campaign-intelligence.md @@ -169,6 +169,7 @@ EwoooC 目前已有 MOMO EDM / 節慶活動資料、`promo_products`、PChome - 2026-05-18 追加 alert review queue contract:`/api/market_intel/opportunity_alert_plan` 補上 `market_alert_review_queue` 的 preview contract、required / audit / forbidden fields、priority lanes 與索引規劃。此階段只定義資料契約,不建立 review table、不寫 queue contract、不執行審核、不派送 Telegram、不呼叫 LLM。 - 2026-05-18 追加 alert review queue migration blueprint:`market_alert_review_queue` 納入 `database/market_intel_models.py`、`migrations/032_market_intel_core_schema.sql` 與 migration blueprint,補齊 additive CREATE TABLE / index / grant / rollback draft。此階段仍不執行 migration、不連 DB、不建立 review queue、不寫入審核資料。 - 2026-05-18 追加 migration apply drill preview:`services.market_intel.migration_drill` 與 `/api/market_intel/migration_apply_drill` 集中正式 migration 前的只讀 schema probe、platform seed diff、人工套用清單、post-apply smoke、回滾演練與風險清單。預設 `execute=false` 不連 DB;人工 smoke 可用 `execute=true` 觸發只讀 catalog / seed diff probe,但仍不執行 psql、不跑 rollback、不寫 DB、不重啟容器、不掛 scheduler。 +- 2026-05-18 追加 migration catalog review preview:`services.market_intel.migration_catalog_review` 與 `/api/market_intel/migration_catalog_review` 將 schema_db_probe 與 platform_seed_db_diff 的只讀結果歸納為 `planned_no_probe` / `not_applied` / `partial_schema` / `already_applied` / `probe_error`,並輸出 risk level、apply path、finding 與下一步。預設 `execute=false` 不連 DB;`execute=true` 仍只查 catalog / seed diff,不執行 psql、不寫 DB、不跑 rollback、不掛 scheduler。 ### Phase 4:Coupang / Shopee Adapter diff --git a/routes/README.md b/routes/README.md index aa85a3e..9c462e6 100644 --- a/routes/README.md +++ b/routes/README.md @@ -19,7 +19,7 @@ | `edm_routes.py` | EDM 與節慶儀表板 | `/edm`, `/festival` | | `monthly_routes.py` | 月結分析 | `/monthly_summary_analysis`, `/api/monthly_summary_data` | | `daily_sales_routes.py` | 當日業績 | `/daily_sales`, `/daily_sales/export*` | -| `market_intel_routes.py` | 市場情報 Phase 43 migration apply drill preview | `/market_intel`, `/market_intel/*`, `/api/market_intel/status`, `/api/market_intel/schema`, `/api/market_intel/schema_smoke`, `/api/market_intel/schema_db_probe`, `/api/market_intel/platform_seed_db_diff`, `/api/market_intel/legacy_source_bridge`, `/api/market_intel/mcp_readiness`, `/api/market_intel/mcp_tool_contract`, `/api/market_intel/mcp_deploy_preflight`, `/api/market_intel/mcp_activation_runbook`, `/api/market_intel/mcp_fetch_gate`, `/api/market_intel/scheduler_plan`, `/api/market_intel/match_review_plan`, `/api/market_intel/opportunity_plan`, `/api/market_intel/opportunity_scoring_plan`, `/api/market_intel/opportunity_evidence_plan`, `/api/market_intel/opportunity_alert_plan`, `/api/market_intel/adapters`, `/api/market_intel/dry_run_plan`, `/api/market_intel/discovery_plan`, `/api/market_intel/manual_discovery`, `/api/market_intel/candidate_preview`, `/api/market_intel/platform_seed_plan`, `/api/market_intel/platform_seed_write_guard`, `/api/market_intel/platform_seed_writer_plan`, `/api/market_intel/migration_blueprint`, `/api/market_intel/migration_apply_drill`, `/api/market_intel/seed_writer_cli_status`, `/api/market_intel/write_approval_runbook`, `/api/market_intel/deployment_readiness` | +| `market_intel_routes.py` | 市場情報 Phase 44 migration catalog review preview | `/market_intel`, `/market_intel/*`, `/api/market_intel/status`, `/api/market_intel/schema`, `/api/market_intel/schema_smoke`, `/api/market_intel/schema_db_probe`, `/api/market_intel/platform_seed_db_diff`, `/api/market_intel/legacy_source_bridge`, `/api/market_intel/mcp_readiness`, `/api/market_intel/mcp_tool_contract`, `/api/market_intel/mcp_deploy_preflight`, `/api/market_intel/mcp_activation_runbook`, `/api/market_intel/mcp_fetch_gate`, `/api/market_intel/scheduler_plan`, `/api/market_intel/match_review_plan`, `/api/market_intel/opportunity_plan`, `/api/market_intel/opportunity_scoring_plan`, `/api/market_intel/opportunity_evidence_plan`, `/api/market_intel/opportunity_alert_plan`, `/api/market_intel/adapters`, `/api/market_intel/dry_run_plan`, `/api/market_intel/discovery_plan`, `/api/market_intel/manual_discovery`, `/api/market_intel/candidate_preview`, `/api/market_intel/platform_seed_plan`, `/api/market_intel/platform_seed_write_guard`, `/api/market_intel/platform_seed_writer_plan`, `/api/market_intel/migration_blueprint`, `/api/market_intel/migration_apply_drill`, `/api/market_intel/migration_catalog_review`, `/api/market_intel/seed_writer_cli_status`, `/api/market_intel/write_approval_runbook`, `/api/market_intel/deployment_readiness` | | `api_routes.py` | 通用任務與查詢 API | `/api/run_task`, `/api/history/*` | | `export_routes.py` | 匯出功能 | `/api/export/*` | | `import_routes.py` | 匯入功能 | `/api/import_excel`, `/api/import/monthly_summary` | diff --git a/routes/market_intel_routes.py b/routes/market_intel_routes.py index 7d9970b..238730a 100644 --- a/routes/market_intel_routes.py +++ b/routes/market_intel_routes.py @@ -271,6 +271,17 @@ def market_intel_migration_apply_drill(): ) +@market_intel_bp.route("/api/market_intel/migration_catalog_review") +@login_required +def market_intel_migration_catalog_review(): + execute_requested = request.args.get("execute", "false").lower() == "true" + return jsonify( + _service().build_migration_catalog_review( + execute_requested=execute_requested, + ) + ) + + @market_intel_bp.route("/api/market_intel/seed_writer_cli_status") @login_required def market_intel_seed_writer_cli_status(): diff --git a/services/market_intel/deployment_readiness.py b/services/market_intel/deployment_readiness.py index 9b94fc6..7f66502 100644 --- a/services/market_intel/deployment_readiness.py +++ b/services/market_intel/deployment_readiness.py @@ -24,6 +24,7 @@ def build_deployment_readiness_preview( opportunity_evidence_plan = service.build_opportunity_evidence_plan() opportunity_alert_plan = service.build_opportunity_alert_plan() migration_apply_drill = service.build_migration_apply_drill() + migration_catalog_review = service.build_migration_catalog_review() checks = { "schema_smoke_passed": bool(schema_smoke["passed"]), "feature_flags_default_safe": bool( @@ -120,6 +121,15 @@ def build_deployment_readiness_preview( and not migration_apply_drill["api_executes_migration"] and not migration_apply_drill["api_executes_rollback"] ), + "migration_catalog_review_preview_safe": bool( + migration_catalog_review["mode"] == "migration_catalog_review_preview" + and not migration_catalog_review["migration_executed"] + and not migration_catalog_review["rollback_executed"] + and not migration_catalog_review["database_write_executed"] + and not migration_catalog_review["database_commit_executed"] + and not migration_catalog_review["api_executes_migration"] + and not migration_catalog_review["api_executes_rollback"] + ), } ready_for_production_deploy = all(checks.values()) blocked_reasons = [ @@ -255,6 +265,7 @@ def build_deployment_readiness_preview( "/api/market_intel/opportunity_evidence_plan", "/api/market_intel/opportunity_alert_plan", "/api/market_intel/migration_apply_drill", + "/api/market_intel/migration_catalog_review", ], "status": status.to_dict(), "schema_smoke": schema_smoke, @@ -266,6 +277,7 @@ def build_deployment_readiness_preview( "write_approval_runbook": service.build_write_approval_runbook(), "migration_blueprint": service.build_migration_blueprint(), "migration_apply_drill": migration_apply_drill, + "migration_catalog_review": migration_catalog_review, "seed_writer_cli_status": service.build_seed_writer_cli_status(), "schema_db_probe": service.build_schema_db_probe(), "platform_seed_db_diff": service.build_platform_seed_db_diff(), diff --git a/services/market_intel/migration_catalog_review.py b/services/market_intel/migration_catalog_review.py new file mode 100644 index 0000000..b852ad5 --- /dev/null +++ b/services/market_intel/migration_catalog_review.py @@ -0,0 +1,331 @@ +"""市場情報正式 DB catalog 判讀。 + +本模組只整合 schema_db_probe 與 platform_seed_db_diff 的只讀結果,產生 +operator 可讀的風險判定;不執行 migration、不寫 DB、不建立 ORM session。 +""" + + +def _catalog_state(schema_db_probe): + mode = schema_db_probe.get("mode") + if mode == "schema_db_probe_planned": + return "planned_no_probe" + if mode == "schema_db_probe_error": + return "probe_error" + if not schema_db_probe.get("read_only_query_executed"): + return "planned_no_probe" + if schema_db_probe.get("schema_tables_exist"): + return "already_applied" + if schema_db_probe.get("existing_tables"): + return "partial_schema" + return "not_applied" + + +def _risk_for_state(catalog_state, seed_state): + if catalog_state == "probe_error": + return "high" + if catalog_state == "partial_schema": + return "high" + if seed_state == "differs": + return "medium" + if catalog_state == "not_applied": + return "low" + if catalog_state == "already_applied": + return "low" + return "info" + + +def _apply_path_for_state(catalog_state, *, execute_requested): + if catalog_state == "planned_no_probe": + return "run_execute_true_read_only_probe_first" + if catalog_state == "probe_error": + return "blocked_until_probe_error_reviewed" + if catalog_state == "partial_schema": + return "blocked_until_partial_schema_reviewed" + if catalog_state == "already_applied": + return "skip_apply_verify_post_apply_state" + if catalog_state == "not_applied" and execute_requested: + return "manual_apply_review_candidate" + return "run_execute_true_read_only_probe_first" + + +def _seed_state(platform_seed_db_diff): + mode = platform_seed_db_diff.get("mode") + if mode == "platform_seed_db_diff_planned": + return "planned_no_probe" + if mode == "platform_seed_db_diff_error": + return "probe_error" + if platform_seed_db_diff.get("changed_codes"): + return "differs" + if platform_seed_db_diff.get("missing_codes"): + return "missing" + if platform_seed_db_diff.get("seed_rows_ready"): + return "ready" + return "unknown" + + +def _finding(key, severity, label, status): + return { + "key": key, + "severity": severity, + "label": label, + "status": status, + } + + +def _build_findings(*, catalog_state, schema_db_probe, seed_state, platform_seed_db_diff): + findings = [] + missing_tables = schema_db_probe.get("missing_tables") or [] + existing_tables = schema_db_probe.get("existing_tables") or [] + if catalog_state == "planned_no_probe": + findings.append( + _finding( + "catalog_probe_not_executed", + "info", + "尚未執行正式 DB catalog 只讀探測,維持 planned 狀態。", + "planned", + ) + ) + elif catalog_state == "not_applied": + findings.append( + _finding( + "market_schema_not_applied", + "low", + "正式 DB 尚未建立 market_* schema,可進入人工 migration review。", + "review", + ) + ) + elif catalog_state == "partial_schema": + findings.append( + _finding( + "partial_market_schema_detected", + "high", + f"正式 DB 只存在 {len(existing_tables)} 張 market_* 表,缺少 {len(missing_tables)} 張。", + "blocked", + ) + ) + elif catalog_state == "already_applied": + findings.append( + _finding( + "market_schema_already_applied", + "low", + "正式 DB 已存在所有 market_* 表,migration apply 可能不需要重跑。", + "verify", + ) + ) + elif catalog_state == "probe_error": + findings.append( + _finding( + "catalog_probe_error", + "high", + "正式 DB catalog 只讀探測失敗,需人工檢查連線與權限。", + "blocked", + ) + ) + + if seed_state == "probe_error": + findings.append( + _finding( + "seed_diff_probe_error", + "medium", + "platform seed DB diff 只讀探測失敗;若 market_platforms 尚未建立,需等 migration 後重跑。", + "review", + ) + ) + elif seed_state == "planned_no_probe": + findings.append( + _finding( + "seed_diff_not_executed", + "info", + "尚未執行 platform seed DB diff,只能停留在 preview 判讀。", + "planned", + ) + ) + elif seed_state == "missing": + findings.append( + _finding( + "platform_seed_missing", + "low", + f"platform seed 尚缺 {len(platform_seed_db_diff.get('missing_codes') or [])} 筆,需等 migration 後獨立批准 seed writer。", + "review", + ) + ) + elif seed_state == "differs": + findings.append( + _finding( + "platform_seed_differs", + "medium", + f"platform seed 有 {len(platform_seed_db_diff.get('changed_codes') or [])} 筆與預期不同,需人工比對。", + "review", + ) + ) + elif seed_state == "ready": + findings.append( + _finding( + "platform_seed_ready", + "low", + "platform seed 與目前 DB 內容一致。", + "verify", + ) + ) + return findings + + +def build_migration_catalog_review_preview( + *, + runtime_status, + migration_blueprint, + schema_db_probe, + platform_seed_db_diff, +): + """建立正式 DB catalog 判讀 payload;不執行 migration 或 DB write。""" + execute_requested = bool( + schema_db_probe.get("execute_requested") + or platform_seed_db_diff.get("execute_requested") + ) + catalog_state = _catalog_state(schema_db_probe) + seed_state = _seed_state(platform_seed_db_diff) + risk_level = _risk_for_state(catalog_state, seed_state) + apply_path = _apply_path_for_state(catalog_state, execute_requested=execute_requested) + safe_checks = { + "schema_probe_read_only_or_planned": bool( + not schema_db_probe.get("database_write_executed") + and not schema_db_probe.get("database_commit_executed") + and not schema_db_probe.get("database_session_created") + and not schema_db_probe.get("migration_executed") + ), + "seed_diff_read_only_or_planned": bool( + not platform_seed_db_diff.get("database_write_executed") + and not platform_seed_db_diff.get("database_commit_executed") + and not platform_seed_db_diff.get("database_session_created") + and not platform_seed_db_diff.get("seed_write_executed") + and not platform_seed_db_diff.get("migration_executed") + ), + "migration_file_matches_blueprint": bool( + migration_blueprint.get("file_created") + and migration_blueprint.get("file_matches_blueprint") + ), + "forward_sql_additive_only": bool( + migration_blueprint.get("safety_checks", {}).get("forward_sql_additive_only") + and not migration_blueprint.get("forward_has_destructive_sql") + ), + "feature_flags_default_safe": bool( + not runtime_status.enabled + and not runtime_status.crawler_enabled + and not runtime_status.write_enabled + ), + "database_write_blocked": bool(not runtime_status.database_write_allowed), + "scheduler_detached": bool(not runtime_status.scheduler_attached), + } + review_ready = all(safe_checks.values()) + ready_for_manual_migration_review = bool( + review_ready + and execute_requested + and catalog_state == "not_applied" + ) + blocked_reasons = [ + "migration_not_executed_by_catalog_review", + "api_never_runs_migration", + "database_write_still_blocked", + ] + blocked_reasons.extend( + key for key, passed in safe_checks.items() + if not passed + ) + if not execute_requested: + blocked_reasons.append("execute_false_planned_only") + if catalog_state == "probe_error": + blocked_reasons.append("catalog_probe_error") + if catalog_state == "partial_schema": + blocked_reasons.append("partial_schema_requires_manual_reconciliation") + if catalog_state == "already_applied": + blocked_reasons.append("market_schema_already_present") + + expected_tables = schema_db_probe.get("expected_tables") or [] + existing_tables = schema_db_probe.get("existing_tables") or [] + missing_tables = schema_db_probe.get("missing_tables") or expected_tables + table_statuses = schema_db_probe.get("table_statuses") or [ + {"table": table_name, "exists": False} + for table_name in expected_tables + ] + + return { + "mode": "migration_catalog_review_preview", + "execute_requested": execute_requested, + "catalog_state": catalog_state, + "seed_state": seed_state, + "risk_level": risk_level, + "apply_path": apply_path, + "review_ready": review_ready, + "ready_for_manual_migration_review": ready_for_manual_migration_review, + "ready_to_apply_migration": False, + "migration_executed": False, + "rollback_executed": False, + "database_connection_opened": bool( + schema_db_probe.get("database_connection_opened") + or platform_seed_db_diff.get("database_connection_opened") + ), + "read_only_query_executed": bool( + schema_db_probe.get("read_only_query_executed") + or platform_seed_db_diff.get("read_only_query_executed") + ), + "database_session_created": False, + "explicit_transaction_opened": False, + "database_write_executed": False, + "database_commit_executed": False, + "external_network_executed": False, + "scheduler_attached": False, + "api_executes_migration": False, + "api_executes_rollback": False, + "safe_checks": safe_checks, + "blocked_reasons": blocked_reasons, + "findings": _build_findings( + catalog_state=catalog_state, + schema_db_probe=schema_db_probe, + seed_state=seed_state, + platform_seed_db_diff=platform_seed_db_diff, + ), + "table_catalog": { + "expected_count": len(expected_tables), + "existing_count": len(existing_tables), + "missing_count": len(missing_tables), + "expected_tables": expected_tables, + "existing_tables": existing_tables, + "missing_tables": missing_tables, + "table_statuses": table_statuses, + }, + "seed_catalog": { + "expected_seed_count": platform_seed_db_diff.get("expected_seed_count", 0), + "existing_seed_count": platform_seed_db_diff.get("existing_seed_count", 0), + "missing_codes": platform_seed_db_diff.get("missing_codes") or [], + "changed_codes": platform_seed_db_diff.get("changed_codes") or [], + "matching_codes": platform_seed_db_diff.get("matching_codes") or [], + "seed_rows_ready": bool(platform_seed_db_diff.get("seed_rows_ready")), + }, + "operator_next_steps": [ + { + "key": "run_catalog_review_execute_true", + "label": "人工 smoke 時呼叫 /api/market_intel/migration_catalog_review?execute=true,只讀判讀正式 DB 狀態", + "status": "required" if not execute_requested else "completed", + }, + { + "key": "review_catalog_state", + "label": "依 catalog_state 決定是否是未套用、部分套用、已套用或探測錯誤", + "status": "required", + }, + { + "key": "manual_migration_apply_window", + "label": "只有 catalog_state=not_applied 且人工批准後,才可在維護窗口手動執行 psql migration", + "status": "blocked_by_operator_approval", + }, + { + "key": "post_apply_read_only_probe", + "label": "套用後重跑 execute=true,只確認所有 market_* 表存在,不自動寫 seed", + "status": "required_after_manual_apply", + }, + ], + "manual_probe_targets": [ + "/api/market_intel/schema_db_probe?execute=true", + "/api/market_intel/platform_seed_db_diff?execute=true", + "/api/market_intel/migration_catalog_review?execute=true", + ], + } diff --git a/services/market_intel/service.py b/services/market_intel/service.py index db958fc..8125751 100644 --- a/services/market_intel/service.py +++ b/services/market_intel/service.py @@ -30,6 +30,9 @@ from services.market_intel.mcp_deploy_preflight import build_mcp_deploy_prefligh from services.market_intel.mcp_fetch_gate import build_mcp_fetch_gate_preview from services.market_intel.mcp_readiness import build_mcp_readiness_plan from services.market_intel.migration_blueprint import build_migration_blueprint +from services.market_intel.migration_catalog_review import ( + build_migration_catalog_review_preview, +) from services.market_intel.migration_drill import build_migration_apply_drill_preview from services.market_intel.opportunity_alerts import ( build_opportunity_alert_plan_preview, @@ -85,7 +88,7 @@ class MarketIntelRuntimeStatus: class MarketIntelService: """市場情報入口服務,先集中 feature gate 與安全狀態。""" - phase = "phase_43_migration_apply_drill_preview" + phase = "phase_44_migration_catalog_review_preview" def get_runtime_status(self) -> MarketIntelRuntimeStatus: return MarketIntelRuntimeStatus( @@ -548,6 +551,37 @@ class MarketIntelService: drill["phase"] = self.phase return drill + def build_migration_catalog_review( + self, + *, + execute_requested=False, + schema_engine=None, + seed_diff_engine=None, + database_url=None, + database_type=None, + ): + """建立正式 DB catalog 判讀;不執行 migration、不寫 DB。""" + schema_db_probe = self.build_schema_db_probe( + execute_requested=execute_requested, + engine=schema_engine, + database_url=database_url, + database_type=database_type, + ) + platform_seed_db_diff = self.build_platform_seed_db_diff( + execute_requested=execute_requested, + engine=seed_diff_engine, + database_url=database_url, + database_type=database_type, + ) + review = build_migration_catalog_review_preview( + runtime_status=self.get_runtime_status(), + migration_blueprint=self.build_migration_blueprint(), + schema_db_probe=schema_db_probe, + platform_seed_db_diff=platform_seed_db_diff, + ) + review["phase"] = self.phase + return review + def build_seed_writer_cli_status( self, platform_code="all", diff --git a/templates/market_intel/disabled.html b/templates/market_intel/disabled.html index a6d57a5..e99abee 100644 --- a/templates/market_intel/disabled.html +++ b/templates/market_intel/disabled.html @@ -630,6 +630,24 @@ +
+
+
+

MIGRATION / CATALOG REVIEW

+

正式 DB catalog 判讀

+
+ +
+
+ loading +
+
+
讀取正式 DB catalog 判讀中...
+
+
+
@@ -689,9 +707,10 @@ const opportunityAlertRoot = document.querySelector('[data-market-intel-opportunity-alert]'); const migrationRoot = document.querySelector('[data-market-intel-migration]'); const migrationDrillRoot = document.querySelector('[data-market-intel-migration-drill]'); + const catalogReviewRoot = document.querySelector('[data-market-intel-catalog-review]'); const approvalRoot = document.querySelector('[data-market-intel-approval]'); const deployRoot = document.querySelector('[data-market-intel-deploy]'); - if (!root && !writerRoot && !cliRoot && !dbProbeRoot && !seedDiffRoot && !legacyBridgeRoot && !mcpReadinessRoot && !mcpPreflightRoot && !mcpActivationRoot && !mcpFetchGateRoot && !schedulerRoot && !matchReviewRoot && !opportunityRoot && !opportunityScoringRoot && !opportunityEvidenceRoot && !opportunityAlertRoot && !migrationRoot && !migrationDrillRoot && !approvalRoot && !deployRoot) return; + if (!root && !writerRoot && !cliRoot && !dbProbeRoot && !seedDiffRoot && !legacyBridgeRoot && !mcpReadinessRoot && !mcpPreflightRoot && !mcpActivationRoot && !mcpFetchGateRoot && !schedulerRoot && !matchReviewRoot && !opportunityRoot && !opportunityScoringRoot && !opportunityEvidenceRoot && !opportunityAlertRoot && !migrationRoot && !migrationDrillRoot && !catalogReviewRoot && !approvalRoot && !deployRoot) return; const meta = root ? root.querySelector('[data-market-intel-preview-meta]') : null; const body = root ? root.querySelector('[data-market-intel-preview-body]') : null; @@ -765,6 +784,10 @@ const migrationDrillBody = migrationDrillRoot ? migrationDrillRoot.querySelector('[data-market-intel-migration-drill-body]') : null; const migrationDrillRefresh = migrationDrillRoot ? migrationDrillRoot.querySelector('[data-market-intel-migration-drill-refresh]') : null; const migrationDrillEndpoint = "{{ url_for('market_intel.market_intel_migration_apply_drill') }}?execute=false"; + const catalogReviewMeta = catalogReviewRoot ? catalogReviewRoot.querySelector('[data-market-intel-catalog-review-meta]') : null; + const catalogReviewBody = catalogReviewRoot ? catalogReviewRoot.querySelector('[data-market-intel-catalog-review-body]') : null; + const catalogReviewRefresh = catalogReviewRoot ? catalogReviewRoot.querySelector('[data-market-intel-catalog-review-refresh]') : null; + const catalogReviewEndpoint = "{{ url_for('market_intel.market_intel_migration_catalog_review') }}?execute=false"; const approvalMeta = approvalRoot ? approvalRoot.querySelector('[data-market-intel-approval-meta]') : null; const approvalBody = approvalRoot ? approvalRoot.querySelector('[data-market-intel-approval-body]') : null; const approvalRefresh = approvalRoot ? approvalRoot.querySelector('[data-market-intel-approval-refresh]') : null; @@ -2433,6 +2456,109 @@ } }; + const renderCatalogReviewMeta = data => { + catalogReviewMeta.innerHTML = [ + `mode=${data.mode || 'unknown'}`, + `catalog=${data.catalog_state || 'unknown'}`, + `risk=${data.risk_level || 'info'}`, + `tables=${data.table_catalog ? `${data.table_catalog.existing_count || 0}/${data.table_catalog.expected_count || 0}` : '0/0'}`, + `apply=${data.apply_path || 'blocked'}` + ].map(item => `${escapeHtml(item)}`).join(''); + }; + + const renderCatalogReviewBody = data => { + const blockers = (data.blocked_reasons || []).join(' / '); + const checks = Object.entries(data.safe_checks || {}); + const tableCatalog = data.table_catalog || {}; + const seedCatalog = data.seed_catalog || {}; + const findings = data.findings || []; + const nextSteps = data.operator_next_steps || []; + const probeTargets = data.manual_probe_targets || []; + const renderCheck = ([name, passed]) => ` +
+
+ ${escapeHtml(name)} +
+ ${passed ? 'PASS' : 'BLOCK'} +
+ `; + const renderNamedItem = item => ` +
+
+ ${escapeHtml(item.key || item.label || 'item')} + ${escapeHtml(item.label || item.key || '')} +
+ ${escapeHtml(item.status || item.severity || 'review').toUpperCase()} +
+ `; + catalogReviewBody.innerHTML = ` +
此卡只判讀正式 DB catalog 與 platform seed diff 的只讀結果;API 不執行 migration、不寫 DB、不跑 rollback、不掛 scheduler。${blockers ? `阻擋:${escapeHtml(blockers)}` : ''}
+
tables=${escapeHtml(tableCatalog.existing_count || 0)}/${escapeHtml(tableCatalog.expected_count || 0)} / missing=${escapeHtml(tableCatalog.missing_count || 0)} / seed_missing=${escapeHtml((seedCatalog.missing_codes || []).length)} / seed_changed=${escapeHtml((seedCatalog.changed_codes || []).length)}
+
+
+

SAFE CHECKS

+
${ + checks.length + ? checks.map(renderCheck).join('') + : '
尚未提供 safety checks。
' + }
+
+
+

CATALOG FINDINGS

+
${ + findings.length + ? findings.map(renderNamedItem).join('') + : '
尚未提供 catalog finding。
' + }
+
+
+

TABLE STATUS

+
${ + (tableCatalog.table_statuses || []).slice(0, 8).map(item => ` +
+
+ ${escapeHtml(item.table)} +
+ ${item.exists ? 'EXISTS' : 'MISSING'} +
+ `).join('') || '
尚未提供 table status。
' + }
+
+
+

NEXT STEPS

+
${ + nextSteps.length + ? nextSteps.map(renderNamedItem).join('') + : '
尚未提供下一步。
' + }
+
+
+

READ-ONLY PROBES

+
${ + probeTargets.length + ? probeTargets.map(item => renderNamedItem({ key: item, label: item, status: 'manual' })).join('') + : '
尚未提供 probe targets。
' + }
+
+
+ `; + }; + + const loadCatalogReview = async () => { + if (!catalogReviewMeta || !catalogReviewBody) return; + catalogReviewBody.innerHTML = '
讀取正式 DB catalog 判讀中...
'; + try { + const response = await fetch(catalogReviewEndpoint, { credentials: 'same-origin' }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const data = await response.json(); + renderCatalogReviewMeta(data); + renderCatalogReviewBody(data); + } catch (error) { + catalogReviewMeta.innerHTML = 'error'; + catalogReviewBody.innerHTML = `
正式 DB catalog 判讀讀取失敗:${escapeHtml(error.message)}
`; + } + }; + const renderApprovalMeta = data => { approvalMeta.innerHTML = [ `mode=${data.mode || 'unknown'}`, @@ -2646,6 +2772,9 @@ if (migrationDrillRefresh) { migrationDrillRefresh.addEventListener('click', loadMigrationDrill); } + if (catalogReviewRefresh) { + catalogReviewRefresh.addEventListener('click', loadCatalogReview); + } if (approvalRefresh) { approvalRefresh.addEventListener('click', loadApproval); } @@ -2670,6 +2799,7 @@ loadOpportunityAlert(); loadMigration(); loadMigrationDrill(); + loadCatalogReview(); loadApproval(); loadDeploy(); })(); diff --git a/tests/test_market_intel_skeleton.py b/tests/test_market_intel_skeleton.py index 70d002a..b20e01e 100644 --- a/tests/test_market_intel_skeleton.py +++ b/tests/test_market_intel_skeleton.py @@ -488,6 +488,10 @@ def test_market_intel_preview_template_uses_safe_fetch_false_endpoint(): assert "data-market-intel-migration-drill-checks" in template assert "data-market-intel-migration-drill-preapply" in template assert "data-market-intel-migration-drill-rollback" in template + assert "data-market-intel-catalog-review" in template + assert "data-market-intel-catalog-review-checks" in template + assert "data-market-intel-catalog-review-findings" in template + assert "data-market-intel-catalog-review-tables" in template assert "data-market-intel-approval" in template assert "data-market-intel-approval-gates" in template assert "data-market-intel-deploy" in template @@ -511,6 +515,7 @@ def test_market_intel_preview_template_uses_safe_fetch_false_endpoint(): assert "market_intel.market_intel_opportunity_alert_plan" in template assert "market_intel.market_intel_migration_blueprint" in template assert "market_intel.market_intel_migration_apply_drill" in template + assert "market_intel.market_intel_migration_catalog_review" in template assert "market_intel.market_intel_write_approval_runbook" in template assert "market_intel.market_intel_deployment_readiness" in template assert "required_manual_steps" in template @@ -535,7 +540,7 @@ def test_legacy_source_bridge_default_is_planned_only(): bridge = MarketIntelService().build_legacy_source_bridge() assert bridge["mode"] == "legacy_source_bridge_planned" - assert bridge["phase"] == "phase_43_migration_apply_drill_preview" + assert bridge["phase"] == "phase_44_migration_catalog_review_preview" assert bridge["execute_requested"] is False assert bridge["read_only_query_executed"] is False assert bridge["database_connection_opened"] is False @@ -693,7 +698,7 @@ def test_mcp_tool_contract_preview_is_read_only_and_whitelisted(): contract = MarketIntelService().build_mcp_tool_contract() assert contract["mode"] == "mcp_tool_contract_preview" - assert contract["phase"] == "phase_43_migration_apply_drill_preview" + assert contract["phase"] == "phase_44_migration_catalog_review_preview" assert contract["caller"] == "market_intel" assert contract["contract_ready"] is True assert contract["blocked_reasons"] == [] @@ -826,7 +831,7 @@ def test_mcp_activation_runbook_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "mcp_activation_runbook_preview" - assert data["phase"] == "phase_43_migration_apply_drill_preview" + assert data["phase"] == "phase_44_migration_catalog_review_preview" assert data["deployment_actions_executed"] is False assert data["docker_command_executed"] is False assert data["ssh_command_executed"] is False @@ -839,7 +844,7 @@ def test_mcp_fetch_gate_default_blocks_external_fetch(): gate = MarketIntelService().build_mcp_fetch_gate(fetch_requested=True) assert gate["mode"] == "mcp_fetch_gate_planned" - assert gate["phase"] == "phase_43_migration_apply_drill_preview" + assert gate["phase"] == "phase_44_migration_catalog_review_preview" assert gate["fetch_requested"] is True assert gate["manual_fetch_gate_open"] is False assert gate["network_request_allowed"] is False @@ -909,7 +914,7 @@ def test_mcp_fetch_gate_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "mcp_fetch_gate_planned" - assert data["phase"] == "phase_43_migration_apply_drill_preview" + assert data["phase"] == "phase_44_migration_catalog_review_preview" assert data["fetch_requested"] is False assert data["network_request_allowed"] is False assert data["external_network_executed"] is False @@ -921,7 +926,7 @@ def test_scheduler_plan_preview_blocks_job_attachment(): plan = MarketIntelService().build_scheduler_plan() assert plan["mode"] == "scheduler_attach_plan_preview" - assert plan["phase"] == "phase_43_migration_apply_drill_preview" + assert plan["phase"] == "phase_44_migration_catalog_review_preview" assert plan["ready_to_attach_scheduler"] is False assert plan["scheduler_attached"] is False assert plan["scheduler_registration_executed"] is False @@ -959,7 +964,7 @@ def test_scheduler_plan_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "scheduler_attach_plan_preview" - assert data["phase"] == "phase_43_migration_apply_drill_preview" + assert data["phase"] == "phase_44_migration_catalog_review_preview" assert data["scheduler_registration_executed"] is False assert data["crawler_job_started"] is False assert data["external_network_executed"] is False @@ -970,7 +975,7 @@ def test_match_review_plan_preview_blocks_auto_confirm(): plan = MarketIntelService().build_match_review_plan() assert plan["mode"] == "match_review_plan_preview" - assert plan["phase"] == "phase_43_migration_apply_drill_preview" + assert plan["phase"] == "phase_44_migration_catalog_review_preview" assert plan["ready_for_review_queue"] is False assert plan["review_queue_created"] is False assert plan["auto_match_executed"] is False @@ -1006,7 +1011,7 @@ def test_match_review_plan_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "match_review_plan_preview" - assert data["phase"] == "phase_43_migration_apply_drill_preview" + assert data["phase"] == "phase_44_migration_catalog_review_preview" assert data["review_queue_created"] is False assert data["auto_confirm_executed"] is False assert data["external_network_executed"] is False @@ -1017,7 +1022,7 @@ def test_opportunity_plan_preview_blocks_alerts_and_ai_summary(): plan = MarketIntelService().build_opportunity_plan() assert plan["mode"] == "opportunity_plan_preview" - assert plan["phase"] == "phase_43_migration_apply_drill_preview" + assert plan["phase"] == "phase_44_migration_catalog_review_preview" assert plan["ready_for_opportunity_queue"] is False assert plan["opportunity_queue_created"] is False assert plan["threat_alert_dispatched"] is False @@ -1058,7 +1063,7 @@ def test_opportunity_plan_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "opportunity_plan_preview" - assert data["phase"] == "phase_43_migration_apply_drill_preview" + assert data["phase"] == "phase_44_migration_catalog_review_preview" assert data["opportunity_queue_created"] is False assert data["threat_alert_dispatched"] is False assert data["ai_summary_generated"] is False @@ -1069,7 +1074,7 @@ def test_opportunity_scoring_plan_preview_blocks_scoring_and_alerts(): plan = MarketIntelService().build_opportunity_scoring_plan() assert plan["mode"] == "opportunity_scoring_plan_preview" - assert plan["phase"] == "phase_43_migration_apply_drill_preview" + assert plan["phase"] == "phase_44_migration_catalog_review_preview" assert plan["ready_for_scoring_job"] is False assert plan["scoring_job_created"] is False assert plan["score_calculation_executed"] is False @@ -1117,7 +1122,7 @@ def test_opportunity_scoring_plan_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "opportunity_scoring_plan_preview" - assert data["phase"] == "phase_43_migration_apply_drill_preview" + assert data["phase"] == "phase_44_migration_catalog_review_preview" assert data["scoring_job_created"] is False assert data["score_calculation_executed"] is False assert data["sample_scores_generated"] is False @@ -1129,7 +1134,7 @@ def test_opportunity_evidence_plan_preview_blocks_queries_and_alerts(): plan = MarketIntelService().build_opportunity_evidence_plan() assert plan["mode"] == "opportunity_evidence_plan_preview" - assert plan["phase"] == "phase_43_migration_apply_drill_preview" + assert plan["phase"] == "phase_44_migration_catalog_review_preview" assert plan["ready_for_evidence_bundle"] is False assert plan["evidence_bundle_created"] is False assert plan["evidence_query_executed"] is False @@ -1175,7 +1180,7 @@ def test_opportunity_evidence_plan_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "opportunity_evidence_plan_preview" - assert data["phase"] == "phase_43_migration_apply_drill_preview" + assert data["phase"] == "phase_44_migration_catalog_review_preview" assert data["evidence_bundle_created"] is False assert data["evidence_query_executed"] is False assert data["sample_evidence_generated"] is False @@ -1188,7 +1193,7 @@ def test_opportunity_alert_plan_preview_blocks_dispatch_and_llm_calls(): plan = MarketIntelService().build_opportunity_alert_plan() assert plan["mode"] == "opportunity_alert_plan_preview" - assert plan["phase"] == "phase_43_migration_apply_drill_preview" + assert plan["phase"] == "phase_44_migration_catalog_review_preview" assert plan["ready_for_alert_candidates"] is False assert plan["alert_candidate_created"] is False assert plan["alert_queue_created"] is False @@ -1273,7 +1278,7 @@ def test_opportunity_alert_plan_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "opportunity_alert_plan_preview" - assert data["phase"] == "phase_43_migration_apply_drill_preview" + assert data["phase"] == "phase_44_migration_catalog_review_preview" assert data["alert_candidate_created"] is False assert data["alert_queue_created"] is False assert data["review_queue_created"] is False @@ -1351,7 +1356,7 @@ def test_mcp_deploy_preflight_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "mcp_external_deploy_preflight_preview" - assert data["phase"] == "phase_43_migration_apply_drill_preview" + assert data["phase"] == "phase_44_migration_catalog_review_preview" assert data["deployment_actions_executed"] is False assert data["docker_command_executed"] is False assert data["ssh_command_executed"] is False @@ -1366,7 +1371,7 @@ def test_mcp_readiness_default_is_planned_only(monkeypatch): readiness = MarketIntelService().build_mcp_readiness() assert readiness["mode"] == "mcp_readiness_planned" - assert readiness["phase"] == "phase_43_migration_apply_drill_preview" + assert readiness["phase"] == "phase_44_migration_catalog_review_preview" assert readiness["execute_requested"] is False assert readiness["router_enabled"] is False assert readiness["external_mcp_complete"] is False @@ -1760,6 +1765,7 @@ def test_deployment_readiness_reports_app_only_release_gate(): assert readiness["checks"]["opportunity_evidence_plan_preview_safe"] is True assert readiness["checks"]["opportunity_alert_plan_preview_safe"] is True assert readiness["checks"]["migration_apply_drill_preview_safe"] is True + assert readiness["checks"]["migration_catalog_review_preview_safe"] is True assert readiness["checks"]["writer_plan_dry_run_only"] is True assert readiness["writer_plan_summary"]["writes_executed"] is False assert "readiness_checks_not_all_passed" not in readiness["blocked_reasons"] @@ -1787,6 +1793,7 @@ def test_deployment_readiness_reports_app_only_release_gate(): assert "/api/market_intel/opportunity_evidence_plan" in readiness["production_smoke_targets"] assert "/api/market_intel/opportunity_alert_plan" in readiness["production_smoke_targets"] assert "/api/market_intel/migration_apply_drill" in readiness["production_smoke_targets"] + assert "/api/market_intel/migration_catalog_review" in readiness["production_smoke_targets"] assert readiness["write_approval_runbook"]["ready_for_real_write"] is False assert readiness["write_approval_runbook"]["writes_executed"] is False assert readiness["migration_blueprint"]["migration_executed"] is False @@ -1797,6 +1804,11 @@ def test_deployment_readiness_reports_app_only_release_gate(): assert readiness["migration_apply_drill"]["rollback_executed"] is False assert readiness["migration_apply_drill"]["database_write_executed"] is False assert readiness["migration_apply_drill"]["api_executes_migration"] is False + assert readiness["migration_catalog_review"]["mode"] == "migration_catalog_review_preview" + assert readiness["migration_catalog_review"]["catalog_state"] == "planned_no_probe" + assert readiness["migration_catalog_review"]["migration_executed"] is False + assert readiness["migration_catalog_review"]["database_write_executed"] is False + assert readiness["migration_catalog_review"]["api_executes_migration"] is False assert readiness["schema_db_probe"]["read_only_query_executed"] is False assert readiness["platform_seed_db_diff"]["read_only_query_executed"] is False assert readiness["legacy_source_bridge"]["read_only_query_executed"] is False @@ -1914,7 +1926,7 @@ def test_migration_apply_drill_planned_is_safe_and_manual_only(): drill = MarketIntelService().build_migration_apply_drill() assert drill["mode"] == "migration_apply_drill_preview" - assert drill["phase"] == "phase_43_migration_apply_drill_preview" + assert drill["phase"] == "phase_44_migration_catalog_review_preview" assert drill["execute_requested"] is False assert drill["schema_state"] == "planned_no_db_probe" assert drill["drill_ready_for_operator_review"] is True @@ -2029,7 +2041,7 @@ def test_migration_apply_drill_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "migration_apply_drill_preview" - assert data["phase"] == "phase_43_migration_apply_drill_preview" + assert data["phase"] == "phase_44_migration_catalog_review_preview" assert data["execute_requested"] is False assert data["migration_executed"] is False assert data["rollback_executed"] is False @@ -2037,6 +2049,134 @@ def test_migration_apply_drill_route_is_preview_only(): assert data["api_executes_migration"] is False +def test_migration_catalog_review_planned_is_safe_and_diagnostic(): + review = MarketIntelService().build_migration_catalog_review() + + assert review["mode"] == "migration_catalog_review_preview" + assert review["phase"] == "phase_44_migration_catalog_review_preview" + assert review["execute_requested"] is False + assert review["catalog_state"] == "planned_no_probe" + assert review["seed_state"] == "planned_no_probe" + assert review["risk_level"] == "info" + assert review["apply_path"] == "run_execute_true_read_only_probe_first" + assert review["review_ready"] is True + assert review["ready_for_manual_migration_review"] is False + assert review["ready_to_apply_migration"] is False + assert review["read_only_query_executed"] is False + assert review["database_connection_opened"] is False + assert review["database_session_created"] is False + assert review["database_write_executed"] is False + assert review["database_commit_executed"] is False + assert review["migration_executed"] is False + assert review["rollback_executed"] is False + assert review["api_executes_migration"] is False + assert review["table_catalog"]["expected_count"] == 8 + assert review["table_catalog"]["existing_count"] == 0 + assert review["table_catalog"]["missing_count"] == 8 + assert "execute_false_planned_only" in review["blocked_reasons"] + assert "migration_not_executed_by_catalog_review" in review["blocked_reasons"] + assert {item["key"] for item in review["findings"]} >= { + "catalog_probe_not_executed", + "seed_diff_not_executed", + } + + +def test_migration_catalog_review_sqlite_read_only_detects_partial_schema(): + service = MarketIntelService() + seed_rows = service.build_platform_seed_plan()["seeds"] + engine = create_engine("sqlite:///:memory:") + with engine.begin() as conn: + conn.execute( + text( + """ + CREATE TABLE market_platforms ( + code TEXT PRIMARY KEY, + name TEXT, + base_url TEXT, + enabled BOOLEAN, + crawl_policy_json TEXT + ) + """ + ) + ) + for seed in seed_rows: + conn.execute( + text( + """ + INSERT INTO market_platforms + (code, name, base_url, enabled, crawl_policy_json) + VALUES + (:code, :name, :base_url, :enabled, :crawl_policy_json) + """ + ), + { + "code": seed["code"], + "name": seed["name"], + "base_url": seed["base_url"], + "enabled": seed["enabled"], + "crawl_policy_json": json.dumps( + seed["crawl_policy_json"], + ensure_ascii=False, + sort_keys=True, + ), + }, + ) + + review = service.build_migration_catalog_review( + execute_requested=True, + schema_engine=engine, + seed_diff_engine=engine, + database_type="sqlite", + ) + + assert review["mode"] == "migration_catalog_review_preview" + assert review["execute_requested"] is True + assert review["read_only_query_executed"] is True + assert review["database_connection_opened"] is True + assert review["catalog_state"] == "partial_schema" + assert review["seed_state"] == "ready" + assert review["risk_level"] == "high" + assert review["apply_path"] == "blocked_until_partial_schema_reviewed" + assert review["ready_for_manual_migration_review"] is False + assert review["table_catalog"]["expected_count"] == 8 + assert review["table_catalog"]["existing_count"] == 1 + assert review["table_catalog"]["missing_count"] == 7 + assert review["seed_catalog"]["seed_rows_ready"] is True + assert "partial_schema_requires_manual_reconciliation" in review["blocked_reasons"] + assert review["database_session_created"] is False + assert review["database_write_executed"] is False + assert review["database_commit_executed"] is False + assert review["migration_executed"] is False + assert {item["key"] for item in review["findings"]} >= { + "partial_market_schema_detected", + "platform_seed_ready", + } + + +def test_migration_catalog_review_route_is_preview_only(): + from routes.market_intel_routes import market_intel_bp + + app = Flask(__name__) + app.secret_key = "test-secret" + app.register_blueprint(market_intel_bp) + client = app.test_client() + with client.session_transaction() as session: + session["logged_in"] = True + + response = client.get("/api/market_intel/migration_catalog_review?execute=false") + data = response.get_json() + + assert response.status_code == 200 + assert data["mode"] == "migration_catalog_review_preview" + assert data["phase"] == "phase_44_migration_catalog_review_preview" + assert data["execute_requested"] is False + assert data["catalog_state"] == "planned_no_probe" + assert data["migration_executed"] is False + assert data["rollback_executed"] is False + assert data["database_write_executed"] is False + assert data["api_executes_migration"] is False + + def test_seed_writer_cli_status_blocks_real_write(): status = MarketIntelService().build_seed_writer_cli_status( platform_code="all",