From 841443f37ca2a4aa17f89c5ed5a4fcba73242160 Mon Sep 17 00:00:00 2001 From: OoO Date: Mon, 18 May 2026 20:05:24 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B8=82=E5=A0=B4=E6=83=85?= =?UTF-8?q?=E5=A0=B1=20migration=20=E5=A5=97=E7=94=A8=E6=BC=94=E7=B7=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 2 +- ...s-platform-market-campaign-intelligence.md | 1 + routes/README.md | 2 +- routes/market_intel_routes.py | 11 + services/market_intel/deployment_readiness.py | 12 + services/market_intel/migration_drill.py | 295 ++++++++++++++++++ services/market_intel/service.py | 59 +++- templates/market_intel/disabled.html | 149 ++++++++- tests/test_market_intel_skeleton.py | 177 +++++++++-- 9 files changed, 683 insertions(+), 25 deletions(-) create mode 100644 services/market_intel/migration_drill.py diff --git a/config.py b/config.py index f2cfeaa..717df16 100644 --- a/config.py +++ b/config.py @@ -320,7 +320,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.203" +SYSTEM_VERSION = "V10.204" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/adr/ADR-035-cross-platform-market-campaign-intelligence.md b/docs/adr/ADR-035-cross-platform-market-campaign-intelligence.md index 81135be..3e41cdb 100644 --- a/docs/adr/ADR-035-cross-platform-market-campaign-intelligence.md +++ b/docs/adr/ADR-035-cross-platform-market-campaign-intelligence.md @@ -168,6 +168,7 @@ EwoooC 目前已有 MOMO EDM / 節慶活動資料、`promo_products`、PChome - 2026-05-18 追加 deployment readiness modularization：將 `/api/market_intel/deployment_readiness` 的大型 app-only release gate 組裝邏輯由 `services.market_intel.service` 拆至 `services.market_intel.deployment_readiness`，主服務保留薄入口，避免後續 crawler / MCP / 審核功能推進時超過 800 行治理線；行為仍維持 preview-only，不執行 git、部署、SSH、migration 或 DB write。 - 2026-05-18 追加 alert review queue contract：`/api/market_intel/opportunity_alert_plan` 補上 `market_alert_review_queue` 的 preview contract、required / audit / forbidden fields、priority lanes 與索引規劃。此階段只定義資料契約，不建立 review table、不寫 queue contract、不執行審核、不派送 Telegram、不呼叫 LLM。 - 2026-05-18 追加 alert review queue migration blueprint：`market_alert_review_queue` 納入 `database/market_intel_models.py`、`migrations/032_market_intel_core_schema.sql` 與 migration blueprint，補齊 additive CREATE TABLE / index / grant / rollback draft。此階段仍不執行 migration、不連 DB、不建立 review queue、不寫入審核資料。 +- 2026-05-18 追加 migration apply drill preview：`services.market_intel.migration_drill` 與 `/api/market_intel/migration_apply_drill` 集中正式 migration 前的只讀 schema probe、platform seed diff、人工套用清單、post-apply smoke、回滾演練與風險清單。預設 `execute=false` 不連 DB；人工 smoke 可用 `execute=true` 觸發只讀 catalog / seed diff probe，但仍不執行 psql、不跑 rollback、不寫 DB、不重啟容器、不掛 scheduler。 ### Phase 4：Coupang / Shopee Adapter diff --git a/routes/README.md b/routes/README.md index d7c8826..aa85a3e 100644 --- a/routes/README.md +++ b/routes/README.md @@ -19,7 +19,7 @@ | `edm_routes.py` | EDM 與節慶儀表板 | `/edm`, `/festival` | | `monthly_routes.py` | 月結分析 | `/monthly_summary_analysis`, `/api/monthly_summary_data` | | `daily_sales_routes.py` | 當日業績 | `/daily_sales`, `/daily_sales/export*` | -| `market_intel_routes.py` | 市場情報 Phase 42 alert review queue migration blueprint | `/market_intel`, `/market_intel/*`, `/api/market_intel/status`, `/api/market_intel/schema`, `/api/market_intel/schema_smoke`, `/api/market_intel/schema_db_probe`, `/api/market_intel/platform_seed_db_diff`, `/api/market_intel/legacy_source_bridge`, `/api/market_intel/mcp_readiness`, `/api/market_intel/mcp_tool_contract`, `/api/market_intel/mcp_deploy_preflight`, `/api/market_intel/mcp_activation_runbook`, `/api/market_intel/mcp_fetch_gate`, `/api/market_intel/scheduler_plan`, `/api/market_intel/match_review_plan`, `/api/market_intel/opportunity_plan`, `/api/market_intel/opportunity_scoring_plan`, `/api/market_intel/opportunity_evidence_plan`, `/api/market_intel/opportunity_alert_plan`, `/api/market_intel/adapters`, `/api/market_intel/dry_run_plan`, `/api/market_intel/discovery_plan`, `/api/market_intel/manual_discovery`, `/api/market_intel/candidate_preview`, `/api/market_intel/platform_seed_plan`, `/api/market_intel/platform_seed_write_guard`, `/api/market_intel/platform_seed_writer_plan`, `/api/market_intel/migration_blueprint`, `/api/market_intel/seed_writer_cli_status`, `/api/market_intel/write_approval_runbook`, `/api/market_intel/deployment_readiness` | +| `market_intel_routes.py` | 市場情報 Phase 43 migration apply drill preview | `/market_intel`, `/market_intel/*`, `/api/market_intel/status`, `/api/market_intel/schema`, `/api/market_intel/schema_smoke`, `/api/market_intel/schema_db_probe`, `/api/market_intel/platform_seed_db_diff`, `/api/market_intel/legacy_source_bridge`, `/api/market_intel/mcp_readiness`, `/api/market_intel/mcp_tool_contract`, `/api/market_intel/mcp_deploy_preflight`, `/api/market_intel/mcp_activation_runbook`, `/api/market_intel/mcp_fetch_gate`, `/api/market_intel/scheduler_plan`, `/api/market_intel/match_review_plan`, `/api/market_intel/opportunity_plan`, `/api/market_intel/opportunity_scoring_plan`, `/api/market_intel/opportunity_evidence_plan`, `/api/market_intel/opportunity_alert_plan`, `/api/market_intel/adapters`, `/api/market_intel/dry_run_plan`, `/api/market_intel/discovery_plan`, `/api/market_intel/manual_discovery`, `/api/market_intel/candidate_preview`, `/api/market_intel/platform_seed_plan`, `/api/market_intel/platform_seed_write_guard`, `/api/market_intel/platform_seed_writer_plan`, `/api/market_intel/migration_blueprint`, `/api/market_intel/migration_apply_drill`, `/api/market_intel/seed_writer_cli_status`, `/api/market_intel/write_approval_runbook`, `/api/market_intel/deployment_readiness` | | `api_routes.py` | 通用任務與查詢 API | `/api/run_task`, `/api/history/*` | | `export_routes.py` | 匯出功能 | `/api/export/*` | | `import_routes.py` | 匯入功能 | `/api/import_excel`, `/api/import/monthly_summary` | diff --git a/routes/market_intel_routes.py b/routes/market_intel_routes.py index be6e2df..7d9970b 100644 --- a/routes/market_intel_routes.py +++ b/routes/market_intel_routes.py @@ -260,6 +260,17 @@ def market_intel_migration_blueprint(): return jsonify(_service().build_migration_blueprint()) +@market_intel_bp.route("/api/market_intel/migration_apply_drill") +@login_required +def market_intel_migration_apply_drill(): + execute_requested = request.args.get("execute", "false").lower() == "true" + return jsonify( + _service().build_migration_apply_drill( + execute_requested=execute_requested, + ) + ) + + @market_intel_bp.route("/api/market_intel/seed_writer_cli_status") @login_required def market_intel_seed_writer_cli_status(): diff --git a/services/market_intel/deployment_readiness.py b/services/market_intel/deployment_readiness.py index 5fec76d..9b94fc6 100644 --- a/services/market_intel/deployment_readiness.py +++ b/services/market_intel/deployment_readiness.py @@ -23,6 +23,7 @@ def build_deployment_readiness_preview( opportunity_scoring_plan = service.build_opportunity_scoring_plan() opportunity_evidence_plan = service.build_opportunity_evidence_plan() opportunity_alert_plan = service.build_opportunity_alert_plan() + migration_apply_drill = service.build_migration_apply_drill() checks = { "schema_smoke_passed": bool(schema_smoke["passed"]), "feature_flags_default_safe": bool( @@ -110,6 +111,15 @@ def build_deployment_readiness_preview( and not opportunity_alert_plan["database_write_executed"] and not opportunity_alert_plan["llm_call_executed"] ), + "migration_apply_drill_preview_safe": bool( + migration_apply_drill["mode"] == "migration_apply_drill_preview" + and not migration_apply_drill["migration_executed"] + and not migration_apply_drill["rollback_executed"] + and not migration_apply_drill["database_write_executed"] + and not migration_apply_drill["database_commit_executed"] + and not migration_apply_drill["api_executes_migration"] + and not migration_apply_drill["api_executes_rollback"] + ), } ready_for_production_deploy = all(checks.values()) blocked_reasons = [ @@ -244,6 +254,7 @@ def build_deployment_readiness_preview( "/api/market_intel/opportunity_scoring_plan", "/api/market_intel/opportunity_evidence_plan", "/api/market_intel/opportunity_alert_plan", + "/api/market_intel/migration_apply_drill", ], "status": status.to_dict(), "schema_smoke": schema_smoke, @@ -254,6 +265,7 @@ def build_deployment_readiness_preview( }, "write_approval_runbook": service.build_write_approval_runbook(), "migration_blueprint": service.build_migration_blueprint(), + "migration_apply_drill": migration_apply_drill, "seed_writer_cli_status": service.build_seed_writer_cli_status(), "schema_db_probe": service.build_schema_db_probe(), "platform_seed_db_diff": service.build_platform_seed_db_diff(), diff --git a/services/market_intel/migration_drill.py b/services/market_intel/migration_drill.py new file mode 100644 index 0000000..6915727 --- /dev/null +++ b/services/market_intel/migration_drill.py @@ -0,0 +1,295 @@ +"""市場情報 migration 套用前演練。 + +本模組只組裝正式 migration 前的檢查、人工步驟與回滾演練，不執行 SQL、 +不建立 ORM session、不寫入資料庫。 +""" + + +def _schema_state(schema_db_probe): + mode = schema_db_probe.get("mode") + if mode == "schema_db_probe_planned": + return "planned_no_db_probe" + if mode == "schema_db_probe_error": + return "probe_error" + if not schema_db_probe.get("read_only_query_executed"): + return "planned_no_db_probe" + if schema_db_probe.get("schema_tables_exist"): + return "already_applied" + existing_tables = schema_db_probe.get("existing_tables") or [] + if existing_tables: + return "partial_schema" + return "not_applied" + + +def _check_item(key, label, passed, status_when_blocked="blocked"): + return { + "key": key, + "label": label, + "passed": bool(passed), + "status": "pass" if passed else status_when_blocked, + } + + +def _manual_step(key, label, status="required"): + return { + "key": key, + "label": label, + "status": status, + } + + +def _build_blocked_reasons(*, checks, schema_state, execute_requested): + blocked_reasons = [ + "migration_not_executed_by_drill", + "api_never_runs_migration", + "backup_not_verified", + "operator_approval_missing", + "production_maintenance_window_required", + ] + blocked_reasons.extend( + key for key, passed in checks.items() + if not passed + ) + if not execute_requested: + blocked_reasons.append("read_only_db_probe_not_executed") + if schema_state == "probe_error": + blocked_reasons.append("schema_probe_error_requires_manual_review") + if schema_state == "partial_schema": + blocked_reasons.append("partial_market_schema_requires_manual_review") + if schema_state == "already_applied": + blocked_reasons.append("market_schema_already_present_apply_not_required") + return blocked_reasons + + +def build_migration_apply_drill_preview( + *, + runtime_status, + migration_blueprint, + schema_db_probe, + platform_seed_db_diff, +): + """建立 migration apply drill payload；不執行 migration 或 DB write。""" + execute_requested = bool( + schema_db_probe.get("execute_requested") + or platform_seed_db_diff.get("execute_requested") + ) + schema_state = _schema_state(schema_db_probe) + checks = { + "migration_file_matches_blueprint": bool( + migration_blueprint.get("file_created") + and migration_blueprint.get("file_matches_blueprint") + ), + "forward_sql_additive_only": bool( + migration_blueprint.get("safety_checks", {}).get("forward_sql_additive_only") + and not migration_blueprint.get("forward_has_destructive_sql") + ), + "schema_probe_read_only_or_planned": bool( + not schema_db_probe.get("database_write_executed") + and not schema_db_probe.get("database_commit_executed") + and not schema_db_probe.get("migration_executed") + and not schema_db_probe.get("database_session_created") + ), + "seed_diff_read_only_or_planned": bool( + not platform_seed_db_diff.get("database_write_executed") + and not platform_seed_db_diff.get("database_commit_executed") + and not platform_seed_db_diff.get("migration_executed") + and not platform_seed_db_diff.get("database_session_created") + and not platform_seed_db_diff.get("seed_write_executed") + ), + "feature_flags_default_safe": bool( + not runtime_status.enabled + and not runtime_status.crawler_enabled + and not runtime_status.write_enabled + ), + "database_write_blocked": bool(not runtime_status.database_write_allowed), + "scheduler_detached": bool(not runtime_status.scheduler_attached), + "external_fetch_blocked": bool(not runtime_status.crawler_enabled), + "rollback_manual_only": bool( + migration_blueprint.get("rollback_requires_manual_approval") + ), + } + drill_ready_for_operator_review = all(checks.values()) + ready_for_manual_apply_review = bool( + drill_ready_for_operator_review + and execute_requested + and schema_state in {"not_applied", "partial_schema"} + ) + ready_to_apply_migration = False + + pre_apply_checklist = [ + _manual_step( + "confirm_worktree_and_release_scope", + "確認本次只包含 market_intel migration / drill 相關變更", + ), + _manual_step( + "run_backup_system", + "套正式 migration 前先完成 python backup_system.py 並確認備份可用", + ), + _manual_step( + "run_schema_probe_execute_true", + "人工呼叫 /api/market_intel/schema_db_probe?execute=true 做只讀 catalog 檢查", + ), + _manual_step( + "run_seed_diff_execute_true", + "人工呼叫 /api/market_intel/platform_seed_db_diff?execute=true 做只讀 seed diff", + ), + _manual_step( + "review_forward_sql", + "確認 forward SQL 只有 CREATE TABLE / CREATE INDEX / GRANT 類 additive 操作", + ), + _manual_step( + "open_maintenance_window", + "確認正式 DB migration 維護窗口與操作員身分", + ), + _manual_step( + "apply_psql_manually", + "由操作員手動執行 migration command；API 不執行 psql", + ), + _manual_step( + "post_apply_smoke", + "套用後驗證 /health、schema_db_probe?execute=true、deployment_readiness", + ), + ] + post_apply_verification = [ + _manual_step("health_endpoint", "驗證 /health healthy 且版本正確"), + _manual_step( + "schema_probe_all_tables", + "確認 schema_db_probe?execute=true 顯示所有 market_* 表存在", + ), + _manual_step( + "seed_writer_cli_status", + "確認 seed writer CLI status 仍未自動寫入，等待下一次獨立批准", + ), + _manual_step( + "flags_still_off", + "確認 MARKET_INTEL_* flags 仍全關，crawler 與 scheduler 未啟用", + ), + ] + rollback_drill = { + "mode": "rollback_drill_preview", + "rollback_sql_available": bool(migration_blueprint.get("rollback_sql")), + "rollback_executed": False, + "database_write_executed": False, + "database_commit_executed": False, + "requires_manual_approval": True, + "requires_backup_before_rollback": True, + "requires_data_loss_review": True, + "rollback_table_order": list(reversed(migration_blueprint.get("expected_tables") or [])), + "manual_command_shape": ( + "將已審核 rollback SQL 寫入臨時檔後，手動執行 " + "psql \"$DATABASE_URL\" -v ON_ERROR_STOP=1 -f " + ), + "fallback_first": [ + "先關閉 MARKET_INTEL_* feature flags", + "確認 scheduler 未掛載市場情報 job", + "app-only 回退上一版並只 recreate momo-app", + "只有在確認 market_* 無需保留資料時才評估 rollback SQL", + ], + } + risk_register = [ + { + "key": "partial_schema", + "label": "若只建立部分 market_* 表，必須人工比對 migration 檔與 catalog，再決定重跑或補修。", + "severity": "medium", + }, + { + "key": "grant_or_sequence_missing", + "label": "表存在不代表 GRANT / sequence 權限完整；post-apply smoke 需檢查 seed writer 仍可用。", + "severity": "medium", + }, + { + "key": "rollback_drops_market_data", + "label": "rollback SQL 會 drop market_* 表，未來有正式資料後不可當一般回退手段。", + "severity": "high", + }, + { + "key": "api_boundary", + "label": "API 只呈現 drill，不得替操作員執行 psql、docker、SSH 或 DB write。", + "severity": "high", + }, + ] + + return { + "mode": "migration_apply_drill_preview", + "execute_requested": execute_requested, + "schema_state": schema_state, + "drill_ready_for_operator_review": drill_ready_for_operator_review, + "ready_for_manual_apply_review": ready_for_manual_apply_review, + "ready_to_apply_migration": ready_to_apply_migration, + "migration_executed": False, + "rollback_executed": False, + "database_connection_opened": bool( + schema_db_probe.get("database_connection_opened") + or platform_seed_db_diff.get("database_connection_opened") + ), + "read_only_query_executed": bool( + schema_db_probe.get("read_only_query_executed") + or platform_seed_db_diff.get("read_only_query_executed") + ), + "database_session_created": False, + "explicit_transaction_opened": False, + "database_write_executed": False, + "database_commit_executed": False, + "external_network_executed": False, + "scheduler_attached": False, + "api_executes_migration": False, + "api_executes_rollback": False, + "api_executes_docker": False, + "api_executes_ssh": False, + "checks": checks, + "check_items": [ + _check_item(key, key, passed) + for key, passed in checks.items() + ], + "blocked_reasons": _build_blocked_reasons( + checks=checks, + schema_state=schema_state, + execute_requested=execute_requested, + ), + "pre_apply_checklist": pre_apply_checklist, + "post_apply_verification": post_apply_verification, + "rollback_drill": rollback_drill, + "risk_register": risk_register, + "manual_commands": { + "schema_probe": "/api/market_intel/schema_db_probe?execute=true", + "platform_seed_db_diff": "/api/market_intel/platform_seed_db_diff?execute=true", + "migration_apply": migration_blueprint.get("command_plan", {}) + .get("migration_apply_command", {}) + .get("command", ""), + }, + "migration_blueprint_summary": { + "suggested_filename": migration_blueprint.get("suggested_filename"), + "file_created": bool(migration_blueprint.get("file_created")), + "file_matches_blueprint": bool( + migration_blueprint.get("file_matches_blueprint") + ), + "table_count": migration_blueprint.get("table_count", 0), + "forward_statement_count": migration_blueprint.get( + "forward_statement_count", + 0, + ), + "rollback_statement_count": migration_blueprint.get( + "rollback_statement_count", + 0, + ), + }, + "schema_db_probe_summary": { + "mode": schema_db_probe.get("mode"), + "read_only_query_executed": bool( + schema_db_probe.get("read_only_query_executed") + ), + "schema_tables_exist": bool(schema_db_probe.get("schema_tables_exist")), + "missing_tables": schema_db_probe.get("missing_tables") or [], + "existing_tables": schema_db_probe.get("existing_tables") or [], + }, + "platform_seed_db_diff_summary": { + "mode": platform_seed_db_diff.get("mode"), + "read_only_query_executed": bool( + platform_seed_db_diff.get("read_only_query_executed") + ), + "seed_rows_ready": bool(platform_seed_db_diff.get("seed_rows_ready")), + "missing_codes": platform_seed_db_diff.get("missing_codes") or [], + "changed_codes": platform_seed_db_diff.get("changed_codes") or [], + "matching_codes": platform_seed_db_diff.get("matching_codes") or [], + }, + } diff --git a/services/market_intel/service.py b/services/market_intel/service.py index 5155f34..db958fc 100644 --- a/services/market_intel/service.py +++ b/services/market_intel/service.py @@ -30,6 +30,7 @@ from services.market_intel.mcp_deploy_preflight import build_mcp_deploy_prefligh from services.market_intel.mcp_fetch_gate import build_mcp_fetch_gate_preview from services.market_intel.mcp_readiness import build_mcp_readiness_plan from services.market_intel.migration_blueprint import build_migration_blueprint +from services.market_intel.migration_drill import build_migration_apply_drill_preview from services.market_intel.opportunity_alerts import ( build_opportunity_alert_plan_preview, ) @@ -84,7 +85,7 @@ class MarketIntelRuntimeStatus: class MarketIntelService: """市場情報入口服務，先集中 feature gate 與安全狀態。""" - phase = "phase_42_alert_review_queue_migration_blueprint" + phase = "phase_43_migration_apply_drill_preview" def get_runtime_status(self) -> MarketIntelRuntimeStatus: return MarketIntelRuntimeStatus( @@ -288,21 +289,42 @@ class MarketIntelService: "expected_tables": self.get_schema_tables(), } - def build_schema_db_probe(self, *, execute_requested=False): + def build_schema_db_probe( + self, + *, + execute_requested=False, + engine=None, + database_url=None, + database_type=None, + ): """回報正式 DB schema 只讀探針；預設不連 DB。""" probe = build_schema_db_probe_plan( MARKET_INTEL_TABLES, execute_requested=execute_requested, + engine=engine, + database_url=database_url, + database_type=database_type, ) probe["phase"] = self.phase return probe - def build_platform_seed_db_diff(self, platform_code="all", *, execute_requested=False): + def build_platform_seed_db_diff( + self, + platform_code="all", + *, + execute_requested=False, + engine=None, + database_url=None, + database_type=None, + ): """回報 platform seed 與 DB 的只讀差異；預設不連 DB。""" seed_plan = self.build_platform_seed_plan(platform_code=platform_code) diff = build_platform_seed_db_diff_plan( seed_plan, execute_requested=execute_requested, + engine=engine, + database_url=database_url, + database_type=database_type, ) diff["phase"] = self.phase diff["platform_code"] = platform_code or "all" @@ -495,6 +517,37 @@ class MarketIntelService: blueprint["phase"] = self.phase return blueprint + def build_migration_apply_drill( + self, + *, + execute_requested=False, + schema_engine=None, + seed_diff_engine=None, + database_url=None, + database_type=None, + ): + """建立正式 migration 前的只讀演練；不執行 migration 或 rollback。""" + schema_db_probe = self.build_schema_db_probe( + execute_requested=execute_requested, + engine=schema_engine, + database_url=database_url, + database_type=database_type, + ) + platform_seed_db_diff = self.build_platform_seed_db_diff( + execute_requested=execute_requested, + engine=seed_diff_engine, + database_url=database_url, + database_type=database_type, + ) + drill = build_migration_apply_drill_preview( + runtime_status=self.get_runtime_status(), + migration_blueprint=self.build_migration_blueprint(), + schema_db_probe=schema_db_probe, + platform_seed_db_diff=platform_seed_db_diff, + ) + drill["phase"] = self.phase + return drill + def build_seed_writer_cli_status( self, platform_code="all", diff --git a/templates/market_intel/disabled.html b/templates/market_intel/disabled.html index 460fa0a..a6d57a5 100644 --- a/templates/market_intel/disabled.html +++ b/templates/market_intel/disabled.html @@ -612,6 +612,24 @@ +
+
+
+

MIGRATION / APPLY DRILL

+

Migration 套用演練

+
+ +
+
+ loading +
+
+
讀取 migration 套用演練中...
+
+
+
@@ -670,9 +688,10 @@ const opportunityEvidenceRoot = document.querySelector('[data-market-intel-opportunity-evidence]'); const opportunityAlertRoot = document.querySelector('[data-market-intel-opportunity-alert]'); const migrationRoot = document.querySelector('[data-market-intel-migration]'); + const migrationDrillRoot = document.querySelector('[data-market-intel-migration-drill]'); const approvalRoot = document.querySelector('[data-market-intel-approval]'); const deployRoot = document.querySelector('[data-market-intel-deploy]'); - if (!root && !writerRoot && !cliRoot && !dbProbeRoot && !seedDiffRoot && !legacyBridgeRoot && !mcpReadinessRoot && !mcpPreflightRoot && !mcpActivationRoot && !mcpFetchGateRoot && !schedulerRoot && !matchReviewRoot && !opportunityRoot && !opportunityScoringRoot && !opportunityEvidenceRoot && !opportunityAlertRoot && !migrationRoot && !approvalRoot && !deployRoot) return; + if (!root && !writerRoot && !cliRoot && !dbProbeRoot && !seedDiffRoot && !legacyBridgeRoot && !mcpReadinessRoot && !mcpPreflightRoot && !mcpActivationRoot && !mcpFetchGateRoot && !schedulerRoot && !matchReviewRoot && !opportunityRoot && !opportunityScoringRoot && !opportunityEvidenceRoot && !opportunityAlertRoot && !migrationRoot && !migrationDrillRoot && !approvalRoot && !deployRoot) return; const meta = root ? root.querySelector('[data-market-intel-preview-meta]') : null; const body = root ? root.querySelector('[data-market-intel-preview-body]') : null; @@ -742,6 +761,10 @@ const migrationBody = migrationRoot ? migrationRoot.querySelector('[data-market-intel-migration-body]') : null; const migrationRefresh = migrationRoot ? migrationRoot.querySelector('[data-market-intel-migration-refresh]') : null; const migrationEndpoint = "{{ url_for('market_intel.market_intel_migration_blueprint') }}"; + const migrationDrillMeta = migrationDrillRoot ? migrationDrillRoot.querySelector('[data-market-intel-migration-drill-meta]') : null; + const migrationDrillBody = migrationDrillRoot ? migrationDrillRoot.querySelector('[data-market-intel-migration-drill-body]') : null; + const migrationDrillRefresh = migrationDrillRoot ? migrationDrillRoot.querySelector('[data-market-intel-migration-drill-refresh]') : null; + const migrationDrillEndpoint = "{{ url_for('market_intel.market_intel_migration_apply_drill') }}?execute=false"; const approvalMeta = approvalRoot ? approvalRoot.querySelector('[data-market-intel-approval-meta]') : null; const approvalBody = approvalRoot ? approvalRoot.querySelector('[data-market-intel-approval-body]') : null; const approvalRefresh = approvalRoot ? approvalRoot.querySelector('[data-market-intel-approval-refresh]') : null; @@ -2290,6 +2313,126 @@ } }; + const renderMigrationDrillMeta = data => { + migrationDrillMeta.innerHTML = [ + `mode=${data.mode || 'unknown'}`, + `schema=${data.schema_state || 'unknown'}`, + `probe=${data.read_only_query_executed ? 'read-only' : 'planned'}`, + `review=${data.drill_ready_for_operator_review ? 'ready' : 'blocked'}`, + `apply=${data.ready_to_apply_migration ? 'yes' : 'manual'}` + ].map(item => `${escapeHtml(item)}`).join(''); + }; + + const renderMigrationDrillBody = data => { + const blockers = (data.blocked_reasons || []).join(' / '); + const checks = Object.entries(data.checks || {}); + const preApply = data.pre_apply_checklist || []; + const postApply = data.post_apply_verification || []; + const rollback = data.rollback_drill || {}; + const risks = data.risk_register || []; + const commands = data.manual_commands || {}; + const schema = data.schema_db_probe_summary || {}; + const seedDiff = data.platform_seed_db_diff_summary || {}; + const renderStep = item => ` +
+
+ ${escapeHtml(item.key || item.label || 'step')} + ${escapeHtml(item.label || item.key || '')} +
+ ${escapeHtml(item.status || 'required').toUpperCase()} +
+ `; + migrationDrillBody.innerHTML = ` +
此演練只集中正式 migration 前的只讀探測、人工套用清單與回滾演練；API 不執行 psql、不寫 DB、不跑 rollback、不重啟容器。${blockers ? `阻擋：${escapeHtml(blockers)}` : ''}
+
schema_missing=${escapeHtml((schema.missing_tables || []).length)} / seed_missing=${escapeHtml((seedDiff.missing_codes || []).length)} / command=${escapeHtml(commands.migration_apply || '')}
+
+
+

DRILL CHECKS

+
${ + checks.length + ? checks.map(([name, passed]) => ` +
+
+ ${escapeHtml(name)} +
+ ${passed ? 'PASS' : 'BLOCK'} +
+ `).join('') + : '
尚未提供 drill check。
' + }
+
+
+

PRE-APPLY

+
${ + preApply.length + ? preApply.map(renderStep).join('') + : '
尚未提供套用前清單。
' + }
+
+
+

POST-APPLY

+
${ + postApply.length + ? postApply.map(renderStep).join('') + : '
尚未提供套用後驗證。
' + }
+
+
+

ROLLBACK DRILL

+
+
+
+ ${escapeHtml(rollback.mode || 'rollback_drill')} + ${escapeHtml(rollback.manual_command_shape || '')} +
+ ${rollback.rollback_executed ? 'EXECUTED' : 'MANUAL'} +
+ ${(rollback.fallback_first || []).map((item, index) => ` +
+
+ ${escapeHtml(`fallback_${index + 1}`)} + ${escapeHtml(item)} +
+ FIRST +
+ `).join('')} +
+
+
+

RISK REGISTER

+
${ + risks.length + ? risks.map(item => ` +
+
+ ${escapeHtml(item.key)} + ${escapeHtml(item.label)} +
+ ${escapeHtml(item.severity || 'medium').toUpperCase()} +
+ `).join('') + : '
尚未提供風險清單。
' + }
+
+
+ `; + }; + + const loadMigrationDrill = async () => { + if (!migrationDrillMeta || !migrationDrillBody) return; + migrationDrillBody.innerHTML = '
讀取 migration 套用演練中...
'; + try { + const response = await fetch(migrationDrillEndpoint, { credentials: 'same-origin' }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const data = await response.json(); + renderMigrationDrillMeta(data); + renderMigrationDrillBody(data); + } catch (error) { + migrationDrillMeta.innerHTML = 'error'; + migrationDrillBody.innerHTML = `
migration 套用演練讀取失敗：${escapeHtml(error.message)}
`; + } + }; + const renderApprovalMeta = data => { approvalMeta.innerHTML = [ `mode=${data.mode || 'unknown'}`, @@ -2500,6 +2643,9 @@ if (migrationRefresh) { migrationRefresh.addEventListener('click', loadMigration); } + if (migrationDrillRefresh) { + migrationDrillRefresh.addEventListener('click', loadMigrationDrill); + } if (approvalRefresh) { approvalRefresh.addEventListener('click', loadApproval); } @@ -2523,6 +2669,7 @@ loadOpportunityEvidence(); loadOpportunityAlert(); loadMigration(); + loadMigrationDrill(); loadApproval(); loadDeploy(); })(); diff --git a/tests/test_market_intel_skeleton.py b/tests/test_market_intel_skeleton.py index 57d7409..70d002a 100644 --- a/tests/test_market_intel_skeleton.py +++ b/tests/test_market_intel_skeleton.py @@ -484,6 +484,10 @@ def test_market_intel_preview_template_uses_safe_fetch_false_endpoint(): assert "data-market-intel-opportunity-alert-sequence" in template assert "data-market-intel-migration" in template assert "data-market-intel-migration-tables" in template + assert "data-market-intel-migration-drill" in template + assert "data-market-intel-migration-drill-checks" in template + assert "data-market-intel-migration-drill-preapply" in template + assert "data-market-intel-migration-drill-rollback" in template assert "data-market-intel-approval" in template assert "data-market-intel-approval-gates" in template assert "data-market-intel-deploy" in template @@ -506,6 +510,7 @@ def test_market_intel_preview_template_uses_safe_fetch_false_endpoint(): assert "market_intel.market_intel_opportunity_evidence_plan" in template assert "market_intel.market_intel_opportunity_alert_plan" in template assert "market_intel.market_intel_migration_blueprint" in template + assert "market_intel.market_intel_migration_apply_drill" in template assert "market_intel.market_intel_write_approval_runbook" in template assert "market_intel.market_intel_deployment_readiness" in template assert "required_manual_steps" in template @@ -530,7 +535,7 @@ def test_legacy_source_bridge_default_is_planned_only(): bridge = MarketIntelService().build_legacy_source_bridge() assert bridge["mode"] == "legacy_source_bridge_planned" - assert bridge["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert bridge["phase"] == "phase_43_migration_apply_drill_preview" assert bridge["execute_requested"] is False assert bridge["read_only_query_executed"] is False assert bridge["database_connection_opened"] is False @@ -688,7 +693,7 @@ def test_mcp_tool_contract_preview_is_read_only_and_whitelisted(): contract = MarketIntelService().build_mcp_tool_contract() assert contract["mode"] == "mcp_tool_contract_preview" - assert contract["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert contract["phase"] == "phase_43_migration_apply_drill_preview" assert contract["caller"] == "market_intel" assert contract["contract_ready"] is True assert contract["blocked_reasons"] == [] @@ -821,7 +826,7 @@ def test_mcp_activation_runbook_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "mcp_activation_runbook_preview" - assert data["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert data["phase"] == "phase_43_migration_apply_drill_preview" assert data["deployment_actions_executed"] is False assert data["docker_command_executed"] is False assert data["ssh_command_executed"] is False @@ -834,7 +839,7 @@ def test_mcp_fetch_gate_default_blocks_external_fetch(): gate = MarketIntelService().build_mcp_fetch_gate(fetch_requested=True) assert gate["mode"] == "mcp_fetch_gate_planned" - assert gate["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert gate["phase"] == "phase_43_migration_apply_drill_preview" assert gate["fetch_requested"] is True assert gate["manual_fetch_gate_open"] is False assert gate["network_request_allowed"] is False @@ -904,7 +909,7 @@ def test_mcp_fetch_gate_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "mcp_fetch_gate_planned" - assert data["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert data["phase"] == "phase_43_migration_apply_drill_preview" assert data["fetch_requested"] is False assert data["network_request_allowed"] is False assert data["external_network_executed"] is False @@ -916,7 +921,7 @@ def test_scheduler_plan_preview_blocks_job_attachment(): plan = MarketIntelService().build_scheduler_plan() assert plan["mode"] == "scheduler_attach_plan_preview" - assert plan["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert plan["phase"] == "phase_43_migration_apply_drill_preview" assert plan["ready_to_attach_scheduler"] is False assert plan["scheduler_attached"] is False assert plan["scheduler_registration_executed"] is False @@ -954,7 +959,7 @@ def test_scheduler_plan_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "scheduler_attach_plan_preview" - assert data["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert data["phase"] == "phase_43_migration_apply_drill_preview" assert data["scheduler_registration_executed"] is False assert data["crawler_job_started"] is False assert data["external_network_executed"] is False @@ -965,7 +970,7 @@ def test_match_review_plan_preview_blocks_auto_confirm(): plan = MarketIntelService().build_match_review_plan() assert plan["mode"] == "match_review_plan_preview" - assert plan["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert plan["phase"] == "phase_43_migration_apply_drill_preview" assert plan["ready_for_review_queue"] is False assert plan["review_queue_created"] is False assert plan["auto_match_executed"] is False @@ -1001,7 +1006,7 @@ def test_match_review_plan_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "match_review_plan_preview" - assert data["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert data["phase"] == "phase_43_migration_apply_drill_preview" assert data["review_queue_created"] is False assert data["auto_confirm_executed"] is False assert data["external_network_executed"] is False @@ -1012,7 +1017,7 @@ def test_opportunity_plan_preview_blocks_alerts_and_ai_summary(): plan = MarketIntelService().build_opportunity_plan() assert plan["mode"] == "opportunity_plan_preview" - assert plan["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert plan["phase"] == "phase_43_migration_apply_drill_preview" assert plan["ready_for_opportunity_queue"] is False assert plan["opportunity_queue_created"] is False assert plan["threat_alert_dispatched"] is False @@ -1053,7 +1058,7 @@ def test_opportunity_plan_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "opportunity_plan_preview" - assert data["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert data["phase"] == "phase_43_migration_apply_drill_preview" assert data["opportunity_queue_created"] is False assert data["threat_alert_dispatched"] is False assert data["ai_summary_generated"] is False @@ -1064,7 +1069,7 @@ def test_opportunity_scoring_plan_preview_blocks_scoring_and_alerts(): plan = MarketIntelService().build_opportunity_scoring_plan() assert plan["mode"] == "opportunity_scoring_plan_preview" - assert plan["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert plan["phase"] == "phase_43_migration_apply_drill_preview" assert plan["ready_for_scoring_job"] is False assert plan["scoring_job_created"] is False assert plan["score_calculation_executed"] is False @@ -1112,7 +1117,7 @@ def test_opportunity_scoring_plan_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "opportunity_scoring_plan_preview" - assert data["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert data["phase"] == "phase_43_migration_apply_drill_preview" assert data["scoring_job_created"] is False assert data["score_calculation_executed"] is False assert data["sample_scores_generated"] is False @@ -1124,7 +1129,7 @@ def test_opportunity_evidence_plan_preview_blocks_queries_and_alerts(): plan = MarketIntelService().build_opportunity_evidence_plan() assert plan["mode"] == "opportunity_evidence_plan_preview" - assert plan["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert plan["phase"] == "phase_43_migration_apply_drill_preview" assert plan["ready_for_evidence_bundle"] is False assert plan["evidence_bundle_created"] is False assert plan["evidence_query_executed"] is False @@ -1170,7 +1175,7 @@ def test_opportunity_evidence_plan_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "opportunity_evidence_plan_preview" - assert data["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert data["phase"] == "phase_43_migration_apply_drill_preview" assert data["evidence_bundle_created"] is False assert data["evidence_query_executed"] is False assert data["sample_evidence_generated"] is False @@ -1183,7 +1188,7 @@ def test_opportunity_alert_plan_preview_blocks_dispatch_and_llm_calls(): plan = MarketIntelService().build_opportunity_alert_plan() assert plan["mode"] == "opportunity_alert_plan_preview" - assert plan["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert plan["phase"] == "phase_43_migration_apply_drill_preview" assert plan["ready_for_alert_candidates"] is False assert plan["alert_candidate_created"] is False assert plan["alert_queue_created"] is False @@ -1268,7 +1273,7 @@ def test_opportunity_alert_plan_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "opportunity_alert_plan_preview" - assert data["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert data["phase"] == "phase_43_migration_apply_drill_preview" assert data["alert_candidate_created"] is False assert data["alert_queue_created"] is False assert data["review_queue_created"] is False @@ -1346,7 +1351,7 @@ def test_mcp_deploy_preflight_route_is_preview_only(): assert response.status_code == 200 assert data["mode"] == "mcp_external_deploy_preflight_preview" - assert data["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert data["phase"] == "phase_43_migration_apply_drill_preview" assert data["deployment_actions_executed"] is False assert data["docker_command_executed"] is False assert data["ssh_command_executed"] is False @@ -1361,7 +1366,7 @@ def test_mcp_readiness_default_is_planned_only(monkeypatch): readiness = MarketIntelService().build_mcp_readiness() assert readiness["mode"] == "mcp_readiness_planned" - assert readiness["phase"] == "phase_42_alert_review_queue_migration_blueprint" + assert readiness["phase"] == "phase_43_migration_apply_drill_preview" assert readiness["execute_requested"] is False assert readiness["router_enabled"] is False assert readiness["external_mcp_complete"] is False @@ -1754,6 +1759,7 @@ def test_deployment_readiness_reports_app_only_release_gate(): assert readiness["checks"]["opportunity_scoring_plan_preview_safe"] is True assert readiness["checks"]["opportunity_evidence_plan_preview_safe"] is True assert readiness["checks"]["opportunity_alert_plan_preview_safe"] is True + assert readiness["checks"]["migration_apply_drill_preview_safe"] is True assert readiness["checks"]["writer_plan_dry_run_only"] is True assert readiness["writer_plan_summary"]["writes_executed"] is False assert "readiness_checks_not_all_passed" not in readiness["blocked_reasons"] @@ -1780,11 +1786,17 @@ def test_deployment_readiness_reports_app_only_release_gate(): assert "/api/market_intel/opportunity_scoring_plan" in readiness["production_smoke_targets"] assert "/api/market_intel/opportunity_evidence_plan" in readiness["production_smoke_targets"] assert "/api/market_intel/opportunity_alert_plan" in readiness["production_smoke_targets"] + assert "/api/market_intel/migration_apply_drill" in readiness["production_smoke_targets"] assert readiness["write_approval_runbook"]["ready_for_real_write"] is False assert readiness["write_approval_runbook"]["writes_executed"] is False assert readiness["migration_blueprint"]["migration_executed"] is False assert readiness["migration_blueprint"]["file_created"] is True assert readiness["migration_blueprint"]["file_matches_blueprint"] is True + assert readiness["migration_apply_drill"]["mode"] == "migration_apply_drill_preview" + assert readiness["migration_apply_drill"]["migration_executed"] is False + assert readiness["migration_apply_drill"]["rollback_executed"] is False + assert readiness["migration_apply_drill"]["database_write_executed"] is False + assert readiness["migration_apply_drill"]["api_executes_migration"] is False assert readiness["schema_db_probe"]["read_only_query_executed"] is False assert readiness["platform_seed_db_diff"]["read_only_query_executed"] is False assert readiness["legacy_source_bridge"]["read_only_query_executed"] is False @@ -1898,6 +1910,133 @@ def test_migration_blueprint_is_additive_preview_only(): assert "DROP TABLE IF EXISTS market_alert_review_queue" in blueprint["rollback_sql"] +def test_migration_apply_drill_planned_is_safe_and_manual_only(): + drill = MarketIntelService().build_migration_apply_drill() + + assert drill["mode"] == "migration_apply_drill_preview" + assert drill["phase"] == "phase_43_migration_apply_drill_preview" + assert drill["execute_requested"] is False + assert drill["schema_state"] == "planned_no_db_probe" + assert drill["drill_ready_for_operator_review"] is True + assert drill["ready_for_manual_apply_review"] is False + assert drill["ready_to_apply_migration"] is False + assert drill["read_only_query_executed"] is False + assert drill["database_connection_opened"] is False + assert drill["database_session_created"] is False + assert drill["database_write_executed"] is False + assert drill["database_commit_executed"] is False + assert drill["migration_executed"] is False + assert drill["rollback_executed"] is False + assert drill["api_executes_migration"] is False + assert drill["api_executes_rollback"] is False + assert drill["checks"]["migration_file_matches_blueprint"] is True + assert drill["checks"]["forward_sql_additive_only"] is True + assert "read_only_db_probe_not_executed" in drill["blocked_reasons"] + assert "api_never_runs_migration" in drill["blocked_reasons"] + assert drill["rollback_drill"]["rollback_executed"] is False + assert drill["rollback_drill"]["requires_manual_approval"] is True + assert "market_alert_review_queue" in drill["rollback_drill"]["rollback_table_order"] + assert len(drill["pre_apply_checklist"]) >= 6 + assert len(drill["post_apply_verification"]) >= 4 + assert len(drill["risk_register"]) >= 4 + + +def test_migration_apply_drill_sqlite_read_only_reports_already_applied(): + service = MarketIntelService() + seed_rows = service.build_platform_seed_plan()["seeds"] + engine = create_engine("sqlite:///:memory:") + with engine.begin() as conn: + conn.execute( + text( + """ + CREATE TABLE market_platforms ( + code TEXT PRIMARY KEY, + name TEXT, + base_url TEXT, + enabled BOOLEAN, + crawl_policy_json TEXT + ) + """ + ) + ) + for table_name in MARKET_INTEL_TABLES: + if table_name == "market_platforms": + continue + conn.execute(text(f"CREATE TABLE {table_name} (id INTEGER PRIMARY KEY)")) + for seed in seed_rows: + conn.execute( + text( + """ + INSERT INTO market_platforms + (code, name, base_url, enabled, crawl_policy_json) + VALUES + (:code, :name, :base_url, :enabled, :crawl_policy_json) + """ + ), + { + "code": seed["code"], + "name": seed["name"], + "base_url": seed["base_url"], + "enabled": seed["enabled"], + "crawl_policy_json": json.dumps( + seed["crawl_policy_json"], + ensure_ascii=False, + sort_keys=True, + ), + }, + ) + + drill = service.build_migration_apply_drill( + execute_requested=True, + schema_engine=engine, + seed_diff_engine=engine, + database_type="sqlite", + ) + + assert drill["mode"] == "migration_apply_drill_preview" + assert drill["execute_requested"] is True + assert drill["read_only_query_executed"] is True + assert drill["database_connection_opened"] is True + assert drill["schema_state"] == "already_applied" + assert drill["schema_db_probe_summary"]["schema_tables_exist"] is True + assert drill["schema_db_probe_summary"]["missing_tables"] == [] + assert drill["platform_seed_db_diff_summary"]["seed_rows_ready"] is True + assert set(drill["platform_seed_db_diff_summary"]["matching_codes"]) == { + "momo", + "pchome", + "coupang", + "shopee", + } + assert drill["database_session_created"] is False + assert drill["database_write_executed"] is False + assert drill["database_commit_executed"] is False + assert drill["migration_executed"] is False + assert "market_schema_already_present_apply_not_required" in drill["blocked_reasons"] + + +def test_migration_apply_drill_route_is_preview_only(): + from routes.market_intel_routes import market_intel_bp + + app = Flask(__name__) + app.secret_key = "test-secret" + app.register_blueprint(market_intel_bp) + client = app.test_client() + with client.session_transaction() as session: + session["logged_in"] = True + + response = client.get("/api/market_intel/migration_apply_drill?execute=false") + data = response.get_json() + + assert response.status_code == 200 + assert data["mode"] == "migration_apply_drill_preview" + assert data["phase"] == "phase_43_migration_apply_drill_preview" + assert data["execute_requested"] is False + assert data["migration_executed"] is False + assert data["rollback_executed"] is False + assert data["database_write_executed"] is False + assert data["api_executes_migration"] is False + + def test_seed_writer_cli_status_blocks_real_write(): status = MarketIntelService().build_seed_writer_cli_status( platform_code="all",