From 9260cc1740cf17715aea241a9ab8f927c11f0d94 Mon Sep 17 00:00:00 2001 From: OoO Date: Mon, 15 Jun 2026 16:19:03 +0800 Subject: [PATCH] =?UTF-8?q?V10.607=20=E5=BB=BA=E7=AB=8B=E5=A4=96=E9=83=A8?= =?UTF-8?q?=E5=B8=82=E5=A0=B4=E4=BE=86=E6=BA=90=E6=AD=A3=E8=A6=8F=E5=8C=96?= =?UTF-8?q?=E5=B1=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 2 +- database/external_market_models.py | 107 +++++ database/manager.py | 1 + docs/AI_INTELLIGENCE_MODULE_SOT.md | 5 +- .../current_execution_queue_20260524.md | 7 + ...44_external_market_offer_normalization.sql | 175 ++++++++ routes/README.md | 1 + routes/ai_routes.py | 23 ++ services/external_market_offer_service.py | 389 ++++++++++++++++++ services/pchome_revenue_growth_service.py | 4 + templates/ai_intelligence.html | 90 ++++ tests/test_external_market_offer_service.py | 89 ++++ tests/test_pchome_revenue_growth_service.py | 6 + 13 files changed, 896 insertions(+), 3 deletions(-) create mode 100644 database/external_market_models.py create mode 100644 migrations/044_external_market_offer_normalization.sql create mode 100644 services/external_market_offer_service.py create mode 100644 tests/test_external_market_offer_service.py diff --git a/config.py b/config.py index dcbfa42..85f2463 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.606" +SYSTEM_VERSION = "V10.607" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/database/external_market_models.py b/database/external_market_models.py new file mode 100644 index 0000000..039d419 --- /dev/null +++ b/database/external_market_models.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""外部市場來源與報價的正規化 ORM models。""" + +from datetime import datetime, timedelta, timezone + +from sqlalchemy import ( + Boolean, + Column, + DateTime, + Float, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.orm import relationship + +from database.models import Base + + +TAIPEI_TZ = timezone(timedelta(hours=8)) + + +def taipei_now(): + return datetime.now(TAIPEI_TZ).replace(tzinfo=None) + + +class ExternalMarketSource(Base): + """外部市場資料來源，例如 MOMO 參考價、蝦皮 API、酷澎 CSV。""" + + __tablename__ = "external_market_sources" + + id = Column(Integer, primary_key=True, autoincrement=True) + code = Column(String(80), unique=True, nullable=False, index=True) + display_name = Column(String(160), nullable=False) + platform_code = Column(String(80), nullable=False, index=True) + source_kind = Column(String(60), nullable=False, index=True) + status = Column(String(40), default="paused", nullable=False, index=True) + enabled = Column(Boolean, default=False, nullable=False) + write_enabled = Column(Boolean, default=False, nullable=False) + allowed_input_methods_json = Column(Text) + quality_policy_json = Column(Text) + plain_note = Column(Text) + created_at = Column(DateTime, default=taipei_now, nullable=False) + updated_at = Column(DateTime, default=taipei_now, onupdate=taipei_now, nullable=False) + + offers = relationship("ExternalOffer", back_populates="source") + + __table_args__ = ( + Index("idx_external_market_sources_status", "status", "enabled"), + ) + + +class ExternalOffer(Base): + """正規化後的外部商品報價。""" + + __tablename__ = "external_offers" + + id = Column(Integer, primary_key=True, autoincrement=True) + source_code = Column(String(80), ForeignKey("external_market_sources.code"), nullable=False, index=True) + platform_code = Column(String(80), nullable=False, index=True) + source_product_id = Column(String(220), nullable=False, index=True) + source_offer_key = Column(String(260), nullable=False) + title = Column(Text, nullable=False) + brand = Column(String(180), index=True) + category_text = Column(String(320), index=True) + product_url = Column(Text) + image_url = Column(Text) + price = Column(Float) + original_price = Column(Float) + currency = Column(String(12), default="TWD", nullable=False) + stock_status = Column(String(80), index=True) + sold_count = Column(Integer) + rating = Column(Float) + review_count = Column(Integer) + observed_at = Column(DateTime, default=taipei_now, nullable=False, index=True) + expires_at = Column(DateTime, index=True) + ingestion_method = Column(String(60), nullable=False, index=True) + connector_key = Column(String(120), index=True) + pchome_product_id = Column(String(120), index=True) + momo_sku = Column(String(80), index=True) + match_status = Column(String(40), default="unmatched", nullable=False, index=True) + quality_score = Column(Float, default=0.0, nullable=False) + data_quality_status = Column(String(40), default="needs_review", nullable=False, index=True) + quality_notes_json = Column(Text) + raw_payload_json = Column(Text) + created_at = Column(DateTime, default=taipei_now, nullable=False) + updated_at = Column(DateTime, default=taipei_now, onupdate=taipei_now, nullable=False) + + source = relationship("ExternalMarketSource", back_populates="offers") + + __table_args__ = ( + UniqueConstraint( + "source_code", + "source_product_id", + "observed_at", + "ingestion_method", + name="uq_external_offer_source_product_observed", + ), + Index("idx_external_offers_source_seen", "source_code", "observed_at"), + Index("idx_external_offers_platform_product", "platform_code", "source_product_id"), + Index("idx_external_offers_pchome_product", "pchome_product_id", "source_code"), + Index("idx_external_offers_match_quality", "match_status", "data_quality_status", "quality_score"), + ) diff --git a/database/manager.py b/database/manager.py index 694dc4c..863efa3 100644 --- a/database/manager.py +++ b/database/manager.py @@ -48,6 +48,7 @@ from .market_intel_models import ( # noqa: F401 - ADR-035 market_* 表 MarketCrawlerRun, MarketAlertReviewQueue, ) +from .external_market_models import ExternalMarketSource, ExternalOffer # noqa: F401 - 外部市場正規化表 # 🚩 導入優化後的日誌管理模組 from utils.logger_manager import SystemLogger diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index c226173..771cfbb 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,8 +1,8 @@ # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth > **最後更新**: 2026-06-15 (台北時間) -> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」 -> **適用版本**: V10.606 +> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層已建立 +> **適用版本**: V10.607 --- @@ -53,6 +53,7 @@ - `services/pchome_revenue_growth_service.py` 是第一版只讀作戰清單：讀 PChome 後台業績與已驗證 MOMO 外部價格參考，輸出 `/api/ai/pchome-growth/opportunities`。此服務不呼叫 LLM、不抓外站、不寫 DB。 - 2026-06-15 只讀盤點確認：`daily_sales_snapshot."商品ID"` 與 `competitor_prices.competitor_product_id` 在正式資料中直接重疊為 0。因此第一版作戰清單不得硬接兩邊 ID；若沒有可驗證對應，只能輸出「先補商品對應」任務。 - 蝦皮與酷澎暫停接入，不進作戰清單、不發告警；後續只可透過 official API / provider API / manual CSV 進 `external_offers` 類正規化層，並清楚標示資料品質。 +- V10.607 新增 `external_market_sources` / `external_offers` 正規化層與 `/api/ai/pchome-growth/source-contract` 只讀 API。MOMO 先以既有比價快取橋接進來源狀態；蝦皮與酷澎只保留 official API、provider API、manual CSV contract，預設暫停且不進告警。 ## 零之一、12 Agent 決策信封（2026-05-24） diff --git a/docs/memory/current_execution_queue_20260524.md b/docs/memory/current_execution_queue_20260524.md index 5fac5db..17df6c7 100644 --- a/docs/memory/current_execution_queue_20260524.md +++ b/docs/memory/current_execution_queue_20260524.md @@ -185,3 +185,10 @@ - 更新 SOT / memory / TODO。 - 推 Gitea，正式部署，確認 `/health` 版本。 - 記錄未完成與下一輪入口。 + +## 10. 2026-06-15 V10.607 外部市場來源正規化 + +- 新增 `external_market_sources` / `external_offers`，作為 MOMO、未來蝦皮、未來酷澎、供應商 API 與手動 CSV 的共同資料入口。 +- `/api/ai/pchome-growth/source-contract` 提供只讀來源狀態與欄位 contract；UI 只顯示白話狀態，例如「正在使用」「先暫停」「可用資料」。 +- MOMO 目前先橋接既有已確認同款的比價快取；蝦皮與酷澎只保留 contract，預設暫停、不進告警。 +- 下一步：做手動 CSV 匯入 dry-run 與外部報價品質檢查頁，讓未來無論官方 API 或 provider API 都能先經過同一套品質門檻。 diff --git a/migrations/044_external_market_offer_normalization.sql b/migrations/044_external_market_offer_normalization.sql new file mode 100644 index 0000000..a278c25 --- /dev/null +++ b/migrations/044_external_market_offer_normalization.sql @@ -0,0 +1,175 @@ +-- ============================================================================= +-- Migration 044: external market source / offer normalization +-- MOMO PRO / PChome revenue growth automation +-- 2026-06-15 Taipei +-- ============================================================================= +-- Notes: +-- Additive only. This migration does not drop, truncate, or rewrite existing +-- competitor_prices / competitor_price_history / market_* tables. +-- ============================================================================= + +CREATE TABLE IF NOT EXISTS external_market_sources ( + id BIGSERIAL PRIMARY KEY, + code VARCHAR(80) NOT NULL UNIQUE, + display_name VARCHAR(160) NOT NULL, + platform_code VARCHAR(80) NOT NULL, + source_kind VARCHAR(60) NOT NULL, + status VARCHAR(40) NOT NULL DEFAULT 'paused', + enabled BOOLEAN NOT NULL DEFAULT FALSE, + write_enabled BOOLEAN NOT NULL DEFAULT FALSE, + allowed_input_methods_json TEXT, + quality_policy_json TEXT, + plain_note TEXT, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_external_market_sources_code + ON external_market_sources (code); +CREATE INDEX IF NOT EXISTS idx_external_market_sources_platform_code + ON external_market_sources (platform_code); +CREATE INDEX IF NOT EXISTS idx_external_market_sources_source_kind + ON external_market_sources (source_kind); +CREATE INDEX IF NOT EXISTS idx_external_market_sources_status + ON external_market_sources (status, enabled); + +CREATE TABLE IF NOT EXISTS external_offers ( + id BIGSERIAL PRIMARY KEY, + source_code VARCHAR(80) NOT NULL REFERENCES external_market_sources(code), + platform_code VARCHAR(80) NOT NULL, + source_product_id VARCHAR(220) NOT NULL, + source_offer_key VARCHAR(260) NOT NULL, + title TEXT NOT NULL, + brand VARCHAR(180), + category_text VARCHAR(320), + product_url TEXT, + image_url TEXT, + price DOUBLE PRECISION, + original_price DOUBLE PRECISION, + currency VARCHAR(12) NOT NULL DEFAULT 'TWD', + stock_status VARCHAR(80), + sold_count INTEGER, + rating DOUBLE PRECISION, + review_count INTEGER, + observed_at TIMESTAMP NOT NULL DEFAULT NOW(), + expires_at TIMESTAMP, + ingestion_method VARCHAR(60) NOT NULL, + connector_key VARCHAR(120), + pchome_product_id VARCHAR(120), + momo_sku VARCHAR(80), + match_status VARCHAR(40) NOT NULL DEFAULT 'unmatched', + quality_score DOUBLE PRECISION NOT NULL DEFAULT 0, + data_quality_status VARCHAR(40) NOT NULL DEFAULT 'needs_review', + quality_notes_json TEXT, + raw_payload_json TEXT, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP NOT NULL DEFAULT NOW(), + CONSTRAINT uq_external_offer_source_product_observed + UNIQUE (source_code, source_product_id, observed_at, ingestion_method) +); + +CREATE INDEX IF NOT EXISTS idx_external_offers_source_code + ON external_offers (source_code); +CREATE INDEX IF NOT EXISTS idx_external_offers_platform_code + ON external_offers (platform_code); +CREATE INDEX IF NOT EXISTS idx_external_offers_source_product_id + ON external_offers (source_product_id); +CREATE INDEX IF NOT EXISTS idx_external_offers_brand + ON external_offers (brand); +CREATE INDEX IF NOT EXISTS idx_external_offers_category_text + ON external_offers (category_text); +CREATE INDEX IF NOT EXISTS idx_external_offers_stock_status + ON external_offers (stock_status); +CREATE INDEX IF NOT EXISTS idx_external_offers_observed_at + ON external_offers (observed_at); +CREATE INDEX IF NOT EXISTS idx_external_offers_expires_at + ON external_offers (expires_at); +CREATE INDEX IF NOT EXISTS idx_external_offers_ingestion_method + ON external_offers (ingestion_method); +CREATE INDEX IF NOT EXISTS idx_external_offers_connector_key + ON external_offers (connector_key); +CREATE INDEX IF NOT EXISTS idx_external_offers_pchome_product_id + ON external_offers (pchome_product_id); +CREATE INDEX IF NOT EXISTS idx_external_offers_momo_sku + ON external_offers (momo_sku); +CREATE INDEX IF NOT EXISTS idx_external_offers_match_status + ON external_offers (match_status); +CREATE INDEX IF NOT EXISTS idx_external_offers_data_quality_status + ON external_offers (data_quality_status); +CREATE INDEX IF NOT EXISTS idx_external_offers_source_seen + ON external_offers (source_code, observed_at); +CREATE INDEX IF NOT EXISTS idx_external_offers_platform_product + ON external_offers (platform_code, source_product_id); +CREATE INDEX IF NOT EXISTS idx_external_offers_pchome_product + ON external_offers (pchome_product_id, source_code); +CREATE INDEX IF NOT EXISTS idx_external_offers_match_quality + ON external_offers (match_status, data_quality_status, quality_score); + +INSERT INTO external_market_sources ( + code, + display_name, + platform_code, + source_kind, + status, + enabled, + write_enabled, + allowed_input_methods_json, + quality_policy_json, + plain_note +) VALUES + ( + 'momo_reference', + 'MOMO 外部價格參考', + 'momo', + 'legacy_bridge', + 'active', + TRUE, + FALSE, + '["legacy_competitor_cache","manual_csv","provider_api"]', + '{"minimum_match_status":"verified","minimum_quality_score":76}', + '目前只採用已確認同款的 MOMO 參考價。' + ), + ( + 'shopee', + '蝦皮', + 'shopee', + 'connector_contract', + 'paused', + FALSE, + FALSE, + '["official_api","provider_api","manual_csv"]', + '{"minimum_match_status":"verified","manual_review_required":true}', + '先暫停，不進告警；未來可接官方 API、供應商資料或手動 CSV。' + ), + ( + 'coupang', + '酷澎', + 'coupang', + 'connector_contract', + 'paused', + FALSE, + FALSE, + '["official_api","provider_api","manual_csv"]', + '{"minimum_match_status":"verified","manual_review_required":true}', + '先暫停，不進告警；未來可接官方 API、供應商資料或手動 CSV。' + ) +ON CONFLICT (code) DO UPDATE SET + display_name = EXCLUDED.display_name, + platform_code = EXCLUDED.platform_code, + source_kind = EXCLUDED.source_kind, + status = EXCLUDED.status, + enabled = EXCLUDED.enabled, + allowed_input_methods_json = EXCLUDED.allowed_input_methods_json, + quality_policy_json = EXCLUDED.quality_policy_json, + plain_note = EXCLUDED.plain_note, + updated_at = NOW(); + +GRANT ALL PRIVILEGES ON external_market_sources TO momo; +GRANT ALL PRIVILEGES ON external_offers TO momo; +GRANT USAGE, SELECT ON SEQUENCE external_market_sources_id_seq TO momo; +GRANT USAGE, SELECT ON SEQUENCE external_offers_id_seq TO momo; + +DO $$ +BEGIN + RAISE NOTICE 'Migration 044 complete: external market source and offer normalization is ready'; +END $$; diff --git a/routes/README.md b/routes/README.md index 553d57b..4a3ef95 100644 --- a/routes/README.md +++ b/routes/README.md @@ -25,6 +25,7 @@ | `market_intel_review_post_ai_routes.py` | 市場情報 AI summary persistence / Telegram dispatch 後續只讀延伸 API（掛在 `market_intel_review_bp`） | `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_run_readiness`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_run_receipt`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_run_closeout`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_gate`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_run_package`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_run_readiness`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_run_receipt`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_closeout`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_archive`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_archive_summary` | | `market_intel_review_report_routes.py` | 市場情報 report input / report run package / report run readiness / report run receipt / report closeout / report archive / report catalog handoff 後續只讀延伸 API（掛在 `market_intel_review_bp`） | `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_report_input`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_report_run_package`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_report_run_readiness`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_report_run_receipt`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_report_closeout`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_report_archive`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_report_archive_summary`, `/api/market_intel/manual_sample_review/candidate_queue_review_ai_summary_persistence_telegram_dispatch_report_catalog_handoff` | | `api_routes.py` | 通用任務與查詢 API | `/api/run_task`, `/api/history/*` | +| `ai_routes.py` | AI 推薦、競情儀表板與 PChome 成長作戰 API | `/ai_recommend`, `/ai_intelligence`, `/api/ai/status`, `/api/ai/icaim/dashboard`, `/api/ai/pchome-growth/opportunities`, `/api/ai/pchome-growth/source-contract` | | `export_routes.py` | 匯出功能 | `/api/export/*` | | `import_routes.py` | 匯入功能 | `/api/import_excel`, `/api/import/monthly_summary` | diff --git a/routes/ai_routes.py b/routes/ai_routes.py index 8e62cef..de11d38 100644 --- a/routes/ai_routes.py +++ b/routes/ai_routes.py @@ -1654,6 +1654,29 @@ def api_pchome_growth_opportunities(): }), 500 +@ai_bp.route('/api/ai/pchome-growth/source-contract') +@login_required +def api_pchome_growth_source_contract(): + """外部市場來源與報價欄位規格，只讀。""" + try: + from config import DATABASE_PATH + from services.external_market_offer_service import build_external_source_readiness + + engine = _create_icaim_dashboard_engine(DATABASE_PATH) + try: + payload = build_external_source_readiness(engine) + finally: + engine.dispose() + + return jsonify(payload) + except Exception as exc: + logger.error("[PChomeGrowth] 外部來源規格讀取失敗: %s", exc, exc_info=True) + return jsonify({ + "success": False, + "error": "外部資料來源狀態暫時無法讀取，請稍後再試。", + }), 500 + + @ai_bp.route('/api/ai/icaim/dashboard') @login_required def api_icaim_dashboard(): diff --git a/services/external_market_offer_service.py b/services/external_market_offer_service.py new file mode 100644 index 0000000..51f0269 --- /dev/null +++ b/services/external_market_offer_service.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""外部市場來源與報價的正規化服務。 + +第一版只做資料規格、來源狀態與只讀統計，不主動抓資料、不寫 DB。 +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +from sqlalchemy import inspect, text + +logger = logging.getLogger(__name__) + + +SOURCE_CONTRACTS = [ + { + "code": "momo_reference", + "display_name": "MOMO 外部價格參考", + "platform_code": "momo", + "status_code": "active", + "status_label": "正在使用", + "source_kind": "legacy_bridge", + "input_methods": ["既有比價快取", "手動 CSV", "供應商 API"], + "data_quality_label": "只採用已確認同款", + "plain_note": "目前用已確認同款的 MOMO 參考價，協助判斷 PChome 商品是否需要調整售價或曝光。", + }, + { + "code": "shopee", + "display_name": "蝦皮", + "platform_code": "shopee", + "status_code": "paused", + "status_label": "先暫停", + "source_kind": "connector_contract", + "input_methods": ["官方 API", "供應商 API", "手動 CSV"], + "data_quality_label": "暫不進告警", + "plain_note": "先保留資料接口，等有穩定合法來源後再啟用，不會影響目前作戰清單。", + }, + { + "code": "coupang", + "display_name": "酷澎", + "platform_code": "coupang", + "status_code": "paused", + "status_label": "先暫停", + "source_kind": "connector_contract", + "input_methods": ["官方 API", "供應商 API", "手動 CSV"], + "data_quality_label": "暫不進告警", + "plain_note": "先保留資料接口，等有穩定合法來源後再啟用，不會影響目前作戰清單。", + }, +] + +NORMALIZED_OFFER_FIELDS = [ + { + "name": "source_code", + "label": "資料來源", + "required": True, + "plain_note": "例如 momo_reference、shopee、coupang。", + }, + { + "name": "source_product_id", + "label": "外部商品 ID", + "required": True, + "plain_note": "外部平台或資料供應商給的商品編號。", + }, + { + "name": "title", + "label": "商品名稱", + "required": True, + "plain_note": "用來做人工確認與名稱比對。", + }, + { + "name": "price", + "label": "售價", + "required": True, + "plain_note": "只填可直接比較的成交或頁面售價。", + }, + { + "name": "observed_at", + "label": "資料時間", + "required": True, + "plain_note": "這筆價格看到的時間。", + }, + { + "name": "ingestion_method", + "label": "取得方式", + "required": True, + "plain_note": "official_api、provider_api、manual_csv 或 legacy_competitor_cache。", + }, + { + "name": "pchome_product_id", + "label": "PChome 商品 ID", + "required": False, + "plain_note": "若已確認同款才填，未確認就留空。", + }, + { + "name": "quality_score", + "label": "資料可信度", + "required": False, + "plain_note": "0 到 100，低於 76 不進自動告警。", + }, +] + + +@dataclass(frozen=True) +class ExternalOfferPayload: + source_code: str + platform_code: str + source_product_id: str + title: str + price: float | None + observed_at: str + ingestion_method: str + currency: str = "TWD" + original_price: float | None = None + product_url: str | None = None + brand: str | None = None + category_text: str | None = None + pchome_product_id: str | None = None + momo_sku: str | None = None + match_status: str = "unmatched" + quality_score: float = 0.0 + data_quality_status: str = "needs_review" + quality_notes: list[str] = field(default_factory=list) + + def to_record(self) -> dict[str, Any]: + return { + "source_code": self.source_code, + "platform_code": self.platform_code, + "source_product_id": self.source_product_id, + "source_offer_key": f"{self.source_code}:{self.source_product_id}", + "title": self.title, + "price": self.price, + "currency": self.currency or "TWD", + "original_price": self.original_price, + "product_url": self.product_url, + "brand": self.brand, + "category_text": self.category_text, + "observed_at": self.observed_at, + "ingestion_method": self.ingestion_method, + "pchome_product_id": self.pchome_product_id, + "momo_sku": self.momo_sku, + "match_status": self.match_status, + "quality_score": self.quality_score, + "data_quality_status": self.data_quality_status, + "quality_notes_json": json.dumps(self.quality_notes, ensure_ascii=False), + } + + +def _to_float(value: Any) -> float | None: + if value is None or value == "": + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _load_json_list(value: Any) -> list[Any]: + if not value: + return [] + if isinstance(value, list): + return value + try: + parsed = json.loads(value) + return parsed if isinstance(parsed, list) else [] + except Exception: + return [] + + +def _has_table(conn, table_name: str) -> bool: + try: + return inspect(conn).has_table(table_name) + except Exception: + logger.warning("[ExternalMarket] table probe failed: %s", table_name, exc_info=True) + return False + + +def normalize_external_offer_payload(payload: dict[str, Any]) -> tuple[ExternalOfferPayload | None, list[str]]: + """把 official API / provider API / manual CSV 的資料轉成同一份欄位。""" + errors: list[str] = [] + source_code = str(payload.get("source_code") or "").strip() + platform_code = str(payload.get("platform_code") or source_code or "").strip() + source_product_id = str(payload.get("source_product_id") or "").strip() + title = str(payload.get("title") or payload.get("name") or "").strip() + ingestion_method = str(payload.get("ingestion_method") or "").strip() + observed_at = str(payload.get("observed_at") or "").strip() + price = _to_float(payload.get("price")) + + required_values = { + "資料來源": source_code, + "外部商品 ID": source_product_id, + "商品名稱": title, + "售價": price, + "資料時間": observed_at, + "取得方式": ingestion_method, + } + for label, value in required_values.items(): + if value is None or value == "": + errors.append(f"缺少{label}") + + if price is not None and price <= 0: + errors.append("售價必須大於 0") + + if observed_at: + try: + datetime.fromisoformat(observed_at.replace("Z", "+00:00")) + except ValueError: + errors.append("資料時間格式需為 ISO 格式，例如 2026-06-15T10:00:00") + + if errors: + return None, errors + + quality_score = _to_float(payload.get("quality_score")) + if quality_score is None: + quality_score = 0.0 + quality_notes = _load_json_list(payload.get("quality_notes")) + if not quality_notes and payload.get("quality_note"): + quality_notes = [str(payload.get("quality_note"))] + + record = ExternalOfferPayload( + source_code=source_code, + platform_code=platform_code, + source_product_id=source_product_id, + title=title, + price=price, + observed_at=observed_at, + ingestion_method=ingestion_method, + currency=str(payload.get("currency") or "TWD").strip() or "TWD", + original_price=_to_float(payload.get("original_price")), + product_url=payload.get("product_url"), + brand=payload.get("brand"), + category_text=payload.get("category_text"), + pchome_product_id=payload.get("pchome_product_id"), + momo_sku=payload.get("momo_sku"), + match_status=str(payload.get("match_status") or "unmatched"), + quality_score=max(0.0, min(100.0, quality_score)), + data_quality_status=str(payload.get("data_quality_status") or "needs_review"), + quality_notes=[str(item) for item in quality_notes], + ) + return record, [] + + +def _legacy_momo_reference_stats(conn) -> dict[str, Any]: + if not _has_table(conn, "competitor_prices"): + return {"usable_offer_count": 0, "last_seen_at": None} + + if conn.dialect.name == "postgresql": + sql = """ + SELECT COUNT(*) AS usable_offer_count, MAX(crawled_at) AS last_seen_at + FROM competitor_prices + WHERE source = 'pchome' + AND competitor_product_id IS NOT NULL + AND price IS NOT NULL + AND price > 0 + AND COALESCE(match_score, 0) >= 0.76 + AND (expires_at IS NULL OR expires_at > CURRENT_TIMESTAMP) + AND COALESCE(tags, '[]'::jsonb) ? 'identity_v2' + """ + else: + sql = """ + SELECT COUNT(*) AS usable_offer_count, MAX(crawled_at) AS last_seen_at + FROM competitor_prices + WHERE source = 'pchome' + AND competitor_product_id IS NOT NULL + AND price IS NOT NULL + AND price > 0 + AND COALESCE(match_score, 0) >= 0.76 + AND COALESCE(tags, '') LIKE '%identity_v2%' + """ + row = conn.execute(text(sql)).mappings().first() or {} + return { + "usable_offer_count": int(row.get("usable_offer_count") or 0), + "last_seen_at": str(row.get("last_seen_at") or "") or None, + } + + +def _normalized_offer_stats(conn) -> dict[str, dict[str, Any]]: + if not all(_has_table(conn, table) for table in {"external_market_sources", "external_offers"}): + return {} + + rows = conn.execute(text(""" + SELECT + s.code, + s.status, + s.enabled, + COUNT(o.id) AS offer_count, + SUM(CASE + WHEN o.data_quality_status IN ('verified', 'usable', 'reviewed') + AND COALESCE(o.quality_score, 0) >= 76 + THEN 1 ELSE 0 END + ) AS usable_offer_count, + MAX(o.observed_at) AS last_seen_at + FROM external_market_sources s + LEFT JOIN external_offers o ON o.source_code = s.code + GROUP BY s.code, s.status, s.enabled + """)).mappings().all() + + return { + str(row["code"]): { + "status": row.get("status"), + "enabled": bool(row.get("enabled")), + "offer_count": int(row.get("offer_count") or 0), + "usable_offer_count": int(row.get("usable_offer_count") or 0), + "last_seen_at": str(row.get("last_seen_at") or "") or None, + } + for row in rows + } + + +def build_connector_contracts() -> dict[str, Any]: + """回傳 connector 與手動 CSV 共同遵守的欄位規格。""" + return { + "success": True, + "version": "2026-06-15", + "plain_summary": "所有外部市場資料都先轉成同一份商品報價格式，再進作戰清單。", + "sources": SOURCE_CONTRACTS, + "normalized_offer_fields": NORMALIZED_OFFER_FIELDS, + "manual_csv": { + "encoding": "utf-8-sig", + "required_headers": [ + field["name"] for field in NORMALIZED_OFFER_FIELDS if field["required"] + ], + "optional_headers": [ + field["name"] for field in NORMALIZED_OFFER_FIELDS if not field["required"] + ], + "plain_rule": "低可信度或未確認同款的資料只進待補資料清單，不自動發告警。", + }, + } + + +def build_external_source_readiness(engine=None) -> dict[str, Any]: + """建立畫面可用的外部資料來源狀態。""" + sources = [dict(source) for source in SOURCE_CONTRACTS] + normalized_stats: dict[str, dict[str, Any]] = {} + legacy_stats: dict[str, Any] = {"usable_offer_count": 0, "last_seen_at": None} + schema_ready = False + + if engine is not None: + try: + with engine.connect() as conn: + schema_ready = all( + _has_table(conn, table) + for table in {"external_market_sources", "external_offers"} + ) + normalized_stats = _normalized_offer_stats(conn) + legacy_stats = _legacy_momo_reference_stats(conn) + except Exception: + logger.warning("[ExternalMarket] source readiness failed", exc_info=True) + + for source in sources: + stats = normalized_stats.get(source["code"], {}) + if source["code"] == "momo_reference": + usable = max( + int(stats.get("usable_offer_count") or 0), + int(legacy_stats.get("usable_offer_count") or 0), + ) + last_seen_at = stats.get("last_seen_at") or legacy_stats.get("last_seen_at") + else: + usable = int(stats.get("usable_offer_count") or 0) + last_seen_at = stats.get("last_seen_at") + + source["schema_ready"] = schema_ready + source["usable_offer_count"] = usable + source["last_seen_at"] = last_seen_at + source["can_alert"] = source["status_code"] == "active" and usable > 0 + if source["status_code"] == "active": + source["plain_state"] = "已接入，可進作戰清單" if usable else "已接入，等待可用資料" + else: + source["plain_state"] = "先保留接口，不進告警" + + active_count = sum(1 for source in sources if source["status_code"] == "active") + paused_count = sum(1 for source in sources if source["status_code"] == "paused") + usable_count = sum(int(source["usable_offer_count"]) for source in sources) + + return { + "success": True, + "schema_ready": schema_ready, + "active_count": active_count, + "paused_count": paused_count, + "usable_offer_count": usable_count, + "sources": sources, + "connector_contract": build_connector_contracts(), + "plain_summary": "MOMO 先用；蝦皮與酷澎先保留接口，暫不進告警。", + } diff --git a/services/pchome_revenue_growth_service.py b/services/pchome_revenue_growth_service.py index 54b104f..d78a766 100644 --- a/services/pchome_revenue_growth_service.py +++ b/services/pchome_revenue_growth_service.py @@ -11,6 +11,8 @@ from typing import Any from sqlalchemy import bindparam, inspect, text +from services.external_market_offer_service import build_external_source_readiness + logger = logging.getLogger(__name__) SYSTEM_DISPLAY_NAME = "PChome 業績成長自動化作戰系統" @@ -404,12 +406,14 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] """讀取 PChome 業績與已驗證外部價格，產生營運用作戰清單。""" limit = max(5, min(int(limit or 20), 50)) generated_at = datetime.now().isoformat(timespec="seconds") + source_readiness = build_external_source_readiness(engine) source_scope = { "primary_goal": "提升 PChome 業績", "primary_sales_source": PRIMARY_SALES_SOURCE, "active_external_sources": list(ACTIVE_EXTERNAL_SOURCES), "paused_external_sources": list(PAUSED_EXTERNAL_SOURCES), "plain_note": "蝦皮與酷澎先暫停，不進作戰清單，也不發告警。", + "source_readiness": source_readiness, } if not _table_exists(engine, "daily_sales_snapshot"): diff --git a/templates/ai_intelligence.html b/templates/ai_intelligence.html index 18af13b..3cffc2f 100644 --- a/templates/ai_intelligence.html +++ b/templates/ai_intelligence.html @@ -273,6 +273,56 @@ line-height: 1.55; } + .growth-source-list { + display: grid; + gap: 8px; + margin-top: 10px; + } + + .growth-source-chip { + border: 1px solid rgba(42, 37, 32, 0.1); + border-radius: 8px; + background: rgba(255, 255, 255, 0.72); + padding: 8px 10px; + } + + .growth-source-chip.is-active { + border-color: rgba(42, 134, 96, 0.24); + background: rgba(235, 248, 241, 0.78); + } + + .growth-source-name { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + color: var(--momo-text-strong); + font-size: 0.8rem; + font-weight: 900; + } + + .growth-source-status { + border-radius: 999px; + background: rgba(42, 37, 32, 0.08); + color: var(--momo-text-muted); + font-size: 0.68rem; + font-weight: 900; + padding: 3px 7px; + white-space: nowrap; + } + + .growth-source-chip.is-active .growth-source-status { + background: rgba(42, 134, 96, 0.14); + color: #1f6d4c; + } + + .growth-source-detail { + margin: 5px 0 0; + color: var(--momo-text-muted); + font-size: 0.74rem; + line-height: 1.4; + } + .growth-list { display: grid; gap: 8px; @@ -506,6 +556,15 @@

來源整理中...

+
+
+
+ 外部資料來源 + 整理中 +
+

正在確認哪些來源可進作戰清單。

+
+
@@ -741,6 +800,7 @@ async function loadGrowthOps(forceRefresh = false) { document.getElementById('growthSourceNote').textContent = `業績來源：${scope.primary_sales_source || 'PChome 後台業績'}。外部價格先看：${active}。暫停來源：${paused}。`; + renderGrowthSourceReadiness((scope.source_readiness || {}).sources || []); renderGrowthOps(data.opportunities || []); } catch (error) { console.error(error); @@ -751,6 +811,36 @@ async function loadGrowthOps(forceRefresh = false) { } } +function renderGrowthSourceReadiness(sources) { + const box = document.getElementById('growthSourceReadiness'); + if (!box) return; + if (!sources.length) { + box.innerHTML = `
+
+ 外部資料來源 + 未確認 +
+

目前還沒有可顯示的來源狀態。

+
`; + return; + } + + box.innerHTML = sources.slice(0, 3).map((source) => { + const usable = Number(source.usable_offer_count || 0); + const detail = usable > 0 + ? `${source.data_quality_label || '資料可用'}，目前有 ${usable.toLocaleString()} 筆可用資料。` + : `${source.plain_state || source.plain_note || '等待資料接入。'}`; + const activeClass = source.status_code === 'active' ? ' is-active' : ''; + return `
+
+ ${escapeHtml(source.display_name || '未命名來源')} + ${escapeHtml(source.status_label || '待確認')} +
+

${escapeHtml(detail)}

+
`; + }).join(''); +} + function renderGrowthOps(rows) { const list = document.getElementById('growthOpsList'); if (!rows.length) { diff --git a/tests/test_external_market_offer_service.py b/tests/test_external_market_offer_service.py new file mode 100644 index 0000000..0e41f64 --- /dev/null +++ b/tests/test_external_market_offer_service.py @@ -0,0 +1,89 @@ +from sqlalchemy import create_engine, text + + +def test_connector_contract_keeps_shopee_and_coupang_paused_with_manual_csv_path(): + from services.external_market_offer_service import build_connector_contracts + + payload = build_connector_contracts() + + assert payload["success"] is True + assert payload["plain_summary"] == "所有外部市場資料都先轉成同一份商品報價格式，再進作戰清單。" + sources = {source["code"]: source for source in payload["sources"]} + assert sources["momo_reference"]["status_label"] == "正在使用" + assert sources["shopee"]["status_label"] == "先暫停" + assert sources["coupang"]["status_label"] == "先暫停" + assert "手動 CSV" in sources["shopee"]["input_methods"] + assert "官方 API" in sources["coupang"]["input_methods"] + assert "price" in payload["manual_csv"]["required_headers"] + + +def test_normalized_offer_payload_validates_plain_required_fields(): + from services.external_market_offer_service import normalize_external_offer_payload + + record, errors = normalize_external_offer_payload({ + "source_code": "momo_reference", + "source_product_id": "MOMO-1", + "title": "外部參考商品", + "price": "899", + "observed_at": "2026-06-15T10:00:00", + "ingestion_method": "manual_csv", + "quality_score": 82, + "quality_note": "人工確認同款", + }) + + assert errors == [] + assert record is not None + assert record.to_record()["source_offer_key"] == "momo_reference:MOMO-1" + assert record.to_record()["data_quality_status"] == "needs_review" + + missing_record, missing_errors = normalize_external_offer_payload({ + "source_code": "shopee", + "price": 0, + }) + + assert missing_record is None + assert "缺少外部商品 ID" in missing_errors + assert "售價必須大於 0" in missing_errors + + +def test_external_source_readiness_uses_legacy_momo_reference_cache(): + from services.external_market_offer_service import build_external_source_readiness + + engine = create_engine("sqlite:///:memory:") + with engine.begin() as conn: + conn.execute(text( + "CREATE TABLE competitor_prices (" + "source TEXT, competitor_product_id TEXT, price REAL, match_score REAL, " + "tags TEXT, crawled_at TEXT, expires_at TEXT)" + )) + conn.execute(text(""" + INSERT INTO competitor_prices + (source, competitor_product_id, price, match_score, tags, crawled_at, expires_at) + VALUES + ('pchome', 'PCH-1', 1000, 0.91, '["identity_v2"]', '2026-06-15 10:00:00', NULL), + ('pchome', 'PCH-2', 500, 0.50, '["identity_v2"]', '2026-06-15 10:00:00', NULL) + """)) + + payload = build_external_source_readiness(engine) + + assert payload["success"] is True + assert payload["schema_ready"] is False + sources = {source["code"]: source for source in payload["sources"]} + assert sources["momo_reference"]["usable_offer_count"] == 1 + assert sources["momo_reference"]["plain_state"] == "已接入，可進作戰清單" + assert sources["shopee"]["plain_state"] == "先保留接口，不進告警" + assert payload["plain_summary"] == "MOMO 先用；蝦皮與酷澎先保留接口，暫不進告警。" + + +def test_external_market_migration_creates_source_and_offer_tables(): + from pathlib import Path + + migration = Path("migrations/044_external_market_offer_normalization.sql").read_text(encoding="utf-8") + + assert "CREATE TABLE IF NOT EXISTS external_market_sources" in migration + assert "CREATE TABLE IF NOT EXISTS external_offers" in migration + assert "momo_reference" in migration + assert "shopee" in migration + assert "coupang" in migration + assert "DROP " not in migration.upper() + assert "TRUNCATE " not in migration.upper() diff --git a/tests/test_pchome_revenue_growth_service.py b/tests/test_pchome_revenue_growth_service.py index 299ab13..8f36470 100644 --- a/tests/test_pchome_revenue_growth_service.py +++ b/tests/test_pchome_revenue_growth_service.py @@ -59,6 +59,11 @@ def test_pchome_growth_opportunities_use_plain_language_and_pause_shopee_coupang assert payload["system_name"] == "PChome 業績成長自動化作戰系統" assert payload["source_scope"]["active_external_sources"] == ["MOMO 外部價格參考"] assert payload["source_scope"]["paused_external_sources"] == ["蝦皮", "酷澎"] + readiness = payload["source_scope"]["source_readiness"] + sources = {source["code"]: source for source in readiness["sources"]} + assert sources["momo_reference"]["status_label"] == "正在使用" + assert sources["shopee"]["status_label"] == "先暫停" + assert sources["coupang"]["status_label"] == "先暫停" assert payload["stats"]["candidate_count"] == 2 assert payload["stats"]["mapped_count"] == 1 assert payload["stats"]["needs_mapping_count"] == 1 @@ -84,4 +89,5 @@ def test_ai_intelligence_template_uses_pchome_growth_name_and_endpoint(): assert "PChome 業績成長自動化作戰系統" in template assert "/api/ai/pchome-growth/opportunities" in template + assert "growthSourceReadiness" in template assert "待補對應" in template