From 2ac7410d4041d0e9a7df5ce3740a5419731929ee Mon Sep 17 00:00:00 2001 From: OoO Date: Fri, 1 May 2026 16:34:13 +0800 Subject: [PATCH] fix(dashboard): prewarm cache and expose pick evidence --- app.py | 4 +- config.py | 2 +- gunicorn.conf.py | 23 +++++++ routes/dashboard_routes.py | 107 ++++++++++++++++++++++-------- routes/export_routes.py | 14 ++++ services/ai_product_pick_agent.py | 25 +++++++ services/cache_manager.py | 7 +- tests/test_cache_manager.py | 3 + tests/test_frontend_v2_assets.py | 2 + tests/test_gunicorn_config.py | 11 +++ 10 files changed, 165 insertions(+), 33 deletions(-) diff --git a/app.py b/app.py index 9e734e1..9fd7f0b 100644 --- a/app.py +++ b/app.py @@ -95,8 +95,8 @@ except Exception as e: sys_log.error(f"無法檢測磁碟空間: {e}") # 🚩 系統版本定義 (備份與顯示用) -# 🚩 2026-05-01 V10.64: Keep only latest 50 AI product picks pending -SYSTEM_VERSION = "V10.64" +# 🚩 2026-05-01 V10.65: Prewarm dashboard cache and expose AI pick evidence gaps +SYSTEM_VERSION = "V10.65" # ========================================== # 🔒 SQL Injection 防護函數 diff --git a/config.py b/config.py index 2974e07..49b71ba 100644 --- a/config.py +++ b/config.py @@ -254,7 +254,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.64" +SYSTEM_VERSION = "V10.65" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/gunicorn.conf.py b/gunicorn.conf.py index 04db811..8a8d5a5 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -7,6 +7,7 @@ restart workers but keep the preloaded app object from the old master process. import os import sys +import threading from sqlalchemy.engine import Engine @@ -89,3 +90,25 @@ def post_fork(server, worker): worker.pid, disposed_count, ) + + +def post_worker_init(worker): + """Warm the expensive dashboard cache after each worker is ready.""" + enabled = os.getenv("DASHBOARD_PREWARM_ON_WORKER_INIT", "1").lower() + if enabled in {"0", "false", "no"}: + return + + def _warm_dashboard_cache(): + try: + from routes.dashboard_routes import warm_full_dashboard_cache + warm_full_dashboard_cache(reason=f"gunicorn-worker-{worker.pid}") + except Exception as exc: + worker.log.warning("Dashboard cache prewarm failed in worker %s: %s", worker.pid, exc) + + thread = threading.Thread( + target=_warm_dashboard_cache, + name=f"dashboard-prewarm-{worker.pid}", + daemon=True, + ) + thread.start() + worker.log.info("Started dashboard cache prewarm thread in worker %s", worker.pid) diff --git a/routes/dashboard_routes.py b/routes/dashboard_routes.py index 3f8ef21..bca49e9 100644 --- a/routes/dashboard_routes.py +++ b/routes/dashboard_routes.py @@ -25,6 +25,7 @@ from services.cache_manager import ( _DASHBOARD_DATA_CACHE, _DASHBOARD_CACHE_TTL, _DASHBOARD_SHARED_CACHE_FILE, + _DASHBOARD_STALE_CACHE_FILE, ) # 時區設定 @@ -518,12 +519,16 @@ class FileLock: self.fd = None -_DASHBOARD_FILE_LOCK = FileLock(_DASHBOARD_LOCK_FILE) +_DASHBOARD_STALE_CACHE_MAX_AGE = 86400 -def _load_shared_full_dashboard_cache(now): - """讀取跨 worker 共享的商品看板深度快取。""" - cache_file = str(_DASHBOARD_SHARED_CACHE_FILE) +def _new_dashboard_file_lock(): + return FileLock(_DASHBOARD_LOCK_FILE) + + +def _load_dashboard_cache_file(now, cache_path, *, allow_stale=False, label='共享'): + """讀取跨 worker 商品看板深度快取；必要時可讀舊快取救援首屏。""" + cache_file = str(cache_path) if not os.path.exists(cache_file): return None @@ -537,7 +542,8 @@ def _load_shared_full_dashboard_cache(now): return None age = now.timestamp() - full_timestamp - if age >= _DASHBOARD_CACHE_TTL: + max_age = _DASHBOARD_STALE_CACHE_MAX_AGE if allow_stale else _DASHBOARD_CACHE_TTL + if age >= max_age: return None _DASHBOARD_DATA_CACHE['full_data'] = full_data @@ -545,13 +551,35 @@ def _load_shared_full_dashboard_cache(now): _DASHBOARD_DATA_CACHE['consolidated_data'] = payload.get('consolidated_data') _DASHBOARD_DATA_CACHE['consolidated_timestamp'] = payload.get('consolidated_timestamp') _DASHBOARD_DATA_CACHE['today_start'] = payload.get('today_start') - sys_log.debug(f"[Dashboard] [Cache] ✅ 使用共享完整看板快取 | 快取年齡: {age:.0f}秒") + level = sys_log.info if allow_stale and age >= _DASHBOARD_CACHE_TTL else sys_log.debug + level(f"[Dashboard] [Cache] ✅ 使用{label}完整看板快取 | 快取年齡: {age:.0f}秒") return full_data except Exception as exc: - sys_log.warning(f"[Dashboard] [Cache] 共享快取讀取失敗，改走資料庫重建: {exc}") + sys_log.warning(f"[Dashboard] [Cache] {label}快取讀取失敗，改走資料庫重建: {exc}") return None +def _load_shared_full_dashboard_cache(now): + return _load_dashboard_cache_file(now, _DASHBOARD_SHARED_CACHE_FILE) + + +def _load_stale_full_dashboard_cache(now): + stale_data = _load_dashboard_cache_file( + now, + _DASHBOARD_STALE_CACHE_FILE, + allow_stale=True, + label='舊版救援', + ) + if stale_data: + return stale_data + return _load_dashboard_cache_file( + now, + _DASHBOARD_SHARED_CACHE_FILE, + allow_stale=True, + label='過期救援', + ) + + def _write_shared_full_dashboard_cache(full_data): """原子寫入跨 worker 共享的商品看板深度快取。""" cache_file = str(_DASHBOARD_SHARED_CACHE_FILE) @@ -576,6 +604,18 @@ def _write_shared_full_dashboard_cache(full_data): except OSError: pass + +def warm_full_dashboard_cache(reason='manual'): + """供 API、排程與 Gunicorn worker 啟動時預熱商品看板完整快取。""" + started = time.time() + data = get_full_dashboard_data(force_rebuild=True) + duration_ms = (time.time() - started) * 1000 + sys_log.info( + f"[Dashboard] [Cache] ✅ 預熱完成 | reason={reason} | " + f"items={len(data.get('unique_items', [])) if data else 0} | 耗時={duration_ms:.0f}ms" + ) + return data + # 慢查詢監控 _SLOW_QUERY_STATS = { 'total_queries': 0, @@ -793,53 +833,62 @@ def get_consolidated_data(): session.close() -def get_full_dashboard_data(): +def get_full_dashboard_data(force_rebuild=False): """獲取完整的看板資料，包含快取清單與全部 KPIs (深度快取)""" global _DASHBOARD_DATA_CACHE now = datetime.now(TAIPEI_TZ) # V-Opt: 先檢查快取（無需鎖） - if _DASHBOARD_DATA_CACHE.get('full_data') and _DASHBOARD_DATA_CACHE.get('full_timestamp'): + if not force_rebuild and _DASHBOARD_DATA_CACHE.get('full_data') and _DASHBOARD_DATA_CACHE.get('full_timestamp'): age = now.timestamp() - _DASHBOARD_DATA_CACHE['full_timestamp'] if age < _DASHBOARD_CACHE_TTL: sys_log.debug(f"[Dashboard] [Cache] ✅ 使用完整看板快取 | 快取年齡: {age:.0f}秒") return _DASHBOARD_DATA_CACHE['full_data'] - shared_full_data = _load_shared_full_dashboard_cache(now) + shared_full_data = None if force_rebuild else _load_shared_full_dashboard_cache(now) if shared_full_data: return shared_full_data # V-Opt: 使用檔案鎖避免多 gunicorn worker 同時計算 - lock_acquired = _DASHBOARD_FILE_LOCK.acquire(blocking=False) + dashboard_lock = _new_dashboard_file_lock() + lock_acquired = dashboard_lock.acquire(blocking=False) if not lock_acquired: - # 如果無法取得鎖，表示其他 worker 正在重建，等待並使用更新後的快取 - sys_log.debug("[Dashboard] [Cache] ⏳ 等待其他 worker 重建快取...") - _DASHBOARD_FILE_LOCK.acquire() # 等待取得鎖 - _DASHBOARD_FILE_LOCK.release() # 立即釋放 - shared_full_data = _load_shared_full_dashboard_cache(now) - if shared_full_data: - return shared_full_data - if _DASHBOARD_DATA_CACHE.get('full_data') and _DASHBOARD_DATA_CACHE.get('full_timestamp'): - age = now.timestamp() - _DASHBOARD_DATA_CACHE['full_timestamp'] - if age < _DASHBOARD_CACHE_TTL: - return _DASHBOARD_DATA_CACHE['full_data'] - lock_acquired = _DASHBOARD_FILE_LOCK.acquire() - if not lock_acquired: - sys_log.warning("[Dashboard] [Cache] 共享鎖取得失敗，改用無鎖重建") + # 其他 worker 正在重建時，先用舊快取救首屏，避免使用者第一次打開卡住。 + stale_full_data = None if force_rebuild else _load_stale_full_dashboard_cache(now) + if stale_full_data: + return stale_full_data - shared_full_data = _load_shared_full_dashboard_cache(now) + sys_log.info("[Dashboard] [Cache] ⏳ 等待其他 worker 重建快取...") + deadline = time.time() + 5 + while time.time() < deadline: + time.sleep(0.25) + shared_full_data = None if force_rebuild else _load_shared_full_dashboard_cache(now) + if shared_full_data: + return shared_full_data + dashboard_lock = _new_dashboard_file_lock() + lock_acquired = dashboard_lock.acquire(blocking=False) + if lock_acquired: + break + + shared_full_data = None if force_rebuild else _load_shared_full_dashboard_cache(now) if shared_full_data: return shared_full_data + if not lock_acquired: + stale_full_data = None if force_rebuild else _load_stale_full_dashboard_cache(now) + if stale_full_data: + return stale_full_data + dashboard_lock = _new_dashboard_file_lock() + lock_acquired = dashboard_lock.acquire(blocking=True) try: # 再次檢查快取（可能其他 worker 已經更新） - if _DASHBOARD_DATA_CACHE.get('full_data') and _DASHBOARD_DATA_CACHE.get('full_timestamp'): + if not force_rebuild and _DASHBOARD_DATA_CACHE.get('full_data') and _DASHBOARD_DATA_CACHE.get('full_timestamp'): age = now.timestamp() - _DASHBOARD_DATA_CACHE['full_timestamp'] if age < _DASHBOARD_CACHE_TTL: sys_log.debug(f"[Dashboard] [Cache] ✅ 使用完整看板快取 (其他 worker 已更新) | 快取年齡: {age:.0f}秒") return _DASHBOARD_DATA_CACHE['full_data'] - shared_full_data = _load_shared_full_dashboard_cache(now) + shared_full_data = None if force_rebuild else _load_shared_full_dashboard_cache(now) if shared_full_data: return shared_full_data @@ -981,7 +1030,7 @@ def get_full_dashboard_data(): finally: # V-Opt: 確保釋放檔案鎖 if lock_acquired: - _DASHBOARD_FILE_LOCK.release() + dashboard_lock.release() def get_dashboard_stats(): diff --git a/routes/export_routes.py b/routes/export_routes.py index b0b2cf8..78d5195 100644 --- a/routes/export_routes.py +++ b/routes/export_routes.py @@ -7,6 +7,7 @@ import os import io +import json from datetime import datetime, timezone, timedelta from flask import Blueprint, request, send_file, redirect, url_for, flash from auth import login_required @@ -153,6 +154,7 @@ def export_excel_ai_picks(): ar.confidence, ar.sales_7d_delta, ar.reason, + ar.model_footprint, ar.created_at, p.url AS momo_url, vc.competitor_product_id, @@ -179,6 +181,14 @@ def export_excel_ai_picks(): pchome_id = row.get('competitor_product_id') or '' momo_url = row.get('momo_url') or f"https://www.momoshop.com.tw/goods/GoodsDetail.jsp?i_code={sku}" pchome_url = f"https://24h.pchome.com.tw/prod/{str(pchome_id).strip()}" if pchome_id else '' + footprint = row.get('model_footprint') or {} + if isinstance(footprint, str): + try: + footprint = json.loads(footprint) + except Exception: + footprint = {} + agent_footprint = footprint.get('agent', {}) if isinstance(footprint, dict) else {} + missing_evidence = agent_footprint.get('missing_evidence') or [] export_rows.append({ 'AI排名': int(row.get('rank') or 0), 'MOMO商品ID': sku, @@ -188,6 +198,10 @@ def export_excel_ai_picks(): 'PChome價格': float(row.get('pchome_price') or 0), '價差百分比': float(row.get('gap_pct') or 0), 'AI信心百分比': round(float(row.get('confidence') or 0) * 100, 1), + '機會分數': float(agent_footprint.get('opportunity_score') or 0), + '證據完整度': float(agent_footprint.get('evidence_quality') or 0), + '信心分層': agent_footprint.get('confidence_band') or '', + '待補證據': '、'.join(str(item) for item in missing_evidence), '近7日銷售變化': float(row.get('sales_7d_delta') or 0), 'PChome商品ID': pchome_id, 'PChome商品名稱': row.get('competitor_product_name') or '', diff --git a/services/ai_product_pick_agent.py b/services/ai_product_pick_agent.py index f125b6f..da554b2 100644 --- a/services/ai_product_pick_agent.py +++ b/services/ai_product_pick_agent.py @@ -328,6 +328,25 @@ def _score_candidate(row: Dict[str, Any]) -> Dict[str, Any]: score = round(min(100, opportunity_score + evidence_quality * 0.35), 1) confidence = round(max(0.45, min(0.98, (score * 0.65 + evidence_quality * 0.35) / 100)), 3) + missing_evidence = [] + if history_points < 3: + missing_evidence.append("PChome 價格歷史不足 3 筆") + if sales_7d <= 0: + missing_evidence.append("近 7 天銷售額缺口") + if qty_7d <= 0: + missing_evidence.append("近 7 天銷量缺口") + if margin_rate is None: + missing_evidence.append("毛利/成本缺口") + if not row.get("competitor_product_id"): + missing_evidence.append("PChome 商品 ID 缺口") + + if confidence >= 0.78 and evidence_quality >= 70: + confidence_band = "high" + elif confidence >= 0.65 and evidence_quality >= 55: + confidence_band = "medium" + else: + confidence_band = "needs_evidence" + if gap_pct >= 10: angle = "PChome 價格優勢明顯" elif gap_pct >= 3: @@ -354,6 +373,8 @@ def _score_candidate(row: Dict[str, Any]) -> Dict[str, Any]: reason_parts.append("PChome 商品評價訊號佳") if "low_stock" in tags: reason_parts.append("PChome 庫存偏低，需留意供貨") + if missing_evidence: + reason_parts.append("待補證據：" + "、".join(missing_evidence[:3])) return { **row, @@ -364,6 +385,8 @@ def _score_candidate(row: Dict[str, Any]) -> Dict[str, Any]: "evidence_quality": round(evidence_quality, 1), "opportunity_score": round(opportunity_score, 1), "margin_rate": round(margin_rate, 1) if margin_rate is not None else None, + "confidence_band": confidence_band, + "missing_evidence": missing_evidence, "reason": "；".join(reason_parts), } @@ -381,6 +404,8 @@ def _write_pick(conn, pick: Dict[str, Any]) -> None: "opportunity_score": pick.get("opportunity_score"), "evidence_quality": pick.get("evidence_quality"), "margin_rate": pick.get("margin_rate"), + "confidence_band": pick.get("confidence_band"), + "missing_evidence": pick.get("missing_evidence", []), }, "competitor": { "source": "pchome", diff --git a/services/cache_manager.py b/services/cache_manager.py index 8ae36cd..385ea4a 100644 --- a/services/cache_manager.py +++ b/services/cache_manager.py @@ -63,6 +63,7 @@ _DASHBOARD_DATA_CACHE = { _DASHBOARD_CACHE_TTL = 1800 _BASE_DIR = Path(__file__).resolve().parents[1] _DASHBOARD_SHARED_CACHE_FILE = _BASE_DIR / "data" / "dashboard_full_cache.pkl" +_DASHBOARD_STALE_CACHE_FILE = _BASE_DIR / "data" / "dashboard_full_cache_stale.pkl" def cleanup_sales_cache(): @@ -128,7 +129,11 @@ def clear_dashboard_cache(): 'full_timestamp': None, }) try: - os.remove(_DASHBOARD_SHARED_CACHE_FILE) + if os.path.exists(_DASHBOARD_SHARED_CACHE_FILE): + try: + os.replace(_DASHBOARD_SHARED_CACHE_FILE, _DASHBOARD_STALE_CACHE_FILE) + except OSError: + os.remove(_DASHBOARD_SHARED_CACHE_FILE) except FileNotFoundError: pass except OSError: diff --git a/tests/test_cache_manager.py b/tests/test_cache_manager.py index 0cab97a..b29cd7e 100644 --- a/tests/test_cache_manager.py +++ b/tests/test_cache_manager.py @@ -72,8 +72,10 @@ def test_dashboard_cache_clear_restores_expected_shape(tmp_path, monkeypatch): from services import cache_manager shared_cache = tmp_path / "dashboard_full_cache.pkl" + stale_cache = tmp_path / "dashboard_full_cache_stale.pkl" shared_cache.write_bytes(b"stale") monkeypatch.setattr(cache_manager, "_DASHBOARD_SHARED_CACHE_FILE", shared_cache) + monkeypatch.setattr(cache_manager, "_DASHBOARD_STALE_CACHE_FILE", stale_cache) cache_manager._DASHBOARD_DATA_CACHE["consolidated_data"] = ["stale"] cache_manager._DASHBOARD_DATA_CACHE["full_data"] = ["stale"] @@ -88,6 +90,7 @@ def test_dashboard_cache_clear_restores_expected_shape(tmp_path, monkeypatch): "full_timestamp": None, } assert not shared_cache.exists() + assert stale_cache.exists() def test_cache_dicts_are_only_defined_in_cache_manager(): diff --git a/tests/test_frontend_v2_assets.py b/tests/test_frontend_v2_assets.py index 79d4f18..5a4cfe2 100644 --- a/tests/test_frontend_v2_assets.py +++ b/tests/test_frontend_v2_assets.py @@ -48,7 +48,9 @@ def test_dashboard_v2_is_production_default_and_uses_real_dashboard_data(): assert "template_name = 'dashboard.html' if request.args.get('ui') == 'legacy' else 'dashboard_v2.html'" in route_source assert "get_full_dashboard_data()" in route_source assert "_load_shared_full_dashboard_cache(now)" in route_source + assert "_load_stale_full_dashboard_cache(now)" in route_source assert "_write_shared_full_dashboard_cache(full_data)" in route_source + assert "warm_full_dashboard_cache" in route_source assert "_load_competitor_decision_overview(session)" in route_source assert "ai_price_recommendations" in route_source assert "pending_match_count" in route_source diff --git a/tests/test_gunicorn_config.py b/tests/test_gunicorn_config.py index cffd48e..7043b55 100644 --- a/tests/test_gunicorn_config.py +++ b/tests/test_gunicorn_config.py @@ -14,6 +14,9 @@ class _Log: def info(self, *_args, **_kwargs): return None + def warning(self, *_args, **_kwargs): + return None + def exception(self, *_args, **_kwargs): return None @@ -80,6 +83,14 @@ def test_gunicorn_disables_preload_for_hup_hot_reload(): assert config.preload_app is False +def test_gunicorn_starts_dashboard_cache_prewarm_thread(): + source = GUNICORN_CONFIG_PATH.read_text(encoding="utf-8") + + assert "def post_worker_init" in source + assert "warm_full_dashboard_cache" in source + assert "DASHBOARD_PREWARM_ON_WORKER_INIT" in source + + def test_gunicorn_uses_thread_worker_for_health_resilience(monkeypatch): monkeypatch.delenv("GUNICORN_WORKER_CLASS", raising=False) monkeypatch.delenv("GUNICORN_THREADS", raising=False)