wooo/ewoooc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(dashboard): prewarm cache and expose pick evidence
All checks were successful
CD Pipeline / deploy (push) Successful in 2m20s
Details

Browse Source
This commit is contained in:
OoO
2026-05-01 16:34:13 +08:00
parent 9e2337764b
commit 2ac7410d40
10 changed files with 165 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app.py
View File

@@ -95,8 +95,8 @@ except Exception as e:
sys_log.error(f"無法檢測磁碟空間: {e}")
# 🚩 系統版本定義 (備份與顯示用)
# 🚩 2026-05-01 V10.64: Keep only latest 50 AI product picks pending
SYSTEM_VERSION = "V10.64"
# 🚩 2026-05-01 V10.65: Prewarm dashboard cache and expose AI pick evidence gaps
SYSTEM_VERSION = "V10.65"
# ==========================================
# 🔒 SQL Injection 防護函數

2
config.py
View File

@@ -254,7 +254,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '')
# ==========================================
# 系統版本與路徑
# ==========================================
SYSTEM_VERSION = "V10.64"
SYSTEM_VERSION = "V10.65"
LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log')
public_url = PUBLIC_URL # 用於模板顯示

23
gunicorn.conf.py
View File

@@ -7,6 +7,7 @@ restart workers but keep the preloaded app object from the old master process.
import os
import sys
import threading
from sqlalchemy.engine import Engine
@@ -89,3 +90,25 @@ def post_fork(server, worker):
worker.pid,
disposed_count,
)
def post_worker_init(worker):
"""Warm the expensive dashboard cache after each worker is ready."""
enabled = os.getenv("DASHBOARD_PREWARM_ON_WORKER_INIT", "1").lower()
if enabled in {"0", "false", "no"}:
return
def _warm_dashboard_cache():
try:
from routes.dashboard_routes import warm_full_dashboard_cache
warm_full_dashboard_cache(reason=f"gunicorn-worker-{worker.pid}")
except Exception as exc:
worker.log.warning("Dashboard cache prewarm failed in worker %s: %s", worker.pid, exc)
thread = threading.Thread(
target=_warm_dashboard_cache,
name=f"dashboard-prewarm-{worker.pid}",
daemon=True,
)
thread.start()
worker.log.info("Started dashboard cache prewarm thread in worker %s", worker.pid)

107
routes/dashboard_routes.py
View File

@@ -25,6 +25,7 @@ from services.cache_manager import (
_DASHBOARD_DATA_CACHE,
_DASHBOARD_CACHE_TTL,
_DASHBOARD_SHARED_CACHE_FILE,
_DASHBOARD_STALE_CACHE_FILE,
)
# 時區設定
@@ -518,12 +519,16 @@ class FileLock:
self.fd = None
_DASHBOARD_FILE_LOCK = FileLock(_DASHBOARD_LOCK_FILE)
_DASHBOARD_STALE_CACHE_MAX_AGE = 86400
def _load_shared_full_dashboard_cache(now):
"""讀取跨 worker 共享的商品看板深度快取。"""
cache_file = str(_DASHBOARD_SHARED_CACHE_FILE)
def _new_dashboard_file_lock():
return FileLock(_DASHBOARD_LOCK_FILE)
def _load_dashboard_cache_file(now, cache_path, *, allow_stale=False, label='共享'):
"""讀取跨 worker 商品看板深度快取；必要時可讀舊快取救援首屏。"""
cache_file = str(cache_path)
if not os.path.exists(cache_file):
return None
@@ -537,7 +542,8 @@ def _load_shared_full_dashboard_cache(now):
return None
age = now.timestamp() - full_timestamp
if age >= _DASHBOARD_CACHE_TTL:
max_age = _DASHBOARD_STALE_CACHE_MAX_AGE if allow_stale else _DASHBOARD_CACHE_TTL
if age >= max_age:
return None
_DASHBOARD_DATA_CACHE['full_data'] = full_data
@@ -545,13 +551,35 @@ def _load_shared_full_dashboard_cache(now):
_DASHBOARD_DATA_CACHE['consolidated_data'] = payload.get('consolidated_data')
_DASHBOARD_DATA_CACHE['consolidated_timestamp'] = payload.get('consolidated_timestamp')
_DASHBOARD_DATA_CACHE['today_start'] = payload.get('today_start')
sys_log.debug(f"[Dashboard] [Cache] ✅ 使用共享完整看板快取 | 快取年齡: {age:.0f}秒")
level = sys_log.info if allow_stale and age >= _DASHBOARD_CACHE_TTL else sys_log.debug
level(f"[Dashboard] [Cache] ✅ 使用{label}完整看板快取 | 快取年齡: {age:.0f}秒")
return full_data
except Exception as exc:
sys_log.warning(f"[Dashboard] [Cache] 共享快取讀取失敗，改走資料庫重建: {exc}")
sys_log.warning(f"[Dashboard] [Cache] {label}快取讀取失敗，改走資料庫重建: {exc}")
return None
def _load_shared_full_dashboard_cache(now):
return _load_dashboard_cache_file(now, _DASHBOARD_SHARED_CACHE_FILE)
def _load_stale_full_dashboard_cache(now):
stale_data = _load_dashboard_cache_file(
now,
_DASHBOARD_STALE_CACHE_FILE,
allow_stale=True,
label='舊版救援',
)
if stale_data:
return stale_data
return _load_dashboard_cache_file(
now,
_DASHBOARD_SHARED_CACHE_FILE,
allow_stale=True,
label='過期救援',
)
def _write_shared_full_dashboard_cache(full_data):
"""原子寫入跨 worker 共享的商品看板深度快取。"""
cache_file = str(_DASHBOARD_SHARED_CACHE_FILE)
@@ -576,6 +604,18 @@ def _write_shared_full_dashboard_cache(full_data):
except OSError:
pass
def warm_full_dashboard_cache(reason='manual'):
"""供 API、排程與 Gunicorn worker 啟動時預熱商品看板完整快取。"""
started = time.time()
data = get_full_dashboard_data(force_rebuild=True)
duration_ms = (time.time() - started) * 1000
sys_log.info(
f"[Dashboard] [Cache] ✅ 預熱完成 | reason={reason} | "
f"items={len(data.get('unique_items', [])) if data else 0} | 耗時={duration_ms:.0f}ms"
)
return data
# 慢查詢監控
_SLOW_QUERY_STATS = {
'total_queries': 0,
@@ -793,53 +833,62 @@ def get_consolidated_data():
session.close()
def get_full_dashboard_data():
def get_full_dashboard_data(force_rebuild=False):
"""獲取完整的看板資料，包含快取清單與全部 KPIs (深度快取)"""
global _DASHBOARD_DATA_CACHE
now = datetime.now(TAIPEI_TZ)
# V-Opt: 先檢查快取（無需鎖）
if _DASHBOARD_DATA_CACHE.get('full_data') and _DASHBOARD_DATA_CACHE.get('full_timestamp'):
if not force_rebuild and _DASHBOARD_DATA_CACHE.get('full_data') and _DASHBOARD_DATA_CACHE.get('full_timestamp'):
age = now.timestamp() - _DASHBOARD_DATA_CACHE['full_timestamp']
if age < _DASHBOARD_CACHE_TTL:
sys_log.debug(f"[Dashboard] [Cache] ✅ 使用完整看板快取 | 快取年齡: {age:.0f}秒")
return _DASHBOARD_DATA_CACHE['full_data']
shared_full_data = _load_shared_full_dashboard_cache(now)
shared_full_data = None if force_rebuild else _load_shared_full_dashboard_cache(now)
if shared_full_data:
return shared_full_data
# V-Opt: 使用檔案鎖避免多 gunicorn worker 同時計算
lock_acquired = _DASHBOARD_FILE_LOCK.acquire(blocking=False)
dashboard_lock = _new_dashboard_file_lock()
lock_acquired = dashboard_lock.acquire(blocking=False)
if not lock_acquired:
# 如果無法取得鎖，表示其他 worker 正在重建，等待並使用更新後的快取
sys_log.debug("[Dashboard] [Cache] ⏳ 等待其他 worker 重建快取...")
_DASHBOARD_FILE_LOCK.acquire() # 等待取得鎖
_DASHBOARD_FILE_LOCK.release() # 立即釋放
shared_full_data = _load_shared_full_dashboard_cache(now)
if shared_full_data:
return shared_full_data
if _DASHBOARD_DATA_CACHE.get('full_data') and _DASHBOARD_DATA_CACHE.get('full_timestamp'):
age = now.timestamp() - _DASHBOARD_DATA_CACHE['full_timestamp']
if age < _DASHBOARD_CACHE_TTL:
return _DASHBOARD_DATA_CACHE['full_data']
lock_acquired = _DASHBOARD_FILE_LOCK.acquire()
if not lock_acquired:
sys_log.warning("[Dashboard] [Cache] 共享鎖取得失敗，改用無鎖重建")
# 其他 worker 正在重建時，先用舊快取救首屏，避免使用者第一次打開卡住。
stale_full_data = None if force_rebuild else _load_stale_full_dashboard_cache(now)
if stale_full_data:
return stale_full_data
shared_full_data = _load_shared_full_dashboard_cache(now)
sys_log.info("[Dashboard] [Cache] ⏳ 等待其他 worker 重建快取...")
deadline = time.time() + 5
while time.time() < deadline:
time.sleep(0.25)
shared_full_data = None if force_rebuild else _load_shared_full_dashboard_cache(now)
if shared_full_data:
return shared_full_data
dashboard_lock = _new_dashboard_file_lock()
lock_acquired = dashboard_lock.acquire(blocking=False)
if lock_acquired:
break
shared_full_data = None if force_rebuild else _load_shared_full_dashboard_cache(now)
if shared_full_data:
return shared_full_data
if not lock_acquired:
stale_full_data = None if force_rebuild else _load_stale_full_dashboard_cache(now)
if stale_full_data:
return stale_full_data
dashboard_lock = _new_dashboard_file_lock()
lock_acquired = dashboard_lock.acquire(blocking=True)
try:
# 再次檢查快取（可能其他 worker 已經更新）
if _DASHBOARD_DATA_CACHE.get('full_data') and _DASHBOARD_DATA_CACHE.get('full_timestamp'):
if not force_rebuild and _DASHBOARD_DATA_CACHE.get('full_data') and _DASHBOARD_DATA_CACHE.get('full_timestamp'):
age = now.timestamp() - _DASHBOARD_DATA_CACHE['full_timestamp']
if age < _DASHBOARD_CACHE_TTL:
sys_log.debug(f"[Dashboard] [Cache] ✅ 使用完整看板快取 (其他 worker 已更新) | 快取年齡: {age:.0f}秒")
return _DASHBOARD_DATA_CACHE['full_data']
shared_full_data = _load_shared_full_dashboard_cache(now)
shared_full_data = None if force_rebuild else _load_shared_full_dashboard_cache(now)
if shared_full_data:
return shared_full_data
@@ -981,7 +1030,7 @@ def get_full_dashboard_data():
finally:
# V-Opt: 確保釋放檔案鎖
if lock_acquired:
_DASHBOARD_FILE_LOCK.release()
dashboard_lock.release()
def get_dashboard_stats():

14
routes/export_routes.py
View File

@@ -7,6 +7,7 @@
import os
import io
import json
from datetime import datetime, timezone, timedelta
from flask import Blueprint, request, send_file, redirect, url_for, flash
from auth import login_required
@@ -153,6 +154,7 @@ def export_excel_ai_picks():
ar.confidence,
ar.sales_7d_delta,
ar.reason,
ar.model_footprint,
ar.created_at,
p.url AS momo_url,
vc.competitor_product_id,
@@ -179,6 +181,14 @@ def export_excel_ai_picks():
pchome_id = row.get('competitor_product_id') or ''
momo_url = row.get('momo_url') or f"https://www.momoshop.com.tw/goods/GoodsDetail.jsp?i_code={sku}"
pchome_url = f"https://24h.pchome.com.tw/prod/{str(pchome_id).strip()}" if pchome_id else ''
footprint = row.get('model_footprint') or {}
if isinstance(footprint, str):
try:
footprint = json.loads(footprint)
except Exception:
footprint = {}
agent_footprint = footprint.get('agent', {}) if isinstance(footprint, dict) else {}
missing_evidence = agent_footprint.get('missing_evidence') or []
export_rows.append({
'AI排名': int(row.get('rank') or 0),
'MOMO商品ID': sku,
@@ -188,6 +198,10 @@ def export_excel_ai_picks():
'PChome價格': float(row.get('pchome_price') or 0),
'價差百分比': float(row.get('gap_pct') or 0),
'AI信心百分比': round(float(row.get('confidence') or 0) * 100, 1),
'機會分數': float(agent_footprint.get('opportunity_score') or 0),
'證據完整度': float(agent_footprint.get('evidence_quality') or 0),
'信心分層': agent_footprint.get('confidence_band') or '',
'待補證據': '、'.join(str(item) for item in missing_evidence),
'近7日銷售變化': float(row.get('sales_7d_delta') or 0),
'PChome商品ID': pchome_id,
'PChome商品名稱': row.get('competitor_product_name') or '',

25
services/ai_product_pick_agent.py
View File

@@ -328,6 +328,25 @@ def _score_candidate(row: Dict[str, Any]) -> Dict[str, Any]:
score = round(min(100, opportunity_score + evidence_quality * 0.35), 1)
confidence = round(max(0.45, min(0.98, (score * 0.65 + evidence_quality * 0.35) / 100)), 3)
missing_evidence = []
if history_points < 3:
missing_evidence.append("PChome 價格歷史不足 3 筆")
if sales_7d <= 0:
missing_evidence.append("近 7 天銷售額缺口")
if qty_7d <= 0:
missing_evidence.append("近 7 天銷量缺口")
if margin_rate is None:
missing_evidence.append("毛利/成本缺口")
if not row.get("competitor_product_id"):
missing_evidence.append("PChome 商品 ID 缺口")
if confidence >= 0.78 and evidence_quality >= 70:
confidence_band = "high"
elif confidence >= 0.65 and evidence_quality >= 55:
confidence_band = "medium"
else:
confidence_band = "needs_evidence"
if gap_pct >= 10:
angle = "PChome 價格優勢明顯"
elif gap_pct >= 3:
@@ -354,6 +373,8 @@ def _score_candidate(row: Dict[str, Any]) -> Dict[str, Any]:
reason_parts.append("PChome 商品評價訊號佳")
if "low_stock" in tags:
reason_parts.append("PChome 庫存偏低，需留意供貨")
if missing_evidence:
reason_parts.append("待補證據：" + "、".join(missing_evidence[:3]))
return {
**row,
@@ -364,6 +385,8 @@ def _score_candidate(row: Dict[str, Any]) -> Dict[str, Any]:
"evidence_quality": round(evidence_quality, 1),
"opportunity_score": round(opportunity_score, 1),
"margin_rate": round(margin_rate, 1) if margin_rate is not None else None,
"confidence_band": confidence_band,
"missing_evidence": missing_evidence,
"reason": "；".join(reason_parts),
}
@@ -381,6 +404,8 @@ def _write_pick(conn, pick: Dict[str, Any]) -> None:
"opportunity_score": pick.get("opportunity_score"),
"evidence_quality": pick.get("evidence_quality"),
"margin_rate": pick.get("margin_rate"),
"confidence_band": pick.get("confidence_band"),
"missing_evidence": pick.get("missing_evidence", []),
},
"competitor": {
"source": "pchome",

7
services/cache_manager.py
View File

@@ -63,6 +63,7 @@ _DASHBOARD_DATA_CACHE = {
_DASHBOARD_CACHE_TTL = 1800
_BASE_DIR = Path(__file__).resolve().parents[1]
_DASHBOARD_SHARED_CACHE_FILE = _BASE_DIR / "data" / "dashboard_full_cache.pkl"
_DASHBOARD_STALE_CACHE_FILE = _BASE_DIR / "data" / "dashboard_full_cache_stale.pkl"
def cleanup_sales_cache():
@@ -128,7 +129,11 @@ def clear_dashboard_cache():
'full_timestamp': None,
})
try:
os.remove(_DASHBOARD_SHARED_CACHE_FILE)
if os.path.exists(_DASHBOARD_SHARED_CACHE_FILE):
try:
os.replace(_DASHBOARD_SHARED_CACHE_FILE, _DASHBOARD_STALE_CACHE_FILE)
except OSError:
os.remove(_DASHBOARD_SHARED_CACHE_FILE)
except FileNotFoundError:
pass
except OSError:

3
tests/test_cache_manager.py
View File

@@ -72,8 +72,10 @@ def test_dashboard_cache_clear_restores_expected_shape(tmp_path, monkeypatch):
from services import cache_manager
shared_cache = tmp_path / "dashboard_full_cache.pkl"
stale_cache = tmp_path / "dashboard_full_cache_stale.pkl"
shared_cache.write_bytes(b"stale")
monkeypatch.setattr(cache_manager, "_DASHBOARD_SHARED_CACHE_FILE", shared_cache)
monkeypatch.setattr(cache_manager, "_DASHBOARD_STALE_CACHE_FILE", stale_cache)
cache_manager._DASHBOARD_DATA_CACHE["consolidated_data"] = ["stale"]
cache_manager._DASHBOARD_DATA_CACHE["full_data"] = ["stale"]
@@ -88,6 +90,7 @@ def test_dashboard_cache_clear_restores_expected_shape(tmp_path, monkeypatch):
"full_timestamp": None,
}
assert not shared_cache.exists()
assert stale_cache.exists()
def test_cache_dicts_are_only_defined_in_cache_manager():

2
tests/test_frontend_v2_assets.py
View File

@@ -48,7 +48,9 @@ def test_dashboard_v2_is_production_default_and_uses_real_dashboard_data():
assert "template_name = 'dashboard.html' if request.args.get('ui') == 'legacy' else 'dashboard_v2.html'" in route_source
assert "get_full_dashboard_data()" in route_source
assert "_load_shared_full_dashboard_cache(now)" in route_source
assert "_load_stale_full_dashboard_cache(now)" in route_source
assert "_write_shared_full_dashboard_cache(full_data)" in route_source
assert "warm_full_dashboard_cache" in route_source
assert "_load_competitor_decision_overview(session)" in route_source
assert "ai_price_recommendations" in route_source
assert "pending_match_count" in route_source

11
tests/test_gunicorn_config.py
View File

@@ -14,6 +14,9 @@ class _Log:
def info(self, *_args, **_kwargs):
return None
def warning(self, *_args, **_kwargs):
return None
def exception(self, *_args, **_kwargs):
return None
@@ -80,6 +83,14 @@ def test_gunicorn_disables_preload_for_hup_hot_reload():
assert config.preload_app is False
def test_gunicorn_starts_dashboard_cache_prewarm_thread():
source = GUNICORN_CONFIG_PATH.read_text(encoding="utf-8")
assert "def post_worker_init" in source
assert "warm_full_dashboard_cache" in source
assert "DASHBOARD_PREWARM_ON_WORKER_INIT" in source
def test_gunicorn_uses_thread_worker_for_health_resilience(monkeypatch):
monkeypatch.delenv("GUNICORN_WORKER_CLASS", raising=False)
monkeypatch.delenv("GUNICORN_THREADS", raising=False)