wooo/ewoooc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(cache): 統一 cache SOT 並啟用 gunicorn preload

Browse Source 
ADR-017 Phase 3f-2：新增 services/cache_manager.py，讓 sales/import/export/daily/dashboard 共用同一份 in-memory cache；cache_service 改為相容 shim；Dockerfile/docker-compose 啟用 gunicorn --preload。
This commit is contained in:
OoO
2026-04-29 21:35:56 +08:00
parent 2550ab45b1
commit 13fa165ee2
10 changed files with 172 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@@ -65,4 +65,4 @@ ENV FLASK_APP=app.py
EXPOSE 80
# 啟動應用（production 用 gunicorn，4 workers + 300s timeout + 啟用 access/error log）
CMD ["gunicorn", "--bind", "0.0.0.0:80", "--workers", "4", "--timeout", "300", "--access-logfile", "-", "--error-logfile", "-", "app:app"]
CMD ["gunicorn", "--bind", "0.0.0.0:80", "--workers", "4", "--timeout", "300", "--access-logfile", "-", "--error-logfile", "-", "--preload", "app:app"]

41
app.py
View File

@@ -81,14 +81,8 @@ for _warn in validate_critical_config():
# 🚩 V-New: 商品看板資料快取 (用於加速首頁載入)
_DASHBOARD_DATA_CACHE = {
'consolidated_data': None, # get_consolidated_data() 結果
'consolidated_timestamp': None, # 快取時間戳記
'stats_data': None, # 統計資料
'stats_timestamp': None # 統計資料時間戳記
}
_DASHBOARD_CACHE_TTL = 300 # 快取有效期 5 分鐘（秒）
# 商品看板 cache 單一來源。實際路由已在 routes/dashboard_routes.py。
from services.cache_manager import _DASHBOARD_DATA_CACHE, _DASHBOARD_CACHE_TTL # noqa: E402
# 🚩 檢查磁碟空間 (V9.52 新增)
try:
@@ -600,34 +594,9 @@ def get_consolidated_data():
session.close()
def get_dashboard_stats():
"""計算看板統計數據 (供通知使用)"""
db = DatabaseManager()
session = db.get_session()
try:
unique_items, today_start = get_consolidated_data()
today_start_db = today_start.replace(tzinfo=None)
# 1. 漲跌
increase_count = sum(1 for item in unique_items if item['yesterday_diff'] > 0)
decrease_count = sum(1 for item in unique_items if item['yesterday_diff'] < 0)
# 2. 今日新增 (使用與 index 路由相同的邏輯)
new_pids_query = session.query(PriceRecord.product_id).group_by(PriceRecord.product_id).having(func.min(PriceRecord.timestamp) >= today_start_db)
new_product_ids = {r[0] for r in new_pids_query.all()}
new_count = len(new_product_ids)
# 3. 今日下架
today_delisted_count = session.query(Product).filter(
Product.status == 'INACTIVE',
Product.updated_at >= today_start_db
).count()
return {'new': new_count, 'up': increase_count, 'down': decrease_count, 'delisted': today_delisted_count}
except Exception as e:
sys_log.error(f"[Stats] ❌ 計算統計失敗: {e}")
return {'new': 0, 'up': 0, 'down': 0, 'delisted': 0}
finally:
session.close()
"""計算看板統計數據 (供通知使用) — backward-compat wrapper."""
from services.dashboard_service import get_dashboard_stats as _get_dashboard_stats
return _get_dashboard_stats()
# ================= 🛣️ 4. Flask 路由 =================

2
docker-compose.yml
View File

@@ -48,7 +48,7 @@ services:
ports:
- "127.0.0.1:5003:80" # 僅本地連線，透過 Nginx 反向代理（nginx 反代 5003）
# 強制使用 gunicorn 綁定 port 80 (覆蓋 Dockerfile CMD)
command: ["gunicorn", "--bind", "0.0.0.0:80", "--workers", "4", "--timeout", "300", "--access-logfile", "-", "--error-logfile", "-", "app:app"]
command: ["gunicorn", "--bind", "0.0.0.0:80", "--workers", "4", "--timeout", "300", "--access-logfile", "-", "--error-logfile", "-", "--preload", "app:app"]
volumes:
# 持久化數據
- ./data:/app/data

12
routes/daily_sales_routes.py
View File

@@ -23,6 +23,10 @@ from services.daily_sales_service import (
prepare_calendar_data,
prepare_marketing_summary,
)
from services.cache_manager import (
_DAILY_SALES_PROCESSED_CACHE as _SALES_PROCESSED_CACHE,
clear_daily_sales_cache as _clear_daily_sales_cache,
)
# 時區設定
TAIPEI_TZ = timezone(timedelta(hours=8))
@@ -33,15 +37,12 @@ sys_log = SystemLogger("DailySalesRoutes").get_logger()
# Blueprint 定義
daily_sales_bp = Blueprint('daily_sales', __name__)
# 快取（帶過期時間）
_SALES_PROCESSED_CACHE = {}
_CACHE_EXPIRY_SECONDS = 300 # 5 分鐘緩存過期
def clear_daily_sales_cache():
"""清除當日業績緩存（供匯入服務調用）"""
global _SALES_PROCESSED_CACHE
_SALES_PROCESSED_CACHE.clear()
_clear_daily_sales_cache()
sys_log.info("已清除當日業績緩存")
@@ -50,9 +51,8 @@ def clear_daily_sales_cache():
@login_required
def api_clear_daily_sales_cache():
"""手動清除快取（保留為 ops escape hatch；正常失效靠 DB fingerprint）"""
global _SALES_PROCESSED_CACHE
cache_size = len(_SALES_PROCESSED_CACHE)
_SALES_PROCESSED_CACHE.clear()
_clear_daily_sales_cache()
sys_log.info(f"[API] 已清除當日業績緩存 (原有 {cache_size} 個緩存項目)")
return {'success': True, 'message': f'已清除 {cache_size} 個緩存項目'}

9
routes/dashboard_routes.py
View File

@@ -20,6 +20,7 @@ from config import BASE_DIR, SYSTEM_VERSION, public_url
from database.manager import DatabaseManager
from database.models import Product, PriceRecord
from services.logger_manager import SystemLogger
from services.cache_manager import _DASHBOARD_DATA_CACHE, _DASHBOARD_CACHE_TTL
# 時區設定
TAIPEI_TZ = timezone(timedelta(hours=8))
@@ -36,14 +37,6 @@ dashboard_bp = Blueprint('dashboard', __name__)
# ==========================================
import fcntl
_DASHBOARD_DATA_CACHE = {
'consolidated_data': None, # get_consolidated_data() 結果
'consolidated_timestamp': None,
'today_start': None,
'full_data': None, # 包含統計數據的完整結果
'full_timestamp': None
}
_DASHBOARD_CACHE_TTL = 1800 # 快取有效期 30 分鐘
_DASHBOARD_LOCK_FILE = os.path.join(BASE_DIR, 'data', '.dashboard_cache.lock') # V-Opt: 檔案鎖（跨進程）

7
routes/export_routes.py
View File

@@ -41,8 +41,8 @@ def _get_consolidated_data():
def _get_sales_cache():
"""從 cache_service 導入業績分析快取"""
from services.cache_service import _SALES_PROCESSED_CACHE
"""從 cache_manager 導入業績分析快取。"""
from services.cache_manager import _SALES_PROCESSED_CACHE
return _SALES_PROCESSED_CACHE
@@ -646,7 +646,8 @@ def export_vendor_analysis():
def export_seasonality_detail():
"""匯出淡旺季熱力圖的詳細資料。"""
try:
from routes.sales_routes import _SALES_PROCESSED_CACHE, _get_filtered_sales_data
from services.cache_manager import _SALES_PROCESSED_CACHE
from routes.sales_routes import _get_filtered_sales_data
table_name = 'realtime_sales_monthly'
data_range_months = int(request.args.get('data_range', '1') or '1')

33
routes/import_routes.py
View File

@@ -38,9 +38,13 @@ import_bp = Blueprint('import', __name__)
# ==========================================
def _get_cache_refs():
"""從 cache_service 導入快取變數"""
from services.cache_service import _SALES_DF_CACHE, _SALES_PROCESSED_CACHE
return _SALES_DF_CACHE, _SALES_PROCESSED_CACHE
"""從 cache_manager 導入快取變數與清除 helper。"""
from services.cache_manager import (
_SALES_DF_CACHE,
_SALES_PROCESSED_CACHE,
clear_sales_cache_for_table,
)
return _SALES_DF_CACHE, _SALES_PROCESSED_CACHE, clear_sales_cache_for_table
def _extract_snapshot_date_from_filename(filename):
@@ -162,7 +166,7 @@ def import_excel():
sys_log.info(f"[Web] [Import] 正在寫入資料庫: {engine.url}")
# 取得快取引用
_SALES_DF_CACHE, _SALES_PROCESSED_CACHE = _get_cache_refs()
_SALES_DF_CACHE, _SALES_PROCESSED_CACHE, clear_sales_cache_for_table = _get_cache_refs()
if table_name in ['realtime_sales_monthly', 'daily_sales_snapshot']:
try:
@@ -237,15 +241,8 @@ def import_excel():
rows_imported = 0
message = '匯入完成，但所有資料皆已存在 (重複)，無新增數據。'
# 清除快取
if table_name in _SALES_DF_CACHE:
del _SALES_DF_CACHE[table_name]
sys_log.info(f"[Web] [Cache] 已清除資料表快取: {table_name}")
cache_keys_to_delete = [key for key in _SALES_PROCESSED_CACHE.keys() if key.startswith(table_name)]
for cache_key in cache_keys_to_delete:
del _SALES_PROCESSED_CACHE[cache_key]
sys_log.info(f"[Web] [Cache] 已清除處理後快取: {cache_key}")
clear_sales_cache_for_table(table_name)
sys_log.info(f"[Web] [Cache] 已清除業績分析快取: {table_name}")
return jsonify({'status': 'success', 'message': message, 'rows': rows_imported, 'table': table_name})
@@ -257,14 +254,8 @@ def import_excel():
sys_log.info(f"[Web] [Import] 使用覆蓋模式 (replace)寫入資料表: {table_name}")
df.to_sql(table_name, con=engine, if_exists='replace', index=False)
if table_name in _SALES_DF_CACHE:
del _SALES_DF_CACHE[table_name]
sys_log.info(f"[Web] [Cache] 已清除資料表快取: {table_name}")
cache_keys_to_delete = [key for key in _SALES_PROCESSED_CACHE.keys() if key.startswith(table_name)]
for cache_key in cache_keys_to_delete:
del _SALES_PROCESSED_CACHE[cache_key]
sys_log.info(f"[Web] [Cache] 已清除處理後快取: {cache_key}")
clear_sales_cache_for_table(table_name)
sys_log.info(f"[Web] [Cache] 已清除業績分析快取: {table_name}")
return jsonify({
'status': 'success',

47
routes/sales_routes.py
View File

@@ -24,6 +24,10 @@ from config import BASE_DIR, DATABASE_TYPE
from database.manager import DatabaseManager
from services.logger_manager import SystemLogger
from services.daily_sales_service import prepare_marketing_summary
from services.cache_manager import (
_SALES_PROCESSED_CACHE,
set_sales_processed_cache,
)
from utils.text_helpers import get_color_for_string
# 時區設定
@@ -35,12 +39,6 @@ sys_log = SystemLogger("SalesRoutes").get_logger()
# Blueprint 定義
sales_bp = Blueprint('sales', __name__)
# 快取
_SALES_DF_CACHE = {}
_SALES_PROCESSED_CACHE = {}
_SALES_OPTIONS_CACHE = {}
_SALES_CACHE_MAX_ENTRIES = 10
_SALES_CACHE_TTL = 600
_TABLE_DATA_CACHE = {}
_TABLE_DATA_CACHE_TTL = 60
@@ -54,34 +52,6 @@ from utils.df_helpers import find_col # noqa: E402, F401
from utils.security import validate_table_name, safe_read_sql # noqa: E402, F401
def _cleanup_sales_cache():
"""清理過期和過多的快取條目"""
global _SALES_PROCESSED_CACHE
current_time = time.time()
# 1. 清理過期條目
expired_keys = [
k for k, v in _SALES_PROCESSED_CACHE.items()
if v.get('time') and current_time - v['time'] > _SALES_CACHE_TTL
]
for k in expired_keys:
del _SALES_PROCESSED_CACHE[k]
# 2. 如果仍超過限制，刪除最舊的條目
if len(_SALES_PROCESSED_CACHE) > _SALES_CACHE_MAX_ENTRIES:
sorted_items = sorted(
[(k, v.get('time', 0)) for k, v in _SALES_PROCESSED_CACHE.items()],
key=lambda x: x[1]
)
# 保留最新的 _SALES_CACHE_MAX_ENTRIES 條
keys_to_delete = [k for k, _ in sorted_items[:-_SALES_CACHE_MAX_ENTRIES]]
for k in keys_to_delete:
del _SALES_PROCESSED_CACHE[k]
if expired_keys or len(_SALES_PROCESSED_CACHE) > _SALES_CACHE_MAX_ENTRIES - 2:
sys_log.debug(f"[Cache] 清理快取: 移除 {len(expired_keys)} 條過期, 剩餘 {len(_SALES_PROCESSED_CACHE)} 條")
def _get_filtered_sales_data(cache_key):
"""
🚩 共用函式：從快取讀取資料並根據 request.args 進行篩選
@@ -689,13 +659,10 @@ def sales_analysis():
'return_qty': col_return_qty,
'pid': col_pid # V-New: 儲存商品ID欄位
},
'pid': col_pid # V-New: 儲存商品ID欄位
'pid': col_pid, # V-New: 儲存商品ID欄位
'time': time.time()
}
_SALES_PROCESSED_CACHE[cache_key] = cache_entry
# V-Fix: 同時使用固定的 table_name 作為 key 儲存副本，供其他路由使用
_SALES_PROCESSED_CACHE[table_name] = cache_entry
# V-Opt (2026-01-23): 定期清理過期快取
_cleanup_sales_cache()
set_sales_processed_cache(cache_key, cache_entry, aliases=(table_name,))
# 🚩 V-Opt: 使用共用篩選函式
target_df, cols_map, err = _get_filtered_sales_data(cache_key)

125
services/cache_manager.py Normal file
View File

@@ -0,0 +1,125 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
快取單一來源模組。
ADR-017 Phase 3f-2: 將 sales/import/export/daily 會共同碰到的
module-level cache 收斂到這裡，避免各 route 各自持有一份 dict。
"""
import time
class FingerprintCache:
"""TTL + fingerprint 的小型 in-memory cache。"""
def __init__(self, name, fingerprint_fn):
self.name = name
self._store = {}
self._fp_fn = fingerprint_fn
def get(self, key, ttl=300):
entry = self._store.get(key)
if not entry:
return None
if time.time() - entry['ts'] > ttl:
return None
try:
if self._fp_fn() != entry['fp']:
return None
except Exception:
pass
return entry['data']
def set(self, key, data):
try:
fp = self._fp_fn()
except Exception:
fp = None
self._store[key] = {'data': data, 'ts': time.time(), 'fp': fp}
def clear(self):
self._store.clear()
_SALES_DF_CACHE = {}
_SALES_PROCESSED_CACHE = {}
_SALES_OPTIONS_CACHE = {}
_SALES_ANALYSIS_RESULT_CACHE = {}
_SALES_CACHE_MAX_ENTRIES = 10
_SALES_CACHE_TTL = 600
_DAILY_SALES_PROCESSED_CACHE = {}
_DASHBOARD_DATA_CACHE = {
'consolidated_data': None,
'consolidated_timestamp': None,
'today_start': None,
'full_data': None,
'full_timestamp': None,
}
_DASHBOARD_CACHE_TTL = 1800
def cleanup_sales_cache():
"""清理 sales 處理後快取的過期與超額條目。"""
current_time = time.time()
expired_keys = [
key for key, value in _SALES_PROCESSED_CACHE.items()
if value.get('time') and current_time - value['time'] > _SALES_CACHE_TTL
]
for key in expired_keys:
_SALES_PROCESSED_CACHE.pop(key, None)
if len(_SALES_PROCESSED_CACHE) > _SALES_CACHE_MAX_ENTRIES:
sorted_items = sorted(
[(key, value.get('time', 0)) for key, value in _SALES_PROCESSED_CACHE.items()],
key=lambda item: item[1],
)
for key, _ in sorted_items[:-_SALES_CACHE_MAX_ENTRIES]:
_SALES_PROCESSED_CACHE.pop(key, None)
def set_sales_processed_cache(key, entry, aliases=()):
"""寫入 sales cache，並可同步寫入別名 key。"""
entry.setdefault('time', time.time())
_SALES_PROCESSED_CACHE[key] = entry
for alias in aliases:
_SALES_PROCESSED_CACHE[alias] = entry
cleanup_sales_cache()
def clear_sales_cache_for_table(table_name):
"""匯入資料後清除指定表對應的所有 sales cache key。"""
_SALES_DF_CACHE.pop(table_name, None)
keys_to_delete = [
key for key in list(_SALES_PROCESSED_CACHE.keys())
if key == table_name or key.startswith(f"{table_name}_")
]
for key in keys_to_delete:
_SALES_PROCESSED_CACHE.pop(key, None)
def clear_sales_cache():
"""清除所有 sales cache。"""
_SALES_DF_CACHE.clear()
_SALES_PROCESSED_CACHE.clear()
_SALES_OPTIONS_CACHE.clear()
_SALES_ANALYSIS_RESULT_CACHE.clear()
def clear_daily_sales_cache():
"""清除當日業績 cache。"""
_DAILY_SALES_PROCESSED_CACHE.clear()
def clear_dashboard_cache():
"""清除商品看板 cache。"""
_DASHBOARD_DATA_CACHE.clear()
_DASHBOARD_DATA_CACHE.update({
'consolidated_data': None,
'consolidated_timestamp': None,
'today_start': None,
'full_data': None,
'full_timestamp': None,
})

49
services/cache_service.py
View File

@@ -6,34 +6,25 @@
"""
from datetime import datetime, timezone, timedelta
from services.cache_manager import (
_SALES_DF_CACHE,
_SALES_PROCESSED_CACHE,
_SALES_OPTIONS_CACHE,
_SALES_ANALYSIS_RESULT_CACHE,
_DASHBOARD_DATA_CACHE,
_DASHBOARD_CACHE_TTL,
clear_sales_cache,
clear_dashboard_cache,
)
# 台北時區
TAIPEI_TZ = timezone(timedelta(hours=8))
# ==========================================
# 業績分析快取
# ==========================================
_SALES_DF_CACHE = {}
_SALES_PROCESSED_CACHE = {} # 處理後資料快取 (二級快取)
_SALES_OPTIONS_CACHE = {} # 下拉選單選項 (類別、品牌、廠商等)
_SALES_ANALYSIS_RESULT_CACHE = {} # 過濾後的分析結果集
# 快取 TTL 設定
_SALES_CACHE_TTL = 3600 # 業績分析快取: 60 分鐘
_SALES_OPTIONS_TTL = 21600 # 選項快取: 6 小時
_SALES_RESULT_TTL = 3600 # 結果快取: 60 分鐘
# ==========================================
# 商品看板快取
# ==========================================
_DASHBOARD_DATA_CACHE = {
'consolidated_data': None,
'consolidated_timestamp': None,
'full_data': None,
'full_timestamp': None
}
_DASHBOARD_CACHE_TTL = 1800 # 商品看板快取: 30 分鐘
# ==========================================
# 成長分析快取
# ==========================================
@@ -81,26 +72,6 @@ def get_slow_query_stats():
return _SLOW_QUERY_STATS.copy()
def clear_sales_cache():
"""清除業績分析快取"""
global _SALES_DF_CACHE, _SALES_PROCESSED_CACHE, _SALES_OPTIONS_CACHE, _SALES_ANALYSIS_RESULT_CACHE
_SALES_DF_CACHE.clear()
_SALES_PROCESSED_CACHE.clear()
_SALES_OPTIONS_CACHE.clear()
_SALES_ANALYSIS_RESULT_CACHE.clear()
def clear_dashboard_cache():
"""清除商品看板快取"""
global _DASHBOARD_DATA_CACHE
_DASHBOARD_DATA_CACHE = {
'consolidated_data': None,
'consolidated_timestamp': None,
'full_data': None,
'full_timestamp': None
}
def clear_growth_cache():
"""清除成長分析快取"""
global _GROWTH_ANALYSIS_CACHE