From 13fa165ee291301c5f1526ca09729b31de24f1c2 Mon Sep 17 00:00:00 2001 From: OoO Date: Wed, 29 Apr 2026 21:35:56 +0800 Subject: [PATCH] =?UTF-8?q?refactor(cache):=20=E7=B5=B1=E4=B8=80=20cache?= =?UTF-8?q?=20SOT=20=E4=B8=A6=E5=95=9F=E7=94=A8=20gunicorn=20preload?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-017 Phase 3f-2：新增 services/cache_manager.py，讓 sales/import/export/daily/dashboard 共用同一份 in-memory cache；cache_service 改為相容 shim；Dockerfile/docker-compose 啟用 gunicorn --preload。 --- Dockerfile | 2 +- app.py | 41 ++---------- docker-compose.yml | 2 +- routes/daily_sales_routes.py | 12 ++-- routes/dashboard_routes.py | 9 +-- routes/export_routes.py | 7 +- routes/import_routes.py | 33 ++++----- routes/sales_routes.py | 47 ++----------- services/cache_manager.py | 125 +++++++++++++++++++++++++++++++++++ services/cache_service.py | 49 +++----------- 10 files changed, 172 insertions(+), 155 deletions(-) create mode 100644 services/cache_manager.py diff --git a/Dockerfile b/Dockerfile index 86d5db3..0ce0977 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,4 +65,4 @@ ENV FLASK_APP=app.py EXPOSE 80 # 啟動應用（production 用 gunicorn，4 workers + 300s timeout + 啟用 access/error log） -CMD ["gunicorn", "--bind", "0.0.0.0:80", "--workers", "4", "--timeout", "300", "--access-logfile", "-", "--error-logfile", "-", "app:app"] +CMD ["gunicorn", "--bind", "0.0.0.0:80", "--workers", "4", "--timeout", "300", "--access-logfile", "-", "--error-logfile", "-", "--preload", "app:app"] diff --git a/app.py b/app.py index 6aa779c..2a83c03 100644 --- a/app.py +++ b/app.py @@ -81,14 +81,8 @@ for _warn in validate_critical_config(): -# 🚩 V-New: 商品看板資料快取 (用於加速首頁載入) -_DASHBOARD_DATA_CACHE = { - 'consolidated_data': None, # get_consolidated_data() 結果 - 'consolidated_timestamp': None, # 快取時間戳記 - 'stats_data': None, # 統計資料 - 'stats_timestamp': None # 統計資料時間戳記 -} -_DASHBOARD_CACHE_TTL = 300 # 快取有效期 5 分鐘（秒） +# 商品看板 cache 單一來源。實際路由已在 routes/dashboard_routes.py。 +from services.cache_manager import _DASHBOARD_DATA_CACHE, _DASHBOARD_CACHE_TTL # noqa: E402 # 🚩 檢查磁碟空間 (V9.52 新增) try: @@ -600,34 +594,9 @@ def get_consolidated_data(): session.close() def get_dashboard_stats(): - """計算看板統計數據 (供通知使用)""" - db = DatabaseManager() - session = db.get_session() - try: - unique_items, today_start = get_consolidated_data() - today_start_db = today_start.replace(tzinfo=None) - - # 1. 漲跌 - increase_count = sum(1 for item in unique_items if item['yesterday_diff'] > 0) - decrease_count = sum(1 for item in unique_items if item['yesterday_diff'] < 0) - - # 2. 今日新增 (使用與 index 路由相同的邏輯) - new_pids_query = session.query(PriceRecord.product_id).group_by(PriceRecord.product_id).having(func.min(PriceRecord.timestamp) >= today_start_db) - new_product_ids = {r[0] for r in new_pids_query.all()} - new_count = len(new_product_ids) - - # 3. 今日下架 - today_delisted_count = session.query(Product).filter( - Product.status == 'INACTIVE', - Product.updated_at >= today_start_db - ).count() - - return {'new': new_count, 'up': increase_count, 'down': decrease_count, 'delisted': today_delisted_count} - except Exception as e: - sys_log.error(f"[Stats] ❌ 計算統計失敗: {e}") - return {'new': 0, 'up': 0, 'down': 0, 'delisted': 0} - finally: - session.close() + """計算看板統計數據 (供通知使用) — backward-compat wrapper.""" + from services.dashboard_service import get_dashboard_stats as _get_dashboard_stats + return _get_dashboard_stats() # ================= 🛣️ 4. Flask 路由 ================= diff --git a/docker-compose.yml b/docker-compose.yml index 1b7895a..f547882 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,7 @@ services: ports: - "127.0.0.1:5003:80" # 僅本地連線，透過 Nginx 反向代理（nginx 反代 5003） # 強制使用 gunicorn 綁定 port 80 (覆蓋 Dockerfile CMD) - command: ["gunicorn", "--bind", "0.0.0.0:80", "--workers", "4", "--timeout", "300", "--access-logfile", "-", "--error-logfile", "-", "app:app"] + command: ["gunicorn", "--bind", "0.0.0.0:80", "--workers", "4", "--timeout", "300", "--access-logfile", "-", "--error-logfile", "-", "--preload", "app:app"] volumes: # 持久化數據 - ./data:/app/data diff --git a/routes/daily_sales_routes.py b/routes/daily_sales_routes.py index 07e62a0..c78d130 100644 --- a/routes/daily_sales_routes.py +++ b/routes/daily_sales_routes.py @@ -23,6 +23,10 @@ from services.daily_sales_service import ( prepare_calendar_data, prepare_marketing_summary, ) +from services.cache_manager import ( + _DAILY_SALES_PROCESSED_CACHE as _SALES_PROCESSED_CACHE, + clear_daily_sales_cache as _clear_daily_sales_cache, +) # 時區設定 TAIPEI_TZ = timezone(timedelta(hours=8)) @@ -33,15 +37,12 @@ sys_log = SystemLogger("DailySalesRoutes").get_logger() # Blueprint 定義 daily_sales_bp = Blueprint('daily_sales', __name__) -# 快取（帶過期時間） -_SALES_PROCESSED_CACHE = {} _CACHE_EXPIRY_SECONDS = 300 # 5 分鐘緩存過期 def clear_daily_sales_cache(): """清除當日業績緩存（供匯入服務調用）""" - global _SALES_PROCESSED_CACHE - _SALES_PROCESSED_CACHE.clear() + _clear_daily_sales_cache() sys_log.info("已清除當日業績緩存") @@ -50,9 +51,8 @@ def clear_daily_sales_cache(): @login_required def api_clear_daily_sales_cache(): """手動清除快取（保留為 ops escape hatch；正常失效靠 DB fingerprint）""" - global _SALES_PROCESSED_CACHE cache_size = len(_SALES_PROCESSED_CACHE) - _SALES_PROCESSED_CACHE.clear() + _clear_daily_sales_cache() sys_log.info(f"[API] 已清除當日業績緩存 (原有 {cache_size} 個緩存項目)") return {'success': True, 'message': f'已清除 {cache_size} 個緩存項目'} diff --git a/routes/dashboard_routes.py b/routes/dashboard_routes.py index 8c47a53..9bb61a6 100644 --- a/routes/dashboard_routes.py +++ b/routes/dashboard_routes.py @@ -20,6 +20,7 @@ from config import BASE_DIR, SYSTEM_VERSION, public_url from database.manager import DatabaseManager from database.models import Product, PriceRecord from services.logger_manager import SystemLogger +from services.cache_manager import _DASHBOARD_DATA_CACHE, _DASHBOARD_CACHE_TTL # 時區設定 TAIPEI_TZ = timezone(timedelta(hours=8)) @@ -36,14 +37,6 @@ dashboard_bp = Blueprint('dashboard', __name__) # ========================================== import fcntl -_DASHBOARD_DATA_CACHE = { - 'consolidated_data': None, # get_consolidated_data() 結果 - 'consolidated_timestamp': None, - 'today_start': None, - 'full_data': None, # 包含統計數據的完整結果 - 'full_timestamp': None -} -_DASHBOARD_CACHE_TTL = 1800 # 快取有效期 30 分鐘 _DASHBOARD_LOCK_FILE = os.path.join(BASE_DIR, 'data', '.dashboard_cache.lock') # V-Opt: 檔案鎖（跨進程） diff --git a/routes/export_routes.py b/routes/export_routes.py index 7f180e1..7e7ef0d 100644 --- a/routes/export_routes.py +++ b/routes/export_routes.py @@ -41,8 +41,8 @@ def _get_consolidated_data(): def _get_sales_cache(): - """從 cache_service 導入業績分析快取""" - from services.cache_service import _SALES_PROCESSED_CACHE + """從 cache_manager 導入業績分析快取。""" + from services.cache_manager import _SALES_PROCESSED_CACHE return _SALES_PROCESSED_CACHE @@ -646,7 +646,8 @@ def export_vendor_analysis(): def export_seasonality_detail(): """匯出淡旺季熱力圖的詳細資料。""" try: - from routes.sales_routes import _SALES_PROCESSED_CACHE, _get_filtered_sales_data + from services.cache_manager import _SALES_PROCESSED_CACHE + from routes.sales_routes import _get_filtered_sales_data table_name = 'realtime_sales_monthly' data_range_months = int(request.args.get('data_range', '1') or '1') diff --git a/routes/import_routes.py b/routes/import_routes.py index 109a634..83bafd5 100644 --- a/routes/import_routes.py +++ b/routes/import_routes.py @@ -38,9 +38,13 @@ import_bp = Blueprint('import', __name__) # ========================================== def _get_cache_refs(): - """從 cache_service 導入快取變數""" - from services.cache_service import _SALES_DF_CACHE, _SALES_PROCESSED_CACHE - return _SALES_DF_CACHE, _SALES_PROCESSED_CACHE + """從 cache_manager 導入快取變數與清除 helper。""" + from services.cache_manager import ( + _SALES_DF_CACHE, + _SALES_PROCESSED_CACHE, + clear_sales_cache_for_table, + ) + return _SALES_DF_CACHE, _SALES_PROCESSED_CACHE, clear_sales_cache_for_table def _extract_snapshot_date_from_filename(filename): @@ -162,7 +166,7 @@ def import_excel(): sys_log.info(f"[Web] [Import] 正在寫入資料庫: {engine.url}") # 取得快取引用 - _SALES_DF_CACHE, _SALES_PROCESSED_CACHE = _get_cache_refs() + _SALES_DF_CACHE, _SALES_PROCESSED_CACHE, clear_sales_cache_for_table = _get_cache_refs() if table_name in ['realtime_sales_monthly', 'daily_sales_snapshot']: try: @@ -237,15 +241,8 @@ def import_excel(): rows_imported = 0 message = '匯入完成，但所有資料皆已存在 (重複)，無新增數據。' - # 清除快取 - if table_name in _SALES_DF_CACHE: - del _SALES_DF_CACHE[table_name] - sys_log.info(f"[Web] [Cache] 已清除資料表快取: {table_name}") - - cache_keys_to_delete = [key for key in _SALES_PROCESSED_CACHE.keys() if key.startswith(table_name)] - for cache_key in cache_keys_to_delete: - del _SALES_PROCESSED_CACHE[cache_key] - sys_log.info(f"[Web] [Cache] 已清除處理後快取: {cache_key}") + clear_sales_cache_for_table(table_name) + sys_log.info(f"[Web] [Cache] 已清除業績分析快取: {table_name}") return jsonify({'status': 'success', 'message': message, 'rows': rows_imported, 'table': table_name}) @@ -257,14 +254,8 @@ def import_excel(): sys_log.info(f"[Web] [Import] 使用覆蓋模式 (replace)寫入資料表: {table_name}") df.to_sql(table_name, con=engine, if_exists='replace', index=False) - if table_name in _SALES_DF_CACHE: - del _SALES_DF_CACHE[table_name] - sys_log.info(f"[Web] [Cache] 已清除資料表快取: {table_name}") - - cache_keys_to_delete = [key for key in _SALES_PROCESSED_CACHE.keys() if key.startswith(table_name)] - for cache_key in cache_keys_to_delete: - del _SALES_PROCESSED_CACHE[cache_key] - sys_log.info(f"[Web] [Cache] 已清除處理後快取: {cache_key}") + clear_sales_cache_for_table(table_name) + sys_log.info(f"[Web] [Cache] 已清除業績分析快取: {table_name}") return jsonify({ 'status': 'success', diff --git a/routes/sales_routes.py b/routes/sales_routes.py index a6d6d25..aace760 100644 --- a/routes/sales_routes.py +++ b/routes/sales_routes.py @@ -24,6 +24,10 @@ from config import BASE_DIR, DATABASE_TYPE from database.manager import DatabaseManager from services.logger_manager import SystemLogger from services.daily_sales_service import prepare_marketing_summary +from services.cache_manager import ( + _SALES_PROCESSED_CACHE, + set_sales_processed_cache, +) from utils.text_helpers import get_color_for_string # 時區設定 @@ -35,12 +39,6 @@ sys_log = SystemLogger("SalesRoutes").get_logger() # Blueprint 定義 sales_bp = Blueprint('sales', __name__) -# 快取 -_SALES_DF_CACHE = {} -_SALES_PROCESSED_CACHE = {} -_SALES_OPTIONS_CACHE = {} -_SALES_CACHE_MAX_ENTRIES = 10 -_SALES_CACHE_TTL = 600 _TABLE_DATA_CACHE = {} _TABLE_DATA_CACHE_TTL = 60 @@ -54,34 +52,6 @@ from utils.df_helpers import find_col # noqa: E402, F401 from utils.security import validate_table_name, safe_read_sql # noqa: E402, F401 -def _cleanup_sales_cache(): - """清理過期和過多的快取條目""" - global _SALES_PROCESSED_CACHE - current_time = time.time() - - # 1. 清理過期條目 - expired_keys = [ - k for k, v in _SALES_PROCESSED_CACHE.items() - if v.get('time') and current_time - v['time'] > _SALES_CACHE_TTL - ] - for k in expired_keys: - del _SALES_PROCESSED_CACHE[k] - - # 2. 如果仍超過限制，刪除最舊的條目 - if len(_SALES_PROCESSED_CACHE) > _SALES_CACHE_MAX_ENTRIES: - sorted_items = sorted( - [(k, v.get('time', 0)) for k, v in _SALES_PROCESSED_CACHE.items()], - key=lambda x: x[1] - ) - # 保留最新的 _SALES_CACHE_MAX_ENTRIES 條 - keys_to_delete = [k for k, _ in sorted_items[:-_SALES_CACHE_MAX_ENTRIES]] - for k in keys_to_delete: - del _SALES_PROCESSED_CACHE[k] - - if expired_keys or len(_SALES_PROCESSED_CACHE) > _SALES_CACHE_MAX_ENTRIES - 2: - sys_log.debug(f"[Cache] 清理快取: 移除 {len(expired_keys)} 條過期, 剩餘 {len(_SALES_PROCESSED_CACHE)} 條") - - def _get_filtered_sales_data(cache_key): """ 🚩 共用函式：從快取讀取資料並根據 request.args 進行篩選 @@ -689,13 +659,10 @@ def sales_analysis(): 'return_qty': col_return_qty, 'pid': col_pid # V-New: 儲存商品ID欄位 }, - 'pid': col_pid # V-New: 儲存商品ID欄位 + 'pid': col_pid, # V-New: 儲存商品ID欄位 + 'time': time.time() } - _SALES_PROCESSED_CACHE[cache_key] = cache_entry - # V-Fix: 同時使用固定的 table_name 作為 key 儲存副本，供其他路由使用 - _SALES_PROCESSED_CACHE[table_name] = cache_entry - # V-Opt (2026-01-23): 定期清理過期快取 - _cleanup_sales_cache() + set_sales_processed_cache(cache_key, cache_entry, aliases=(table_name,)) # 🚩 V-Opt: 使用共用篩選函式 target_df, cols_map, err = _get_filtered_sales_data(cache_key) diff --git a/services/cache_manager.py b/services/cache_manager.py new file mode 100644 index 0000000..723699e --- /dev/null +++ b/services/cache_manager.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +快取單一來源模組。 + +ADR-017 Phase 3f-2: 將 sales/import/export/daily 會共同碰到的 +module-level cache 收斂到這裡，避免各 route 各自持有一份 dict。 +""" + +import time + + +class FingerprintCache: + """TTL + fingerprint 的小型 in-memory cache。""" + + def __init__(self, name, fingerprint_fn): + self.name = name + self._store = {} + self._fp_fn = fingerprint_fn + + def get(self, key, ttl=300): + entry = self._store.get(key) + if not entry: + return None + if time.time() - entry['ts'] > ttl: + return None + try: + if self._fp_fn() != entry['fp']: + return None + except Exception: + pass + return entry['data'] + + def set(self, key, data): + try: + fp = self._fp_fn() + except Exception: + fp = None + self._store[key] = {'data': data, 'ts': time.time(), 'fp': fp} + + def clear(self): + self._store.clear() + + +_SALES_DF_CACHE = {} +_SALES_PROCESSED_CACHE = {} +_SALES_OPTIONS_CACHE = {} +_SALES_ANALYSIS_RESULT_CACHE = {} +_SALES_CACHE_MAX_ENTRIES = 10 +_SALES_CACHE_TTL = 600 + +_DAILY_SALES_PROCESSED_CACHE = {} + +_DASHBOARD_DATA_CACHE = { + 'consolidated_data': None, + 'consolidated_timestamp': None, + 'today_start': None, + 'full_data': None, + 'full_timestamp': None, +} +_DASHBOARD_CACHE_TTL = 1800 + + +def cleanup_sales_cache(): + """清理 sales 處理後快取的過期與超額條目。""" + current_time = time.time() + expired_keys = [ + key for key, value in _SALES_PROCESSED_CACHE.items() + if value.get('time') and current_time - value['time'] > _SALES_CACHE_TTL + ] + for key in expired_keys: + _SALES_PROCESSED_CACHE.pop(key, None) + + if len(_SALES_PROCESSED_CACHE) > _SALES_CACHE_MAX_ENTRIES: + sorted_items = sorted( + [(key, value.get('time', 0)) for key, value in _SALES_PROCESSED_CACHE.items()], + key=lambda item: item[1], + ) + for key, _ in sorted_items[:-_SALES_CACHE_MAX_ENTRIES]: + _SALES_PROCESSED_CACHE.pop(key, None) + + +def set_sales_processed_cache(key, entry, aliases=()): + """寫入 sales cache，並可同步寫入別名 key。""" + entry.setdefault('time', time.time()) + _SALES_PROCESSED_CACHE[key] = entry + for alias in aliases: + _SALES_PROCESSED_CACHE[alias] = entry + cleanup_sales_cache() + + +def clear_sales_cache_for_table(table_name): + """匯入資料後清除指定表對應的所有 sales cache key。""" + _SALES_DF_CACHE.pop(table_name, None) + keys_to_delete = [ + key for key in list(_SALES_PROCESSED_CACHE.keys()) + if key == table_name or key.startswith(f"{table_name}_") + ] + for key in keys_to_delete: + _SALES_PROCESSED_CACHE.pop(key, None) + + +def clear_sales_cache(): + """清除所有 sales cache。""" + _SALES_DF_CACHE.clear() + _SALES_PROCESSED_CACHE.clear() + _SALES_OPTIONS_CACHE.clear() + _SALES_ANALYSIS_RESULT_CACHE.clear() + + +def clear_daily_sales_cache(): + """清除當日業績 cache。""" + _DAILY_SALES_PROCESSED_CACHE.clear() + + +def clear_dashboard_cache(): + """清除商品看板 cache。""" + _DASHBOARD_DATA_CACHE.clear() + _DASHBOARD_DATA_CACHE.update({ + 'consolidated_data': None, + 'consolidated_timestamp': None, + 'today_start': None, + 'full_data': None, + 'full_timestamp': None, + }) diff --git a/services/cache_service.py b/services/cache_service.py index 742bc35..a96089f 100644 --- a/services/cache_service.py +++ b/services/cache_service.py @@ -6,34 +6,25 @@ """ from datetime import datetime, timezone, timedelta +from services.cache_manager import ( + _SALES_DF_CACHE, + _SALES_PROCESSED_CACHE, + _SALES_OPTIONS_CACHE, + _SALES_ANALYSIS_RESULT_CACHE, + _DASHBOARD_DATA_CACHE, + _DASHBOARD_CACHE_TTL, + clear_sales_cache, + clear_dashboard_cache, +) # 台北時區 TAIPEI_TZ = timezone(timedelta(hours=8)) -# ========================================== -# 業績分析快取 -# ========================================== -_SALES_DF_CACHE = {} -_SALES_PROCESSED_CACHE = {} # 處理後資料快取 (二級快取) -_SALES_OPTIONS_CACHE = {} # 下拉選單選項 (類別、品牌、廠商等) -_SALES_ANALYSIS_RESULT_CACHE = {} # 過濾後的分析結果集 - # 快取 TTL 設定 _SALES_CACHE_TTL = 3600 # 業績分析快取: 60 分鐘 _SALES_OPTIONS_TTL = 21600 # 選項快取: 6 小時 _SALES_RESULT_TTL = 3600 # 結果快取: 60 分鐘 -# ========================================== -# 商品看板快取 -# ========================================== -_DASHBOARD_DATA_CACHE = { - 'consolidated_data': None, - 'consolidated_timestamp': None, - 'full_data': None, - 'full_timestamp': None -} -_DASHBOARD_CACHE_TTL = 1800 # 商品看板快取: 30 分鐘 - # ========================================== # 成長分析快取 # ========================================== @@ -81,26 +72,6 @@ def get_slow_query_stats(): return _SLOW_QUERY_STATS.copy() -def clear_sales_cache(): - """清除業績分析快取""" - global _SALES_DF_CACHE, _SALES_PROCESSED_CACHE, _SALES_OPTIONS_CACHE, _SALES_ANALYSIS_RESULT_CACHE - _SALES_DF_CACHE.clear() - _SALES_PROCESSED_CACHE.clear() - _SALES_OPTIONS_CACHE.clear() - _SALES_ANALYSIS_RESULT_CACHE.clear() - - -def clear_dashboard_cache(): - """清除商品看板快取""" - global _DASHBOARD_DATA_CACHE - _DASHBOARD_DATA_CACHE = { - 'consolidated_data': None, - 'consolidated_timestamp': None, - 'full_data': None, - 'full_timestamp': None - } - - def clear_growth_cache(): """清除成長分析快取""" global _GROWTH_ANALYSIS_CACHE