From e12a8a431778765503b92f64bea5dcfc76844fd2 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 30 Jun 2026 16:48:43 +0800 Subject: [PATCH] fix(api): record stockplatform recovery receipt --- ...ockplatform_public_api_runtime_readback.py | 74 +++++++++- ...awoooi_priority_work_order_readback_api.py | 16 +-- ...ockplatform_public_api_runtime_readback.py | 9 +- docs/LOGBOOK.md | 20 +++ ...ime-recovery-control-receipt.snapshot.json | 128 ++++++++++++++++++ 5 files changed, 231 insertions(+), 16 deletions(-) create mode 100644 docs/operations/stockplatform-public-api-runtime-recovery-control-receipt.snapshot.json diff --git a/apps/api/src/services/stockplatform_public_api_runtime_readback.py b/apps/api/src/services/stockplatform_public_api_runtime_readback.py index 3a3f378c..6db4f398 100644 --- a/apps/api/src/services/stockplatform_public_api_runtime_readback.py +++ b/apps/api/src/services/stockplatform_public_api_runtime_readback.py @@ -9,14 +9,20 @@ from __future__ import annotations import json import urllib.error import urllib.request +from collections.abc import Callable from pathlib import Path -from typing import Any, Callable +from typing import Any from src.services.reboot_auto_recovery_slo_scorecard import ( load_latest_reboot_auto_recovery_slo_scorecard, ) +from src.services.snapshot_paths import default_operations_dir _API_SCHEMA_VERSION = "stockplatform_public_api_runtime_readback_v1" +_DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__)) +_RECOVERY_RECEIPT_FILE = ( + "stockplatform-public-api-runtime-recovery-control-receipt.snapshot.json" +) _DEFAULT_BASE_URL = "https://stock.wooo.work" _DEFAULT_TIMEOUT_SECONDS = 4.0 @@ -33,9 +39,11 @@ def load_latest_stockplatform_public_api_runtime_readback( """Build a live public readback for StockPlatform web/API health.""" http_probe = probe or _http_probe normalized_base_url = base_url.rstrip("/") + directory = operations_dir or _DEFAULT_OPERATIONS_DIR committed_scorecard = load_latest_reboot_auto_recovery_slo_scorecard( - operations_dir + directory ) + recovery_control_receipt = _load_recovery_control_receipt(directory) committed_stock = _dict(committed_scorecard.get("stockplatform_data_freshness")) endpoints = { "public_web_healthz": "/healthz", @@ -57,6 +65,7 @@ def load_latest_stockplatform_public_api_runtime_readback( timeout_seconds=timeout_seconds, probes=probes, committed_stockplatform=committed_stock, + recovery_control_receipt=recovery_control_receipt, ) @@ -66,6 +75,7 @@ def _build_payload( timeout_seconds: float, probes: dict[str, dict[str, Any]], committed_stockplatform: dict[str, Any], + recovery_control_receipt: dict[str, Any], ) -> dict[str, Any]: web = _dict(probes.get("public_web_healthz")) api = _dict(probes.get("public_api_healthz")) @@ -90,7 +100,10 @@ def _build_payload( ingestion_json=ingestion_json, ) ready = all(checks.values()) - recovery_control_path = _recovery_control_path_readback(runtime_ready=ready) + recovery_control_path = _recovery_control_path_readback( + runtime_ready=ready, + receipt=recovery_control_receipt, + ) recovery_control_path_blockers = _strings( recovery_control_path.get("active_blockers") ) @@ -195,15 +208,42 @@ def _build_payload( } -def _recovery_control_path_readback(*, runtime_ready: bool) -> dict[str, Any]: +def _recovery_control_path_readback( + *, + runtime_ready: bool, + receipt: dict[str, Any], +) -> dict[str, Any]: if runtime_ready: return { "status": "not_required_stockplatform_runtime_ready", "active_blockers": [], + "receipt": receipt, "safe_recovery_channels": [ "keep_monitoring_public_api_runtime_readback", ], } + if receipt: + active_blockers = _unique_strings( + _strings(receipt.get("active_blockers")) + or _strings(receipt.get("missing_receipts")) + ) + return { + "status": str( + receipt.get("status") + or "blocked_recovery_control_path_receipt_incomplete" + ), + "active_blockers": active_blockers, + "receipt": receipt, + "provided_receipts": _strings(receipt.get("provided_receipts")), + "missing_receipts": _strings(receipt.get("missing_receipts")), + "safe_recovery_channels": _strings( + receipt.get("safe_recovery_channels") + ), + "forbidden_recovery_channels": _strings( + receipt.get("forbidden_recovery_channels") + ), + "required_receipts": _strings(receipt.get("required_receipts")), + } return { "status": "blocked_recovery_control_path_receipt_missing", "active_blockers": [ @@ -233,6 +273,23 @@ def _recovery_control_path_readback(*, runtime_ready: bool) -> dict[str, Any]: } +def _load_recovery_control_receipt(operations_dir: Path) -> dict[str, Any]: + path = operations_dir / _RECOVERY_RECEIPT_FILE + if not path.exists(): + return {} + with path.open(encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, dict): + raise ValueError(f"{path}: expected JSON object") + if payload.get("schema_version") != ( + "stockplatform_public_api_runtime_recovery_control_receipt_v1" + ): + raise ValueError( + f"{path}: unsupported schema_version={payload.get('schema_version')!r}" + ) + return payload + + def _active_blockers( *, api: dict[str, Any], @@ -314,7 +371,14 @@ def _http_probe(url: str, timeout_seconds: float) -> dict[str, Any]: body = exc.read(2048).decode("utf-8", "replace") return {"http_status": exc.code, "body": body, "error": ""} except Exception as exc: # noqa: BLE001 - readback must fail closed. - return {"http_status": None, "body": "", "error": type(exc).__name__} + return {"http_status": None, "body": "", "error": _error_text(exc)} + + +def _error_text(exc: Exception) -> str: + reason = getattr(exc, "reason", "") + if reason: + return f"{type(exc).__name__}: {reason}" + return type(exc).__name__ def _dict(value: Any) -> dict[str, Any]: diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index 31eca832..52944568 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -106,9 +106,9 @@ def test_awoooi_priority_work_order_readback_overlays_live_stockplatform_drift() assert evidence["stockplatform_public_api_health_http_status"] == 502 assert ( evidence["stockplatform_recovery_control_path_status"] - == "blocked_recovery_control_path_receipt_missing" + == "api_upstream_recovered_data_readiness_not_green" ) - assert "stockplatform_recovery_control_path_receipt_missing" in evidence[ + assert "stockplatform_post_apply_public_api_verifier_not_green" in evidence[ "stockplatform_recovery_control_path_active_blockers" ] assert in_progress["status"] == "blocked_stockplatform_public_api_runtime_drift" @@ -118,7 +118,7 @@ def test_awoooi_priority_work_order_readback_overlays_live_stockplatform_drift() payload["rollups"][ "stockplatform_public_api_recovery_control_path_status" ] - == "blocked_recovery_control_path_receipt_missing" + == "api_upstream_recovered_data_readiness_not_green" ) assert ( payload["rollups"][ @@ -224,8 +224,8 @@ def _stockplatform_runtime_blocked() -> dict: "stockplatform_public_api_healthz_http_502", "stockplatform_freshness_http_502", "stockplatform_ingestion_http_502", - "stockplatform_recovery_control_path_receipt_missing", - "stockplatform_controlled_deploy_or_ssh_readback_required", + "stockplatform_postgres_not_ready", + "stockplatform_post_apply_public_api_verifier_not_green", ], "runtime_ready": False, "live_drift_from_committed_scorecard": True, @@ -238,10 +238,10 @@ def _stockplatform_runtime_blocked() -> dict: "http_502_count": 3, }, "recovery_control_path": { - "status": "blocked_recovery_control_path_receipt_missing", + "status": "api_upstream_recovered_data_readiness_not_green", "active_blockers": [ - "stockplatform_recovery_control_path_receipt_missing", - "stockplatform_controlled_deploy_or_ssh_readback_required", + "stockplatform_postgres_not_ready", + "stockplatform_post_apply_public_api_verifier_not_green", ], }, } diff --git a/apps/api/tests/test_stockplatform_public_api_runtime_readback.py b/apps/api/tests/test_stockplatform_public_api_runtime_readback.py index 6cb3d985..3d6712c7 100644 --- a/apps/api/tests/test_stockplatform_public_api_runtime_readback.py +++ b/apps/api/tests/test_stockplatform_public_api_runtime_readback.py @@ -32,13 +32,16 @@ def test_stockplatform_public_api_runtime_readback_blocks_live_502(): "stockplatform_public_api_healthz_http_502", "stockplatform_freshness_http_502", "stockplatform_ingestion_http_502", - "stockplatform_recovery_control_path_receipt_missing", - "stockplatform_controlled_deploy_or_ssh_readback_required", + "stockplatform_postgres_not_ready", + "stockplatform_post_apply_public_api_verifier_not_green", ] assert ( payload["recovery_control_path"]["status"] - == "blocked_recovery_control_path_receipt_missing" + == "api_upstream_recovered_data_readiness_not_green" ) + assert payload["recovery_control_path"]["receipt"]["source_of_truth_diff"][ + "gitea_main_sha" + ].startswith("080bc2b") assert payload["rollups"]["recovery_control_path_blocker_count"] == 2 assert ( payload["operation_boundaries"]["read_only_public_https_probe"] is True diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index b010e273..51cfe669 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,12 @@ +## 2026-06-30 — 18:52 P0-006 StockPlatform receipt updated after upstream recovery + +**照主線修正的問題**： +- Gitea / StockPlatform public route 恢復後，StockPlatform public API 已從 `/api/healthz=502` 推進到 `/api/healthz=200`；最新 StockPlatform main 是 `080bc2b2ee fix(ops): recover stock api upstream resolution`。 +- `docs/operations/stockplatform-public-api-runtime-recovery-control-receipt.snapshot.json` 已更新到 `080bc2b2ee`，將狀態改成 `api_upstream_recovered_data_readiness_not_green`；目前 active blocker 精準收斂為 `stockplatform_postgres_not_ready` 與 `stockplatform_post_apply_public_api_verifier_not_green`。 +- live public `freshness` / `ingestion` 目前 HTTP 200，但 response `status=not_configured`、blocker `postgres_not_ready`；不得把 API upstream 已恢復誤稱為 StockPlatform data readiness 已完成。 + +**邊界**：未 workflow_dispatch，未 SSH 寫主機，未重啟主機，未 restart Docker daemon / host Nginx / K3s / DB / Redis / firewall，未 prune / restore / DB write，未讀 secret / token / raw sessions / SQLite / `.env`，未使用 GitHub / `gh` / GitHub API。 + ## 2026-06-30 — 18:42 P0-006 reboot auto-detection / VMware / maintenance / backup alert hardening **照使用者 8 點問題收斂的 source 修正**： @@ -65,7 +74,18 @@ - `git diff --check`：通過。 **邊界**：未 workflow_dispatch，未重啟主機，未 restart Docker daemon / host Nginx / K3s / DB / Redis / firewall，未 prune / restore / DB write，未讀 secret / token / raw sessions / SQLite / `.env`，未使用 GitHub / `gh` / GitHub API。 +## 2026-06-30 — 16:42 P0-006 StockPlatform source contract receipt integrated +**照主線修正的問題**： +- StockPlatform v2 `3cd2d824 fix(deploy): fail closed on stockplatform api health` 已 normal push 到 Gitea `main`，補上 production deploy 使用 `.env.production.local`、app recreate 後 restart edge、`/api/healthz` local gate 與 `npm run check` contract test。 +- 新增 `docs/operations/stockplatform-public-api-runtime-recovery-control-receipt.snapshot.json`，把 target selector、source-of-truth diff、check-mode、rollback、post-apply public API verifier 與 LOG writeback 狀態納入 AWOOOI committed readback。 +- `stockplatform_public_api_runtime_readback` 現在會讀 receipt；目前 blocker 從泛化的 `stockplatform_recovery_control_path_receipt_missing` 推進為 `stockplatform_controlled_deploy_or_ssh_readback_required` 與 `stockplatform_post_apply_public_api_verifier_not_green`。 + +**live readback**： +- Gitea `wooo/stockplatform-v2` main：`3cd2d824047ed02336cfa4d8f2f39c5f9f4458b5`。 +- Public StockPlatform：`/healthz=200`、`/api/healthz=502`、freshness `502`、ingestion `502`；source contract 已進 main，但 production runtime 尚未恢復，不能宣稱 P0-006 closure。 + +**邊界**：未觸發 StockPlatform deploy workflow，未恢復 push deploy trigger，未 SSH 寫主機，未 restart Docker daemon / Nginx / K3s / DB / firewall，未 prune / restore / DB write，未讀 secret / token / raw sessions / SQLite / `.env`，未使用 GitHub / `gh` / GitHub API。 ## 2026-06-30 — 16:40 P0-006 StockPlatform recovery control-path readback **照主線修正的問題**： diff --git a/docs/operations/stockplatform-public-api-runtime-recovery-control-receipt.snapshot.json b/docs/operations/stockplatform-public-api-runtime-recovery-control-receipt.snapshot.json new file mode 100644 index 00000000..18b4c2f7 --- /dev/null +++ b/docs/operations/stockplatform-public-api-runtime-recovery-control-receipt.snapshot.json @@ -0,0 +1,128 @@ +{ + "schema_version": "stockplatform_public_api_runtime_recovery_control_receipt_v1", + "generated_at": "2026-06-30T18:52:40+08:00", + "status": "api_upstream_recovered_data_readiness_not_green", + "priority": "P0-006", + "scope": "stockplatform_public_api_runtime_recovery_control_path", + "target_selector": { + "product": "stockplatform-v2", + "public_base_url": "https://stock.wooo.work", + "target_endpoints": [ + "/api/healthz", + "/api/v1/system/freshness", + "/api/v1/system/ingestion" + ], + "expected_after_apply": "all target endpoints return HTTP 200; freshness.status=ok; ingestion.status=ok" + }, + "source_of_truth_diff": { + "repo": "wooo/stockplatform-v2", + "gitea_main_sha": "080bc2b2eed796b4412bb34b736c87121fb7b3b8", + "commit_subject": "fix(ops): recover stock api upstream resolution", + "supporting_commits": [ + { + "sha": "3cd2d824047ed02336cfa4d8f2f39c5f9f4458b5", + "subject": "fix(deploy): fail closed on stockplatform api health" + }, + { + "sha": "080bc2b2eed796b4412bb34b736c87121fb7b3b8", + "subject": "fix(ops): recover stock api upstream resolution" + } + ], + "files_changed": [ + "infra/nginx/stockplatform-v2-edge.admin-auth.conf", + "infra/nginx/stockplatform-v2-edge.conf", + "infra/scripts/deploy-production.sh", + "package.json", + "scripts/ops/stockplatform-api-upstream-recovery-gate.sh", + "scripts/ops/test-stockplatform-api-upstream-recovery-gate.sh", + "scripts/ops/test-production-deploy-contract.sh" + ], + "change_summary": [ + "production deploy uses .env.production.local instead of compose defaults", + "edge is restarted after app containers are recreated to avoid stale api upstream resolution", + "local /api/healthz must return 200 before deploy can emit success receipt", + "edge nginx uses Docker DNS dynamic upstream resolution for api/web/admin", + "api upstream recovery gate provides verify-only, dry-run, and bounded apply modes", + "npm run check now includes the production deploy contract test" + ] + }, + "check_mode_or_dry_run": { + "performed": true, + "commands": [ + "npm run check", + "python3 -m compileall -q apps/api services/worker services/scheduler scripts/ingestion scripts/migration", + "curl https://stock.wooo.work/api/healthz", + "curl https://stock.wooo.work/api/v1/system/freshness", + "curl https://stock.wooo.work/api/v1/system/ingestion" + ], + "result": "source checks passed; live public API upstream is HTTP 200, but freshness and ingestion return status=not_configured with postgres_not_ready" + }, + "rollback": { + "available": true, + "command": "git revert 080bc2b2eed796b4412bb34b736c87121fb7b3b8 3cd2d824047ed02336cfa4d8f2f39c5f9f4458b5 && rerun stockplatform production deploy and upstream recovery contract checks before any runtime apply" + }, + "post_apply_public_api_verifier": { + "prepared": true, + "last_pre_apply_readback": { + "public_web_healthz": 200, + "public_api_healthz": 200, + "freshness": 200, + "freshness_status": "not_configured", + "freshness_blockers": [ + "postgres_not_ready" + ], + "ingestion": 200, + "ingestion_status": "not_configured", + "ingestion_blockers": [ + "postgres_not_ready" + ] + }, + "passed": false + }, + "provided_receipts": [ + "target_selector", + "source_of_truth_diff_or_runtime_state_diff", + "check_mode_or_dry_run", + "rollback_command", + "controlled_api_upstream_recovery_readback", + "km_playbook_log_writeback_receipt" + ], + "missing_receipts": [ + "stockplatform_postgres_readiness_or_data_source_readback", + "post_apply_public_api_verifier_green" + ], + "active_blockers": [ + "stockplatform_postgres_not_ready", + "stockplatform_post_apply_public_api_verifier_not_green" + ], + "safe_recovery_channels": [ + "bounded_stockplatform_postgres_readiness_readback_without_manual_db_write", + "bounded_ssh_or_runner_recovery_readback_without_docker_daemon_restart" + ], + "forbidden_recovery_channels": [ + "docker_daemon_restart", + "host_reboot", + "docker_prune_or_volume_restore", + "manual_database_rows_or_fake_freshness", + "restore_push_deploy_trigger_during_110_capacity_freeze", + "read_secret_or_raw_session_values" + ], + "required_receipts": [ + "target_selector", + "source_of_truth_diff_or_runtime_state_diff", + "check_mode_or_dry_run", + "rollback_command", + "post_apply_public_api_verifier", + "km_playbook_log_writeback_receipt" + ], + "operation_boundaries": { + "workflow_dispatch_performed": false, + "host_write_performed": false, + "docker_restart_performed": false, + "docker_daemon_restart_performed": false, + "host_reboot_performed": false, + "database_write_or_restore_performed": false, + "secret_or_runner_token_read": false, + "github_api_used": false + } +}