fix(api): record stockplatform recovery receipt
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 36s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped

This commit is contained in:
Your Name
2026-06-30 16:48:43 +08:00
parent 98cad13459
commit e12a8a4317
5 changed files with 231 additions and 16 deletions

View File

@@ -9,14 +9,20 @@ from __future__ import annotations
import json
import urllib.error
import urllib.request
from collections.abc import Callable
from pathlib import Path
from typing import Any, Callable
from typing import Any
from src.services.reboot_auto_recovery_slo_scorecard import (
load_latest_reboot_auto_recovery_slo_scorecard,
)
from src.services.snapshot_paths import default_operations_dir
_API_SCHEMA_VERSION = "stockplatform_public_api_runtime_readback_v1"
_DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__))
_RECOVERY_RECEIPT_FILE = (
"stockplatform-public-api-runtime-recovery-control-receipt.snapshot.json"
)
_DEFAULT_BASE_URL = "https://stock.wooo.work"
_DEFAULT_TIMEOUT_SECONDS = 4.0
@@ -33,9 +39,11 @@ def load_latest_stockplatform_public_api_runtime_readback(
"""Build a live public readback for StockPlatform web/API health."""
http_probe = probe or _http_probe
normalized_base_url = base_url.rstrip("/")
directory = operations_dir or _DEFAULT_OPERATIONS_DIR
committed_scorecard = load_latest_reboot_auto_recovery_slo_scorecard(
operations_dir
directory
)
recovery_control_receipt = _load_recovery_control_receipt(directory)
committed_stock = _dict(committed_scorecard.get("stockplatform_data_freshness"))
endpoints = {
"public_web_healthz": "/healthz",
@@ -57,6 +65,7 @@ def load_latest_stockplatform_public_api_runtime_readback(
timeout_seconds=timeout_seconds,
probes=probes,
committed_stockplatform=committed_stock,
recovery_control_receipt=recovery_control_receipt,
)
@@ -66,6 +75,7 @@ def _build_payload(
timeout_seconds: float,
probes: dict[str, dict[str, Any]],
committed_stockplatform: dict[str, Any],
recovery_control_receipt: dict[str, Any],
) -> dict[str, Any]:
web = _dict(probes.get("public_web_healthz"))
api = _dict(probes.get("public_api_healthz"))
@@ -90,7 +100,10 @@ def _build_payload(
ingestion_json=ingestion_json,
)
ready = all(checks.values())
recovery_control_path = _recovery_control_path_readback(runtime_ready=ready)
recovery_control_path = _recovery_control_path_readback(
runtime_ready=ready,
receipt=recovery_control_receipt,
)
recovery_control_path_blockers = _strings(
recovery_control_path.get("active_blockers")
)
@@ -195,15 +208,42 @@ def _build_payload(
}
def _recovery_control_path_readback(*, runtime_ready: bool) -> dict[str, Any]:
def _recovery_control_path_readback(
*,
runtime_ready: bool,
receipt: dict[str, Any],
) -> dict[str, Any]:
if runtime_ready:
return {
"status": "not_required_stockplatform_runtime_ready",
"active_blockers": [],
"receipt": receipt,
"safe_recovery_channels": [
"keep_monitoring_public_api_runtime_readback",
],
}
if receipt:
active_blockers = _unique_strings(
_strings(receipt.get("active_blockers"))
or _strings(receipt.get("missing_receipts"))
)
return {
"status": str(
receipt.get("status")
or "blocked_recovery_control_path_receipt_incomplete"
),
"active_blockers": active_blockers,
"receipt": receipt,
"provided_receipts": _strings(receipt.get("provided_receipts")),
"missing_receipts": _strings(receipt.get("missing_receipts")),
"safe_recovery_channels": _strings(
receipt.get("safe_recovery_channels")
),
"forbidden_recovery_channels": _strings(
receipt.get("forbidden_recovery_channels")
),
"required_receipts": _strings(receipt.get("required_receipts")),
}
return {
"status": "blocked_recovery_control_path_receipt_missing",
"active_blockers": [
@@ -233,6 +273,23 @@ def _recovery_control_path_readback(*, runtime_ready: bool) -> dict[str, Any]:
}
def _load_recovery_control_receipt(operations_dir: Path) -> dict[str, Any]:
path = operations_dir / _RECOVERY_RECEIPT_FILE
if not path.exists():
return {}
with path.open(encoding="utf-8") as handle:
payload = json.load(handle)
if not isinstance(payload, dict):
raise ValueError(f"{path}: expected JSON object")
if payload.get("schema_version") != (
"stockplatform_public_api_runtime_recovery_control_receipt_v1"
):
raise ValueError(
f"{path}: unsupported schema_version={payload.get('schema_version')!r}"
)
return payload
def _active_blockers(
*,
api: dict[str, Any],
@@ -314,7 +371,14 @@ def _http_probe(url: str, timeout_seconds: float) -> dict[str, Any]:
body = exc.read(2048).decode("utf-8", "replace")
return {"http_status": exc.code, "body": body, "error": ""}
except Exception as exc: # noqa: BLE001 - readback must fail closed.
return {"http_status": None, "body": "", "error": type(exc).__name__}
return {"http_status": None, "body": "", "error": _error_text(exc)}
def _error_text(exc: Exception) -> str:
reason = getattr(exc, "reason", "")
if reason:
return f"{type(exc).__name__}: {reason}"
return type(exc).__name__
def _dict(value: Any) -> dict[str, Any]:

View File

@@ -106,9 +106,9 @@ def test_awoooi_priority_work_order_readback_overlays_live_stockplatform_drift()
assert evidence["stockplatform_public_api_health_http_status"] == 502
assert (
evidence["stockplatform_recovery_control_path_status"]
== "blocked_recovery_control_path_receipt_missing"
== "api_upstream_recovered_data_readiness_not_green"
)
assert "stockplatform_recovery_control_path_receipt_missing" in evidence[
assert "stockplatform_post_apply_public_api_verifier_not_green" in evidence[
"stockplatform_recovery_control_path_active_blockers"
]
assert in_progress["status"] == "blocked_stockplatform_public_api_runtime_drift"
@@ -118,7 +118,7 @@ def test_awoooi_priority_work_order_readback_overlays_live_stockplatform_drift()
payload["rollups"][
"stockplatform_public_api_recovery_control_path_status"
]
== "blocked_recovery_control_path_receipt_missing"
== "api_upstream_recovered_data_readiness_not_green"
)
assert (
payload["rollups"][
@@ -224,8 +224,8 @@ def _stockplatform_runtime_blocked() -> dict:
"stockplatform_public_api_healthz_http_502",
"stockplatform_freshness_http_502",
"stockplatform_ingestion_http_502",
"stockplatform_recovery_control_path_receipt_missing",
"stockplatform_controlled_deploy_or_ssh_readback_required",
"stockplatform_postgres_not_ready",
"stockplatform_post_apply_public_api_verifier_not_green",
],
"runtime_ready": False,
"live_drift_from_committed_scorecard": True,
@@ -238,10 +238,10 @@ def _stockplatform_runtime_blocked() -> dict:
"http_502_count": 3,
},
"recovery_control_path": {
"status": "blocked_recovery_control_path_receipt_missing",
"status": "api_upstream_recovered_data_readiness_not_green",
"active_blockers": [
"stockplatform_recovery_control_path_receipt_missing",
"stockplatform_controlled_deploy_or_ssh_readback_required",
"stockplatform_postgres_not_ready",
"stockplatform_post_apply_public_api_verifier_not_green",
],
},
}

View File

@@ -32,13 +32,16 @@ def test_stockplatform_public_api_runtime_readback_blocks_live_502():
"stockplatform_public_api_healthz_http_502",
"stockplatform_freshness_http_502",
"stockplatform_ingestion_http_502",
"stockplatform_recovery_control_path_receipt_missing",
"stockplatform_controlled_deploy_or_ssh_readback_required",
"stockplatform_postgres_not_ready",
"stockplatform_post_apply_public_api_verifier_not_green",
]
assert (
payload["recovery_control_path"]["status"]
== "blocked_recovery_control_path_receipt_missing"
== "api_upstream_recovered_data_readiness_not_green"
)
assert payload["recovery_control_path"]["receipt"]["source_of_truth_diff"][
"gitea_main_sha"
].startswith("080bc2b")
assert payload["rollups"]["recovery_control_path_blocker_count"] == 2
assert (
payload["operation_boundaries"]["read_only_public_https_probe"] is True

View File

@@ -1,3 +1,12 @@
## 2026-06-30 — 18:52 P0-006 StockPlatform receipt updated after upstream recovery
**照主線修正的問題**
- Gitea / StockPlatform public route 恢復後StockPlatform public API 已從 `/api/healthz=502` 推進到 `/api/healthz=200`;最新 StockPlatform main 是 `080bc2b2ee fix(ops): recover stock api upstream resolution`
- `docs/operations/stockplatform-public-api-runtime-recovery-control-receipt.snapshot.json` 已更新到 `080bc2b2ee`,將狀態改成 `api_upstream_recovered_data_readiness_not_green`;目前 active blocker 精準收斂為 `stockplatform_postgres_not_ready``stockplatform_post_apply_public_api_verifier_not_green`
- live public `freshness` / `ingestion` 目前 HTTP 200但 response `status=not_configured`、blocker `postgres_not_ready`;不得把 API upstream 已恢復誤稱為 StockPlatform data readiness 已完成。
**邊界**:未 workflow_dispatch未 SSH 寫主機,未重啟主機,未 restart Docker daemon / host Nginx / K3s / DB / Redis / firewall未 prune / restore / DB write未讀 secret / token / raw sessions / SQLite / `.env`,未使用 GitHub / `gh` / GitHub API。
## 2026-06-30 — 18:42 P0-006 reboot auto-detection / VMware / maintenance / backup alert hardening
**照使用者 8 點問題收斂的 source 修正**
@@ -65,7 +74,18 @@
- `git diff --check`:通過。
**邊界**:未 workflow_dispatch未重啟主機未 restart Docker daemon / host Nginx / K3s / DB / Redis / firewall未 prune / restore / DB write未讀 secret / token / raw sessions / SQLite / `.env`,未使用 GitHub / `gh` / GitHub API。
## 2026-06-30 — 16:42 P0-006 StockPlatform source contract receipt integrated
**照主線修正的問題**
- StockPlatform v2 `3cd2d824 fix(deploy): fail closed on stockplatform api health` 已 normal push 到 Gitea `main`,補上 production deploy 使用 `.env.production.local`、app recreate 後 restart edge、`/api/healthz` local gate 與 `npm run check` contract test。
- 新增 `docs/operations/stockplatform-public-api-runtime-recovery-control-receipt.snapshot.json`,把 target selector、source-of-truth diff、check-mode、rollback、post-apply public API verifier 與 LOG writeback 狀態納入 AWOOOI committed readback。
- `stockplatform_public_api_runtime_readback` 現在會讀 receipt目前 blocker 從泛化的 `stockplatform_recovery_control_path_receipt_missing` 推進為 `stockplatform_controlled_deploy_or_ssh_readback_required``stockplatform_post_apply_public_api_verifier_not_green`
**live readback**
- Gitea `wooo/stockplatform-v2` main`3cd2d824047ed02336cfa4d8f2f39c5f9f4458b5`
- Public StockPlatform`/healthz=200``/api/healthz=502`、freshness `502`、ingestion `502`source contract 已進 main但 production runtime 尚未恢復,不能宣稱 P0-006 closure。
**邊界**:未觸發 StockPlatform deploy workflow未恢復 push deploy trigger未 SSH 寫主機,未 restart Docker daemon / Nginx / K3s / DB / firewall未 prune / restore / DB write未讀 secret / token / raw sessions / SQLite / `.env`,未使用 GitHub / `gh` / GitHub API。
## 2026-06-30 — 16:40 P0-006 StockPlatform recovery control-path readback
**照主線修正的問題**

View File

@@ -0,0 +1,128 @@
{
"schema_version": "stockplatform_public_api_runtime_recovery_control_receipt_v1",
"generated_at": "2026-06-30T18:52:40+08:00",
"status": "api_upstream_recovered_data_readiness_not_green",
"priority": "P0-006",
"scope": "stockplatform_public_api_runtime_recovery_control_path",
"target_selector": {
"product": "stockplatform-v2",
"public_base_url": "https://stock.wooo.work",
"target_endpoints": [
"/api/healthz",
"/api/v1/system/freshness",
"/api/v1/system/ingestion"
],
"expected_after_apply": "all target endpoints return HTTP 200; freshness.status=ok; ingestion.status=ok"
},
"source_of_truth_diff": {
"repo": "wooo/stockplatform-v2",
"gitea_main_sha": "080bc2b2eed796b4412bb34b736c87121fb7b3b8",
"commit_subject": "fix(ops): recover stock api upstream resolution",
"supporting_commits": [
{
"sha": "3cd2d824047ed02336cfa4d8f2f39c5f9f4458b5",
"subject": "fix(deploy): fail closed on stockplatform api health"
},
{
"sha": "080bc2b2eed796b4412bb34b736c87121fb7b3b8",
"subject": "fix(ops): recover stock api upstream resolution"
}
],
"files_changed": [
"infra/nginx/stockplatform-v2-edge.admin-auth.conf",
"infra/nginx/stockplatform-v2-edge.conf",
"infra/scripts/deploy-production.sh",
"package.json",
"scripts/ops/stockplatform-api-upstream-recovery-gate.sh",
"scripts/ops/test-stockplatform-api-upstream-recovery-gate.sh",
"scripts/ops/test-production-deploy-contract.sh"
],
"change_summary": [
"production deploy uses .env.production.local instead of compose defaults",
"edge is restarted after app containers are recreated to avoid stale api upstream resolution",
"local /api/healthz must return 200 before deploy can emit success receipt",
"edge nginx uses Docker DNS dynamic upstream resolution for api/web/admin",
"api upstream recovery gate provides verify-only, dry-run, and bounded apply modes",
"npm run check now includes the production deploy contract test"
]
},
"check_mode_or_dry_run": {
"performed": true,
"commands": [
"npm run check",
"python3 -m compileall -q apps/api services/worker services/scheduler scripts/ingestion scripts/migration",
"curl https://stock.wooo.work/api/healthz",
"curl https://stock.wooo.work/api/v1/system/freshness",
"curl https://stock.wooo.work/api/v1/system/ingestion"
],
"result": "source checks passed; live public API upstream is HTTP 200, but freshness and ingestion return status=not_configured with postgres_not_ready"
},
"rollback": {
"available": true,
"command": "git revert 080bc2b2eed796b4412bb34b736c87121fb7b3b8 3cd2d824047ed02336cfa4d8f2f39c5f9f4458b5 && rerun stockplatform production deploy and upstream recovery contract checks before any runtime apply"
},
"post_apply_public_api_verifier": {
"prepared": true,
"last_pre_apply_readback": {
"public_web_healthz": 200,
"public_api_healthz": 200,
"freshness": 200,
"freshness_status": "not_configured",
"freshness_blockers": [
"postgres_not_ready"
],
"ingestion": 200,
"ingestion_status": "not_configured",
"ingestion_blockers": [
"postgres_not_ready"
]
},
"passed": false
},
"provided_receipts": [
"target_selector",
"source_of_truth_diff_or_runtime_state_diff",
"check_mode_or_dry_run",
"rollback_command",
"controlled_api_upstream_recovery_readback",
"km_playbook_log_writeback_receipt"
],
"missing_receipts": [
"stockplatform_postgres_readiness_or_data_source_readback",
"post_apply_public_api_verifier_green"
],
"active_blockers": [
"stockplatform_postgres_not_ready",
"stockplatform_post_apply_public_api_verifier_not_green"
],
"safe_recovery_channels": [
"bounded_stockplatform_postgres_readiness_readback_without_manual_db_write",
"bounded_ssh_or_runner_recovery_readback_without_docker_daemon_restart"
],
"forbidden_recovery_channels": [
"docker_daemon_restart",
"host_reboot",
"docker_prune_or_volume_restore",
"manual_database_rows_or_fake_freshness",
"restore_push_deploy_trigger_during_110_capacity_freeze",
"read_secret_or_raw_session_values"
],
"required_receipts": [
"target_selector",
"source_of_truth_diff_or_runtime_state_diff",
"check_mode_or_dry_run",
"rollback_command",
"post_apply_public_api_verifier",
"km_playbook_log_writeback_receipt"
],
"operation_boundaries": {
"workflow_dispatch_performed": false,
"host_write_performed": false,
"docker_restart_performed": false,
"docker_daemon_restart_performed": false,
"host_reboot_performed": false,
"database_write_or_restore_performed": false,
"secret_or_runner_token_read": false,
"github_api_used": false
}
}