fix(api): record stockplatform recovery receipt
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 36s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 36s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
This commit is contained in:
@@ -9,14 +9,20 @@ from __future__ import annotations
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
from typing import Any
|
||||
|
||||
from src.services.reboot_auto_recovery_slo_scorecard import (
|
||||
load_latest_reboot_auto_recovery_slo_scorecard,
|
||||
)
|
||||
from src.services.snapshot_paths import default_operations_dir
|
||||
|
||||
_API_SCHEMA_VERSION = "stockplatform_public_api_runtime_readback_v1"
|
||||
_DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__))
|
||||
_RECOVERY_RECEIPT_FILE = (
|
||||
"stockplatform-public-api-runtime-recovery-control-receipt.snapshot.json"
|
||||
)
|
||||
_DEFAULT_BASE_URL = "https://stock.wooo.work"
|
||||
_DEFAULT_TIMEOUT_SECONDS = 4.0
|
||||
|
||||
@@ -33,9 +39,11 @@ def load_latest_stockplatform_public_api_runtime_readback(
|
||||
"""Build a live public readback for StockPlatform web/API health."""
|
||||
http_probe = probe or _http_probe
|
||||
normalized_base_url = base_url.rstrip("/")
|
||||
directory = operations_dir or _DEFAULT_OPERATIONS_DIR
|
||||
committed_scorecard = load_latest_reboot_auto_recovery_slo_scorecard(
|
||||
operations_dir
|
||||
directory
|
||||
)
|
||||
recovery_control_receipt = _load_recovery_control_receipt(directory)
|
||||
committed_stock = _dict(committed_scorecard.get("stockplatform_data_freshness"))
|
||||
endpoints = {
|
||||
"public_web_healthz": "/healthz",
|
||||
@@ -57,6 +65,7 @@ def load_latest_stockplatform_public_api_runtime_readback(
|
||||
timeout_seconds=timeout_seconds,
|
||||
probes=probes,
|
||||
committed_stockplatform=committed_stock,
|
||||
recovery_control_receipt=recovery_control_receipt,
|
||||
)
|
||||
|
||||
|
||||
@@ -66,6 +75,7 @@ def _build_payload(
|
||||
timeout_seconds: float,
|
||||
probes: dict[str, dict[str, Any]],
|
||||
committed_stockplatform: dict[str, Any],
|
||||
recovery_control_receipt: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
web = _dict(probes.get("public_web_healthz"))
|
||||
api = _dict(probes.get("public_api_healthz"))
|
||||
@@ -90,7 +100,10 @@ def _build_payload(
|
||||
ingestion_json=ingestion_json,
|
||||
)
|
||||
ready = all(checks.values())
|
||||
recovery_control_path = _recovery_control_path_readback(runtime_ready=ready)
|
||||
recovery_control_path = _recovery_control_path_readback(
|
||||
runtime_ready=ready,
|
||||
receipt=recovery_control_receipt,
|
||||
)
|
||||
recovery_control_path_blockers = _strings(
|
||||
recovery_control_path.get("active_blockers")
|
||||
)
|
||||
@@ -195,15 +208,42 @@ def _build_payload(
|
||||
}
|
||||
|
||||
|
||||
def _recovery_control_path_readback(*, runtime_ready: bool) -> dict[str, Any]:
|
||||
def _recovery_control_path_readback(
|
||||
*,
|
||||
runtime_ready: bool,
|
||||
receipt: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
if runtime_ready:
|
||||
return {
|
||||
"status": "not_required_stockplatform_runtime_ready",
|
||||
"active_blockers": [],
|
||||
"receipt": receipt,
|
||||
"safe_recovery_channels": [
|
||||
"keep_monitoring_public_api_runtime_readback",
|
||||
],
|
||||
}
|
||||
if receipt:
|
||||
active_blockers = _unique_strings(
|
||||
_strings(receipt.get("active_blockers"))
|
||||
or _strings(receipt.get("missing_receipts"))
|
||||
)
|
||||
return {
|
||||
"status": str(
|
||||
receipt.get("status")
|
||||
or "blocked_recovery_control_path_receipt_incomplete"
|
||||
),
|
||||
"active_blockers": active_blockers,
|
||||
"receipt": receipt,
|
||||
"provided_receipts": _strings(receipt.get("provided_receipts")),
|
||||
"missing_receipts": _strings(receipt.get("missing_receipts")),
|
||||
"safe_recovery_channels": _strings(
|
||||
receipt.get("safe_recovery_channels")
|
||||
),
|
||||
"forbidden_recovery_channels": _strings(
|
||||
receipt.get("forbidden_recovery_channels")
|
||||
),
|
||||
"required_receipts": _strings(receipt.get("required_receipts")),
|
||||
}
|
||||
return {
|
||||
"status": "blocked_recovery_control_path_receipt_missing",
|
||||
"active_blockers": [
|
||||
@@ -233,6 +273,23 @@ def _recovery_control_path_readback(*, runtime_ready: bool) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _load_recovery_control_receipt(operations_dir: Path) -> dict[str, Any]:
|
||||
path = operations_dir / _RECOVERY_RECEIPT_FILE
|
||||
if not path.exists():
|
||||
return {}
|
||||
with path.open(encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"{path}: expected JSON object")
|
||||
if payload.get("schema_version") != (
|
||||
"stockplatform_public_api_runtime_recovery_control_receipt_v1"
|
||||
):
|
||||
raise ValueError(
|
||||
f"{path}: unsupported schema_version={payload.get('schema_version')!r}"
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _active_blockers(
|
||||
*,
|
||||
api: dict[str, Any],
|
||||
@@ -314,7 +371,14 @@ def _http_probe(url: str, timeout_seconds: float) -> dict[str, Any]:
|
||||
body = exc.read(2048).decode("utf-8", "replace")
|
||||
return {"http_status": exc.code, "body": body, "error": ""}
|
||||
except Exception as exc: # noqa: BLE001 - readback must fail closed.
|
||||
return {"http_status": None, "body": "", "error": type(exc).__name__}
|
||||
return {"http_status": None, "body": "", "error": _error_text(exc)}
|
||||
|
||||
|
||||
def _error_text(exc: Exception) -> str:
|
||||
reason = getattr(exc, "reason", "")
|
||||
if reason:
|
||||
return f"{type(exc).__name__}: {reason}"
|
||||
return type(exc).__name__
|
||||
|
||||
|
||||
def _dict(value: Any) -> dict[str, Any]:
|
||||
|
||||
@@ -106,9 +106,9 @@ def test_awoooi_priority_work_order_readback_overlays_live_stockplatform_drift()
|
||||
assert evidence["stockplatform_public_api_health_http_status"] == 502
|
||||
assert (
|
||||
evidence["stockplatform_recovery_control_path_status"]
|
||||
== "blocked_recovery_control_path_receipt_missing"
|
||||
== "api_upstream_recovered_data_readiness_not_green"
|
||||
)
|
||||
assert "stockplatform_recovery_control_path_receipt_missing" in evidence[
|
||||
assert "stockplatform_post_apply_public_api_verifier_not_green" in evidence[
|
||||
"stockplatform_recovery_control_path_active_blockers"
|
||||
]
|
||||
assert in_progress["status"] == "blocked_stockplatform_public_api_runtime_drift"
|
||||
@@ -118,7 +118,7 @@ def test_awoooi_priority_work_order_readback_overlays_live_stockplatform_drift()
|
||||
payload["rollups"][
|
||||
"stockplatform_public_api_recovery_control_path_status"
|
||||
]
|
||||
== "blocked_recovery_control_path_receipt_missing"
|
||||
== "api_upstream_recovered_data_readiness_not_green"
|
||||
)
|
||||
assert (
|
||||
payload["rollups"][
|
||||
@@ -224,8 +224,8 @@ def _stockplatform_runtime_blocked() -> dict:
|
||||
"stockplatform_public_api_healthz_http_502",
|
||||
"stockplatform_freshness_http_502",
|
||||
"stockplatform_ingestion_http_502",
|
||||
"stockplatform_recovery_control_path_receipt_missing",
|
||||
"stockplatform_controlled_deploy_or_ssh_readback_required",
|
||||
"stockplatform_postgres_not_ready",
|
||||
"stockplatform_post_apply_public_api_verifier_not_green",
|
||||
],
|
||||
"runtime_ready": False,
|
||||
"live_drift_from_committed_scorecard": True,
|
||||
@@ -238,10 +238,10 @@ def _stockplatform_runtime_blocked() -> dict:
|
||||
"http_502_count": 3,
|
||||
},
|
||||
"recovery_control_path": {
|
||||
"status": "blocked_recovery_control_path_receipt_missing",
|
||||
"status": "api_upstream_recovered_data_readiness_not_green",
|
||||
"active_blockers": [
|
||||
"stockplatform_recovery_control_path_receipt_missing",
|
||||
"stockplatform_controlled_deploy_or_ssh_readback_required",
|
||||
"stockplatform_postgres_not_ready",
|
||||
"stockplatform_post_apply_public_api_verifier_not_green",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -32,13 +32,16 @@ def test_stockplatform_public_api_runtime_readback_blocks_live_502():
|
||||
"stockplatform_public_api_healthz_http_502",
|
||||
"stockplatform_freshness_http_502",
|
||||
"stockplatform_ingestion_http_502",
|
||||
"stockplatform_recovery_control_path_receipt_missing",
|
||||
"stockplatform_controlled_deploy_or_ssh_readback_required",
|
||||
"stockplatform_postgres_not_ready",
|
||||
"stockplatform_post_apply_public_api_verifier_not_green",
|
||||
]
|
||||
assert (
|
||||
payload["recovery_control_path"]["status"]
|
||||
== "blocked_recovery_control_path_receipt_missing"
|
||||
== "api_upstream_recovered_data_readiness_not_green"
|
||||
)
|
||||
assert payload["recovery_control_path"]["receipt"]["source_of_truth_diff"][
|
||||
"gitea_main_sha"
|
||||
].startswith("080bc2b")
|
||||
assert payload["rollups"]["recovery_control_path_blocker_count"] == 2
|
||||
assert (
|
||||
payload["operation_boundaries"]["read_only_public_https_probe"] is True
|
||||
|
||||
@@ -1,3 +1,12 @@
|
||||
## 2026-06-30 — 18:52 P0-006 StockPlatform receipt updated after upstream recovery
|
||||
|
||||
**照主線修正的問題**:
|
||||
- Gitea / StockPlatform public route 恢復後,StockPlatform public API 已從 `/api/healthz=502` 推進到 `/api/healthz=200`;最新 StockPlatform main 是 `080bc2b2ee fix(ops): recover stock api upstream resolution`。
|
||||
- `docs/operations/stockplatform-public-api-runtime-recovery-control-receipt.snapshot.json` 已更新到 `080bc2b2ee`,將狀態改成 `api_upstream_recovered_data_readiness_not_green`;目前 active blocker 精準收斂為 `stockplatform_postgres_not_ready` 與 `stockplatform_post_apply_public_api_verifier_not_green`。
|
||||
- live public `freshness` / `ingestion` 目前 HTTP 200,但 response `status=not_configured`、blocker `postgres_not_ready`;不得把 API upstream 已恢復誤稱為 StockPlatform data readiness 已完成。
|
||||
|
||||
**邊界**:未 workflow_dispatch,未 SSH 寫主機,未重啟主機,未 restart Docker daemon / host Nginx / K3s / DB / Redis / firewall,未 prune / restore / DB write,未讀 secret / token / raw sessions / SQLite / `.env`,未使用 GitHub / `gh` / GitHub API。
|
||||
|
||||
## 2026-06-30 — 18:42 P0-006 reboot auto-detection / VMware / maintenance / backup alert hardening
|
||||
|
||||
**照使用者 8 點問題收斂的 source 修正**:
|
||||
@@ -65,7 +74,18 @@
|
||||
- `git diff --check`:通過。
|
||||
|
||||
**邊界**:未 workflow_dispatch,未重啟主機,未 restart Docker daemon / host Nginx / K3s / DB / Redis / firewall,未 prune / restore / DB write,未讀 secret / token / raw sessions / SQLite / `.env`,未使用 GitHub / `gh` / GitHub API。
|
||||
## 2026-06-30 — 16:42 P0-006 StockPlatform source contract receipt integrated
|
||||
|
||||
**照主線修正的問題**:
|
||||
- StockPlatform v2 `3cd2d824 fix(deploy): fail closed on stockplatform api health` 已 normal push 到 Gitea `main`,補上 production deploy 使用 `.env.production.local`、app recreate 後 restart edge、`/api/healthz` local gate 與 `npm run check` contract test。
|
||||
- 新增 `docs/operations/stockplatform-public-api-runtime-recovery-control-receipt.snapshot.json`,把 target selector、source-of-truth diff、check-mode、rollback、post-apply public API verifier 與 LOG writeback 狀態納入 AWOOOI committed readback。
|
||||
- `stockplatform_public_api_runtime_readback` 現在會讀 receipt;目前 blocker 從泛化的 `stockplatform_recovery_control_path_receipt_missing` 推進為 `stockplatform_controlled_deploy_or_ssh_readback_required` 與 `stockplatform_post_apply_public_api_verifier_not_green`。
|
||||
|
||||
**live readback**:
|
||||
- Gitea `wooo/stockplatform-v2` main:`3cd2d824047ed02336cfa4d8f2f39c5f9f4458b5`。
|
||||
- Public StockPlatform:`/healthz=200`、`/api/healthz=502`、freshness `502`、ingestion `502`;source contract 已進 main,但 production runtime 尚未恢復,不能宣稱 P0-006 closure。
|
||||
|
||||
**邊界**:未觸發 StockPlatform deploy workflow,未恢復 push deploy trigger,未 SSH 寫主機,未 restart Docker daemon / Nginx / K3s / DB / firewall,未 prune / restore / DB write,未讀 secret / token / raw sessions / SQLite / `.env`,未使用 GitHub / `gh` / GitHub API。
|
||||
## 2026-06-30 — 16:40 P0-006 StockPlatform recovery control-path readback
|
||||
|
||||
**照主線修正的問題**:
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
{
|
||||
"schema_version": "stockplatform_public_api_runtime_recovery_control_receipt_v1",
|
||||
"generated_at": "2026-06-30T18:52:40+08:00",
|
||||
"status": "api_upstream_recovered_data_readiness_not_green",
|
||||
"priority": "P0-006",
|
||||
"scope": "stockplatform_public_api_runtime_recovery_control_path",
|
||||
"target_selector": {
|
||||
"product": "stockplatform-v2",
|
||||
"public_base_url": "https://stock.wooo.work",
|
||||
"target_endpoints": [
|
||||
"/api/healthz",
|
||||
"/api/v1/system/freshness",
|
||||
"/api/v1/system/ingestion"
|
||||
],
|
||||
"expected_after_apply": "all target endpoints return HTTP 200; freshness.status=ok; ingestion.status=ok"
|
||||
},
|
||||
"source_of_truth_diff": {
|
||||
"repo": "wooo/stockplatform-v2",
|
||||
"gitea_main_sha": "080bc2b2eed796b4412bb34b736c87121fb7b3b8",
|
||||
"commit_subject": "fix(ops): recover stock api upstream resolution",
|
||||
"supporting_commits": [
|
||||
{
|
||||
"sha": "3cd2d824047ed02336cfa4d8f2f39c5f9f4458b5",
|
||||
"subject": "fix(deploy): fail closed on stockplatform api health"
|
||||
},
|
||||
{
|
||||
"sha": "080bc2b2eed796b4412bb34b736c87121fb7b3b8",
|
||||
"subject": "fix(ops): recover stock api upstream resolution"
|
||||
}
|
||||
],
|
||||
"files_changed": [
|
||||
"infra/nginx/stockplatform-v2-edge.admin-auth.conf",
|
||||
"infra/nginx/stockplatform-v2-edge.conf",
|
||||
"infra/scripts/deploy-production.sh",
|
||||
"package.json",
|
||||
"scripts/ops/stockplatform-api-upstream-recovery-gate.sh",
|
||||
"scripts/ops/test-stockplatform-api-upstream-recovery-gate.sh",
|
||||
"scripts/ops/test-production-deploy-contract.sh"
|
||||
],
|
||||
"change_summary": [
|
||||
"production deploy uses .env.production.local instead of compose defaults",
|
||||
"edge is restarted after app containers are recreated to avoid stale api upstream resolution",
|
||||
"local /api/healthz must return 200 before deploy can emit success receipt",
|
||||
"edge nginx uses Docker DNS dynamic upstream resolution for api/web/admin",
|
||||
"api upstream recovery gate provides verify-only, dry-run, and bounded apply modes",
|
||||
"npm run check now includes the production deploy contract test"
|
||||
]
|
||||
},
|
||||
"check_mode_or_dry_run": {
|
||||
"performed": true,
|
||||
"commands": [
|
||||
"npm run check",
|
||||
"python3 -m compileall -q apps/api services/worker services/scheduler scripts/ingestion scripts/migration",
|
||||
"curl https://stock.wooo.work/api/healthz",
|
||||
"curl https://stock.wooo.work/api/v1/system/freshness",
|
||||
"curl https://stock.wooo.work/api/v1/system/ingestion"
|
||||
],
|
||||
"result": "source checks passed; live public API upstream is HTTP 200, but freshness and ingestion return status=not_configured with postgres_not_ready"
|
||||
},
|
||||
"rollback": {
|
||||
"available": true,
|
||||
"command": "git revert 080bc2b2eed796b4412bb34b736c87121fb7b3b8 3cd2d824047ed02336cfa4d8f2f39c5f9f4458b5 && rerun stockplatform production deploy and upstream recovery contract checks before any runtime apply"
|
||||
},
|
||||
"post_apply_public_api_verifier": {
|
||||
"prepared": true,
|
||||
"last_pre_apply_readback": {
|
||||
"public_web_healthz": 200,
|
||||
"public_api_healthz": 200,
|
||||
"freshness": 200,
|
||||
"freshness_status": "not_configured",
|
||||
"freshness_blockers": [
|
||||
"postgres_not_ready"
|
||||
],
|
||||
"ingestion": 200,
|
||||
"ingestion_status": "not_configured",
|
||||
"ingestion_blockers": [
|
||||
"postgres_not_ready"
|
||||
]
|
||||
},
|
||||
"passed": false
|
||||
},
|
||||
"provided_receipts": [
|
||||
"target_selector",
|
||||
"source_of_truth_diff_or_runtime_state_diff",
|
||||
"check_mode_or_dry_run",
|
||||
"rollback_command",
|
||||
"controlled_api_upstream_recovery_readback",
|
||||
"km_playbook_log_writeback_receipt"
|
||||
],
|
||||
"missing_receipts": [
|
||||
"stockplatform_postgres_readiness_or_data_source_readback",
|
||||
"post_apply_public_api_verifier_green"
|
||||
],
|
||||
"active_blockers": [
|
||||
"stockplatform_postgres_not_ready",
|
||||
"stockplatform_post_apply_public_api_verifier_not_green"
|
||||
],
|
||||
"safe_recovery_channels": [
|
||||
"bounded_stockplatform_postgres_readiness_readback_without_manual_db_write",
|
||||
"bounded_ssh_or_runner_recovery_readback_without_docker_daemon_restart"
|
||||
],
|
||||
"forbidden_recovery_channels": [
|
||||
"docker_daemon_restart",
|
||||
"host_reboot",
|
||||
"docker_prune_or_volume_restore",
|
||||
"manual_database_rows_or_fake_freshness",
|
||||
"restore_push_deploy_trigger_during_110_capacity_freeze",
|
||||
"read_secret_or_raw_session_values"
|
||||
],
|
||||
"required_receipts": [
|
||||
"target_selector",
|
||||
"source_of_truth_diff_or_runtime_state_diff",
|
||||
"check_mode_or_dry_run",
|
||||
"rollback_command",
|
||||
"post_apply_public_api_verifier",
|
||||
"km_playbook_log_writeback_receipt"
|
||||
],
|
||||
"operation_boundaries": {
|
||||
"workflow_dispatch_performed": false,
|
||||
"host_write_performed": false,
|
||||
"docker_restart_performed": false,
|
||||
"docker_daemon_restart_performed": false,
|
||||
"host_reboot_performed": false,
|
||||
"database_write_or_restore_performed": false,
|
||||
"secret_or_runner_token_read": false,
|
||||
"github_api_used": false
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user