wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(backup): add alert receipt contract
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 54s
Details
CD Pipeline / post-deploy-checks (push) Has been cancelled
Details
CD Pipeline / build-and-deploy (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Your Name
2026-07-03 07:32:54 +08:00
parent 9d493ee869
commit 0acda9f728
14 changed files with 439 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
scripts/ops/backup-alert-label-contract-check.py
View File

@@ -125,6 +125,39 @@ def static_check(path: Path, baseline_path: Path) -> list[str]:
_require_contains(_annotation_text(rule), "$labels.item", "BackupCredentialEscrowEvidenceMissing annotations")
lines.append("OK BackupCredentialEscrowEvidenceMissing uses item label")
rule = _require_alert(alerts, "BackupAlertReceiptMetricMissing")
_require_contains(
str(rule.get("expr", "")),
"awoooi_backup_alert_receipt_expected_info",
"BackupAlertReceiptMetricMissing expr",
)
lines.append("OK BackupAlertReceiptMetricMissing checks receipt requirement metric")
rule = _require_alert(alerts, "BackupAlertReceiptMetricMissing188")
_require_contains(
str(rule.get("expr", "")),
"awoooi_backup_alert_receipt_expected_info",
"BackupAlertReceiptMetricMissing188 expr",
)
lines.append("OK BackupAlertReceiptMetricMissing188 checks 188 receipt requirement metric")
rule = _require_alert(alerts, "BackupAlertReceiptStageMissing")
_require_contains(
str(rule.get("expr", "")),
"awoooi_backup_alert_receipt_stage_fresh == 0",
"BackupAlertReceiptStageMissing expr",
)
text = _annotation_text(rule)
for required_label in [
"$labels.required_alert",
"$labels.scope",
"$labels.stage",
"$labels.required_notification_type",
"$labels.receipt_channel",
]:
_require_contains(text, required_label, "BackupAlertReceiptStageMissing annotations")
lines.append("OK BackupAlertReceiptStageMissing uses required_alert/scope/stage/receipt labels")
return lines
@@ -230,6 +263,11 @@ def live_check(base_url: str, baseline_path: Path) -> list[str]:
'awoooi_backup_config_capture_ok{host="110"}',
{"host", "target", "source", "critical"},
),
_require_live_label(
base_url,
'awoooi_backup_alert_receipt_stage_expected_info{host="110"}',
{"host", "required_alert", "scope", "stage", "required_notification_type", "receipt_channel"},
),
]
lines.extend(_require_live_rules(base_url, _expected_backup_alerts(baseline_path)))
return lines

62
scripts/ops/backup-alert-live-visibility-check.py
View File

@@ -187,6 +187,54 @@ def _escrow_required_alerts(prometheus_url: str, host: str, timeout: int) -> lis
return sorted(required, key=lambda alert: alert.labels["item"])
def _receipt_stage_required_alerts(prometheus_url: str, host: str, timeout: int) -> list[RequiredAlert]:
expr = f'awoooi_backup_alert_receipt_stage_fresh{{host="{host}"}} == 0'
rows = _prom_query(prometheus_url, expr, timeout)
required: list[RequiredAlert] = []
for row in rows:
labels = _metric_labels(row)
required_alert = labels.get("required_alert")
scope = labels.get("scope")
stage = labels.get("stage")
receipt_channel = labels.get("receipt_channel")
required_notification_type = labels.get("required_notification_type")
missing = [
label
for label, value in {
"required_alert": required_alert,
"scope": scope,
"stage": stage,
"receipt_channel": receipt_channel,
"required_notification_type": required_notification_type,
}.items()
if not value
]
if missing:
raise VisibilityError(f"Backup alert receipt gap metric missing labels {missing}: {row}")
required.append(
RequiredAlert(
"BackupAlertReceiptStageMissing",
{
**COMMON_LABELS,
"host": host,
"required_alert": str(required_alert),
"scope": str(scope),
"stage": str(stage),
"receipt_channel": str(receipt_channel),
"required_notification_type": str(required_notification_type),
},
)
)
return sorted(
required,
key=lambda alert: (
alert.labels["required_alert"],
alert.labels["scope"],
alert.labels["stage"],
),
)
def live_check(prometheus_url: str, alertmanager_url: str, host: str, timeout: int) -> list[str]:
required_alerts: list[RequiredAlert] = []
lines: list[str] = []
@@ -205,6 +253,20 @@ def live_check(prometheus_url: str, alertmanager_url: str, host: str, timeout: i
else:
lines.append("OK credential escrow markers are fresh; no escrow gap alert required")
receipt_stage_alerts = _receipt_stage_required_alerts(prometheus_url, host, timeout)
required_alerts.extend(receipt_stage_alerts)
if receipt_stage_alerts:
samples = ", ".join(
f"{alert.labels['required_alert']}:{alert.labels['stage']}"
for alert in receipt_stage_alerts[:8]
)
suffix = "" if len(receipt_stage_alerts) <= 8 else f", ... +{len(receipt_stage_alerts) - 8}"
lines.append(
f"OK backup alert receipt gap metrics require {len(receipt_stage_alerts)} alert(s): {samples}{suffix}"
)
else:
lines.append("OK backup alert receipt markers are fresh; no receipt stage alert required")
prom_alerts = _prom_alerts(prometheus_url, timeout)
for required in required_alerts:
_require_prom_alert(prom_alerts, required)

93
scripts/ops/backup-health-textfile-exporter.py
View File

@@ -34,6 +34,8 @@ BACKUP_OFFSITE_ENV = Path(os.environ.get("AIOPS_BACKUP_OFFSITE_ENV", "/backup/sc
OFFSITE_STATUS_DIR = Path(os.environ.get("AIOPS_OFFSITE_STATUS_DIR", "/backup/offsite"))
ESCROW_EVIDENCE_DIR = Path(os.environ.get("AIOPS_ESCROW_EVIDENCE_DIR", "/backup/escrow-evidence"))
CONFIG_CAPTURE_STATUS_FILE = Path(os.environ.get("AIOPS_CONFIG_CAPTURE_STATUS_FILE", "/backup/status/backup-configs-last-status.json"))
BACKUP_ALERT_RECEIPT_DIR = Path(os.environ.get("AIOPS_BACKUP_ALERT_RECEIPT_DIR", "/backup/alert-receipts"))
BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS = float(os.environ.get("AIOPS_BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS", "168"))
DEFAULT_GITEA_BUNDLE_EXPECTED_REPOS = [
"wooo/awoooi",
"wooo/ewoooc",
@@ -55,6 +57,34 @@ ESCROW_ITEMS = [
"dns_registrar_recovery",
"oauth_ai_provider_recovery",
]
BACKUP_ALERT_RECEIPT_STAGES = ("sent", "received", "dedup", "escalation")
BACKUP_ALERT_RECEIPT_REQUIREMENTS = [
{"host": "110", "alertname": "BackupHealthMonitorMissing110", "scope": "monitor"},
{"host": "188", "alertname": "BackupHealthMonitorMissing188", "scope": "monitor"},
{"host": "110", "alertname": "BackupHealthMonitorStale", "scope": "monitor"},
{"host": "110", "alertname": "BackupExpectedJobMissing", "scope": "schedule"},
{"host": "110", "alertname": "BackupScriptMissing", "scope": "script"},
{"host": "110", "alertname": "BackupJobStale", "scope": "freshness"},
{"host": "110", "alertname": "BackupCoverageDomainStale", "scope": "coverage"},
{"host": "188", "alertname": "GiteaPrivateBundleBackupIncomplete", "scope": "gitea_bundle"},
{"host": "188", "alertname": "GiteaPrivateBundleRestoreDryRunMissing", "scope": "gitea_bundle_restore"},
{"host": "110", "alertname": "BackupAggregateRunFailed", "scope": "aggregate"},
{"host": "110", "alertname": "BackupConfigCapturePartial", "scope": "config_capture"},
{"host": "110", "alertname": "BackupConfigCaptureStatusStale", "scope": "config_capture"},
{"host": "110", "alertname": "BackupIntegrityCheckMissingOrFailed", "scope": "integrity"},
{"host": "110", "alertname": "BackupRestoreDrillMissingOrFailed", "scope": "restore_drill"},
{"host": "110", "alertname": "BackupRestoreTestMissing", "scope": "velero_restore_test"},
{"host": "110", "alertname": "BackupRestoreTestCronMissing", "scope": "velero_restore_test"},
{"host": "110", "alertname": "BackupRestoreTestFailed", "scope": "velero_restore_test"},
{"host": "110", "alertname": "BackupRestoreTestStale", "scope": "velero_restore_test"},
{"host": "110", "alertname": "BackupOffsiteCopyNotConfigured", "scope": "offsite"},
{"host": "110", "alertname": "BackupOffsiteCopyStale", "scope": "offsite"},
{"host": "110", "alertname": "BackupRetentionPolicyNotLatestOnly", "scope": "retention"},
{"host": "110", "alertname": "BackupSnapshotRetentionExceeded", "scope": "retention"},
{"host": "110", "alertname": "BackupOffsiteFullVerifyFailed", "scope": "offsite_verify"},
{"host": "110", "alertname": "BackupOffsiteRemoteSnapshotRetentionExceeded", "scope": "offsite_retention"},
{"host": "110", "alertname": "BackupCredentialEscrowEvidenceMissing", "scope": "credential_escrow"},
]
def _escape_label(value: str) -> str:
@@ -106,6 +136,55 @@ def _marker_timestamp(paths: list[Path]) -> int:
return 0
def _backup_alert_receipt_marker_timestamp(alertname: str, stage: str) -> int:
return _marker_timestamp(
[
BACKUP_ALERT_RECEIPT_DIR / f"{alertname}.{stage}.last_success",
BACKUP_ALERT_RECEIPT_DIR / alertname / f"{stage}.last_success",
BACKUP_ALERT_RECEIPT_DIR / f"{alertname}.last_success",
BACKUP_ALERT_RECEIPT_DIR / alertname / "last_success",
]
)
def _backup_alert_receipt_metric_lines(host: str) -> list[str]:
now = int(time.time())
lines: list[str] = []
for requirement in BACKUP_ALERT_RECEIPT_REQUIREMENTS:
if requirement["host"] != host:
continue
alertname = requirement["alertname"]
scope = requirement["scope"]
base_labels = (
f'host="{_escape_label(host)}",'
f'required_alert="{_escape_label(alertname)}",'
f'scope="{_escape_label(scope)}",'
'required_notification_type="TYPE-3",'
'receipt_channel="telegram_awooop",'
f'max_age_hours="{BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS:g}"'
)
lines.append(f"awoooi_backup_alert_receipt_expected_info{{{base_labels}}} 1")
stage_fresh_values: list[int] = []
for stage in BACKUP_ALERT_RECEIPT_STAGES:
timestamp = _backup_alert_receipt_marker_timestamp(alertname, stage)
age = now - timestamp if timestamp else 0
fresh = 1 if timestamp and age <= int(BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS * 3600) else 0
stage_fresh_values.append(fresh)
stage_labels = f'{base_labels},stage="{_escape_label(stage)}"'
lines.extend(
[
f"awoooi_backup_alert_receipt_stage_expected_info{{{stage_labels}}} 1",
f"awoooi_backup_alert_receipt_stage_last_success_timestamp{{{stage_labels}}} {timestamp}",
f"awoooi_backup_alert_receipt_stage_age_seconds{{{stage_labels}}} {age}",
f"awoooi_backup_alert_receipt_stage_fresh{{{stage_labels}}} {fresh}",
]
)
lines.append(f"awoooi_backup_alert_receipt_fresh{{{base_labels}}} {1 if all(stage_fresh_values) else 0}")
return lines
def _shell_export_value(path: Path, key: str) -> str:
try:
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
@@ -860,6 +939,18 @@ def _base_lines(host: str) -> list[str]:
"# TYPE awoooi_backup_coverage_domain_expected_info gauge",
"# HELP awoooi_backup_coverage_domain_fresh Whether every required evidence job for a backup coverage domain is fresh.",
"# TYPE awoooi_backup_coverage_domain_fresh gauge",
"# HELP awoooi_backup_alert_receipt_expected_info Backup alert receipt requirements that must have Telegram/AwoooP evidence.",
"# TYPE awoooi_backup_alert_receipt_expected_info gauge",
"# HELP awoooi_backup_alert_receipt_fresh Whether every required receipt stage for a backup alert is fresh.",
"# TYPE awoooi_backup_alert_receipt_fresh gauge",
"# HELP awoooi_backup_alert_receipt_stage_expected_info Required sent/received/dedup/escalation receipt stage for a backup alert.",
"# TYPE awoooi_backup_alert_receipt_stage_expected_info gauge",
"# HELP awoooi_backup_alert_receipt_stage_last_success_timestamp Unix timestamp of the backup alert receipt stage evidence marker.",
"# TYPE awoooi_backup_alert_receipt_stage_last_success_timestamp gauge",
"# HELP awoooi_backup_alert_receipt_stage_age_seconds Age of the backup alert receipt stage evidence marker.",
"# TYPE awoooi_backup_alert_receipt_stage_age_seconds gauge",
"# HELP awoooi_backup_alert_receipt_stage_fresh Whether a backup alert receipt stage marker is fresh.",
"# TYPE awoooi_backup_alert_receipt_stage_fresh gauge",
"# HELP awoooi_backup_job_configured Whether the expected backup cron/config is present.",
"# TYPE awoooi_backup_job_configured gauge",
"# HELP awoooi_backup_script_present Whether the backup script exists on this host.",
@@ -1107,6 +1198,7 @@ def _collect_110(host: str) -> list[str]:
lines.extend(_retention_metric_lines(host))
lines.extend(_cron_duplicate_metric_lines(host, cron))
lines.extend(_velero_metric_lines(host))
lines.extend(_backup_alert_receipt_metric_lines(host))
return lines
@@ -1183,6 +1275,7 @@ def _collect_188(host: str) -> list[str]:
sample_count=1 if momo_ts else 0,
)
)
lines.extend(_backup_alert_receipt_metric_lines(host))
return lines

62
scripts/ops/tests/test_backup_health_textfile_exporter.py
View File

@@ -165,3 +165,65 @@ def test_dr_phase_ready_when_full_offsite_is_fresh_and_escrow_is_complete(
)
assert 'awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0' in rendered
assert 'awoooi_backup_dr_phase{host="110",next_step="offsite_and_escrow_ready"} 5' in rendered
def test_backup_alert_receipt_metrics_require_all_stages(tmp_path: Path, monkeypatch) -> None:
exporter = load_exporter()
receipt_dir = tmp_path / "alert-receipts"
receipt_dir.mkdir()
now = 1_782_900_000
monkeypatch.setattr(exporter, "BACKUP_ALERT_RECEIPT_DIR", receipt_dir)
monkeypatch.setattr(exporter, "BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS", 168)
monkeypatch.setattr(exporter.time, "time", lambda: now)
for stage in exporter.BACKUP_ALERT_RECEIPT_STAGES[:-1]:
(receipt_dir / f"BackupJobStale.{stage}.last_success").write_text(
str(now - 60),
encoding="utf-8",
)
metrics = exporter._backup_alert_receipt_metric_lines("110")
rendered = "\n".join(metrics)
assert 'awoooi_backup_alert_receipt_expected_info{host="110",required_alert="BackupJobStale"' in rendered
assert 'awoooi_backup_alert_receipt_stage_expected_info{host="110",required_alert="BackupJobStale"' in rendered
assert 'stage="sent"' in rendered
assert 'stage="received"' in rendered
assert 'stage="dedup"' in rendered
assert 'stage="escalation"' in rendered
assert (
'awoooi_backup_alert_receipt_stage_fresh{host="110",required_alert="BackupJobStale",scope="freshness",'
'required_notification_type="TYPE-3",receipt_channel="telegram_awooop",max_age_hours="168",stage="escalation"} 0'
in rendered
)
assert (
'awoooi_backup_alert_receipt_fresh{host="110",required_alert="BackupJobStale",scope="freshness",'
'required_notification_type="TYPE-3",receipt_channel="telegram_awooop",max_age_hours="168"} 0'
in rendered
)
def test_backup_alert_receipt_shared_marker_can_satisfy_all_stages(tmp_path: Path, monkeypatch) -> None:
exporter = load_exporter()
receipt_dir = tmp_path / "alert-receipts"
receipt_dir.mkdir()
now = 1_782_900_000
monkeypatch.setattr(exporter, "BACKUP_ALERT_RECEIPT_DIR", receipt_dir)
monkeypatch.setattr(exporter, "BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS", 168)
monkeypatch.setattr(exporter.time, "time", lambda: now)
(receipt_dir / "GiteaPrivateBundleRestoreDryRunMissing.last_success").write_text(
str(now - 60),
encoding="utf-8",
)
metrics = exporter._backup_alert_receipt_metric_lines("188")
rendered = "\n".join(metrics)
assert (
'awoooi_backup_alert_receipt_fresh{host="188",required_alert="GiteaPrivateBundleRestoreDryRunMissing",'
'scope="gitea_bundle_restore",required_notification_type="TYPE-3",receipt_channel="telegram_awooop",'
'max_age_hours="168"} 1'
in rendered
)

20
scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py
View File

@@ -333,10 +333,23 @@ def test_backup_exporter_emits_domain_level_backup_coverage() -> None:
assert "awoooi_backup_coverage_domain_expected_info" in exporter
assert "awoooi_backup_coverage_domain_fresh" in exporter
assert "awoooi_backup_alert_receipt_expected_info" in exporter
assert "awoooi_backup_alert_receipt_stage_fresh" in exporter
assert "BACKUP_ALERT_RECEIPT_STAGES = (\"sent\", \"received\", \"dedup\", \"escalation\")" in exporter
assert "AIOPS_BACKUP_ALERT_RECEIPT_DIR" in exporter
assert "gitea_repo_mirror_from_110" in exporter
assert "/home/ollama/backup/110/gitea" in exporter
for domain in ["host", "database", "website", "service", "package", "tool", "log"]:
assert f'"{domain}"' in exporter
for alertname in [
"BackupJobStale",
"BackupCoverageDomainStale",
"GiteaPrivateBundleBackupIncomplete",
"GiteaPrivateBundleRestoreDryRunMissing",
"BackupRestoreDrillMissingOrFailed",
"BackupCredentialEscrowEvidenceMissing",
]:
assert alertname in exporter
def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None:
@@ -347,6 +360,13 @@ def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None:
assert "awoooi_backup_coverage_domain_fresh == 0" in alerts
assert "GiteaPrivateBundleRestoreDryRunMissing" in alerts
assert "awoooi_gitea_bundle_sample_restore_dry_run_ok" in alerts
assert "BackupAlertReceiptMetricMissing" in alerts
assert "BackupAlertReceiptStageMissing" in alerts
assert "awoooi_backup_alert_receipt_expected_info" in alerts
assert "awoooi_backup_alert_receipt_stage_fresh == 0" in alerts
assert "$labels.required_alert" in alerts
assert "$labels.required_notification_type" in alerts
assert "telegram_awooop" in alerts
def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None: