Compare commits
1 Commits
codex/host
...
codex/host
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f6e2dee8bf |
@@ -225,6 +225,7 @@ $agentFiles = @(
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-host188-backup-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
|
||||
@@ -21,6 +21,7 @@ $requiredFiles = @(
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-host188-backup-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
@@ -133,6 +134,7 @@ $control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane.
|
||||
$dbBoundedExecutor = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-db-bounded-executor.ps1"), [Text.Encoding]::UTF8)
|
||||
$signozMetadataExecutor = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-signoz-metadata-executor.ps1"), [Text.Encoding]::UTF8)
|
||||
$host110BackupRuntimeBroker = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-host110-backup-runtime-broker.ps1"), [Text.Encoding]::UTF8)
|
||||
$host188BackupBroker = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-host188-backup-broker.ps1"), [Text.Encoding]::UTF8)
|
||||
$inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8)
|
||||
$telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8)
|
||||
$bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8)
|
||||
@@ -192,7 +194,7 @@ $signozMetadataExecutorContract = [bool](
|
||||
$signozMetadataExecutor.Contains('agent99_active:') -and
|
||||
$signozMetadataExecutor.Contains('$RemoteResult.completed -ne $true') -and
|
||||
$signozMetadataExecutor.Contains('Get-Agent99RuntimeSourceBinding') -and
|
||||
$signozMetadataExecutor.Contains('$ExpectedRuntimeFileCount = 19') -and
|
||||
$signozMetadataExecutor.Contains('$ExpectedRuntimeFileCount = 20') -and
|
||||
$signozMetadataExecutor.Contains('$result.fileCount -eq $ExpectedRuntimeFileCount') -and
|
||||
$signozMetadataExecutor.Contains('pass_export_verified_restore_pending') -and
|
||||
$signozMetadataExecutor.Contains('productionRuntimeMutationPerformed = $false') -and
|
||||
@@ -229,6 +231,42 @@ $host110BackupRuntimeContract = [bool](
|
||||
-not $host110BackupRuntimeBroker.Contains('Invoke-Expression')
|
||||
)
|
||||
Add-Check "backup:host110_runtime_fixed_broker" $host110BackupRuntimeContract "Windows99 fetches one exact Gitea revision, binds executor and verifier digests, performs one fixed process-bounded Host110 transaction, and preserves an independent no-write verifier readback"
|
||||
$host188BackupBrokerContract = [bool](
|
||||
$host188BackupBroker.Contains('$TargetHost = "192.168.0.110"') -and
|
||||
$host188BackupBroker.Contains('$TargetUser = "wooo"') -and
|
||||
$host188BackupBroker.Contains('agent99-host110-backup-runtime-broker.ps1') -and
|
||||
$host188BackupBroker.Contains('/backup/scripts/backup-host188-products.sh') -and
|
||||
$host188BackupBroker.Contains('/backup/scripts/verify-host188-products-backup.sh') -and
|
||||
$host188BackupBroker.Contains('BACKUP_NOTIFY_SUCCESS=0') -and
|
||||
$host188BackupBroker.Contains('new_snapshot_not_observed') -and
|
||||
$host188BackupBroker.Contains('durableReadbackVerified') -and
|
||||
$host188BackupBroker.Contains('agent99_host188_backup_broker_receipt_v2') -and
|
||||
$host188BackupBroker.Contains('Assert-Agent99ExistingBrokerEvidence') -and
|
||||
$host188BackupBroker.Contains('Assert-Agent99ExactJsonFields') -and
|
||||
$host188BackupBroker.Contains('agent99_host188_backup_transport_reconciliation_v1') -and
|
||||
$host188BackupBroker.Contains('committed_receipt_recovered') -and
|
||||
$host188BackupBroker.Contains('committed_unknown') -and
|
||||
$host188BackupBroker.Contains('retryApplyAllowed = $false') -and
|
||||
$host188BackupBroker.Contains('$ApplyReceiptReadbackScript') -and
|
||||
$host188BackupBroker.Contains('agent99_host188_backup_operation_gate_v1') -and
|
||||
$host188BackupBroker.Contains('Write-Agent99Host188OperationGate') -and
|
||||
$host188BackupBroker.Contains('Get-Agent99Host188OperationGateState') -and
|
||||
$host188BackupBroker.Contains('Invoke-Agent99Host188SameRunReconciliation') -and
|
||||
$host188BackupBroker.Contains('read_only_same_run_reconciliation') -and
|
||||
$host188BackupBroker.Contains('Invoke-Agent99Host188FreshnessReadback') -and
|
||||
$host188BackupBroker.Contains('agent99_host188_backup_fresh_skip_v1') -and
|
||||
$host188BackupBroker.Contains('Assert-Agent99Host188GateResolution') -and
|
||||
$host188BackupBroker.Contains('Assert-Agent99ExistingBrokerEvidence $verifiedReceipt') -and
|
||||
$host188BackupBroker.Contains('sourceReadbackRequired') -and
|
||||
$host188BackupBroker.Contains('productionRestorePerformed') -and
|
||||
$host188BackupBroker.Contains('decisionProvider = "deterministic_only"') -and
|
||||
$host188BackupBroker.Contains('arbitraryCommandAllowed = $false') -and
|
||||
-not $host188BackupBroker.Contains('Invoke-Expression') -and
|
||||
-not $host188BackupBroker.Contains('docker stop') -and
|
||||
-not $host188BackupBroker.Contains('rclone ')
|
||||
)
|
||||
Add-Check "backup:host188_fixed_run_broker" $host188BackupBrokerContract "Windows99 runs one fixed Host188 product-data backup through Host110 only after exact runtime identity readback and closes on the same snapshot with an independent non-production restore verifier"
|
||||
Add-Check "backup:host188_outer_receipt_fail_closed" ($control.Contains('Test-AgentHost188ExactReceiptFields') -and $control.Contains('Test-AgentHost188RuntimeIdentityEvidence') -and $control.Contains('Test-AgentHost188ApplyReconciliationEvidence') -and $control.Contains('Test-AgentHost188OperationGateEvidence') -and $control.Contains('Test-AgentHost188GateResolutionEvidence') -and $control.Contains('Test-AgentHost188FreshSkipEvidence') -and $control.Contains('same_run_reconciliation_started') -and $control.Contains('broker_timeout_operation_gate_persisted') -and $control.Contains('if ($targetRow.ok -and -not $reconcileOnly)') -and $control.Contains('agent99_host188_backup_broker_receipt_v2')) "BackupCheck accepts only exact typed Host188 receipts, persists uncertain operations before Apply, reconciles gates before fresh-target return, and rechecks freshness under the broker mutex"
|
||||
Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("Enter-AgentRecoveryTriggerReservation") -and $control.Contains('"recovery-trigger-reservation.lock"') -and $control.Contains('[System.IO.FileShare]::None') -and $control.Contains('reason = "recovery_trigger_reservation_busy"') -and $control.Contains("recovery_already_queued")) "boot and host-down recovery reserve one atomic single-flight queue decision"
|
||||
Add-Check "recovery:auto_trigger_controlled_identity" ($control.Contains("New-AgentObservedRecoveryDispatchIdentity") -and $control.Contains('-RouteId "agent99_local_observer"') -and $control.Contains('schema_version = "agent99_controlled_dispatch_identity_v1"') -and $control.Contains('correlationKey = $dispatchIdentity.idempotencyKey') -and $control.Contains('canonicalDigest = $dispatchIdentity.canonicalDigest') -and $control.Contains("Invoke-AgentControlledDispatchIdentitySelfTest")) "observer-created Recover requests carry a stable canonical controlled dispatch identity before queue processing"
|
||||
Add-Check "recovery:queue_identity_fail_closed" ($control.Contains('dispatchIdentitySchemaVersion = $dispatchIdentity.schemaVersion') -and $control.Contains('Test-AgentControlledDispatchIdentity $dispatchIdentityEnvelope') -and $control.Contains('"controlled_dispatch_identity_invalid"') -and $control.IndexOf('"controlled_dispatch_identity_invalid"') -lt $control.IndexOf('$process = Start-Process -FilePath "powershell.exe" -ArgumentList $args')) "queue consumer rebuilds and validates canonical identity before any controlled executor starts"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -36,6 +36,7 @@ $runtimeFiles = @(
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-host188-backup-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
|
||||
2078
agent99-host188-backup-broker.ps1
Normal file
2078
agent99-host188-backup-broker.ps1
Normal file
File diff suppressed because it is too large
Load Diff
@@ -34,6 +34,12 @@ $serviceSettings = New-ScheduledTaskSettingsSet `
|
||||
-StartWhenAvailable `
|
||||
-MultipleInstances IgnoreNew `
|
||||
-ExecutionTimeLimit ([TimeSpan]::Zero)
|
||||
$backupSettings = New-ScheduledTaskSettingsSet `
|
||||
-AllowStartIfOnBatteries `
|
||||
-DontStopIfGoingOnBatteries `
|
||||
-StartWhenAvailable `
|
||||
-MultipleInstances IgnoreNew `
|
||||
-ExecutionTimeLimit (New-TimeSpan -Minutes 125)
|
||||
|
||||
$recoverInstruction = "Windows99 startup detected; run the allowlisted all-host, VM, service, website, freshness, verifier, callback, and learning recovery chain."
|
||||
$recoverArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$submitRequest`" -Instruction `"$recoverInstruction`" -Mode Recover -ControlledApply -RunNow -Source agent99-startup-recovery -ExternalId windows99-startup"
|
||||
@@ -161,7 +167,7 @@ Register-ScheduledTask `
|
||||
-Description "Agent99 self-health guard for scheduled tasks, evidence freshness, host perf readback, and Telegram delivery." `
|
||||
-Force | Out-Null
|
||||
|
||||
$backupArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$launcher`" -Mode BackupCheck"
|
||||
$backupArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$launcher`" -Mode BackupCheck -ControlledApply"
|
||||
$backupAction = New-ScheduledTaskAction -Execute $ps -Argument $backupArgs
|
||||
$backupTrigger = New-ScheduledTaskTrigger `
|
||||
-Once `
|
||||
@@ -173,8 +179,8 @@ Register-ScheduledTask `
|
||||
-Action $backupAction `
|
||||
-Trigger $backupTrigger `
|
||||
-Principal $principal `
|
||||
-Settings $settings `
|
||||
-Description "Agent99 backup freshness, integrity-status, cron-contract, and Telegram alert guard." `
|
||||
-Settings $backupSettings `
|
||||
-Description "Agent99 backup freshness guard with the fixed Host188 product backup and independent snapshot restore verifier lane." `
|
||||
-Force | Out-Null
|
||||
|
||||
# Superseded by the canonical all-host performance guard. An older Host188-only
|
||||
|
||||
@@ -21,7 +21,7 @@ $IdentityFile = Join-Path $AgentRoot "keys\agent99_ed25519"
|
||||
$EvidenceDir = Join-Path $AgentRoot "evidence"
|
||||
$RuntimeManifestPath = Join-Path $AgentRoot "state\runtime-manifest.json"
|
||||
$EntrypointPath = [string]$MyInvocation.MyCommand.Path
|
||||
$ExpectedRuntimeFileCount = 19
|
||||
$ExpectedRuntimeFileCount = 20
|
||||
$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"
|
||||
$BundleId = "bb6d78b67f506072b2e45e92bc1d415364e25852282a9b068900665c578df011"
|
||||
$ToolchainRoot = "/backup/toolchains/signoz-metadata/$BundleId"
|
||||
|
||||
205
docs/proposals/AWOOOI-AG99-H188-FIRST-BACKUP-RUNNER-V1.md
Normal file
205
docs/proposals/AWOOOI-AG99-H188-FIRST-BACKUP-RUNNER-V1.md
Normal file
@@ -0,0 +1,205 @@
|
||||
# AWOOOI Agent99 Host188 First Backup Runner V1
|
||||
|
||||
Status: proposed. Central review and explicit owner approval are required before
|
||||
any production apply. This candidate must not be pushed to `main`, sent through
|
||||
CD, or applied to Windows99, Host110, or Host188 while it remains proposed.
|
||||
|
||||
This is the first approval identity for this Host188 runner. Earlier local WIP,
|
||||
test output, review messages, stashes, branch names, or untracked bytes grant no
|
||||
production execution authority.
|
||||
|
||||
## Exact Scope
|
||||
|
||||
- Agent99 runtime bundle: 20 exact ordered files. The new twentieth runtime file
|
||||
is `agent99-host188-backup-broker.ps1`; every producer and consumer validates
|
||||
the same count, order, basename, digest, source revision, and runtime readback.
|
||||
- Windows99 is the only production dispatcher. Agent99 `BackupCheck` evaluates
|
||||
only the required `host188-products` target and may enter this lane only for
|
||||
`backup_snapshot_missing` or `backup_snapshot_stale`.
|
||||
- Fixed controller: `wooo@192.168.0.110`. Fixed source: `192.168.0.188`.
|
||||
- Fixed data sources: VibeWork PostgreSQL, FIFA PostgreSQL, FIFA Redis, and n8n.
|
||||
The existing bounded online backup script remains the only backup executor.
|
||||
- Host110 backup payload remains 18 ordered files and its remote stage remains
|
||||
21 unique basenames. This candidate updates one existing payload member,
|
||||
`backup-host188-products.sh`; it does not change the 18/21 count contracts.
|
||||
- `BackupCheck` runs every 15 minutes with `IgnoreNew` and a bounded 125-minute
|
||||
task limit. It does not introduce a second Host188-only performance task.
|
||||
- No arbitrary remote command, provider switch, paid model call, offsite sync,
|
||||
snapshot prune, production restore, database mutation, container lifecycle,
|
||||
VM operation, network/firewall change, or secret-value return is in scope.
|
||||
|
||||
## Controlled Flow
|
||||
|
||||
1. Agent99 reads backup freshness and scans durable Host188 operation gates
|
||||
before it may return `target_already_fresh`.
|
||||
2. An unresolved exact RunId gate permits read-only same-run reconciliation
|
||||
only. A different run is blocked and `retryApplyAllowed=false`.
|
||||
3. Windows99 acquires one global Host188 broker mutex, validates the exact
|
||||
Agent99 runtime revision twice, and performs a fixed Host110/Host188 preflight.
|
||||
4. Before remote Apply, Agent99 writes an immutable operation gate. Transport
|
||||
timeout, SSH loss, missing stdout, malformed receipt, or process termination
|
||||
therefore cannot erase the uncertain-write state or trigger a blind retry.
|
||||
5. Host110 executes only `/backup/scripts/backup-host188-products.sh`, writes a
|
||||
durable exact-RunId executor receipt, and returns no raw session or secret.
|
||||
6. Agent99 verifies the exact resulting snapshot with the independent,
|
||||
snapshot-specific data restore verifier. Recovery readiness and credential
|
||||
escrow remain separate gates and are never inferred from data restore alone.
|
||||
7. A digest-bound broker receipt and gate-resolution marker close the run. The
|
||||
control plane revalidates the complete referenced receipt before trusting the
|
||||
marker, then performs a final backup-health readback.
|
||||
|
||||
## Concurrency And Freshness
|
||||
|
||||
- The Windows global mutex serializes Agent99 runs and prevents two Agent99
|
||||
owners from applying concurrently.
|
||||
- The Host110 executor has an exact run-bound lock and immutable receipt.
|
||||
- The production backup entrypoint acquires the shared runtime-promotion lock
|
||||
and then `/tmp/host188-products-backup.lock` before inspecting freshness or
|
||||
reading source data.
|
||||
- The opt-in Agent99 path rechecks restic freshness only after the product lock
|
||||
is held. This closes the race with an independent Host110 cron that may finish
|
||||
between the Windows-side freshness readback and source execution.
|
||||
- If the locked snapshot is already within 1800 minutes, the executor returns
|
||||
`result=fresh_skip`, creates no snapshot, and Agent99 still runs the same
|
||||
snapshot restore verifier before writing `fresh_skip_verified`.
|
||||
- If an Agent99 waiter observes a fresh snapshot before Apply, it returns a
|
||||
typed no-write skip without canonical success evidence. The outer event says
|
||||
only that another execution updated freshness; it does not claim this run
|
||||
created or restore-verified a snapshot.
|
||||
- Absolute, relative, and symlink production entrypoints share the exact same
|
||||
runtime lock. A valid inherited FD 197 is reused without close/reopen; an
|
||||
absent FD is opened normally; a wrong FD or forged environment value fails
|
||||
closed.
|
||||
|
||||
## Receipt And Failure Truth
|
||||
|
||||
- Executor `result` is an exact required string: `snapshot_created`,
|
||||
`fresh_skip`, or `failed`.
|
||||
- `snapshot_created` requires a valid snapshot different from the before ID.
|
||||
- `fresh_skip` requires a valid snapshot equal to the before ID.
|
||||
- `fresh_skip_verified` requires `result=fresh_skip`, the matching independent
|
||||
verifier receipt, `runtimeWritePerformed=false`, and `runtimeWriteState=none`.
|
||||
- `verified` requires `result=snapshot_created`. `reconciled_verified` is
|
||||
read-only same-run closure and may verify either exact executor result.
|
||||
- Missing, extra, wrong-type, conflicting, malformed, reparse-point, oversized,
|
||||
wrong-path, wrong-digest, wrong-run, or wrong-revision evidence fails closed.
|
||||
- `committed_unknown` and unresolved operation gates are non-success and cannot
|
||||
be converted into a new Apply. Only exact same-run receipt readback plus the
|
||||
independent verifier may resolve them.
|
||||
|
||||
## Apply Effects
|
||||
|
||||
- Windows99 runtime promotion writes the exact 20 Agent99 files, runtime
|
||||
manifest, and bounded deployment receipts.
|
||||
- Host110 runtime promotion replaces the existing ordered 18-file payload,
|
||||
including the updated Host188 backup entrypoint, through the already-reviewed
|
||||
exclusive filesystem transaction and verifier.
|
||||
- Windows99 task registration updates the canonical `BackupCheck` task to the
|
||||
15-minute, `IgnoreNew`, 125-minute bounded contract.
|
||||
- A later eligible `BackupCheck` may read the four fixed Host188 data sources,
|
||||
create one restic snapshot on Host110, and write bounded receipts. It does not
|
||||
stop the source services or write production data on Host188.
|
||||
- A fresh-skip execution reads restic metadata and the selected snapshot for
|
||||
verification but creates no new snapshot.
|
||||
|
||||
## Reproducible Validation
|
||||
|
||||
Broad focused suite:
|
||||
|
||||
```sh
|
||||
pytest -q \
|
||||
scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \
|
||||
scripts/reboot-recovery/tests/test_agent99_host188_backup_broker.py \
|
||||
scripts/backup/tests/test_backup_host188_products_contract.py \
|
||||
scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py \
|
||||
scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py \
|
||||
scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py \
|
||||
scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py
|
||||
```
|
||||
|
||||
Current result: `110 passed, 4 skipped` from 114 collected tests.
|
||||
|
||||
Shared-lock dependency subset:
|
||||
|
||||
```sh
|
||||
pytest -q \
|
||||
scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \
|
||||
-k 'shared_lock or archive_lock'
|
||||
```
|
||||
|
||||
Current result: `17 passed, 32 deselected`. It covers absolute, relative,
|
||||
symlink, exact inherited FD, wrong inherited FD, and forged-environment paths.
|
||||
|
||||
Windows99 no-write validation parses all nine changed PowerShell entrypoints in
|
||||
memory with Windows PowerShell. Current result: `fileCount=9`, every
|
||||
`parseErrors=0`, `remoteWritePerformed=false`, `secretValuesRead=false`.
|
||||
|
||||
The broker replay reports `ok=true`, 41 existing-evidence rejections,
|
||||
`sourceLockFreshSkipAccepted=true`, `committedUnknownPreserved=true`,
|
||||
`mutexContenderBlocked=true`, `retryApplyAllowed=false`,
|
||||
`remoteWritePerformed=false`, and `secretValuesRead=false`.
|
||||
|
||||
The control-plane replay reports `ok=true`, 6 rejection cases,
|
||||
`sourceLockFreshSkipAccepted=true`,
|
||||
`reconciledSourceLockFreshSkipAccepted=true`,
|
||||
`verifiedRemediationOutcomeMatrixAccepted=true`,
|
||||
`freshTargetStillRequiresGateReconciliation=true`,
|
||||
`newerUnrelatedVerifiedDoesNotResolveGate=true`, `retryApplyAllowed=false`,
|
||||
`remoteWritePerformed=false`, and `secretValuesRead=false`.
|
||||
|
||||
Independent focused review found no remaining material P0/P1 issue after the
|
||||
reconciled fresh-skip event classification was moved from top-level status to
|
||||
the validated executor `apply.result` and covered by the four-state replay.
|
||||
|
||||
Static checks:
|
||||
|
||||
```sh
|
||||
ruff check \
|
||||
scripts/reboot-recovery/tests/test_agent99_host188_backup_broker.py \
|
||||
scripts/backup/tests/test_backup_host188_products_contract.py
|
||||
bash -n \
|
||||
scripts/backup/backup-host188-products.sh \
|
||||
scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Current checks are green. No CD or production apply was triggered.
|
||||
|
||||
## Exact Identity And Diff Serialization
|
||||
|
||||
The central-review intake must independently bind:
|
||||
|
||||
- live Gitea feature ref `codex/host188-first-backup-runner-20260718`;
|
||||
- exact candidate Git SHA;
|
||||
- this tracked artifact path and exact artifact-bytes SHA-256;
|
||||
- exact binary diff SHA-256, path count, insertion/deletion counts, and base;
|
||||
- Agent99 runtime 20-file, Host110 payload 18-file, and remote stage 21-file
|
||||
contracts;
|
||||
- both Windows99 no-write replay receipts and the independent review result.
|
||||
|
||||
Exact diff bytes are serialized with:
|
||||
|
||||
```sh
|
||||
git diff --binary --full-index \
|
||||
2c2dc086b055e2187ac7539f353380bbdbd3de4d..<candidate_git_sha> |
|
||||
shasum -a 256
|
||||
```
|
||||
|
||||
No alternative serialization, untracked byte, local stash, CD result, or
|
||||
runtime observation may substitute for this exact approval identity.
|
||||
|
||||
## Rollback
|
||||
|
||||
- Source rollback reverts the exact candidate diff to
|
||||
`2c2dc086b055e2187ac7539f353380bbdbd3de4d` without force-pushing.
|
||||
- Agent99 runtime rollback restores the prior 19-file runtime and manifest and
|
||||
independently verifies every prior digest.
|
||||
- Host110 rollback uses the reviewed payload transaction to restore the prior
|
||||
backup script content, metadata, hashes, and zero-residue state.
|
||||
- Task rollback restores the previous canonical `BackupCheck` definition and
|
||||
verifies action, trigger, instance policy, and execution limit.
|
||||
- An unresolved operation gate is never deleted merely to permit rollback or a
|
||||
new Apply. It remains evidence until exact same-run reconciliation or an
|
||||
explicitly approved incident procedure resolves it.
|
||||
- Rollback does not run a backup, restore production data, sync offsite data,
|
||||
restart a source service, operate a VM, or read/return a secret value.
|
||||
@@ -15,6 +15,7 @@ LOCK_FILE="/tmp/host188-products-backup.lock"
|
||||
MIN_POSTGRES_DUMP_BYTES="${MIN_POSTGRES_DUMP_BYTES:-4096}"
|
||||
MIN_FREE_BYTES="${HOST188_BACKUP_MIN_FREE_BYTES:-8589934592}"
|
||||
SSH_TIMEOUT_SECONDS="${HOST188_BACKUP_SSH_TIMEOUT_SECONDS:-1800}"
|
||||
SKIP_IF_FRESH_MAX_AGE_MINUTES="${HOST188_BACKUP_SKIP_IF_FRESH_MAX_AGE_MINUTES:-0}"
|
||||
SSH_OPTIONS=(
|
||||
-o BatchMode=yes
|
||||
-o StrictHostKeyChecking=yes
|
||||
@@ -49,7 +50,7 @@ require_commands() {
|
||||
log_error "Restic password reference is unavailable"
|
||||
return 69
|
||||
fi
|
||||
case "${MIN_FREE_BYTES}:${SSH_TIMEOUT_SECONDS}" in
|
||||
case "${MIN_FREE_BYTES}:${SSH_TIMEOUT_SECONDS}:${SKIP_IF_FRESH_MAX_AGE_MINUTES}" in
|
||||
*[!0-9:]*|:*|*:) log_error "Host188 backup numeric limits are invalid"; return 64 ;;
|
||||
esac
|
||||
if [ "${MIN_FREE_BYTES}" -le 0 ] || [ "${SSH_TIMEOUT_SECONDS}" -le 0 ]; then
|
||||
@@ -58,6 +59,31 @@ require_commands() {
|
||||
fi
|
||||
}
|
||||
|
||||
latest_snapshot_freshness() {
|
||||
if [ ! -d "${LOCAL_REPO}" ]; then
|
||||
return 1
|
||||
fi
|
||||
restic -r "${LOCAL_REPO}" snapshots --json \
|
||||
--password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \
|
||||
python3 -c '
|
||||
import datetime as dt
|
||||
import json
|
||||
import sys
|
||||
|
||||
rows = json.load(sys.stdin)
|
||||
if not rows:
|
||||
raise SystemExit(1)
|
||||
def parsed_time(row):
|
||||
return dt.datetime.fromisoformat(str(row["time"]).replace("Z", "+00:00"))
|
||||
latest = max(rows, key=parsed_time)
|
||||
snapshot_id = str(latest.get("short_id") or latest.get("id") or "").lower()
|
||||
if not (8 <= len(snapshot_id) <= 64 and all(c in "0123456789abcdef" for c in snapshot_id)):
|
||||
raise SystemExit(1)
|
||||
age_minutes = max(0, int((dt.datetime.now(dt.timezone.utc) - parsed_time(latest)).total_seconds() // 60))
|
||||
print(f"{snapshot_id} {age_minutes}")
|
||||
'
|
||||
}
|
||||
|
||||
run_remote() {
|
||||
timeout --signal=TERM "${SSH_TIMEOUT_SECONDS}s" \
|
||||
ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "$@"
|
||||
@@ -290,7 +316,7 @@ latest_snapshot_id() {
|
||||
}
|
||||
|
||||
main() {
|
||||
local start_time timestamp snapshot_id duration
|
||||
local start_time timestamp snapshot_id duration freshness fresh_snapshot fresh_age_minutes
|
||||
start_time="$(date +%s)"
|
||||
timestamp="$(date +%Y%m%d_%H%M%S)"
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
@@ -302,6 +328,21 @@ main() {
|
||||
log_error "Host188 product backup is already running"
|
||||
return 75
|
||||
fi
|
||||
if [ "${SKIP_IF_FRESH_MAX_AGE_MINUTES}" -gt 0 ]; then
|
||||
freshness="$(latest_snapshot_freshness || true)"
|
||||
read -r fresh_snapshot fresh_age_minutes <<<"${freshness}"
|
||||
case "${fresh_snapshot:-}" in *[!0-9a-f]*|'') fresh_snapshot="" ;; esac
|
||||
case "${fresh_age_minutes:-}" in *[!0-9]*|'') fresh_age_minutes="" ;; esac
|
||||
if [ -z "${fresh_age_minutes}" ] || [ "${#fresh_snapshot}" -lt 8 ] || [ "${#fresh_snapshot}" -gt 64 ]; then
|
||||
fresh_snapshot=""
|
||||
fi
|
||||
if [ -n "${fresh_snapshot}" ] && \
|
||||
[ "${fresh_age_minutes}" -le "${SKIP_IF_FRESH_MAX_AGE_MINUTES}" ]; then
|
||||
printf 'HOST188_BACKUP_SKIPPED_FRESH=1 snapshot=%s ageMinutes=%s maxAgeMinutes=%s\n' \
|
||||
"${fresh_snapshot}" "${fresh_age_minutes}" "${SKIP_IF_FRESH_MAX_AGE_MINUTES}"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
require_disk_headroom
|
||||
install -d -m 700 "${DUMP_DIR}"
|
||||
log_info "Host188 product backup started"
|
||||
|
||||
@@ -70,7 +70,13 @@ def test_host188_product_backup_preserves_secret_and_runtime_boundaries() -> Non
|
||||
assert "flock -n 9" in source
|
||||
assert "HOST188_BACKUP_MIN_FREE_BYTES" in source
|
||||
assert "HOST188_BACKUP_SSH_TIMEOUT_SECONDS" in source
|
||||
assert 'HOST188_BACKUP_SKIP_IF_FRESH_MAX_AGE_MINUTES:-0' in source
|
||||
assert "HOST188_BACKUP_SKIPPED_FRESH=1 snapshot=%s ageMinutes=%s maxAgeMinutes=%s" in source
|
||||
assert "timeout --signal=TERM" in source
|
||||
main = source.split("main() {", 1)[1]
|
||||
assert main.index("flock -n 9") < main.index("latest_snapshot_freshness")
|
||||
assert main.index("latest_snapshot_freshness") < main.index("require_disk_headroom")
|
||||
assert main.index("HOST188_BACKUP_SKIPPED_FRESH=1") < main.index("dump_postgres_container")
|
||||
|
||||
for forbidden in (
|
||||
"--decrypted",
|
||||
|
||||
@@ -26,7 +26,7 @@ def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None:
|
||||
assert "._+" not in source
|
||||
assert '$EvidenceRunId = if ($RunId -match $SafeIdPattern)' in source
|
||||
assert "Get-Agent99RuntimeSourceBinding" in source
|
||||
assert "$ExpectedRuntimeFileCount = 19" in source
|
||||
assert "$ExpectedRuntimeFileCount = 20" in source
|
||||
assert "$result.fileCount -eq $ExpectedRuntimeFileCount" in source
|
||||
assert "$result.sourceRevision -eq $SourceRevision" in source
|
||||
assert "$result.entrypointHashMatched" in source
|
||||
@@ -294,8 +294,8 @@ def test_entrypoint_is_in_exact_agent99_runtime_bundle() -> None:
|
||||
assert expected in _quoted_array(contract, "$requiredFiles = @(", ")")
|
||||
assert expected in _quoted_array(sender, "RUNTIME_FILES=(", ")")
|
||||
assert expected in _quoted_array(receiver, "$FixedRuntimeFiles = @(", ")")
|
||||
assert 'expectedRuntimeFileCount": 19' in sender
|
||||
assert '$runtimeManifest.fileCount -eq 19' in receiver
|
||||
assert 'expectedRuntimeFileCount": 20' in sender
|
||||
assert '$runtimeManifest.fileCount -eq 20' in receiver
|
||||
|
||||
|
||||
def test_runtime_contract_check_requires_typed_signoz_transport_readback() -> None:
|
||||
|
||||
@@ -122,7 +122,7 @@ def test_agent99_atomic_bundle_owns_the_dispatch_entrypoint() -> None:
|
||||
assert "$dbExecutorRecoveryContract" in contract
|
||||
assert "awoooi_db_bounded_executor_receipt_v2" in contract
|
||||
assert "Receipt.database_sqlstate" in contract
|
||||
assert 'expectedRuntimeFileCount": 19' in transport
|
||||
assert "expectedRuntimeFileCount=19" in transport
|
||||
assert "expectedRuntimeFileCount -ne 19" in receiver
|
||||
assert "fileCount = 19" in receiver
|
||||
assert 'expectedRuntimeFileCount": 20' in transport
|
||||
assert "expectedRuntimeFileCount=20" in transport
|
||||
assert "expectedRuntimeFileCount -ne 20" in receiver
|
||||
assert "fileCount = 20" in receiver
|
||||
|
||||
@@ -486,7 +486,7 @@ $decision = Get-AgentLivePreflightDecision `
|
||||
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
|
||||
-Manifest $manifest `
|
||||
-ManifestCheckRequired ([bool]($ReadinessScope -eq "FullRuntime")) `
|
||||
-ExpectedRuntimeFileCount 19 `
|
||||
-ExpectedRuntimeFileCount 20 `
|
||||
-TaskFailureCount $taskFailures.Count `
|
||||
-RelayListenerCount $relayListeners.Count `
|
||||
-RequiredEvidenceStaleCount $requiredEvidenceStale.Count `
|
||||
|
||||
@@ -35,6 +35,7 @@ $FixedRuntimeFiles = @(
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-host188-backup-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
@@ -952,7 +953,7 @@ try {
|
||||
}
|
||||
$sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant()
|
||||
if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
|
||||
if ([int]$envelope.expectedRuntimeFileCount -ne 19) { throw "invalid_expected_runtime_file_count" }
|
||||
if ([int]$envelope.expectedRuntimeFileCount -ne 20) { throw "invalid_expected_runtime_file_count" }
|
||||
|
||||
$traceId = [string]$envelope.traceId
|
||||
$runId = [string]$envelope.runId
|
||||
@@ -987,7 +988,7 @@ try {
|
||||
$sourceManifest = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_source_manifest_v1"
|
||||
sourceRevision = $sourceRevision
|
||||
fileCount = 19
|
||||
fileCount = 20
|
||||
manifestSha256 = $manifestDigest
|
||||
files = @($FixedRuntimeFiles | ForEach-Object {
|
||||
[pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() }
|
||||
@@ -1024,7 +1025,7 @@ try {
|
||||
status = "check_ready"
|
||||
mode = "check"
|
||||
sourceRevision = $sourceRevision
|
||||
expectedRuntimeFileCount = 19
|
||||
expectedRuntimeFileCount = 20
|
||||
manifestSha256 = $manifestDigest
|
||||
plannedStagingPath = $stagePath
|
||||
runtimeManifest = $currentManifest
|
||||
@@ -1111,7 +1112,7 @@ try {
|
||||
if (
|
||||
[string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or
|
||||
[string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or
|
||||
[int]$existingSourceManifest.fileCount -ne 19 -or
|
||||
[int]$existingSourceManifest.fileCount -ne 20 -or
|
||||
[string]$existingSourceManifest.manifestSha256 -ne $manifestDigest
|
||||
) { throw "existing_staging_manifest_mismatch" }
|
||||
} catch {
|
||||
@@ -1135,7 +1136,7 @@ try {
|
||||
[string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and
|
||||
$live.runtimeMatched -and
|
||||
$live.sourceRevision -eq $sourceRevision -and
|
||||
$live.fileCount -eq 19 -and
|
||||
$live.fileCount -eq 20 -and
|
||||
$live.mismatchCount -eq 0 -and
|
||||
$liveLauncherContract.ok -and
|
||||
[string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256
|
||||
@@ -1448,7 +1449,7 @@ try {
|
||||
$runtimeManifest.exists -and
|
||||
$runtimeManifest.runtimeMatched -and
|
||||
$runtimeManifest.sourceRevision -eq $sourceRevision -and
|
||||
$runtimeManifest.fileCount -eq 19 -and
|
||||
$runtimeManifest.fileCount -eq 20 -and
|
||||
$runtimeManifest.mismatchCount -eq 0 -and
|
||||
$launcherContract.ok -and
|
||||
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and
|
||||
|
||||
@@ -19,6 +19,7 @@ RUNTIME_FILES=(
|
||||
"agent99-db-bounded-executor.ps1"
|
||||
"agent99-signoz-metadata-executor.ps1"
|
||||
"agent99-host110-backup-runtime-broker.ps1"
|
||||
"agent99-host188-backup-broker.ps1"
|
||||
"agent99-deploy.ps1"
|
||||
"agent99-register-tasks.ps1"
|
||||
"agent99-alertmanager-alertchain-poll.ps1"
|
||||
@@ -228,7 +229,7 @@ trace_id, run_id, work_item_id = sys.argv[4:7]
|
||||
output_path = Path(sys.argv[7])
|
||||
runtime_names = sys.argv[8:]
|
||||
|
||||
if len(runtime_names) != 19 or len(set(runtime_names)) != 19:
|
||||
if len(runtime_names) != 20 or len(set(runtime_names)) != 20:
|
||||
raise SystemExit("fixed_runtime_file_contract_failed")
|
||||
|
||||
files: list[dict[str, str]] = []
|
||||
@@ -268,7 +269,7 @@ envelope = {
|
||||
"runId": run_id,
|
||||
"workItemId": work_item_id,
|
||||
"sourceRevision": source_revision,
|
||||
"expectedRuntimeFileCount": 19,
|
||||
"expectedRuntimeFileCount": 20,
|
||||
"manifestSha256": manifest_sha256,
|
||||
"files": files,
|
||||
"livePreflight": {
|
||||
@@ -420,14 +421,14 @@ stage_token = hashlib.sha256(stage_identity.encode("utf-8")).hexdigest()[:20]
|
||||
script = r'''$ErrorActionPreference='Stop';$ProgressPreference='SilentlyContinue'
|
||||
$a='C:\Wooo\Agent99';$b=Join-Path $a 'bin';$p=Join-Path $a 'state\runtime-manifest.json'
|
||||
$r=[ordered]@{exists=$false;sourceRevision='';runtimeMatched=$false;recordedRuntimeMatched=$false;fileCount=0;mismatchCount=-1;recordedMismatchCount=-1;parseError=''}
|
||||
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 19-and$m.mismatchCount-eq 0-and$rows.Count-eq 19-and$seen.Count-eq 19-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
|
||||
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 20-and$m.mismatchCount-eq 0-and$rows.Count-eq 20-and$seen.Count-eq 20-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
|
||||
$c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''}
|
||||
try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name}
|
||||
$z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false}
|
||||
$ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0)
|
||||
$status=if($ready){'check_ready'}else{'check_failed_no_apply'}
|
||||
$next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'}
|
||||
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=19;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
|
||||
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=20;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
|
||||
if(-not$ready){exit 65}'''
|
||||
script = (
|
||||
script.replace("__SOURCE_REVISION__", source_revision)
|
||||
|
||||
@@ -0,0 +1,646 @@
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
if (-not $BrokerGzipBase64) { throw "broker_payload_missing" }
|
||||
$compressed = [Convert]::FromBase64String([string]$BrokerGzipBase64)
|
||||
$inputStream = New-Object IO.MemoryStream(,$compressed)
|
||||
$gzip = New-Object IO.Compression.GzipStream(
|
||||
$inputStream,
|
||||
[IO.Compression.CompressionMode]::Decompress
|
||||
)
|
||||
$reader = New-Object IO.StreamReader($gzip, [Text.Encoding]::UTF8)
|
||||
try {
|
||||
$brokerSource = $reader.ReadToEnd()
|
||||
} finally {
|
||||
$reader.Dispose()
|
||||
$gzip.Dispose()
|
||||
$inputStream.Dispose()
|
||||
}
|
||||
|
||||
$tokens = $null
|
||||
$parseErrors = $null
|
||||
$ast = [Management.Automation.Language.Parser]::ParseInput(
|
||||
$brokerSource,
|
||||
[ref]$tokens,
|
||||
[ref]$parseErrors
|
||||
)
|
||||
if ($parseErrors.Count -ne 0) { throw "broker_parse_failed" }
|
||||
|
||||
$functionNames = @(
|
||||
"Test-Agent99SafeIdentity",
|
||||
"Write-Agent99ImmutableJson",
|
||||
"Test-Agent99JsonField",
|
||||
"Assert-Agent99JsonFields",
|
||||
"Assert-Agent99ExactJsonFields",
|
||||
"Get-Agent99JsonResult",
|
||||
"Assert-Agent99RuntimeIdentityEvidence",
|
||||
"Assert-Agent99Host188PreflightEvidence",
|
||||
"Assert-Agent99Host188FreshnessEvidence",
|
||||
"New-Agent99Host188FreshSkipReceipt",
|
||||
"Assert-Agent99Host188ApplyEvidence",
|
||||
"Assert-Agent99Host188ApplyEvidenceMatch",
|
||||
"Get-Agent99Host188ApplyReceiptReadback",
|
||||
"New-Agent99Host188ApplyReconciliation",
|
||||
"Assert-Agent99Host188ApplyReconciliation",
|
||||
"Assert-Agent99Host188VerifyEvidence",
|
||||
"Assert-Agent99ExistingBrokerEvidence",
|
||||
"Get-Agent99Host188GatePath",
|
||||
"Get-Agent99Host188GateResolutionPath",
|
||||
"Assert-Agent99Host188OperationGate",
|
||||
"Assert-Agent99Host188GateResolution",
|
||||
"Get-Agent99Host188OperationGateState",
|
||||
"Write-Agent99Host188OperationGate",
|
||||
"Write-Agent99Host188GateResolution",
|
||||
"Enter-Agent99Host188BrokerLease",
|
||||
"New-Agent99Host188BusyReceipt",
|
||||
"New-Agent99Host188GateBlockedReceipt",
|
||||
"New-Agent99Host188ReconciliationPendingReceipt"
|
||||
)
|
||||
foreach ($functionName in $functionNames) {
|
||||
$node = $ast.Find(
|
||||
{
|
||||
param($candidate)
|
||||
$candidate -is [Management.Automation.Language.FunctionDefinitionAst] -and
|
||||
$candidate.Name -eq $functionName
|
||||
},
|
||||
$true
|
||||
)
|
||||
if ($null -eq $node) { throw "broker_function_missing_$functionName" }
|
||||
Invoke-Expression $node.Extent.Text
|
||||
}
|
||||
|
||||
$SourceRevision = "a" * 40
|
||||
$RunId = "host188-windows99-replay"
|
||||
$Mode = "Apply"
|
||||
$SnapshotId = ""
|
||||
$TargetHost = "192.168.0.110"
|
||||
$ExpectedAgentRoot = "C:\Wooo\Agent99"
|
||||
$MaxBackupAgeMinutes = 1800
|
||||
$sourceHead = "b" * 40
|
||||
$beforeSnapshotId = "c" * 8
|
||||
$createdSnapshotId = "d" * 8
|
||||
$verifierSha256 = "e" * 64
|
||||
$evidencePath = Join-Path $ExpectedAgentRoot "evidence\host188-backup-run\agent99-host188-backup-$RunId.json"
|
||||
|
||||
$freshness = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_freshness_v1"
|
||||
ok = $true
|
||||
state = "fresh"
|
||||
snapshotId = "9" * 8
|
||||
ageMinutes = 1
|
||||
maxAgeMinutes = 1800
|
||||
sourceHost = "192.168.0.188"
|
||||
controllerHost = "192.168.0.110"
|
||||
credentialReferenceConsumed = $true
|
||||
temporaryReadbackFileUsed = $true
|
||||
remoteWritePerformed = $false
|
||||
secretValueReturned = $false
|
||||
}
|
||||
$freshSkip = New-Agent99Host188FreshSkipReceipt $freshness
|
||||
if (
|
||||
[string]$freshSkip.schemaVersion -ne "agent99_host188_backup_fresh_skip_v1" -or
|
||||
-not [bool]$freshSkip.ok -or
|
||||
[string]$freshSkip.status -ne "fresh_skip" -or
|
||||
[bool]$freshSkip.applyInvoked -or
|
||||
[bool]$freshSkip.canonicalEvidenceWritten -or
|
||||
[bool]$freshSkip.remoteWritePerformed -or
|
||||
[bool]$freshSkip.secretValueReturned
|
||||
) { throw "fresh_waiter_skip_contract_failed" }
|
||||
|
||||
function Copy-ReplayDocument {
|
||||
param([object]$Document)
|
||||
return ($Document | ConvertTo-Json -Compress -Depth 12 | ConvertFrom-Json)
|
||||
}
|
||||
|
||||
function Assert-Rejected {
|
||||
param([scriptblock]$Operation, [string]$Name)
|
||||
$accepted = $false
|
||||
try {
|
||||
& $Operation | Out-Null
|
||||
$accepted = $true
|
||||
} catch {}
|
||||
if ($accepted) { throw "contract_mutation_accepted_$Name" }
|
||||
}
|
||||
|
||||
function New-RuntimeIdentityEvidence {
|
||||
param([string]$Phase)
|
||||
$runtimeRunId = "$RunId-runtime-$Phase"
|
||||
$check = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_preflight_v1"
|
||||
ok = $true
|
||||
sourceRevision = $SourceRevision
|
||||
sourceHead = $sourceHead
|
||||
fileCount = 18
|
||||
backupScriptFileCount = 17
|
||||
backupHealthExporterIncluded = $true
|
||||
remoteWritePerformed = $false
|
||||
}
|
||||
$verify = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_verifier_v1"
|
||||
ok = $true
|
||||
sourceRevision = $SourceRevision
|
||||
runId = $runtimeRunId
|
||||
fileCount = 18
|
||||
backupScriptFileCount = 17
|
||||
backupHealthExporterIncluded = $true
|
||||
destination = "/backup/scripts"
|
||||
exporterDestination = "/home/wooo/scripts/backup-health-textfile-exporter.py"
|
||||
remoteWritePerformed = $false
|
||||
secretValuesRead = $false
|
||||
selfIdentityVerified = $true
|
||||
verifierSha256 = $verifierSha256
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v3"
|
||||
ok = $true
|
||||
status = "readback_ok"
|
||||
mode = "Verify"
|
||||
sourceRevision = $SourceRevision
|
||||
sourceHead = $sourceHead
|
||||
runId = $runtimeRunId
|
||||
targetHost = $TargetHost
|
||||
sourceTransport = "windows99_gitea_exact_revision_manifest"
|
||||
executor = "host110_backup_runtime_executor"
|
||||
verifier = "independent_host110_backup_runtime_python_readback"
|
||||
decisionProvider = "deterministic_only"
|
||||
criticProvider = "deterministic_only"
|
||||
agentAction = "readback"
|
||||
check = $check
|
||||
apply = $null
|
||||
verify = $verify
|
||||
cleanupVerified = $true
|
||||
errorCode = ""
|
||||
elapsedSeconds = 1.25
|
||||
evidence = Join-Path $ExpectedAgentRoot "evidence\host110-backup-runtime\agent99-host110-backup-runtime-$runtimeRunId.json"
|
||||
secretValuesRead = $false
|
||||
rawSessionStored = $false
|
||||
}
|
||||
}
|
||||
|
||||
$runtimePre = New-RuntimeIdentityEvidence "pre"
|
||||
$runtimePost = New-RuntimeIdentityEvidence "post"
|
||||
$preflight = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_preflight_v1"
|
||||
ok = $true
|
||||
targetHost = $TargetHost
|
||||
sourceHost = "192.168.0.188"
|
||||
requiredContainers = 4
|
||||
sourceReadbackRequired = $true
|
||||
sourceReadbackPerformed = $true
|
||||
remoteWritePerformed = $false
|
||||
secretValueReturned = $false
|
||||
}
|
||||
$apply = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_executor_v1"
|
||||
ok = $true
|
||||
mode = "apply"
|
||||
result = "snapshot_created"
|
||||
runId = $RunId
|
||||
sourceRevision = $SourceRevision
|
||||
beforeSnapshotId = $beforeSnapshotId
|
||||
snapshotId = $createdSnapshotId
|
||||
backupExitCode = 0
|
||||
elapsedSeconds = 30
|
||||
errorCode = ""
|
||||
remoteReceipt = "/backup/logs/agent99-host188-backup-run/$RunId.json"
|
||||
remoteLog = "/backup/logs/agent99-host188-backup-run/$RunId.log"
|
||||
sourceHost = "192.168.0.188"
|
||||
controllerHost = $TargetHost
|
||||
serviceStopped = $false
|
||||
offsiteWritePerformed = $false
|
||||
durableReadbackVerified = $true
|
||||
credentialReferenceConsumed = $true
|
||||
secretValueReturned = $false
|
||||
}
|
||||
$receiptReadback = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_receipt_readback_v1"
|
||||
ok = $true
|
||||
state = "present"
|
||||
runId = $RunId
|
||||
sourceRevision = $SourceRevision
|
||||
receiptPath = "/backup/logs/agent99-host188-backup-run/$RunId.json"
|
||||
receipt = $apply
|
||||
remoteWritePerformed = $false
|
||||
secretValueReturned = $false
|
||||
}
|
||||
$transport = [pscustomobject]@{
|
||||
ok = $false
|
||||
reason = "transport_timeout"
|
||||
exitCode = -2
|
||||
stdout = ""
|
||||
}
|
||||
$reconciliation = New-Agent99Host188ApplyReconciliation `
|
||||
$transport $receiptReadback $apply "host188_backup_apply_transport_timeout" ""
|
||||
Assert-Agent99Host188ApplyReconciliation $reconciliation
|
||||
|
||||
$verify = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_verifier_v1"
|
||||
ok = $true
|
||||
runId = $RunId
|
||||
sourceRevision = $SourceRevision
|
||||
snapshotId = $createdSnapshotId
|
||||
applyReceipt = "/backup/logs/agent99-host188-backup-run/$RunId.json"
|
||||
resticSnapshotUnique = $true
|
||||
dataRestoreVerified = $true
|
||||
recoveryReadyClaimed = $false
|
||||
productionRestorePerformed = $false
|
||||
productionDataWritePerformed = $false
|
||||
temporaryReadbackWorkspaceUsed = $true
|
||||
credentialReferenceConsumed = $true
|
||||
secretValueReturned = $false
|
||||
}
|
||||
$brokerSuccess = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_broker_receipt_v2"
|
||||
ok = $true
|
||||
status = "verified"
|
||||
mode = "Apply"
|
||||
sourceRevision = $SourceRevision
|
||||
runId = $RunId
|
||||
requestedSnapshotId = ""
|
||||
targetHost = $TargetHost
|
||||
sourceHost = "192.168.0.188"
|
||||
decisionProvider = "deterministic_only"
|
||||
criticProvider = "deterministic_only"
|
||||
agentAction = "controlled_host188_backup"
|
||||
executor = "fixed_host110_backup_host188_products"
|
||||
verifier = "independent_snapshot_specific_restore_readback"
|
||||
runtimeIdentityPre = $runtimePre
|
||||
preflight = $preflight
|
||||
apply = $apply
|
||||
applyReconciliation = $null
|
||||
runtimeIdentityPost = $runtimePost
|
||||
verify = $verify
|
||||
errorCode = ""
|
||||
elapsedSeconds = 60.5
|
||||
evidence = $evidencePath
|
||||
arbitraryCommandAllowed = $false
|
||||
serviceStopped = $false
|
||||
offsiteWritePerformed = $false
|
||||
secretValueReturned = $false
|
||||
rawSessionStored = $false
|
||||
}
|
||||
|
||||
try {
|
||||
Assert-Agent99ExistingBrokerEvidence $brokerSuccess $evidencePath
|
||||
} catch {
|
||||
$identityChecks = [ordered]@{
|
||||
schemaVersion = [string]$brokerSuccess.schemaVersion -eq "agent99_host188_backup_broker_receipt_v2"
|
||||
mode = [string]$brokerSuccess.mode -eq $Mode
|
||||
sourceRevision = [string]$brokerSuccess.sourceRevision -eq $SourceRevision
|
||||
runId = [string]$brokerSuccess.runId -eq $RunId
|
||||
requestedSnapshotId = [string]$brokerSuccess.requestedSnapshotId -eq $SnapshotId
|
||||
targetHost = [string]$brokerSuccess.targetHost -eq $TargetHost
|
||||
sourceHost = [string]$brokerSuccess.sourceHost -eq "192.168.0.188"
|
||||
decisionProvider = [string]$brokerSuccess.decisionProvider -eq "deterministic_only"
|
||||
criticProvider = [string]$brokerSuccess.criticProvider -eq "deterministic_only"
|
||||
agentAction = [string]$brokerSuccess.agentAction -eq "controlled_host188_backup"
|
||||
executor = [string]$brokerSuccess.executor -eq "fixed_host110_backup_host188_products"
|
||||
verifier = [string]$brokerSuccess.verifier -eq "independent_snapshot_specific_restore_readback"
|
||||
elapsedSeconds = [double]$brokerSuccess.elapsedSeconds -ge 0
|
||||
evidence = [string]$brokerSuccess.evidence -eq $evidencePath
|
||||
arbitraryCommandAllowed = -not [bool]$brokerSuccess.arbitraryCommandAllowed
|
||||
serviceStopped = -not [bool]$brokerSuccess.serviceStopped
|
||||
offsiteWritePerformed = -not [bool]$brokerSuccess.offsiteWritePerformed
|
||||
secretValueReturned = -not [bool]$brokerSuccess.secretValueReturned
|
||||
rawSessionStored = -not [bool]$brokerSuccess.rawSessionStored
|
||||
}
|
||||
$failedIdentityChecks = @($identityChecks.Keys | Where-Object { -not $identityChecks[$_] })
|
||||
throw "valid_broker_fixture_rejected identity=$($failedIdentityChecks -join ',') original=$([string]$_.Exception.Message)"
|
||||
}
|
||||
$recoveredSuccess = Copy-ReplayDocument $brokerSuccess
|
||||
$recoveredSuccess.applyReconciliation = $reconciliation
|
||||
Assert-Agent99ExistingBrokerEvidence $recoveredSuccess $evidencePath
|
||||
$reconciledEvidencePath = Join-Path $ExpectedAgentRoot "evidence\host188-backup-run\agent99-host188-backup-reconcile-$RunId.json"
|
||||
$reconciledSuccess = Copy-ReplayDocument $recoveredSuccess
|
||||
$reconciledSuccess.status = "reconciled_verified"
|
||||
$reconciledSuccess.agentAction = "read_only_same_run_reconciliation"
|
||||
$reconciledSuccess.evidence = $reconciledEvidencePath
|
||||
Assert-Agent99ExistingBrokerEvidence $reconciledSuccess $reconciledEvidencePath
|
||||
|
||||
$sourceLockFreshApply = Copy-ReplayDocument $apply
|
||||
$sourceLockFreshApply.result = "fresh_skip"
|
||||
$sourceLockFreshApply.snapshotId = $beforeSnapshotId
|
||||
$sourceLockFreshVerify = Copy-ReplayDocument $verify
|
||||
$sourceLockFreshVerify.snapshotId = $beforeSnapshotId
|
||||
$sourceLockFreshSuccess = Copy-ReplayDocument $brokerSuccess
|
||||
$sourceLockFreshSuccess.status = "fresh_skip_verified"
|
||||
$sourceLockFreshSuccess.agentAction = "fresh_snapshot_verified_no_apply"
|
||||
$sourceLockFreshSuccess.apply = $sourceLockFreshApply
|
||||
$sourceLockFreshSuccess.verify = $sourceLockFreshVerify
|
||||
Assert-Agent99ExistingBrokerEvidence $sourceLockFreshSuccess $evidencePath
|
||||
$mutated = Copy-ReplayDocument $sourceLockFreshSuccess
|
||||
$mutated.status = "verified"
|
||||
$mutated.agentAction = "controlled_host188_backup"
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $evidencePath } "fresh_result_verified_status_conflict"
|
||||
$existingEvidenceRejections++
|
||||
$mutated = Copy-ReplayDocument $brokerSuccess
|
||||
$mutated.status = "fresh_skip_verified"
|
||||
$mutated.agentAction = "fresh_snapshot_verified_no_apply"
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $evidencePath } "created_result_fresh_status_conflict"
|
||||
$existingEvidenceRejections++
|
||||
|
||||
$topLevelFields = @(
|
||||
"schemaVersion", "ok", "status", "mode", "sourceRevision", "runId",
|
||||
"requestedSnapshotId", "targetHost", "sourceHost", "decisionProvider",
|
||||
"criticProvider", "agentAction", "executor", "verifier",
|
||||
"runtimeIdentityPre", "preflight", "apply", "applyReconciliation",
|
||||
"runtimeIdentityPost", "verify", "errorCode", "elapsedSeconds", "evidence",
|
||||
"arbitraryCommandAllowed", "serviceStopped", "offsiteWritePerformed",
|
||||
"secretValueReturned", "rawSessionStored"
|
||||
)
|
||||
$existingEvidenceRejections = 0
|
||||
foreach ($field in $topLevelFields) {
|
||||
$mutated = Copy-ReplayDocument $brokerSuccess
|
||||
$mutated.PSObject.Properties.Remove($field)
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $evidencePath } "missing_$field"
|
||||
$existingEvidenceRejections++
|
||||
}
|
||||
|
||||
$mutations = @(
|
||||
@{ name = "ok_string_false"; field = "ok"; value = "false" },
|
||||
@{ name = "wrong_status"; field = "status"; value = "ready" },
|
||||
@{ name = "wrong_mode"; field = "mode"; value = "Verify" },
|
||||
@{ name = "wrong_run_id"; field = "runId"; value = "other-run" },
|
||||
@{ name = "wrong_evidence"; field = "evidence"; value = "C:\wrong.json" },
|
||||
@{ name = "unsafe_service_stop"; field = "serviceStopped"; value = $true }
|
||||
)
|
||||
foreach ($mutation in $mutations) {
|
||||
$mutated = Copy-ReplayDocument $brokerSuccess
|
||||
$mutated.($mutation.field) = $mutation.value
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $evidencePath } $mutation.name
|
||||
$existingEvidenceRejections++
|
||||
}
|
||||
|
||||
$mutated = Copy-ReplayDocument $brokerSuccess
|
||||
$mutated | Add-Member -NotePropertyName unexpected -NotePropertyValue $true
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $evidencePath } "extra_top"
|
||||
$existingEvidenceRejections++
|
||||
$mutated = Copy-ReplayDocument $brokerSuccess
|
||||
$mutated.apply.PSObject.Properties.Remove("durableReadbackVerified")
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $evidencePath } "missing_apply_field"
|
||||
$existingEvidenceRejections++
|
||||
$mutated = Copy-ReplayDocument $brokerSuccess
|
||||
$mutated.apply.ok = "true"
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $evidencePath } "wrong_apply_type"
|
||||
$existingEvidenceRejections++
|
||||
$mutated = Copy-ReplayDocument $brokerSuccess
|
||||
$mutated.apply.snapshotId = $mutated.apply.beforeSnapshotId
|
||||
$mutated.verify.snapshotId = $mutated.apply.beforeSnapshotId
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $evidencePath } "snapshot_not_advanced"
|
||||
$existingEvidenceRejections++
|
||||
$mutated = Copy-ReplayDocument $recoveredSuccess
|
||||
$mutated.applyReconciliation.retryApplyAllowed = $true
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $evidencePath } "reconciliation_retry_enabled"
|
||||
$existingEvidenceRejections++
|
||||
$mutated = Copy-ReplayDocument $recoveredSuccess
|
||||
$mutated.applyReconciliation.receiptReadback.receipt.snapshotId = "f" * 8
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $evidencePath } "readback_snapshot_conflict"
|
||||
$existingEvidenceRejections++
|
||||
|
||||
$missingReadback = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_receipt_readback_v1"
|
||||
ok = $true
|
||||
state = "missing"
|
||||
runId = $RunId
|
||||
sourceRevision = $SourceRevision
|
||||
receiptPath = "/backup/logs/agent99-host188-backup-run/$RunId.json"
|
||||
receipt = $null
|
||||
remoteWritePerformed = $false
|
||||
secretValueReturned = $false
|
||||
}
|
||||
$committedUnknown = New-Agent99Host188ApplyReconciliation `
|
||||
$transport $missingReadback $null "host188_backup_apply_transport_timeout" ""
|
||||
Assert-Agent99Host188ApplyReconciliation $committedUnknown
|
||||
if (
|
||||
$committedUnknown.status -ne "committed_unknown" -or
|
||||
$committedUnknown.ok -or
|
||||
$committedUnknown.retryApplyAllowed -or
|
||||
$committedUnknown.remoteWritePerformed -or
|
||||
$committedUnknown.secretValueReturned
|
||||
) { throw "committed_unknown_truth_failed" }
|
||||
|
||||
$brokerFailure = Copy-ReplayDocument $brokerSuccess
|
||||
$brokerFailure.ok = $false
|
||||
$brokerFailure.status = "failed"
|
||||
$brokerFailure.apply = $null
|
||||
$brokerFailure.applyReconciliation = $committedUnknown
|
||||
$brokerFailure.runtimeIdentityPost = $null
|
||||
$brokerFailure.verify = $null
|
||||
$brokerFailure.errorCode = "host188_backup_apply_committed_unknown"
|
||||
Assert-Agent99ExistingBrokerEvidence $brokerFailure $evidencePath
|
||||
$malformedFailure = Copy-ReplayDocument $brokerFailure
|
||||
$malformedFailure.ok = "false"
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $malformedFailure $evidencePath } "malformed_failure_not_upgraded"
|
||||
$existingEvidenceRejections++
|
||||
|
||||
$leaseNode = $ast.Find(
|
||||
{
|
||||
param($candidate)
|
||||
$candidate -is [Management.Automation.Language.FunctionDefinitionAst] -and
|
||||
$candidate.Name -eq "Enter-Agent99Host188BrokerLease"
|
||||
},
|
||||
$true
|
||||
)
|
||||
if ($null -eq $leaseNode) { throw "lease_function_missing" }
|
||||
$mutexName = "Local\Agent99Host188Replay-$([Guid]::NewGuid().ToString('N'))"
|
||||
$createdNew = $false
|
||||
$parentMutex = [Threading.Mutex]::new($true, $mutexName, [ref]$createdNew)
|
||||
if (-not $createdNew) { throw "replay_mutex_identity_conflict" }
|
||||
try {
|
||||
$childTemplate = @'
|
||||
$ErrorActionPreference = "Stop"
|
||||
__LEASE_FUNCTION__
|
||||
$mutex = [Threading.Mutex]::new($false, "__MUTEX_NAME__")
|
||||
$acquired = $false
|
||||
try {
|
||||
$acquired = Enter-Agent99Host188BrokerLease $mutex 250
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_mutex_contender_replay_v1"
|
||||
acquired = [bool]$acquired
|
||||
canonicalEvidenceWritten = $false
|
||||
remoteWritePerformed = $false
|
||||
} | ConvertTo-Json -Compress
|
||||
} finally {
|
||||
if ($acquired) { try { $mutex.ReleaseMutex() } catch {} }
|
||||
$mutex.Dispose()
|
||||
}
|
||||
exit 0
|
||||
'@
|
||||
$childSource = $childTemplate.Replace("__LEASE_FUNCTION__", $leaseNode.Extent.Text).Replace("__MUTEX_NAME__", $mutexName)
|
||||
$encodedChild = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($childSource))
|
||||
$childOutput = & "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" `
|
||||
-NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand $encodedChild 2>$null
|
||||
if ($LASTEXITCODE -ne 0) { throw "mutex_contender_process_failed" }
|
||||
$mutexReplay = [string]$childOutput | ConvertFrom-Json
|
||||
if (
|
||||
[string]$mutexReplay.schemaVersion -ne "agent99_host188_mutex_contender_replay_v1" -or
|
||||
[bool]$mutexReplay.acquired -or
|
||||
[bool]$mutexReplay.canonicalEvidenceWritten -or
|
||||
[bool]$mutexReplay.remoteWritePerformed
|
||||
) { throw "mutex_contender_contract_failed" }
|
||||
} finally {
|
||||
$parentMutex.ReleaseMutex()
|
||||
$parentMutex.Dispose()
|
||||
}
|
||||
|
||||
$originalEvidenceRoot = $EvidenceRoot
|
||||
$originalGateRoot = $GateRoot
|
||||
$gateReplayRoot = Join-Path $env:TEMP "agent99-host188-gate-replay-$([Guid]::NewGuid().ToString('N'))"
|
||||
$EvidenceRoot = Join-Path $gateReplayRoot "evidence\host188-backup-run"
|
||||
$GateRoot = Join-Path $EvidenceRoot "gates"
|
||||
New-Item -ItemType Directory -Force -Path $GateRoot | Out-Null
|
||||
$gateMutexName = "Local\Agent99Host188GateReplay-$([Guid]::NewGuid().ToString('N'))"
|
||||
$gateCreatedNew = $false
|
||||
$gateOwnerMutex = [Threading.Mutex]::new($true, $gateMutexName, [ref]$gateCreatedNew)
|
||||
if (-not $gateCreatedNew) { throw "gate_replay_mutex_identity_conflict" }
|
||||
$gateChildOut = Join-Path $gateReplayRoot "child.out"
|
||||
$gateChildErr = Join-Path $gateReplayRoot "child.err"
|
||||
$gateChild = $null
|
||||
try {
|
||||
$childFunctions = @(
|
||||
"Test-Agent99SafeIdentity",
|
||||
"Test-Agent99JsonField",
|
||||
"Assert-Agent99JsonFields",
|
||||
"Assert-Agent99ExactJsonFields",
|
||||
"Get-Agent99Host188GatePath",
|
||||
"Get-Agent99Host188GateResolutionPath",
|
||||
"Assert-Agent99Host188OperationGate",
|
||||
"Assert-Agent99Host188GateResolution",
|
||||
"Get-Agent99Host188OperationGateState",
|
||||
"Enter-Agent99Host188BrokerLease"
|
||||
)
|
||||
$childFunctionSource = @()
|
||||
foreach ($functionName in $childFunctions) {
|
||||
$node = $ast.Find(
|
||||
{
|
||||
param($candidate)
|
||||
$candidate -is [Management.Automation.Language.FunctionDefinitionAst] -and
|
||||
$candidate.Name -eq $functionName
|
||||
},
|
||||
$true
|
||||
)
|
||||
if ($null -eq $node) { throw "gate_replay_function_missing_$functionName" }
|
||||
$childFunctionSource += $node.Extent.Text
|
||||
}
|
||||
$gateChildTemplate = @'
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
$SourceRevision = "__SOURCE_REVISION__"
|
||||
$RunId = "__RUN_ID__"
|
||||
$TargetHost = "192.168.0.110"
|
||||
$EvidenceRoot = "__EVIDENCE_ROOT__"
|
||||
$GateRoot = "__GATE_ROOT__"
|
||||
__FUNCTIONS__
|
||||
$mutex = [Threading.Mutex]::new($false, "__MUTEX_NAME__")
|
||||
$acquired = $false
|
||||
try {
|
||||
$acquired = Enter-Agent99Host188BrokerLease $mutex 5000
|
||||
if (-not $acquired) { throw "gate_waiter_did_not_acquire" }
|
||||
$state = Get-Agent99Host188OperationGateState
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_gate_waiter_replay_v1"
|
||||
acquiredAfterOwnerRelease = $true
|
||||
blocked = [bool]$state.blocked
|
||||
reason = [string]$state.reason
|
||||
blockingRunId = [string]$state.runId
|
||||
applyInvoked = $false
|
||||
canonicalEvidenceWritten = $false
|
||||
remoteWritePerformed = $false
|
||||
} | ConvertTo-Json -Compress
|
||||
} finally {
|
||||
if ($acquired) { try { $mutex.ReleaseMutex() } catch {} }
|
||||
$mutex.Dispose()
|
||||
}
|
||||
exit 0
|
||||
'@
|
||||
$gateChildSource = $gateChildTemplate.Replace("__SOURCE_REVISION__", $SourceRevision)
|
||||
$gateChildSource = $gateChildSource.Replace("__RUN_ID__", $RunId)
|
||||
$gateChildSource = $gateChildSource.Replace("__EVIDENCE_ROOT__", $EvidenceRoot)
|
||||
$gateChildSource = $gateChildSource.Replace("__GATE_ROOT__", $GateRoot)
|
||||
$gateChildSource = $gateChildSource.Replace("__FUNCTIONS__", ($childFunctionSource -join "`r`n"))
|
||||
$gateChildSource = $gateChildSource.Replace("__MUTEX_NAME__", $gateMutexName)
|
||||
$gateChildEncoded = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($gateChildSource))
|
||||
$gateChild = Start-Process -FilePath "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" `
|
||||
-ArgumentList @("-NoLogo", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-EncodedCommand", $gateChildEncoded) `
|
||||
-NoNewWindow -RedirectStandardOutput $gateChildOut -RedirectStandardError $gateChildErr -PassThru
|
||||
Start-Sleep -Milliseconds 300
|
||||
$publishedGate = Write-Agent99Host188OperationGate
|
||||
Assert-Agent99Host188OperationGate $publishedGate (Get-Agent99Host188GatePath $RunId)
|
||||
$gateOwnerMutex.ReleaseMutex()
|
||||
$gateOwnerMutex.Dispose()
|
||||
$gateOwnerMutex = $null
|
||||
if (-not $gateChild.WaitForExit(10000)) { $gateChild.Kill(); throw "gate_waiter_timeout" }
|
||||
$gateChild.WaitForExit() | Out-Null
|
||||
$gateChild.Refresh()
|
||||
$childError = if (Test-Path -LiteralPath $gateChildErr) { Get-Content -LiteralPath $gateChildErr -Raw } else { "" }
|
||||
$childOutput = if (Test-Path -LiteralPath $gateChildOut) { Get-Content -LiteralPath $gateChildOut -Raw } else { "" }
|
||||
if ($childError -or -not $childOutput) { throw "gate_waiter_failed_stderr=$childError stdout=$childOutput" }
|
||||
$gateWaiter = $childOutput | ConvertFrom-Json
|
||||
if (
|
||||
[string]$gateWaiter.schemaVersion -ne "agent99_host188_gate_waiter_replay_v1" -or
|
||||
-not [bool]$gateWaiter.acquiredAfterOwnerRelease -or
|
||||
-not [bool]$gateWaiter.blocked -or
|
||||
[string]$gateWaiter.reason -ne "operation_gate_unresolved" -or
|
||||
[string]$gateWaiter.blockingRunId -ne $RunId -or
|
||||
[bool]$gateWaiter.applyInvoked -or
|
||||
[bool]$gateWaiter.canonicalEvidenceWritten -or
|
||||
[bool]$gateWaiter.remoteWritePerformed
|
||||
) { throw "gate_waiter_contract_failed" }
|
||||
$malformedResolutionEvidence = Join-Path $EvidenceRoot "agent99-host188-backup-reconcile-$RunId.json"
|
||||
Write-Agent99ImmutableJson $malformedResolutionEvidence ([pscustomobject]@{ schemaVersion = "malformed" })
|
||||
$malformedResolution = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_gate_resolution_v1"
|
||||
ok = $true
|
||||
status = "resolved_verified"
|
||||
sourceRevision = $SourceRevision
|
||||
runId = $RunId
|
||||
operationGate = Get-Agent99Host188GatePath $RunId
|
||||
resolutionEvidence = $malformedResolutionEvidence
|
||||
resolutionSha256 = (Get-FileHash -LiteralPath $malformedResolutionEvidence -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
snapshotId = $createdSnapshotId
|
||||
reconciliationPerformed = $true
|
||||
retryApplyAllowed = $false
|
||||
remoteWritePerformed = $false
|
||||
secretValueRead = $false
|
||||
}
|
||||
Assert-Rejected {
|
||||
Assert-Agent99Host188GateResolution `
|
||||
$malformedResolution `
|
||||
$publishedGate `
|
||||
(Get-Agent99Host188GateResolutionPath $RunId)
|
||||
} "matching_digest_malformed_resolution_receipt"
|
||||
$resolvedOwnerReceipt = Copy-ReplayDocument $brokerSuccess
|
||||
$resolvedOwnerReceipt.evidence = [string]$publishedGate.canonicalEvidence
|
||||
Assert-Agent99ExistingBrokerEvidence $resolvedOwnerReceipt ([string]$publishedGate.canonicalEvidence)
|
||||
Write-Agent99ImmutableJson ([string]$publishedGate.canonicalEvidence) $resolvedOwnerReceipt
|
||||
Write-Agent99Host188GateResolution $publishedGate $resolvedOwnerReceipt $false | Out-Null
|
||||
$resolvedOwnerState = Get-Agent99Host188OperationGateState
|
||||
if ($resolvedOwnerState.blocked -or [bool]$freshSkip.applyInvoked) {
|
||||
throw "resolved_owner_fresh_waiter_would_apply"
|
||||
}
|
||||
} finally {
|
||||
if ($gateOwnerMutex) { try { $gateOwnerMutex.ReleaseMutex() } catch {}; $gateOwnerMutex.Dispose() }
|
||||
if ($gateChild) { $gateChild.Dispose() }
|
||||
$EvidenceRoot = $originalEvidenceRoot
|
||||
$GateRoot = $originalGateRoot
|
||||
Remove-Item -LiteralPath $gateReplayRoot -Recurse -Force -ErrorAction SilentlyContinue | Out-Null
|
||||
}
|
||||
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_broker_contract_replay_v1"
|
||||
ok = $true
|
||||
existingEvidenceRejections = $existingEvidenceRejections
|
||||
directSuccessAccepted = $true
|
||||
recoveredReceiptAccepted = $true
|
||||
sameRunReconciliationAccepted = $true
|
||||
freshWaiterSkipAccepted = $true
|
||||
sourceLockFreshSkipAccepted = $true
|
||||
resolvedOwnerFreshWaiterBlockedFromApply = $true
|
||||
committedUnknownPreserved = $true
|
||||
mutexContenderBlocked = $true
|
||||
waiterBlockedAfterOwnerUnknown = $true
|
||||
malformedResolutionReceiptRejected = $true
|
||||
canonicalEvidenceWrittenByContender = $false
|
||||
retryApplyAllowed = $false
|
||||
remoteWritePerformed = $false
|
||||
secretValuesRead = $false
|
||||
} | ConvertTo-Json -Compress
|
||||
@@ -0,0 +1,605 @@
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
if (-not $ControlSliceGzipBase64) { throw "control_slice_payload_missing" }
|
||||
$compressed = [Convert]::FromBase64String([string]$ControlSliceGzipBase64)
|
||||
$inputStream = New-Object IO.MemoryStream(,$compressed)
|
||||
$gzip = New-Object IO.Compression.GzipStream(
|
||||
$inputStream,
|
||||
[IO.Compression.CompressionMode]::Decompress
|
||||
)
|
||||
$reader = New-Object IO.StreamReader($gzip, [Text.Encoding]::UTF8)
|
||||
try {
|
||||
$controlSource = $reader.ReadToEnd()
|
||||
} finally {
|
||||
$reader.Dispose()
|
||||
$gzip.Dispose()
|
||||
$inputStream.Dispose()
|
||||
}
|
||||
|
||||
$tokens = $null
|
||||
$parseErrors = $null
|
||||
$ast = [Management.Automation.Language.Parser]::ParseInput(
|
||||
$controlSource,
|
||||
[ref]$tokens,
|
||||
[ref]$parseErrors
|
||||
)
|
||||
if ($parseErrors.Count -ne 0) { throw "control_slice_parse_failed" }
|
||||
|
||||
$functionNames = @(
|
||||
"Test-AgentHost188ReceiptField",
|
||||
"Test-AgentHost188ReceiptFields",
|
||||
"Test-AgentHost188ExactReceiptFields",
|
||||
"Test-AgentHost188RuntimeIdentityEvidence",
|
||||
"Test-AgentHost188PreflightEvidence",
|
||||
"Test-AgentHost188ApplyEvidence",
|
||||
"Test-AgentHost188ApplyReconciliationEvidence",
|
||||
"Test-AgentHost188VerifyEvidence",
|
||||
"Test-AgentHost188VerifiedBrokerEvidence",
|
||||
"Test-AgentHost188CommittedUnknownEvidence",
|
||||
"Test-AgentHost188OperationGateEvidence",
|
||||
"Test-AgentHost188GateResolutionEvidence",
|
||||
"Get-AgentHost188CommittedUnknownGate",
|
||||
"Test-AgentHost188BusyEvidence",
|
||||
"Test-AgentHost188FreshSkipEvidence",
|
||||
"Get-AgentHost188VerifiedRemediationOutcome",
|
||||
"Invoke-AgentHost188BackupRemediation"
|
||||
)
|
||||
foreach ($functionName in $functionNames) {
|
||||
$node = $ast.Find(
|
||||
{
|
||||
param($candidate)
|
||||
$candidate -is [Management.Automation.Language.FunctionDefinitionAst] -and
|
||||
$candidate.Name -eq $functionName
|
||||
},
|
||||
$true
|
||||
)
|
||||
if ($null -eq $node) { throw "control_function_missing_$functionName" }
|
||||
Invoke-Expression $node.Extent.Text
|
||||
}
|
||||
|
||||
$sourceRevision = "a" * 40
|
||||
$runId = "host188-control-replay"
|
||||
$agentRoot = "C:\Wooo\Agent99"
|
||||
$sourceHead = "b" * 40
|
||||
$beforeSnapshotId = "c" * 8
|
||||
$createdSnapshotId = "d" * 8
|
||||
$verifierSha256 = "e" * 64
|
||||
$evidencePath = Join-Path $agentRoot "evidence\host188-backup-run\agent99-host188-backup-$runId.json"
|
||||
|
||||
function Copy-ReplayDocument {
|
||||
param([object]$Document)
|
||||
return ($Document | ConvertTo-Json -Compress -Depth 12 | ConvertFrom-Json)
|
||||
}
|
||||
|
||||
function Assert-Rejected {
|
||||
param([scriptblock]$Operation, [string]$Name)
|
||||
$accepted = $false
|
||||
try {
|
||||
$accepted = [bool](& $Operation)
|
||||
} catch {}
|
||||
if ($accepted) { throw "control_contract_mutation_accepted_$Name" }
|
||||
}
|
||||
|
||||
function New-RuntimeEvidence {
|
||||
param([string]$Phase)
|
||||
$runtimeRunId = "$runId-runtime-$Phase"
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v3"
|
||||
ok = $true
|
||||
status = "readback_ok"
|
||||
mode = "Verify"
|
||||
sourceRevision = $sourceRevision
|
||||
sourceHead = $sourceHead
|
||||
runId = $runtimeRunId
|
||||
targetHost = "192.168.0.110"
|
||||
sourceTransport = "windows99_gitea_exact_revision_manifest"
|
||||
executor = "host110_backup_runtime_executor"
|
||||
verifier = "independent_host110_backup_runtime_python_readback"
|
||||
decisionProvider = "deterministic_only"
|
||||
criticProvider = "deterministic_only"
|
||||
agentAction = "readback"
|
||||
check = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_preflight_v1"
|
||||
ok = $true
|
||||
sourceRevision = $sourceRevision
|
||||
sourceHead = $sourceHead
|
||||
fileCount = 18
|
||||
backupScriptFileCount = 17
|
||||
backupHealthExporterIncluded = $true
|
||||
remoteWritePerformed = $false
|
||||
}
|
||||
apply = $null
|
||||
verify = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_verifier_v1"
|
||||
ok = $true
|
||||
sourceRevision = $sourceRevision
|
||||
runId = $runtimeRunId
|
||||
fileCount = 18
|
||||
backupScriptFileCount = 17
|
||||
backupHealthExporterIncluded = $true
|
||||
destination = "/backup/scripts"
|
||||
exporterDestination = "/home/wooo/scripts/backup-health-textfile-exporter.py"
|
||||
remoteWritePerformed = $false
|
||||
secretValuesRead = $false
|
||||
selfIdentityVerified = $true
|
||||
verifierSha256 = $verifierSha256
|
||||
}
|
||||
cleanupVerified = $true
|
||||
errorCode = ""
|
||||
elapsedSeconds = 1.25
|
||||
evidence = Join-Path $agentRoot "evidence\host110-backup-runtime\agent99-host110-backup-runtime-$runtimeRunId.json"
|
||||
secretValuesRead = $false
|
||||
rawSessionStored = $false
|
||||
}
|
||||
}
|
||||
|
||||
$runtimePre = New-RuntimeEvidence "pre"
|
||||
$runtimePost = New-RuntimeEvidence "post"
|
||||
$preflight = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_preflight_v1"
|
||||
ok = $true
|
||||
targetHost = "192.168.0.110"
|
||||
sourceHost = "192.168.0.188"
|
||||
requiredContainers = 4
|
||||
sourceReadbackRequired = $true
|
||||
sourceReadbackPerformed = $true
|
||||
remoteWritePerformed = $false
|
||||
secretValueReturned = $false
|
||||
}
|
||||
$apply = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_executor_v1"
|
||||
ok = $true
|
||||
mode = "apply"
|
||||
result = "snapshot_created"
|
||||
runId = $runId
|
||||
sourceRevision = $sourceRevision
|
||||
beforeSnapshotId = $beforeSnapshotId
|
||||
snapshotId = $createdSnapshotId
|
||||
backupExitCode = 0
|
||||
elapsedSeconds = 30
|
||||
errorCode = ""
|
||||
remoteReceipt = "/backup/logs/agent99-host188-backup-run/$runId.json"
|
||||
remoteLog = "/backup/logs/agent99-host188-backup-run/$runId.log"
|
||||
sourceHost = "192.168.0.188"
|
||||
controllerHost = "192.168.0.110"
|
||||
serviceStopped = $false
|
||||
offsiteWritePerformed = $false
|
||||
durableReadbackVerified = $true
|
||||
credentialReferenceConsumed = $true
|
||||
secretValueReturned = $false
|
||||
}
|
||||
$verify = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_verifier_v1"
|
||||
ok = $true
|
||||
runId = $runId
|
||||
sourceRevision = $sourceRevision
|
||||
snapshotId = $createdSnapshotId
|
||||
applyReceipt = "/backup/logs/agent99-host188-backup-run/$runId.json"
|
||||
resticSnapshotUnique = $true
|
||||
dataRestoreVerified = $true
|
||||
recoveryReadyClaimed = $false
|
||||
productionRestorePerformed = $false
|
||||
productionDataWritePerformed = $false
|
||||
temporaryReadbackWorkspaceUsed = $true
|
||||
credentialReferenceConsumed = $true
|
||||
secretValueReturned = $false
|
||||
}
|
||||
$success = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_broker_receipt_v2"
|
||||
ok = $true
|
||||
status = "verified"
|
||||
mode = "Apply"
|
||||
sourceRevision = $sourceRevision
|
||||
runId = $runId
|
||||
requestedSnapshotId = ""
|
||||
targetHost = "192.168.0.110"
|
||||
sourceHost = "192.168.0.188"
|
||||
decisionProvider = "deterministic_only"
|
||||
criticProvider = "deterministic_only"
|
||||
agentAction = "controlled_host188_backup"
|
||||
executor = "fixed_host110_backup_host188_products"
|
||||
verifier = "independent_snapshot_specific_restore_readback"
|
||||
runtimeIdentityPre = $runtimePre
|
||||
preflight = $preflight
|
||||
apply = $apply
|
||||
applyReconciliation = $null
|
||||
runtimeIdentityPost = $runtimePost
|
||||
verify = $verify
|
||||
errorCode = ""
|
||||
elapsedSeconds = 60.5
|
||||
evidence = $evidencePath
|
||||
arbitraryCommandAllowed = $false
|
||||
serviceStopped = $false
|
||||
offsiteWritePerformed = $false
|
||||
secretValueReturned = $false
|
||||
rawSessionStored = $false
|
||||
}
|
||||
if (-not (Test-AgentHost188VerifiedBrokerEvidence $success $sourceRevision $runId $evidencePath $agentRoot)) {
|
||||
throw "valid_success_rejected"
|
||||
}
|
||||
$sourceLockFreshApply = Copy-ReplayDocument $apply
|
||||
$sourceLockFreshApply.result = "fresh_skip"
|
||||
$sourceLockFreshApply.snapshotId = $beforeSnapshotId
|
||||
$sourceLockFreshVerify = Copy-ReplayDocument $verify
|
||||
$sourceLockFreshVerify.snapshotId = $beforeSnapshotId
|
||||
$sourceLockFreshSuccess = Copy-ReplayDocument $success
|
||||
$sourceLockFreshSuccess.status = "fresh_skip_verified"
|
||||
$sourceLockFreshSuccess.agentAction = "fresh_snapshot_verified_no_apply"
|
||||
$sourceLockFreshSuccess.apply = $sourceLockFreshApply
|
||||
$sourceLockFreshSuccess.verify = $sourceLockFreshVerify
|
||||
if (-not (Test-AgentHost188VerifiedBrokerEvidence $sourceLockFreshSuccess $sourceRevision $runId $evidencePath $agentRoot)) {
|
||||
throw "valid_source_lock_fresh_skip_rejected"
|
||||
}
|
||||
$mutated = Copy-ReplayDocument $sourceLockFreshSuccess
|
||||
$mutated.status = "verified"
|
||||
$mutated.agentAction = "controlled_host188_backup"
|
||||
Assert-Rejected { Test-AgentHost188VerifiedBrokerEvidence $mutated $sourceRevision $runId $evidencePath $agentRoot } "fresh_result_verified_status_conflict"
|
||||
$mutated = Copy-ReplayDocument $success
|
||||
$mutated.status = "fresh_skip_verified"
|
||||
$mutated.agentAction = "fresh_snapshot_verified_no_apply"
|
||||
Assert-Rejected { Test-AgentHost188VerifiedBrokerEvidence $mutated $sourceRevision $runId $evidencePath $agentRoot } "created_result_fresh_status_conflict"
|
||||
$rejections = 0
|
||||
$mutated = Copy-ReplayDocument $success
|
||||
$mutated.evidence = Join-Path $agentRoot "evidence\agent99-host188-backup-$runId.json"
|
||||
Assert-Rejected { Test-AgentHost188VerifiedBrokerEvidence $mutated $sourceRevision $runId $evidencePath $agentRoot } "wrong_evidence_path"
|
||||
$rejections++
|
||||
$mutated = Copy-ReplayDocument $success
|
||||
$mutated.ok = "true"
|
||||
Assert-Rejected { Test-AgentHost188VerifiedBrokerEvidence $mutated $sourceRevision $runId $evidencePath $agentRoot } "success_ok_string"
|
||||
$rejections++
|
||||
|
||||
$missingReadback = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_receipt_readback_v1"
|
||||
ok = $true
|
||||
state = "missing"
|
||||
runId = $runId
|
||||
sourceRevision = $sourceRevision
|
||||
receiptPath = "/backup/logs/agent99-host188-backup-run/$runId.json"
|
||||
receipt = $null
|
||||
remoteWritePerformed = $false
|
||||
secretValueReturned = $false
|
||||
}
|
||||
$unknownReconciliation = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_transport_reconciliation_v1"
|
||||
ok = $false
|
||||
status = "committed_unknown"
|
||||
runId = $runId
|
||||
sourceRevision = $sourceRevision
|
||||
receiptState = "missing"
|
||||
applyReceipt = $null
|
||||
receiptReadback = $missingReadback
|
||||
retryApplyAllowed = $false
|
||||
transportReason = "transport_timeout"
|
||||
transportExitCode = -2
|
||||
directReceiptError = "host188_backup_apply_transport_timeout"
|
||||
readbackError = ""
|
||||
remoteWritePerformed = $false
|
||||
secretValueReturned = $false
|
||||
}
|
||||
$unknown = Copy-ReplayDocument $success
|
||||
$unknown.ok = $false
|
||||
$unknown.status = "failed"
|
||||
$unknown.apply = $null
|
||||
$unknown.applyReconciliation = $unknownReconciliation
|
||||
$unknown.runtimeIdentityPost = $null
|
||||
$unknown.verify = $null
|
||||
$unknown.errorCode = "host188_backup_apply_committed_unknown"
|
||||
if (-not (Test-AgentHost188CommittedUnknownEvidence $unknown $evidencePath $agentRoot)) {
|
||||
throw "valid_committed_unknown_rejected"
|
||||
}
|
||||
$mutated = Copy-ReplayDocument $unknown
|
||||
$mutated.applyReconciliation.retryApplyAllowed = $true
|
||||
Assert-Rejected { Test-AgentHost188CommittedUnknownEvidence $mutated $evidencePath $agentRoot } "unknown_retry_enabled"
|
||||
$rejections++
|
||||
$mutated = Copy-ReplayDocument $unknown
|
||||
$mutated.applyReconciliation.receiptReadback.receipt = $apply
|
||||
Assert-Rejected { Test-AgentHost188CommittedUnknownEvidence $mutated $evidencePath $agentRoot } "unknown_receipt_conflict"
|
||||
$rejections++
|
||||
|
||||
$presentReadback = Copy-ReplayDocument $missingReadback
|
||||
$presentReadback.state = "present"
|
||||
$presentReadback.receipt = $apply
|
||||
$recoveredReconciliation = Copy-ReplayDocument $unknownReconciliation
|
||||
$recoveredReconciliation.ok = $true
|
||||
$recoveredReconciliation.status = "committed_receipt_recovered"
|
||||
$recoveredReconciliation.receiptState = "present"
|
||||
$recoveredReconciliation.applyReceipt = $apply
|
||||
$recoveredReconciliation.receiptReadback = $presentReadback
|
||||
$reconciledEvidencePath = Join-Path $agentRoot "evidence\host188-backup-run\agent99-host188-backup-reconcile-$runId.json"
|
||||
$reconciled = Copy-ReplayDocument $success
|
||||
$reconciled.status = "reconciled_verified"
|
||||
$reconciled.agentAction = "read_only_same_run_reconciliation"
|
||||
$reconciled.applyReconciliation = $recoveredReconciliation
|
||||
$reconciled.evidence = $reconciledEvidencePath
|
||||
if (-not (Test-AgentHost188VerifiedBrokerEvidence $reconciled $sourceRevision $runId $reconciledEvidencePath $agentRoot)) {
|
||||
throw "valid_same_run_reconciliation_rejected"
|
||||
}
|
||||
$freshPresentReadback = Copy-ReplayDocument $presentReadback
|
||||
$freshPresentReadback.receipt = $sourceLockFreshApply
|
||||
$freshRecoveredReconciliation = Copy-ReplayDocument $recoveredReconciliation
|
||||
$freshRecoveredReconciliation.applyReceipt = $sourceLockFreshApply
|
||||
$freshRecoveredReconciliation.receiptReadback = $freshPresentReadback
|
||||
$reconciledFreshSkip = Copy-ReplayDocument $sourceLockFreshSuccess
|
||||
$reconciledFreshSkip.status = "reconciled_verified"
|
||||
$reconciledFreshSkip.agentAction = "read_only_same_run_reconciliation"
|
||||
$reconciledFreshSkip.applyReconciliation = $freshRecoveredReconciliation
|
||||
$reconciledFreshSkip.evidence = $reconciledEvidencePath
|
||||
if (-not (Test-AgentHost188VerifiedBrokerEvidence $reconciledFreshSkip $sourceRevision $runId $reconciledEvidencePath $agentRoot)) {
|
||||
throw "valid_reconciled_source_lock_fresh_skip_rejected"
|
||||
}
|
||||
|
||||
$directCreatedOutcome = Get-AgentHost188VerifiedRemediationOutcome $success $false
|
||||
$directFreshOutcome = Get-AgentHost188VerifiedRemediationOutcome $sourceLockFreshSuccess $false
|
||||
$reconciledCreatedOutcome = Get-AgentHost188VerifiedRemediationOutcome $reconciled $true
|
||||
$reconciledFreshOutcome = Get-AgentHost188VerifiedRemediationOutcome $reconciledFreshSkip $true
|
||||
if (
|
||||
[bool]$directCreatedOutcome.skipped -or
|
||||
-not [bool]$directCreatedOutcome.runtimeWritePerformed -or
|
||||
[string]$directCreatedOutcome.runtimeWriteState -ne "performed" -or
|
||||
[string]$directCreatedOutcome.reason -ne "snapshot_created_and_restore_verified" -or
|
||||
-not [bool]$directFreshOutcome.skipped -or
|
||||
[bool]$directFreshOutcome.runtimeWritePerformed -or
|
||||
[string]$directFreshOutcome.reason -ne "fresh_snapshot_verified_under_source_lock" -or
|
||||
[bool]$reconciledCreatedOutcome.skipped -or
|
||||
[bool]$reconciledCreatedOutcome.runtimeWritePerformed -or
|
||||
[string]$reconciledCreatedOutcome.reason -ne "same_run_commit_reconciled_and_restore_verified" -or
|
||||
-not [bool]$reconciledFreshOutcome.skipped -or
|
||||
[bool]$reconciledFreshOutcome.runtimeWritePerformed -or
|
||||
[string]$reconciledFreshOutcome.runtimeWriteState -ne "none" -or
|
||||
[string]$reconciledFreshOutcome.reason -ne "same_run_fresh_skip_reconciled_and_restore_verified"
|
||||
) { throw "verified_remediation_outcome_matrix_failed" }
|
||||
|
||||
$busy = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_broker_busy_v1"
|
||||
ok = $false
|
||||
status = "busy"
|
||||
mode = "Apply"
|
||||
sourceRevision = $sourceRevision
|
||||
runId = $runId
|
||||
targetHost = "192.168.0.110"
|
||||
retryApplyAllowed = $false
|
||||
canonicalEvidenceWritten = $false
|
||||
remoteWritePerformed = $false
|
||||
secretValueReturned = $false
|
||||
}
|
||||
if (-not (Test-AgentHost188BusyEvidence $busy $sourceRevision $runId)) {
|
||||
throw "valid_busy_receipt_rejected"
|
||||
}
|
||||
$mutated = Copy-ReplayDocument $busy
|
||||
$mutated.canonicalEvidenceWritten = $true
|
||||
Assert-Rejected { Test-AgentHost188BusyEvidence $mutated $sourceRevision $runId } "busy_canonical_write"
|
||||
$rejections++
|
||||
|
||||
$freshSkip = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_fresh_skip_v1"
|
||||
ok = $true
|
||||
status = "fresh_skip"
|
||||
mode = "Apply"
|
||||
sourceRevision = $sourceRevision
|
||||
runId = $runId
|
||||
targetHost = "192.168.0.110"
|
||||
snapshotId = "9" * 8
|
||||
ageMinutes = 1
|
||||
maxAgeMinutes = 1800
|
||||
reason = "snapshot_became_fresh_while_waiting"
|
||||
retryApplyAllowed = $false
|
||||
applyInvoked = $false
|
||||
canonicalEvidenceWritten = $false
|
||||
remoteWritePerformed = $false
|
||||
secretValueReturned = $false
|
||||
}
|
||||
if (-not (Test-AgentHost188FreshSkipEvidence $freshSkip $sourceRevision $runId)) {
|
||||
throw "valid_fresh_skip_rejected"
|
||||
}
|
||||
|
||||
function Set-ReplayReceiptRun {
|
||||
param([object]$Receipt, [string]$OldRunId, [string]$NewRunId, [string]$NewEvidencePath)
|
||||
$Receipt.runId = $NewRunId
|
||||
$Receipt.evidence = $NewEvidencePath
|
||||
foreach ($phase in @("runtimeIdentityPre", "runtimeIdentityPost")) {
|
||||
if ($Receipt.$phase) {
|
||||
$suffix = if ($phase -eq "runtimeIdentityPre") { "pre" } else { "post" }
|
||||
$runtimeRunId = "$NewRunId-runtime-$suffix"
|
||||
$Receipt.$phase.runId = $runtimeRunId
|
||||
$Receipt.$phase.verify.runId = $runtimeRunId
|
||||
$Receipt.$phase.evidence = Join-Path $agentRoot "evidence\host110-backup-runtime\agent99-host110-backup-runtime-$runtimeRunId.json"
|
||||
}
|
||||
}
|
||||
if ($Receipt.apply) {
|
||||
$Receipt.apply.runId = $NewRunId
|
||||
$Receipt.apply.remoteReceipt = "/backup/logs/agent99-host188-backup-run/$NewRunId.json"
|
||||
$Receipt.apply.remoteLog = "/backup/logs/agent99-host188-backup-run/$NewRunId.log"
|
||||
}
|
||||
if ($Receipt.verify) {
|
||||
$Receipt.verify.runId = $NewRunId
|
||||
$Receipt.verify.applyReceipt = "/backup/logs/agent99-host188-backup-run/$NewRunId.json"
|
||||
}
|
||||
}
|
||||
|
||||
$resolvedRunId = "host188-control-replay-resolved"
|
||||
$resolvedEvidencePath = Join-Path $agentRoot "evidence\host188-backup-run\agent99-host188-backup-$resolvedRunId.json"
|
||||
$resolvedSuccess = Copy-ReplayDocument $success
|
||||
Set-ReplayReceiptRun $resolvedSuccess $runId $resolvedRunId $resolvedEvidencePath
|
||||
$script:ReplayEvidenceDocuments = @{
|
||||
$evidencePath = ($unknown | ConvertTo-Json -Compress -Depth 12)
|
||||
$resolvedEvidencePath = ($resolvedSuccess | ConvertTo-Json -Compress -Depth 12)
|
||||
}
|
||||
$gateRoot = Join-Path $agentRoot "evidence\host188-backup-run\gates"
|
||||
$operationGatePath = Join-Path $gateRoot "agent99-host188-backup-gate-$runId.json"
|
||||
$operationGate = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_operation_gate_v1"
|
||||
ok = $false
|
||||
status = "in_flight_or_uncertain"
|
||||
mode = "Apply"
|
||||
sourceRevision = $sourceRevision
|
||||
runId = $runId
|
||||
targetHost = "192.168.0.110"
|
||||
sourceHost = "192.168.0.188"
|
||||
canonicalEvidence = $evidencePath
|
||||
createdAtUtc = "2026-07-18T00:00:00.0000000+00:00"
|
||||
retryApplyAllowed = $false
|
||||
runtimeWriteState = "unknown"
|
||||
remoteOperationMayHaveCommitted = $true
|
||||
stateWritePerformed = $true
|
||||
remoteWritePerformed = $false
|
||||
secretValueRead = $false
|
||||
}
|
||||
$script:ReplayEvidenceDocuments[$operationGatePath] = ($operationGate | ConvertTo-Json -Compress -Depth 8)
|
||||
$baseTime = [datetime]::SpecifyKind([datetime]"2026-07-18T00:00:00", [DateTimeKind]::Utc)
|
||||
$unknownFile = [pscustomobject]@{
|
||||
Name = "agent99-host188-backup-$runId.json"
|
||||
FullName = $evidencePath
|
||||
Length = 4096
|
||||
Attributes = [IO.FileAttributes]::Normal
|
||||
LastWriteTimeUtc = $baseTime
|
||||
}
|
||||
$resolvedFile = [pscustomobject]@{
|
||||
Name = "agent99-host188-backup-$resolvedRunId.json"
|
||||
FullName = $resolvedEvidencePath
|
||||
Length = 4096
|
||||
Attributes = [IO.FileAttributes]::Normal
|
||||
LastWriteTimeUtc = $baseTime.AddMinutes(1)
|
||||
}
|
||||
$operationGateFile = [pscustomobject]@{
|
||||
Name = "agent99-host188-backup-gate-$runId.json"
|
||||
FullName = $operationGatePath
|
||||
Length = 2048
|
||||
Attributes = [IO.FileAttributes]::Normal
|
||||
LastWriteTimeUtc = $baseTime
|
||||
}
|
||||
$script:ReplayEvidenceFiles = @($unknownFile)
|
||||
$script:ReplayGateFiles = @($operationGateFile)
|
||||
function Test-Path {
|
||||
param([string]$LiteralPath, [object]$PathType)
|
||||
return [bool](
|
||||
$LiteralPath -eq (Join-Path $agentRoot "evidence\host188-backup-run") -or
|
||||
$LiteralPath -eq $gateRoot -or
|
||||
$script:ReplayEvidenceDocuments.ContainsKey($LiteralPath)
|
||||
)
|
||||
}
|
||||
function Get-ChildItem {
|
||||
param([string]$LiteralPath, [switch]$File, [string]$Filter, [object]$ErrorAction)
|
||||
if ($Filter -eq "agent99-host188-backup-gate-*.json") { return @($script:ReplayGateFiles) }
|
||||
return @($script:ReplayEvidenceFiles)
|
||||
}
|
||||
function Get-Content {
|
||||
param([string]$LiteralPath, [switch]$Raw)
|
||||
return [string]$script:ReplayEvidenceDocuments[$LiteralPath]
|
||||
}
|
||||
function Get-Item {
|
||||
param([string]$LiteralPath, [object]$ErrorAction)
|
||||
return [pscustomobject]@{
|
||||
FullName = $LiteralPath
|
||||
Length = 4096
|
||||
Attributes = [IO.FileAttributes]::Normal
|
||||
}
|
||||
}
|
||||
function Get-FileHash {
|
||||
param([string]$LiteralPath, [string]$Algorithm)
|
||||
return [pscustomobject]@{ Hash = "f" * 64 }
|
||||
}
|
||||
|
||||
$operationGateState = Get-AgentHost188CommittedUnknownGate $agentRoot
|
||||
if (
|
||||
-not $operationGateState.blocked -or
|
||||
$operationGateState.reason -ne "operation_gate_unresolved" -or
|
||||
$operationGateState.runId -ne $runId
|
||||
) { throw "operation_gate_not_blocked" }
|
||||
$resolutionMarkerPath = Join-Path $gateRoot "agent99-host188-backup-resolution-$runId.json"
|
||||
$resolutionMarker = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_backup_gate_resolution_v1"
|
||||
ok = $true
|
||||
status = "resolved_verified"
|
||||
sourceRevision = $sourceRevision
|
||||
runId = $runId
|
||||
operationGate = $operationGatePath
|
||||
resolutionEvidence = $reconciledEvidencePath
|
||||
resolutionSha256 = "f" * 64
|
||||
snapshotId = $createdSnapshotId
|
||||
reconciliationPerformed = $true
|
||||
retryApplyAllowed = $false
|
||||
remoteWritePerformed = $false
|
||||
secretValueRead = $false
|
||||
}
|
||||
$script:ReplayEvidenceDocuments[$reconciledEvidencePath] = ($reconciled | ConvertTo-Json -Compress -Depth 12)
|
||||
$script:ReplayEvidenceDocuments[$resolutionMarkerPath] = ($resolutionMarker | ConvertTo-Json -Compress -Depth 8)
|
||||
$resolvedOperationGateState = Get-AgentHost188CommittedUnknownGate $agentRoot
|
||||
if ($resolvedOperationGateState.blocked) { throw "verified_resolution_marker_did_not_resolve_operation_gate" }
|
||||
$malformedResolutionMarker = Copy-ReplayDocument $resolutionMarker
|
||||
$malformedResolutionMarker.resolutionEvidence = $evidencePath
|
||||
$malformedResolutionMarker.reconciliationPerformed = $false
|
||||
Assert-Rejected {
|
||||
Test-AgentHost188GateResolutionEvidence `
|
||||
$malformedResolutionMarker `
|
||||
$operationGate `
|
||||
$resolutionMarkerPath `
|
||||
$agentRoot
|
||||
} "matching_digest_unverified_resolution_receipt"
|
||||
$rejections++
|
||||
$script:ReplayGateFiles = @()
|
||||
$unknownGate = Get-AgentHost188CommittedUnknownGate $agentRoot
|
||||
if (
|
||||
-not $unknownGate.blocked -or
|
||||
$unknownGate.reason -ne "committed_unknown_requires_reconciliation" -or
|
||||
$unknownGate.runId -ne $runId
|
||||
) { throw "committed_unknown_gate_not_blocked" }
|
||||
$script:ReplayEvidenceFiles = @($unknownFile, $resolvedFile)
|
||||
$stillUnknownGate = Get-AgentHost188CommittedUnknownGate $agentRoot
|
||||
if (-not $stillUnknownGate.blocked) { throw "newer_unrelated_verified_backup_resolved_unknown" }
|
||||
$unknownFile.LastWriteTimeUtc = $baseTime.AddMinutes(2)
|
||||
$script:ReplayEvidenceFiles = @($resolvedFile, $unknownFile)
|
||||
$newerUnknownGate = Get-AgentHost188CommittedUnknownGate $agentRoot
|
||||
if (-not $newerUnknownGate.blocked) { throw "newer_committed_unknown_not_blocked" }
|
||||
|
||||
function Get-AgentHost188CommittedUnknownGate {
|
||||
param([string]$AgentRoot)
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_committed_unknown_gate_v1"
|
||||
blocked = $true
|
||||
reason = "operation_gate_unresolved"
|
||||
runId = $runId
|
||||
sourceRevision = $sourceRevision
|
||||
evidence = $operationGatePath
|
||||
evidenceCount = 1
|
||||
remoteWritePerformed = $false
|
||||
secretValueRead = $false
|
||||
}
|
||||
}
|
||||
$ControlledApply = $false
|
||||
$Config = [pscustomobject]@{ agentRoot = $agentRoot }
|
||||
$freshHealth = [pscustomobject]@{
|
||||
targets = @([pscustomobject]@{
|
||||
name = "host188-products"
|
||||
ok = $true
|
||||
reason = "healthy"
|
||||
})
|
||||
}
|
||||
$freshWithUnresolvedGate = Invoke-AgentHost188BackupRemediation $freshHealth
|
||||
if (
|
||||
-not [bool]$freshWithUnresolvedGate.blocked -or
|
||||
[string]$freshWithUnresolvedGate.reason -ne "controlled_apply_required_for_reconciliation" -or
|
||||
[string]$freshWithUnresolvedGate.runtimeWriteState -ne "unknown" -or
|
||||
[bool]$freshWithUnresolvedGate.attempted
|
||||
) { throw "fresh_target_bypassed_unresolved_operation_gate" }
|
||||
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_host188_control_plane_contract_replay_v1"
|
||||
ok = $true
|
||||
rejectionCount = $rejections
|
||||
successPathAccepted = $true
|
||||
committedUnknownAccepted = $true
|
||||
committedUnknownGateBlocked = $true
|
||||
operationGateBlocked = $true
|
||||
sameRunReconciliationAccepted = $true
|
||||
reconciledSourceLockFreshSkipAccepted = $true
|
||||
verifiedRemediationOutcomeMatrixAccepted = $true
|
||||
verifiedResolutionMarkerResolvesGate = $true
|
||||
newerUnrelatedVerifiedDoesNotResolveGate = $true
|
||||
busyReceiptAccepted = $true
|
||||
freshWaiterSkipAccepted = $true
|
||||
sourceLockFreshSkipAccepted = $true
|
||||
freshTargetStillRequiresGateReconciliation = $true
|
||||
retryApplyAllowed = $false
|
||||
remoteWritePerformed = $false
|
||||
secretValuesRead = $false
|
||||
} | ConvertTo-Json -Compress
|
||||
@@ -0,0 +1,605 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
BROKER = ROOT / "agent99-host188-backup-broker.ps1"
|
||||
CONTROL_PLANE = ROOT / "agent99-control-plane.ps1"
|
||||
TASKS = ROOT / "agent99-register-tasks.ps1"
|
||||
BROKER_CONTRACT_REPLAY = (
|
||||
ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-host188-broker-contract-replay.ps1"
|
||||
)
|
||||
CONTROL_PLANE_CONTRACT_REPLAY = (
|
||||
ROOT
|
||||
/ "scripts"
|
||||
/ "reboot-recovery"
|
||||
/ "tests"
|
||||
/ "agent99-host188-control-plane-contract-replay.ps1"
|
||||
)
|
||||
|
||||
|
||||
def _embedded_shell(source: str, variable: str) -> str:
|
||||
match = re.search(
|
||||
rf"\${re.escape(variable)}\s*=\s*@'\n(.*?)\n'@",
|
||||
source,
|
||||
re.DOTALL,
|
||||
)
|
||||
assert match is not None, variable
|
||||
return match.group(1) + "\n"
|
||||
|
||||
|
||||
def _powershell_function(source: str, name: str) -> str:
|
||||
start = source.index(f"function {name} {{")
|
||||
next_function = source.find("\nfunction ", start + 1)
|
||||
return source[start:] if next_function < 0 else source[start:next_function]
|
||||
|
||||
|
||||
def test_embedded_remote_scripts_are_valid_bash() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
|
||||
for variable in (
|
||||
"PreflightScript",
|
||||
"FreshnessScript",
|
||||
"ApplyScript",
|
||||
"ApplyReceiptReadbackScript",
|
||||
"VerifyScript",
|
||||
):
|
||||
result = subprocess.run(
|
||||
["bash", "-n"],
|
||||
input=_embedded_shell(source, variable),
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, f"{variable}: {result.stderr}"
|
||||
|
||||
|
||||
def test_embedded_python_heredocs_compile() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
|
||||
for index, python_source in enumerate(
|
||||
re.findall(r"<<'PY'\n(.*?)\nPY", source, re.DOTALL),
|
||||
start=1,
|
||||
):
|
||||
compile(python_source, f"embedded-python-{index}", "exec")
|
||||
|
||||
|
||||
def test_host188_backup_broker_is_fixed_bounded_and_runtime_bound() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
|
||||
for expected in (
|
||||
'[ValidateSet("Check", "Apply", "Verify")]',
|
||||
'$TargetHost = "192.168.0.110"',
|
||||
'$TargetUser = "wooo"',
|
||||
'keys\\agent99_ed25519',
|
||||
'agent99-host110-backup-runtime-broker.ps1',
|
||||
'agent99_host110_backup_runtime_broker_receipt_v3',
|
||||
'"-Mode", "Verify"',
|
||||
'runtime_identity_${Phase}_contract_failed',
|
||||
'StrictHostKeyChecking=yes',
|
||||
'NumberOfPasswordPrompts=0',
|
||||
'$process.WaitForExit($TimeoutSeconds * 1000)',
|
||||
'taskkill.exe /PID $process.Id /T /F',
|
||||
'$ApplyTimeoutSeconds = 3600',
|
||||
'$VerifyTimeoutSeconds = 2100',
|
||||
):
|
||||
assert expected in source
|
||||
|
||||
for forbidden in (
|
||||
"Invoke-Expression",
|
||||
"StrictHostKeyChecking=no",
|
||||
"StrictHostKeyChecking=accept-new",
|
||||
"docker stop",
|
||||
"docker restart",
|
||||
"rclone ",
|
||||
"restic forget",
|
||||
"restic prune",
|
||||
'agent99_host110_backup_runtime_broker_receipt_v2',
|
||||
):
|
||||
assert forbidden not in source
|
||||
|
||||
|
||||
def test_host188_broker_rejects_missing_or_wrong_typed_receipt_fields_before_cast() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
field_guard = _powershell_function(source, "Test-Agent99JsonField")
|
||||
exact_guard = _powershell_function(source, "Assert-Agent99ExactJsonFields")
|
||||
apply_contract = _powershell_function(source, "Assert-Agent99Host188ApplyEvidence")
|
||||
verify_contract = _powershell_function(source, "Assert-Agent99Host188VerifyEvidence")
|
||||
runtime_contract = _powershell_function(source, "Assert-Agent99RuntimeIdentityEvidence")
|
||||
|
||||
assert '$property = $Object.PSObject.Properties[$Name]' in field_guard
|
||||
assert 'if ($null -eq $property) { return $false }' in field_guard
|
||||
assert "$properties.Count -ne $Contract.Count" in exact_guard
|
||||
assert "$Contract.ContainsKey([string]$property.Name)" in exact_guard
|
||||
for field in (
|
||||
"mode",
|
||||
"result",
|
||||
"runId",
|
||||
"sourceRevision",
|
||||
"beforeSnapshotId",
|
||||
"snapshotId",
|
||||
"backupExitCode",
|
||||
"elapsedSeconds",
|
||||
"errorCode",
|
||||
"remoteReceipt",
|
||||
"remoteLog",
|
||||
"sourceHost",
|
||||
"controllerHost",
|
||||
"serviceStopped",
|
||||
"offsiteWritePerformed",
|
||||
"durableReadbackVerified",
|
||||
"credentialReferenceConsumed",
|
||||
"secretValueReturned",
|
||||
):
|
||||
assert re.search(rf"^\s+{field} = \"", apply_contract, re.MULTILINE), field
|
||||
for field in (
|
||||
"runId",
|
||||
"sourceRevision",
|
||||
"snapshotId",
|
||||
"applyReceipt",
|
||||
"resticSnapshotUnique",
|
||||
"dataRestoreVerified",
|
||||
"recoveryReadyClaimed",
|
||||
"productionRestorePerformed",
|
||||
"productionDataWritePerformed",
|
||||
"temporaryReadbackWorkspaceUsed",
|
||||
"credentialReferenceConsumed",
|
||||
"secretValueReturned",
|
||||
):
|
||||
assert re.search(rf"^\s+{field} = \"", verify_contract, re.MULTILINE), field
|
||||
assert 'agent99_host110_backup_runtime_broker_receipt_v3' in runtime_contract
|
||||
assert 'agent99_host110_backup_runtime_broker_receipt_v2' not in runtime_contract
|
||||
for field in ("status", "mode", "sourceRevision", "runId", "verify", "cleanupVerified"):
|
||||
assert re.search(rf"^\s+{field} = \"", runtime_contract, re.MULTILINE), field
|
||||
|
||||
|
||||
def test_host188_backup_apply_is_single_target_idempotent_and_no_offsite() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
|
||||
for expected in (
|
||||
'/backup/scripts/backup-host188-products.sh',
|
||||
'BACKUP_NOTIFY_SUCCESS=0',
|
||||
'HOST188_BACKUP_SKIP_IF_FRESH_MAX_AGE_MINUTES=1800',
|
||||
'HOST188_BACKUP_SKIPPED_FRESH=1 snapshot=',
|
||||
'result="fresh_skip"',
|
||||
'result="snapshot_created"',
|
||||
'/tmp/agent99-host188-backup-run.lock',
|
||||
'os.O_WRONLY | os.O_CREAT | os.O_EXCL',
|
||||
'os.fsync(handle.fileno())',
|
||||
'os.link(temporary, path)',
|
||||
'if path.read_bytes() != payload',
|
||||
'new_snapshot_not_observed',
|
||||
'"sourceHost": "192.168.0.188"',
|
||||
'"controllerHost": "192.168.0.110"',
|
||||
'"serviceStopped": False',
|
||||
'"offsiteWritePerformed": False',
|
||||
'"durableReadbackVerified": True',
|
||||
'"credentialReferenceConsumed": True',
|
||||
'"secretValueReturned": False',
|
||||
'runtimeIdentityPost = $runtimePost',
|
||||
'-not [bool]$Apply.durableReadbackVerified',
|
||||
'Assert-Agent99Host188ApplyEvidence',
|
||||
'snapshotId -eq [string]$Apply.beforeSnapshotId',
|
||||
'test "$(stat -c %a "${receipt_path}")" = "600"',
|
||||
'test "$(stat -c %a "${log_path}")" = "600"',
|
||||
'test ! -L "${lock_path}"',
|
||||
'test "$(stat -c %a "${lock_path}")" = "600"',
|
||||
'output_path="$(mktemp /tmp/agent99-host188-backup-verify.XXXXXXXX)"',
|
||||
"trap cleanup_output EXIT",
|
||||
"trap 'exit 143' TERM",
|
||||
):
|
||||
assert expected in source
|
||||
assert source.index('os.link(temporary, path)') < source.index('print(payload.decode("utf-8"), end="")')
|
||||
assert source.index('if test -f "${receipt_path}"') < source.index('test ! -e "${log_path}"')
|
||||
|
||||
|
||||
def test_host188_backup_verifier_is_snapshot_specific_and_nonproduction() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
|
||||
for expected in (
|
||||
'/backup/scripts/verify-host188-products-backup.sh',
|
||||
'--snapshot "${snapshot_id}" --data-only',
|
||||
'HOST188_RESTORE_DRILL_DATA_VERIFIED=1',
|
||||
'HOST188_RESTORE_DRILL_RECOVERY_READY=0',
|
||||
'"resticSnapshotUnique": True',
|
||||
'"dataRestoreVerified": True',
|
||||
'"recoveryReadyClaimed": False',
|
||||
'"productionRestorePerformed": False',
|
||||
'"productionDataWritePerformed": False',
|
||||
'verify_snapshot_id_required',
|
||||
):
|
||||
assert expected in source
|
||||
|
||||
|
||||
def test_verify_preflight_remains_available_when_source_host_is_down() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
preflight = _embedded_shell(source, "PreflightScript")
|
||||
verify_branch = preflight.split('if [ "${mode}" = "verify" ]; then', 1)[1].split("else", 1)[0]
|
||||
source_branch = preflight.split("else", 1)[1]
|
||||
|
||||
assert "test -d /backup/host188-products" in verify_branch
|
||||
assert "192.168.0.188" not in verify_branch
|
||||
assert "ssh" not in verify_branch
|
||||
assert "ollama@192.168.0.188" in source_branch
|
||||
assert "source_readback_performed=1" in source_branch
|
||||
assert "Invoke-Agent99Host188Preflight $Mode" in source
|
||||
assert 'sourceReadbackRequired = [bool]($safeMode -ne "verify")' in source
|
||||
|
||||
|
||||
def test_host188_backup_broker_writes_one_human_auditable_receipt() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
|
||||
for expected in (
|
||||
'schemaVersion = "agent99_host188_backup_broker_receipt_v2"',
|
||||
'decisionProvider = "deterministic_only"',
|
||||
'criticProvider = "deterministic_only"',
|
||||
'agentAction = if ($terminalStatus -eq "fresh_skip_verified")',
|
||||
'executor = "fixed_host110_backup_host188_products"',
|
||||
'verifier = "independent_snapshot_specific_restore_readback"',
|
||||
'Write-Agent99ImmutableJson $evidencePath $outputReceipt',
|
||||
'[IO.FileOptions]::WriteThrough',
|
||||
'$stream.Flush($true)',
|
||||
'[IO.File]::ReadAllBytes($Path)',
|
||||
'arbitraryCommandAllowed = $false',
|
||||
'rawSessionStored = $false',
|
||||
):
|
||||
assert expected in source
|
||||
assert source.index('$stream.Flush($true)') < source.index('[IO.File]::Move($temporary, $Path)')
|
||||
assert source.index('Enter-Agent99Host188BrokerLease $mutex 30000') < source.index(
|
||||
'Test-Path -LiteralPath $evidencePath -PathType Leaf'
|
||||
)
|
||||
assert source.index('Write-Agent99ImmutableJson $evidencePath $outputReceipt') < source.index(
|
||||
'$mutex.ReleaseMutex()'
|
||||
)
|
||||
busy_branch = source.split('if (-not $acquired) {', 1)[1].split('} else {', 1)[0]
|
||||
busy_receipt = source.split("function New-Agent99Host188BusyReceipt {", 1)[1].split(
|
||||
"\n}\n\nfunction New-Agent99Host188GateBlockedReceipt", 1
|
||||
)[0]
|
||||
assert "New-Agent99Host188BusyReceipt" in busy_branch
|
||||
assert 'canonicalEvidenceWritten = $false' in busy_receipt
|
||||
assert 'Write-Agent99ImmutableJson' not in busy_branch
|
||||
assert 'Write-Agent99ImmutableJson' not in busy_receipt
|
||||
|
||||
|
||||
def test_backupcheck_autoremmediates_only_exact_host188_missing_or_stale() -> None:
|
||||
control = CONTROL_PLANE.read_text(encoding="utf-8")
|
||||
tasks = TASKS.read_text(encoding="utf-8")
|
||||
|
||||
for expected in (
|
||||
"function Invoke-AgentHost188BackupRemediation",
|
||||
"function Test-AgentHost188ReceiptField",
|
||||
"function Test-AgentHost188ReceiptFields",
|
||||
"function Test-AgentHost188ExactReceiptFields",
|
||||
"function Test-AgentHost188RuntimeIdentityEvidence",
|
||||
"function Test-AgentHost188PreflightEvidence",
|
||||
"function Test-AgentHost188ApplyEvidence",
|
||||
"function Test-AgentHost188ApplyReconciliationEvidence",
|
||||
"function Test-AgentHost188VerifyEvidence",
|
||||
"function Test-AgentHost188VerifiedBrokerEvidence",
|
||||
"function Test-AgentHost188CommittedUnknownEvidence",
|
||||
"function Get-AgentHost188CommittedUnknownGate",
|
||||
"function Test-AgentHost188BusyEvidence",
|
||||
"function Get-AgentHost188VerifiedRemediationOutcome",
|
||||
'Where-Object { $_.name -eq "host188-products" }',
|
||||
'@("backup_snapshot_missing", "backup_snapshot_stale")',
|
||||
'reason = "controlled_apply_required"',
|
||||
"Test-AgentRuntimeManifest",
|
||||
'agent99-host188-backup-broker.ps1"',
|
||||
'"-Mode", "Apply"',
|
||||
"$process.WaitForExit(6720 * 1000)",
|
||||
'status -in @("verified", "reconciled_verified", "fresh_skip_verified")',
|
||||
'$topLevelFieldsValid = Test-AgentHost188ExactReceiptFields $Receipt @{',
|
||||
'schemaVersion -eq "agent99_host188_backup_broker_receipt_v2"',
|
||||
'Test-AgentHost188ApplyReconciliationEvidence',
|
||||
'Test-AgentHost188RuntimeIdentityEvidence',
|
||||
'Test-AgentHost188VerifyEvidence',
|
||||
'evidence\\host188-backup-run\\agent99-host188-backup-$runId.json',
|
||||
'[string]$Receipt.evidence -eq $ExpectedEvidence',
|
||||
'-not [bool]$Receipt.arbitraryCommandAllowed',
|
||||
'-not [bool]$Receipt.rawSessionStored',
|
||||
"snapshot_created_and_restore_verified",
|
||||
"fresh_snapshot_verified_under_source_lock",
|
||||
"same_run_fresh_skip_reconciled_and_restore_verified",
|
||||
"Host188 已有新鮮產品備份;已略過重複快照",
|
||||
"Host188 備份快照已由另一個執行更新;本次未重複寫入",
|
||||
'runtimeWriteState = "none"',
|
||||
'$base.runtimeWriteState = "unknown"',
|
||||
'if ($ReconcileOnly -or $sourceLockFreshSkip) { "none" } else { "performed" }',
|
||||
'committed_unknown_requires_reconciliation',
|
||||
'Get-AgentHost188CommittedUnknownGate',
|
||||
"backup_auto_repair_result",
|
||||
"backupRemediation = $backupRemediation",
|
||||
"($backupRemediation -and $backupRemediation.runtimeWritePerformed)",
|
||||
):
|
||||
assert expected in control
|
||||
|
||||
assert '-Mode BackupCheck -ControlledApply' in tasks
|
||||
assert "$backupSettings = New-ScheduledTaskSettingsSet" in tasks
|
||||
assert "New-TimeSpan -Minutes 125" in tasks
|
||||
assert "-MultipleInstances IgnoreNew" in tasks
|
||||
remediation = _powershell_function(control, "Invoke-AgentHost188BackupRemediation")
|
||||
assert remediation.index("Get-Content -LiteralPath $stdoutPath -Raw") < remediation.index(
|
||||
"if ($process.ExitCode -ne 0)"
|
||||
)
|
||||
assert remediation.index("Get-AgentHost188CommittedUnknownGate") < remediation.index(
|
||||
"$runId = if ($reconcileOnly)"
|
||||
)
|
||||
|
||||
|
||||
def test_backupcheck_outer_receipt_contract_is_missing_field_and_type_fail_closed() -> None:
|
||||
control = CONTROL_PLANE.read_text(encoding="utf-8")
|
||||
field_guard = _powershell_function(control, "Test-AgentHost188ReceiptField")
|
||||
remediation = _powershell_function(control, "Invoke-AgentHost188BackupRemediation")
|
||||
verified_contract = _powershell_function(control, "Test-AgentHost188VerifiedBrokerEvidence")
|
||||
|
||||
assert '$property = $Object.PSObject.Properties[$Name]' in field_guard
|
||||
assert 'if ($null -eq $property) { return $false }' in field_guard
|
||||
exact_guard = _powershell_function(control, "Test-AgentHost188ExactReceiptFields")
|
||||
assert "$properties.Count -ne $Contract.Count" in exact_guard
|
||||
assert "$Contract.ContainsKey([string]$property.Name)" in exact_guard
|
||||
for field in (
|
||||
"schemaVersion",
|
||||
"ok",
|
||||
"status",
|
||||
"mode",
|
||||
"sourceRevision",
|
||||
"runId",
|
||||
"requestedSnapshotId",
|
||||
"targetHost",
|
||||
"sourceHost",
|
||||
"decisionProvider",
|
||||
"criticProvider",
|
||||
"agentAction",
|
||||
"executor",
|
||||
"verifier",
|
||||
"runtimeIdentityPre",
|
||||
"preflight",
|
||||
"apply",
|
||||
"applyReconciliation",
|
||||
"runtimeIdentityPost",
|
||||
"verify",
|
||||
"errorCode",
|
||||
"elapsedSeconds",
|
||||
"evidence",
|
||||
"arbitraryCommandAllowed",
|
||||
"serviceStopped",
|
||||
"offsiteWritePerformed",
|
||||
"secretValueReturned",
|
||||
"rawSessionStored",
|
||||
):
|
||||
assert re.search(rf"^\s+{field} = \"", verified_contract, re.MULTILINE), field
|
||||
assert "Test-AgentHost188VerifiedBrokerEvidence" in remediation
|
||||
assert '"broker_receipt_contract_failed_operation_gate_persisted"' in remediation
|
||||
|
||||
|
||||
def test_backupcheck_timeout_budget_exceeds_broker_chain_with_task_reserve() -> None:
|
||||
broker = BROKER.read_text(encoding="utf-8")
|
||||
control = CONTROL_PLANE.read_text(encoding="utf-8")
|
||||
tasks = TASKS.read_text(encoding="utf-8")
|
||||
|
||||
apply_timeout = int(re.search(r"\$ApplyTimeoutSeconds = (\d+)", broker).group(1))
|
||||
verify_timeout = int(re.search(r"\$VerifyTimeoutSeconds = (\d+)", broker).group(1))
|
||||
runtime_timeout = int(
|
||||
re.search(
|
||||
r"Invoke-Agent99BoundedProcess \$WindowsPowerShell \$arguments (\d+)",
|
||||
broker,
|
||||
).group(1)
|
||||
)
|
||||
preflight_timeout = int(
|
||||
re.search(
|
||||
r"Invoke-Agent99RemoteScript \$PreflightScript @\(\$safeMode\) (\d+)",
|
||||
broker,
|
||||
).group(1)
|
||||
)
|
||||
receipt_readback_timeout = int(
|
||||
re.search(
|
||||
r"Invoke-Agent99RemoteScript \$ApplyReceiptReadbackScript @\(\$RunId, \$SourceRevision\) (\d+)",
|
||||
broker,
|
||||
).group(1)
|
||||
)
|
||||
freshness_timeout = int(
|
||||
re.search(
|
||||
r"Invoke-Agent99RemoteScript \$FreshnessScript @\(\) (\d+)",
|
||||
broker,
|
||||
).group(1)
|
||||
)
|
||||
remediation = _powershell_function(control, "Invoke-AgentHost188BackupRemediation")
|
||||
outer_timeout = int(re.search(r"WaitForExit\((\d+) \* 1000\)", remediation).group(1))
|
||||
task_limit_minutes = int(
|
||||
re.search(r"\$backupSettings = .*?-Minutes (\d+)\)", tasks, re.DOTALL).group(1)
|
||||
)
|
||||
|
||||
legal_chain = (
|
||||
runtime_timeout * 2
|
||||
+ preflight_timeout
|
||||
+ freshness_timeout
|
||||
+ apply_timeout
|
||||
+ receipt_readback_timeout
|
||||
+ verify_timeout
|
||||
)
|
||||
assert outer_timeout >= legal_chain + 120
|
||||
assert task_limit_minutes * 60 >= outer_timeout + 600
|
||||
|
||||
|
||||
def test_operation_gate_precedes_apply_and_same_run_reconciliation_never_applies() -> None:
|
||||
broker = BROKER.read_text(encoding="utf-8")
|
||||
control = CONTROL_PLANE.read_text(encoding="utf-8")
|
||||
apply_flow = _powershell_function(broker, "Invoke-Agent99Host188Apply")
|
||||
reconciliation = _powershell_function(
|
||||
broker, "Invoke-Agent99Host188SameRunReconciliation"
|
||||
)
|
||||
remediation = _powershell_function(control, "Invoke-AgentHost188BackupRemediation")
|
||||
main_flow = broker.split("try {\n $acquired = Enter-Agent99Host188BrokerLease", 1)[1]
|
||||
|
||||
assert main_flow.index("Get-Agent99Host188OperationGateState") < main_flow.index(
|
||||
"Test-Path -LiteralPath $evidencePath -PathType Leaf"
|
||||
)
|
||||
freshness_index = main_flow.index("Invoke-Agent99Host188FreshnessReadback")
|
||||
normal_gate_index = main_flow.index(
|
||||
"$operationGate = Write-Agent99Host188OperationGate", freshness_index
|
||||
)
|
||||
assert freshness_index < normal_gate_index
|
||||
assert normal_gate_index < main_flow.index("$applyOutcome = Invoke-Agent99Host188Apply")
|
||||
assert "Invoke-Agent99RemoteScript $ApplyScript" in apply_flow
|
||||
assert "$ApplyScript" not in reconciliation
|
||||
assert "Invoke-Agent99RemoteScript $ApplyReceiptReadbackScript" in reconciliation
|
||||
assert 'status = "reconciled_verified"' in reconciliation
|
||||
assert 'retryApplyAllowed = $false' in broker
|
||||
assert '[string]$reconciliationGate.runId' in remediation
|
||||
assert '"broker_timeout_operation_gate_persisted"' in remediation
|
||||
assert remediation.index("Get-AgentHost188CommittedUnknownGate") < remediation.index(
|
||||
"if ($targetRow.ok -and -not $reconcileOnly)"
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_manifest_gate_requires_exact_ordered_twenty_file_bundle() -> None:
|
||||
control = CONTROL_PLANE.read_text(encoding="utf-8")
|
||||
manifest_check = _powershell_function(control, "Test-AgentRuntimeManifest")
|
||||
expected_block = manifest_check.split("$expectedFiles = @(", 1)[1].split("\n )", 1)[0]
|
||||
expected_names = re.findall(r'^\s+"([^"]+)"', expected_block, re.MULTILINE)
|
||||
|
||||
assert len(expected_names) == len(set(expected_names)) == 20
|
||||
assert "agent99-host188-backup-broker.ps1" in expected_names
|
||||
for expected in (
|
||||
'schemaVersion -ne "agent99_runtime_manifest_v1"',
|
||||
'sourceRevision -notmatch "^[0-9a-f]{40}$"',
|
||||
'fileCount -ne $expectedFiles.Count',
|
||||
'mismatchCount -ne 0',
|
||||
'manifestEntries.Count -ne $expectedFiles.Count',
|
||||
'[string]$entry.name -eq $expectedName',
|
||||
'[string]$entry.sourceSha256 -eq [string]$entry.runtimeSha256',
|
||||
'$contractViolationCount -eq 0',
|
||||
):
|
||||
assert expected in manifest_check
|
||||
|
||||
|
||||
def test_transport_loss_reconciliation_is_read_only_and_never_retries_apply() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
reconciliation = _powershell_function(source, "New-Agent99Host188ApplyReconciliation")
|
||||
apply_flow = _powershell_function(source, "Invoke-Agent99Host188Apply")
|
||||
|
||||
for expected in (
|
||||
'schemaVersion = "agent99_host188_backup_transport_reconciliation_v1"',
|
||||
'"committed_receipt_recovered"',
|
||||
'"failed_receipt_recovered"',
|
||||
'"committed_unknown"',
|
||||
'retryApplyAllowed = $false',
|
||||
'remoteWritePerformed = $false',
|
||||
'secretValueReturned = $false',
|
||||
):
|
||||
assert expected in reconciliation
|
||||
assert apply_flow.count("Invoke-Agent99RemoteScript $ApplyScript") == 1
|
||||
assert "Invoke-Agent99RemoteScript $ApplyReceiptReadbackScript" in apply_flow
|
||||
assert source.index("Invoke-Agent99RemoteScript $ApplyScript") < source.index(
|
||||
"Invoke-Agent99RemoteScript $ApplyReceiptReadbackScript"
|
||||
)
|
||||
|
||||
|
||||
def test_existing_broker_evidence_is_exact_typed_and_state_machine_bound() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
existing = _powershell_function(source, "Assert-Agent99ExistingBrokerEvidence")
|
||||
|
||||
for expected in (
|
||||
'agent99_host188_backup_broker_receipt_v2',
|
||||
'existing_host188_broker_identity_contract_failed',
|
||||
'existing_host188_apply_success_contract_failed',
|
||||
'existing_host188_failure_contract_failed',
|
||||
'Assert-Agent99RuntimeIdentityEvidence',
|
||||
'Assert-Agent99Host188PreflightEvidence',
|
||||
'Assert-Agent99Host188ApplyEvidence',
|
||||
'Assert-Agent99Host188ApplyReconciliation',
|
||||
'Assert-Agent99Host188VerifyEvidence',
|
||||
):
|
||||
assert expected in existing
|
||||
assert "Assert-Agent99ExactJsonFields $Existing" in existing
|
||||
assert '[bool]$Existing.ok' not in existing.split(
|
||||
'Assert-Agent99ExactJsonFields $Existing', 1
|
||||
)[0]
|
||||
|
||||
|
||||
def test_windows99_contract_replay_uses_actual_functions_without_remote_write() -> None:
|
||||
source = BROKER_CONTRACT_REPLAY.read_text(encoding="utf-8")
|
||||
|
||||
assert "FunctionDefinitionAst" in source
|
||||
for function_name in (
|
||||
"Test-Agent99JsonField",
|
||||
"Assert-Agent99JsonFields",
|
||||
"Assert-Agent99ExactJsonFields",
|
||||
"Get-Agent99JsonResult",
|
||||
"Assert-Agent99RuntimeIdentityEvidence",
|
||||
"Assert-Agent99Host188PreflightEvidence",
|
||||
"Assert-Agent99Host188FreshnessEvidence",
|
||||
"New-Agent99Host188FreshSkipReceipt",
|
||||
"Assert-Agent99Host188ApplyEvidence",
|
||||
"Assert-Agent99Host188ApplyEvidenceMatch",
|
||||
"Get-Agent99Host188ApplyReceiptReadback",
|
||||
"New-Agent99Host188ApplyReconciliation",
|
||||
"Assert-Agent99Host188ApplyReconciliation",
|
||||
"Assert-Agent99Host188VerifyEvidence",
|
||||
"Assert-Agent99ExistingBrokerEvidence",
|
||||
"Enter-Agent99Host188BrokerLease",
|
||||
"New-Agent99Host188BusyReceipt",
|
||||
):
|
||||
assert f'"{function_name}"' in source
|
||||
assert 'ok = "false"' in source
|
||||
assert 'status -ne "committed_unknown"' in source
|
||||
assert "malformed_failure_not_upgraded" in source
|
||||
assert "mutex_contender_contract_failed" in source
|
||||
assert "mutexContenderBlocked = $true" in source
|
||||
assert "waiterBlockedAfterOwnerUnknown = $true" in source
|
||||
assert "freshWaiterSkipAccepted = $true" in source
|
||||
assert "sourceLockFreshSkipAccepted = $true" in source
|
||||
assert "resolvedOwnerFreshWaiterBlockedFromApply = $true" in source
|
||||
assert "malformedResolutionReceiptRejected = $true" in source
|
||||
assert "gate_waiter_contract_failed" in source
|
||||
assert "canonicalEvidenceWrittenByContender = $false" in source
|
||||
assert "retryApplyAllowed = $false" in source
|
||||
assert "remoteWritePerformed = $false" in source
|
||||
assert "secretValuesRead = $false" in source
|
||||
for forbidden in ("Set-Content", "WriteAllText", "ssh.exe", "scp.exe"):
|
||||
assert forbidden not in source
|
||||
|
||||
|
||||
def test_windows99_control_plane_replay_covers_path_unknown_and_busy_contracts() -> None:
|
||||
source = CONTROL_PLANE_CONTRACT_REPLAY.read_text(encoding="utf-8")
|
||||
|
||||
assert "FunctionDefinitionAst" in source
|
||||
for function_name in (
|
||||
"Test-AgentHost188ExactReceiptFields",
|
||||
"Test-AgentHost188RuntimeIdentityEvidence",
|
||||
"Test-AgentHost188ApplyEvidence",
|
||||
"Test-AgentHost188ApplyReconciliationEvidence",
|
||||
"Test-AgentHost188VerifiedBrokerEvidence",
|
||||
"Test-AgentHost188CommittedUnknownEvidence",
|
||||
"Test-AgentHost188OperationGateEvidence",
|
||||
"Test-AgentHost188GateResolutionEvidence",
|
||||
"Get-AgentHost188CommittedUnknownGate",
|
||||
"Test-AgentHost188BusyEvidence",
|
||||
"Test-AgentHost188FreshSkipEvidence",
|
||||
"Get-AgentHost188VerifiedRemediationOutcome",
|
||||
"Invoke-AgentHost188BackupRemediation",
|
||||
):
|
||||
assert f'"{function_name}"' in source
|
||||
assert "wrong_evidence_path" in source
|
||||
assert "unknown_retry_enabled" in source
|
||||
assert "unknown_receipt_conflict" in source
|
||||
assert "busy_canonical_write" in source
|
||||
assert "valid_fresh_skip_rejected" in source
|
||||
assert "valid_source_lock_fresh_skip_rejected" in source
|
||||
assert "valid_reconciled_source_lock_fresh_skip_rejected" in source
|
||||
assert "verified_remediation_outcome_matrix_failed" in source
|
||||
assert "matching_digest_unverified_resolution_receipt" in source
|
||||
assert "fresh_target_bypassed_unresolved_operation_gate" in source
|
||||
assert "operation_gate_not_blocked" in source
|
||||
assert "verified_resolution_marker_did_not_resolve_operation_gate" in source
|
||||
assert "committed_unknown_gate_not_blocked" in source
|
||||
assert "newer_unrelated_verified_backup_resolved_unknown" in source
|
||||
assert "newer_committed_unknown_not_blocked" in source
|
||||
assert "retryApplyAllowed = $false" in source
|
||||
assert "remoteWritePerformed = $false" in source
|
||||
assert "secretValuesRead = $false" in source
|
||||
for forbidden in ("New-Item", "Set-Content", "WriteAllText", "ssh.exe", "scp.exe"):
|
||||
assert forbidden not in source
|
||||
@@ -36,7 +36,7 @@ def _base_case() -> dict[str, Any]:
|
||||
"exists": True,
|
||||
"sourceRevision": "a" * 40,
|
||||
"runtimeMatched": True,
|
||||
"fileCount": 19,
|
||||
"fileCount": 20,
|
||||
"mismatchCount": 0,
|
||||
"parseError": "",
|
||||
},
|
||||
@@ -84,7 +84,7 @@ $parameters = @{{
|
||||
AgentRootPresent = [bool]$case.agentRootPresent
|
||||
Manifest = $case.manifest
|
||||
ManifestCheckRequired = [bool]$case.manifestCheckRequired
|
||||
ExpectedRuntimeFileCount = 19
|
||||
ExpectedRuntimeFileCount = 20
|
||||
TaskFailureCount = [int]$case.taskFailureCount
|
||||
RelayListenerCount = [int]$case.relayListenerCount
|
||||
RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount
|
||||
@@ -133,7 +133,7 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
|
||||
)
|
||||
assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision
|
||||
assert "preflightGreen = [bool]$decision.deploymentEligible" in source
|
||||
assert "-ExpectedRuntimeFileCount 19" in source
|
||||
assert "-ExpectedRuntimeFileCount 20" in source
|
||||
assert "warningReasons = $warningReasons" in source
|
||||
assert "selfHealthPerformanceIssues = @(" in source
|
||||
assert 'if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$")' in source
|
||||
|
||||
@@ -22,6 +22,7 @@ BOOTSTRAP = ROOT / "agent99-bootstrap.ps1"
|
||||
SIGNOZ_EXECUTOR = ROOT / "agent99-signoz-metadata-executor.ps1"
|
||||
CONTRACT_CHECK = ROOT / "agent99-contract-check.ps1"
|
||||
LIVE_PREFLIGHT = ROOT / "scripts/reboot-recovery/agent99-live-preflight.ps1"
|
||||
CONTROL_PLANE = ROOT / "agent99-control-plane.ps1"
|
||||
|
||||
EXPECTED_RUNTIME_FILES = (
|
||||
"agent99-bootstrap.ps1",
|
||||
@@ -30,6 +31,7 @@ EXPECTED_RUNTIME_FILES = (
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-host188-backup-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
@@ -82,16 +84,23 @@ def test_runtime_manifest_producers_and_consumers_share_exact_bundle_count() ->
|
||||
signoz_executor = SIGNOZ_EXECUTOR.read_text(encoding="utf-8")
|
||||
contract_check = CONTRACT_CHECK.read_text(encoding="utf-8")
|
||||
live_preflight = LIVE_PREFLIGHT.read_text(encoding="utf-8")
|
||||
control_plane = CONTROL_PLANE.read_text(encoding="utf-8")
|
||||
|
||||
sender_files = _quoted_array(sender, "RUNTIME_FILES=(", ")")
|
||||
receiver_files = _quoted_array(receiver, "$FixedRuntimeFiles = @(", ")")
|
||||
deployer_files = _quoted_array(deployer, "$runtimeFiles = @(", ")")
|
||||
bootstrap_files = _quoted_array(bootstrap, "$agentFiles = @(", ")")
|
||||
control_plane_files = _quoted_array(
|
||||
control_plane,
|
||||
"function Test-AgentRuntimeManifest {\n $expectedFiles = @(",
|
||||
"\n )",
|
||||
)
|
||||
|
||||
assert sender_files == EXPECTED_RUNTIME_FILES
|
||||
assert receiver_files == EXPECTED_RUNTIME_FILES
|
||||
assert deployer_files == EXPECTED_RUNTIME_FILES
|
||||
assert bootstrap_files == EXPECTED_RUNTIME_FILES
|
||||
assert control_plane_files == EXPECTED_RUNTIME_FILES
|
||||
expected_count = len(EXPECTED_RUNTIME_FILES)
|
||||
preflight_count = re.search(
|
||||
r"-ExpectedRuntimeFileCount\s+(\d+)", live_preflight
|
||||
@@ -107,7 +116,7 @@ def test_runtime_manifest_producers_and_consumers_share_exact_bundle_count() ->
|
||||
assert f"[int]$envelope.expectedRuntimeFileCount -ne {expected_count}" in receiver
|
||||
assert f"fileCount = {expected_count}" in receiver
|
||||
assert "$result.fileCount -eq $ExpectedRuntimeFileCount" in signoz_executor
|
||||
assert "$ExpectedRuntimeFileCount = 19" in contract_check
|
||||
assert "$ExpectedRuntimeFileCount = 20" in contract_check
|
||||
assert "$result.fileCount -eq $ExpectedRuntimeFileCount" in contract_check
|
||||
assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver
|
||||
assert "required_runtime_file_missing" in receiver
|
||||
@@ -283,7 +292,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
|
||||
)
|
||||
assert "$livePreflight.sourceRevision -eq $sourceRevision" in source
|
||||
assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source
|
||||
assert '$runtimeManifest.fileCount -eq 19' in source
|
||||
assert '$runtimeManifest.fileCount -eq 20' in source
|
||||
assert "function Invoke-AgentWindowsControlBaseline" in source
|
||||
assert "function Get-AgentRunEvidenceToken" in source
|
||||
assert "$evidenceRunToken = Get-AgentRunEvidenceToken -Value $RunId" in source
|
||||
|
||||
Reference in New Issue
Block a user