Implement M7.4a checkpoint publication and verified recovery
ADR-0061 publishes environmental-v1 full checkpoints. Manifest v1 is a fixed 280-byte record (identity, chain, lease, versions, build, zero RNG/history declarations, durability policy, payload length/SHA-256) that PostgreSQL parses and digests itself. Schema v3 adds region_commits with a digest CHECK and a composite head FK, owner-managed retained builds, fenced publish/load functions and owner-only genesis; it drops the bare v2 head kernel and fails closed while v2 heads exist. Linux evo::region_store adds OpenSSL SHA-256, an immutable content-addressed filesystem object store (temp+fsync+link, no overwrite/delete, O_NOFOLLOW exact-length verified reads), and publication (validate -> upload -> read back -> fenced transaction -> receipt check) and recovery (re-digest, allowlist, verify, full restore). Portable evo::region_checkpoint holds the codec and payload validation. Validated: Windows CTest 56/56; WSL Release, clang-tidy and ASan/UBSan private-PostgreSQL suites; four SQL and one host mutation each caught. M7.4 stays open for 7.4b (lineage archive codec and history binding). No Docker/Gitea CI, remote store, process kill or deployment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+21
-2
@@ -77,6 +77,11 @@ add_library(evo::region_lease ALIAS evo_region_lease)
|
||||
target_include_directories(evo_region_lease PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
|
||||
target_compile_features(evo_region_lease PUBLIC cxx_std_23)
|
||||
evo_owned_target(evo_region_lease)
|
||||
# Portable checkpoint manifest codec and environmental-v1 payload validation.
|
||||
add_library(evo_region_checkpoint STATIC src/persistence/region_checkpoint.cpp)
|
||||
add_library(evo::region_checkpoint ALIAS evo_region_checkpoint)
|
||||
target_link_libraries(evo_region_checkpoint PUBLIC evo::region_lease evo::processes)
|
||||
evo_owned_target(evo_region_checkpoint)
|
||||
|
||||
include(CTest)
|
||||
if(EVO_BUILD_POSTGRES)
|
||||
@@ -90,10 +95,19 @@ if(EVO_BUILD_POSTGRES)
|
||||
target_include_directories(evo_postgres PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
|
||||
target_link_libraries(evo_postgres PUBLIC evo::region_lease PRIVATE PostgreSQL::PostgreSQL)
|
||||
evo_owned_target(evo_postgres)
|
||||
# SHA-256 via system OpenSSL libcrypto (already required by libpq TLS; ADR-0061).
|
||||
find_package(OpenSSL 3 REQUIRED COMPONENTS Crypto)
|
||||
add_library(evo_region_store STATIC src/persistence/sha256.cpp src/persistence/object_store.cpp
|
||||
src/persistence/checkpoint_store.cpp)
|
||||
add_library(evo::region_store ALIAS evo_region_store)
|
||||
target_link_libraries(evo_region_store PUBLIC evo::postgres evo::region_checkpoint
|
||||
PRIVATE OpenSSL::Crypto)
|
||||
evo_owned_target(evo_region_store)
|
||||
if(BUILD_TESTING)
|
||||
# Contention tests use one thread and one connection per contender.
|
||||
add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp)
|
||||
target_link_libraries(evo-postgres-tests PRIVATE evo::postgres Threads::Threads)
|
||||
add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp
|
||||
tests/region_checkpoint_store_tests.cpp)
|
||||
target_link_libraries(evo-postgres-tests PRIVATE evo::region_store Threads::Threads)
|
||||
evo_owned_target(evo-postgres-tests)
|
||||
# Intentionally not an ordinary CTest: tools/ci/postgres-check.sh owns its private DB.
|
||||
endif()
|
||||
@@ -104,6 +118,11 @@ if(BUILD_TESTING)
|
||||
evo_owned_target(evo-region-lease-tests)
|
||||
add_test(NAME region-lease-contracts COMMAND evo-region-lease-tests)
|
||||
set_tests_properties(region-lease-contracts PROPERTIES TIMEOUT 60)
|
||||
add_executable(evo-region-checkpoint-tests tests/region_checkpoint_tests.cpp)
|
||||
target_link_libraries(evo-region-checkpoint-tests PRIVATE evo::region_checkpoint)
|
||||
evo_owned_target(evo-region-checkpoint-tests)
|
||||
add_test(NAME region-checkpoint-contracts COMMAND evo-region-checkpoint-tests)
|
||||
set_tests_properties(region-checkpoint-contracts PROPERTIES TIMEOUT 60)
|
||||
add_executable(evo-environmental-qualification tests/environmental_qualification.cpp)
|
||||
target_link_libraries(evo-environmental-qualification PRIVATE evo::processes)
|
||||
evo_owned_target(evo-environmental-qualification)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: evolution-db-migrate-v2
|
||||
name: evolution-db-migrate-v3
|
||||
namespace: evolution-dev
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
|
||||
@@ -515,3 +515,17 @@ Real-PostgreSQL tests cover the scenarios the roadmap row requires:
|
||||
Timings were selected and measured locally. The director is intentionally deferred
|
||||
because it has no authority and would be a placeholder. Next: M7.4 checkpoint
|
||||
codecs/publication. It must also revoke runtime EXECUTE on the bare head kernel.
|
||||
|
||||
M7.4 split into 7.4a (environmental-v1 publication) and 7.4b (lineage archive
|
||||
codec and history-capable binding). M7.4a is implemented and locally qualified
|
||||
(2026-09-28) under ADR-0061:
|
||||
- Manifest v1 is 280 bytes and is parsed by the database itself.
|
||||
- Schema v3 adds a commit chain with digest and head constraints, retained
|
||||
builds, and fenced publication. The bare v2 kernel is dropped.
|
||||
- SHA-256 comes from OpenSSL, and payloads go to an immutable filesystem store.
|
||||
- Publication validates, uploads, verifies, then commits. Recovery re-digests and
|
||||
verifies.
|
||||
- Missing or corrupt objects, upload and manifest failures, orphans, size caps,
|
||||
retained-build allowlists and restored-trajectory equality are covered.
|
||||
|
||||
Next: M7.4b. M7.4 stays open until it passes.
|
||||
|
||||
@@ -769,3 +769,22 @@ lease check, a disabled busy check) were each detected by a failing test.
|
||||
Windows ran 55/55; Linux Release, clang-tidy and ASan/UBSan PostgreSQL suites
|
||||
passed in WSL. No director, host, gateway, payload publication or deployment
|
||||
exists yet. The v2 head must not carry real world state before M7.4. Next: M7.4.
|
||||
|
||||
## M7.4a checkpoint publication (2026-09-28)
|
||||
|
||||
ADR-0061 and schema v3 publish environmental-v1 checkpoints. A fixed 280-byte
|
||||
manifest is parsed and digested by PostgreSQL. Payloads live in an immutable,
|
||||
content-addressed filesystem store and are hashed with OpenSSL SHA-256. The
|
||||
database enforces the commit chain, retained builds and fixed versions/quantum,
|
||||
and refuses to migrate while v2 heads exist.
|
||||
|
||||
Recovery re-digests the manifest, checks the build allowlist and fully restores the
|
||||
payload. A restored state continues identically to the uninterrupted original.
|
||||
Missing, corrupt or tampered data quarantines the region. Four SQL mutations and
|
||||
one host mutation were each caught by a test.
|
||||
|
||||
Local fixture payloads are 23-37 KiB and publish in about 3.5-10.6 ms across
|
||||
builds. Windows ran 56/56; Linux Release, clang-tidy and ASan/UBSan PostgreSQL
|
||||
suites passed in WSL. M7.4 stays open for M7.4b (lineage archive codec and
|
||||
history-capable binding). No real world was provisioned; there is no deployment,
|
||||
host or process-kill test yet.
|
||||
|
||||
@@ -35,4 +35,6 @@ Record revision/hash, owner, license, build options and removal boundary; mirror
|
||||
|
||||
M7.2's [ADR-0059](0059-postgresql-bootstrap.md) accepts a scoped checksum-pinned
|
||||
source build for the paired PostgreSQL server/client. The general cross-platform
|
||||
package-manager comparison remains open.
|
||||
package-manager comparison remains open. M7.4's [ADR-0061](0061-checkpoint-publication.md)
|
||||
links system OpenSSL 3 libcrypto (already required by libpq TLS) for SHA-256 in the
|
||||
Linux persistence adapter only.
|
||||
|
||||
@@ -97,6 +97,8 @@ binds it to verified payloads, so v2 must not host real world state. Custom
|
||||
SQLSTATEs are project conventions; they are not PostgreSQL-reserved.
|
||||
|
||||
## Consequences
|
||||
Done in M7.4a (ADR-0061): schema v3 drops the bare kernel and publishes through a
|
||||
manifest-parsing function under the same lock order.
|
||||
The M7.4 migration must add the manifest/payload publication function that calls
|
||||
the kernel, and revoke runtime EXECUTE on `advance_region_head`. M7.5 adds
|
||||
commit-ID resolution. Until then, "head equals my commit ID" is only the primitive
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
# ADR-0061: Checkpoint publication and verified recovery
|
||||
|
||||
## Status
|
||||
Accepted for M7.4a, 2026-09-28. Covers the `environmental-v1` profile only. The
|
||||
lineage archive codec and history-capable profile binding are M7.4b; M7.4 is not
|
||||
complete until both parts pass. Decisions, outbox and commit resolution remain M7.5.
|
||||
|
||||
## Context
|
||||
ADR-0058 selects full checkpoint batches: payload bytes go to an immutable object,
|
||||
then a fenced database transaction makes them the committed head. ADR-0060 fences
|
||||
the head, but its v2 kernel accepts any 32-byte digest with no payload behind it.
|
||||
ADR-0059 deferred the SHA-256 provider. No object-store dependency has been selected.
|
||||
The M6 environmental envelope is complete for its profile and already validates
|
||||
itself fully on restore.
|
||||
|
||||
## Decision
|
||||
**Manifest v1** is a fixed 280-byte little-endian record (schemas/region-commit.md)
|
||||
holding:
|
||||
- identity, commit ID and sequence, predecessor ID and digest, and the tick range;
|
||||
- the lease holder and epoch, and the tick quantum;
|
||||
- nine pinned versions and the retained build identity;
|
||||
- explicit declarations that there are zero RNG streams, no history profile, and a
|
||||
single-volume development durability policy;
|
||||
- the payload length and SHA-256.
|
||||
|
||||
The commit digest is SHA-256 of these bytes. The database parses the same bytes
|
||||
itself, so no separately supplied field can disagree with the digested record.
|
||||
|
||||
**Schema v3** makes the following changes:
|
||||
- It adds `region_commits`. A CHECK constraint ties each commit digest to its
|
||||
manifest, and a composite foreign key requires the head to name an existing
|
||||
commit with the same ID and digest.
|
||||
- It adds owner-managed `retained_builds`.
|
||||
- It adds `publish_region_checkpoint(manifest)`, in the same order as the v2 kernel:
|
||||
lease row lock, database-time lease check, then head lock. It requires the
|
||||
exact successor of the committed head, with unchanged versions and tick quantum,
|
||||
a retained build, and a unique commit ID.
|
||||
- It adds `load_region_checkpoint` and owner-only `provision_region_checkpoint`
|
||||
for genesis.
|
||||
- It drops the v2 kernel and v2 provisioning functions.
|
||||
- It refuses to migrate a database that still has v2 heads, because those have no
|
||||
payload binding. Such regions must be re-provisioned.
|
||||
|
||||
The runtime role can execute exactly six functions.
|
||||
|
||||
**SHA-256** comes from the system OpenSSL 3 libcrypto EVP interface. It is linked
|
||||
only into the Linux `evo::region_store` adapter. libpq TLS already requires that
|
||||
library in the same images. Locally observed: Ubuntu `libssl3t64`/`libssl-dev`
|
||||
3.0.13-0ubuntu3.15, Apache-2.0. PostgreSQL's built-in `sha256()` independently
|
||||
computes commit digests, and the host checks that the receipt matches.
|
||||
|
||||
**Object store**: a filesystem store on a trusted mounted volume (Linux only).
|
||||
- Each region has its own `w<world>/r<region>` directory with mode 0700. Objects
|
||||
are named by the hex SHA-256 of their content.
|
||||
- Upload writes a private temporary file, fsyncs it, `link()`s it to the key (a
|
||||
link never replaces an existing name), and then fsyncs the directory.
|
||||
- Existing identical content is reused. Differing content already under a key is
|
||||
reported as corrupt and never overwritten.
|
||||
- Reads use `O_NOFOLLOW`, check an exact-length regular file before allocating,
|
||||
and verify the digest.
|
||||
- There is no delete or GC operation: unreferenced uploads are orphans.
|
||||
|
||||
**Publication order**:
|
||||
1. Validate the candidate the way recovery would: full envelope restore,
|
||||
identity, tick and quantum checks, and a byte-identical re-encode.
|
||||
2. Upload the payload.
|
||||
3. Read it back and verify it.
|
||||
4. Run the fenced transaction.
|
||||
5. Compare the receipt with the local manifest digest.
|
||||
|
||||
Only step 4 has authority. **Recovery** loads the head and manifest, re-digests
|
||||
the manifest, requires the head fields to match and the build to be in the host's
|
||||
retained allowlist, fetches and verifies the object, and fully restores it. It
|
||||
never creates genesis or selects an older head. A missing or corrupt payload
|
||||
quarantines the region.
|
||||
|
||||
## Alternatives Considered
|
||||
- **S3/MinIO clients** add a large network dependency before any measured need.
|
||||
The interface boundary allows one later, with a separate ADR.
|
||||
- **Payload bytes in PostgreSQL** was deferred by ADR-0058 until measured.
|
||||
- **Host-supplied manifest fields as SQL parameters** could disagree with the
|
||||
digested bytes.
|
||||
- **A project-written SHA-256** is forbidden by the constitution.
|
||||
- **rename()** can silently replace an existing object.
|
||||
|
||||
## Benefits
|
||||
The committed head is always backed by a verified, immutable payload whose
|
||||
identity, chain, lease and versions the database has checked. Tampering is
|
||||
blocked by constraints and detected again on recovery. Crashes before the
|
||||
transaction leave the previous head.
|
||||
|
||||
## Costs
|
||||
- A full payload is written per batch. The local fixture's payload is about 23-37
|
||||
KiB, and publication took about 3.5-6.6 ms in non-sanitizer builds.
|
||||
- Orphans accumulate.
|
||||
- The existing v2 regions must be re-provisioned.
|
||||
- Recovery fetches whole payloads.
|
||||
|
||||
## Risks
|
||||
- A single volume is not host-loss durability (M8).
|
||||
- `fsync` semantics depend on the filesystem and mount options. The local test
|
||||
volume does not prove power-loss behavior.
|
||||
- The database cannot validate payload contents: a compromised worker holding a
|
||||
valid lease can publish a self-consistent but illegitimate state (ADR-0058).
|
||||
- The read-back verification path is not exercised by any fault injection.
|
||||
|
||||
## Consequences
|
||||
M7.4b adds the lineage archive codec, organism-to-genome binding and a separately
|
||||
versioned history-capable profile. M7.5 adds decisions, outbox and commit-ID
|
||||
resolution to the same transaction. Operators insert each promoted image digest
|
||||
into `retained_builds` before hosts publish with it, and hosts restore only builds
|
||||
in their own allowlist.
|
||||
@@ -71,3 +71,4 @@ design. No unadopted candidate is a link dependency of the proof.
|
||||
|
||||
- [ADR-0059: PostgreSQL bootstrap](0059-postgresql-bootstrap.md) - accepted for M7.2 optional adapter, pinned server/client, migrations and Secret boundary.
|
||||
- [ADR-0060: Fenced region ownership](0060-fenced-region-ownership.md) - accepted for M7.3 database-time leases, guarded head advance, runtime grants and selected timings.
|
||||
- [ADR-0061: Checkpoint publication](0061-checkpoint-publication.md) - accepted for M7.4a manifest v1, immutable filesystem objects, OpenSSL SHA-256 and verified recovery.
|
||||
|
||||
@@ -168,3 +168,17 @@ rejected, and the head is unchanged. A paused stale worker resuming past its lea
|
||||
is fenced both locally and by the database. A writer queued on the lease lock past
|
||||
expiry is also rejected. These are in-process pauses, not killed pods or network
|
||||
partitions. The 7.6 parts of those rows remain open.
|
||||
|
||||
M7.4a (ADR-0061) executed its rows for environmental-v1 against real PostgreSQL and
|
||||
a local filesystem object store:
|
||||
- A partial upload leaves only an unresolvable temporary name.
|
||||
- A failed upload sends no manifest.
|
||||
- A verified upload whose transaction is rejected stays an unreferenced orphan,
|
||||
and recovery returns the previous head.
|
||||
- Missing, corrupt, symlinked or resized payloads quarantine the region.
|
||||
- Unsupported builds are refused by both the database and recovery.
|
||||
- Recovery never creates genesis or picks an older head.
|
||||
|
||||
Two parts remain open. M7.4b adds the lineage archive codec and a history-capable
|
||||
profile; until then the history rows of this audit stay open. No process-kill
|
||||
test was run yet (7.7).
|
||||
|
||||
@@ -2103,3 +2103,58 @@ Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate
|
||||
overflow (`9223372036854775807 + 2000`).
|
||||
- Reran Windows CTest 55/55, the repository format check, and GCC14 Release,
|
||||
clang-tidy analysis and ASan/UBSan private-PostgreSQL suites: all passed.
|
||||
|
||||
## M7.4a checkpoint publication - 2026-09-28
|
||||
|
||||
- Windows MSVC Release configure/build/CTest passed 56/56, including the new
|
||||
portable `region-checkpoint-contracts`. It covers:
|
||||
- golden byte offsets and round trips;
|
||||
- malformed and unsupported manifests, including u32 values above 255 that
|
||||
must not truncate onto supported enums;
|
||||
- chain rules;
|
||||
- payload identity, tick, quantum, corruption and budget checks;
|
||||
- 24-tick continuation equality after restore.
|
||||
The repository clang-format 19 check passed.
|
||||
- WSL Ubuntu, with GCC 14.2.0, clang 19.1.1, PostgreSQL/libpq 18.6 and OpenSSL
|
||||
3.0.13-0ubuntu3.15, running as uid 1000 so permission faults were really
|
||||
exercised. GCC14 Release, clang19 analysis (clang-tidy warnings-as-errors) and
|
||||
clang19 ASan/UBSan each passed `tools/ci/postgres-check.sh`.
|
||||
- Analysis initially flagged `performance-enum-size` on the u32-based
|
||||
profile/durability enums. They are now u8 with explicit decoding of values
|
||||
above 255 as unknown. All three configurations were rerun.
|
||||
- New real-database/filesystem coverage:
|
||||
- SHA-256 vectors (NIST "", "abc" and the 448-bit message) and agreement with
|
||||
PostgreSQL `sha256()`.
|
||||
- Object store: rejects root symlinks, creates 0700 namespaces, puts are
|
||||
idempotent, never overwrites differing content, ignores temporary leftovers,
|
||||
and detects byte flips, symlinks, truncation and growth. Size bounds and
|
||||
write-permission failure are also covered.
|
||||
- Genesis provisioning and duplicate refusal; three batches (16/64/16 ticks);
|
||||
a 65-tick batch refused before upload.
|
||||
- Recovery from a new store handle is byte-identical and continues 44 ticks
|
||||
identically to an uninterrupted reference.
|
||||
- The build allowlist is enforced by the host and by the database.
|
||||
- Upload failure leaves the head unchanged. A fenced publication leaves a
|
||||
verified orphan, and recovery ignores it.
|
||||
- Missing and corrupt payloads quarantine; restoring the object recovers.
|
||||
- Owner edits to a manifest (23514) and head repointing (23503) are refused.
|
||||
With the constraint dropped, recovery still detects the edit, and the
|
||||
constraint is restored afterwards.
|
||||
- Unknown regions are never created.
|
||||
- Lease tests were ported to manifest publication. They additionally cover
|
||||
malformed manifests, a quantum change, an unretained build, a future rules
|
||||
version and a globally reused commit ID.
|
||||
- Migrations: clean→v3, repeated no-op, v1→v3. v2 with existing heads fails
|
||||
closed and stays at v2. The audit rejects EXECUTE on provisioning and SELECT on
|
||||
commits. Tampered v2/v3 checksums and an unknown v4 are refused.
|
||||
- Mutations, run on scratch copies with the source restored afterwards: removing
|
||||
the predecessor-digest check, the lease check, the version/quantum continuity
|
||||
check or the retained-build check in SQL, or the host manifest re-digest, each
|
||||
failed at the expected assertion.
|
||||
- Local fixture measurements: payloads of 22775 and 37367 bytes.
|
||||
- Release publication took 3.6-4.8 ms per batch; analysis 5.5-6.7 ms; ASan
|
||||
about 10.5 ms.
|
||||
- Manifest publish p50 was about 0.74-0.91 ms.
|
||||
These are local socket and local disk only.
|
||||
- Not executed: Docker/Gitea CI, a remote object store or PVC, power-loss fsync
|
||||
semantics, read-back verification fault injection, process kill, and TSan.
|
||||
|
||||
@@ -68,7 +68,7 @@ every role transitively in it, and rolls back on any violation:
|
||||
`pg_write_all_data`);
|
||||
- any table privilege in `evolution`, including via PUBLIC;
|
||||
- CREATE on the schema, the database or `public`;
|
||||
- EXECUTE outside the five fenced functions.
|
||||
- EXECUTE outside the six fenced functions (schema v3).
|
||||
Attribute checks alone are not enough: a login with `pg_write_all_data` can update
|
||||
lease rows directly. Rerun the migrate Job after adding or changing runtime logins
|
||||
or grants; it is the audit. Grants made later without a rerun are not detected. Bootstrap revokes PUBLIC access to the schema, tables and
|
||||
@@ -77,20 +77,41 @@ never creates logins or passwords, and never grants superuser.
|
||||
|
||||
## Fenced region ownership (schema v2, ADR-0060)
|
||||
|
||||
The runtime role has schema USAGE and EXECUTE on five SECURITY DEFINER functions:
|
||||
`acquire_region_lease`, `renew_region_lease`, `release_region_lease`,
|
||||
`load_region_head` and `advance_region_head`. It cannot read or write tables, create
|
||||
objects, call `check_region_lease` or provision regions. Each mutation locks the lease row
|
||||
In schema v3 the runtime role has schema USAGE and EXECUTE on six SECURITY DEFINER
|
||||
functions: `acquire_region_lease`, `renew_region_lease`, `release_region_lease`,
|
||||
`load_region_head`, `publish_region_checkpoint` and `load_region_checkpoint`. It
|
||||
cannot read or write tables, create objects, call internal check/parse functions or
|
||||
provision regions. (Schema v2's `advance_region_head` is dropped in v3.) Each mutation locks the lease row
|
||||
and checks holder, epoch and expiry with `clock_timestamp()` after the lock wait.
|
||||
Clients never send times. Rejections: EVL01 busy, EVL02 fenced, EVL03 head mismatch,
|
||||
EVL04 not provisioned, EVL05 epoch/sequence exhausted, 22023 invalid arguments.
|
||||
`evo::postgres` maps them to `LeaseStoreError`; a timeout or broken connection is
|
||||
`unknown`.
|
||||
EVL04 not provisioned, EVL05 epoch/sequence exhausted, EVL06 unsupported
|
||||
profile/version/build or a changed rules/quantum, EVL07 reused commit ID, and 22023
|
||||
malformed arguments or manifest. `evo::postgres` maps them to `LeaseStoreError`; a
|
||||
timeout or broken connection is `unknown`.
|
||||
|
||||
Genesis is operator-only (owner login), never done by recovery or runtime:
|
||||
`SELECT evolution.provision_region(world, region, frame, genesis_commit_uuid,
|
||||
'\x<64 hex digest>');`. In schema v2 the head is an identity/digest pointer with no
|
||||
payload binding. Do not provision real world state before M7.4 publication.
|
||||
## Checkpoint publication (schema v3, ADR-0061)
|
||||
|
||||
Each commit row stores the exact 280-byte manifest, and a constraint ties the
|
||||
commit digest to `sha256(manifest)`. The head has a composite foreign key to its
|
||||
commit, so even the owner cannot silently repoint or edit it.
|
||||
|
||||
Payloads are immutable files under a trusted object root:
|
||||
`<root>/w<world>/r<region>/<sha256 hex>`, mode 0700 directories and 0600 files.
|
||||
Mount a dedicated volume that only the region host's user can write. Nothing is
|
||||
ever deleted; orphans from failed transactions are expected.
|
||||
|
||||
Operator steps:
|
||||
1. Record each promoted build (image digest, 32 bytes) before a host uses it:
|
||||
`INSERT INTO evolution.retained_builds(build_identity) VALUES ('\x<64 hex>');`
|
||||
2. Provision genesis only through `provision_environmental_region` on an owner
|
||||
connection. It uploads and verifies the tick-0 payload, then calls the
|
||||
owner-only `provision_region_checkpoint(manifest)`.
|
||||
|
||||
There is no operator CLI for provisioning yet, and no real world has been
|
||||
provisioned. Migrating v2→v3 fails closed if any v2 head exists, because v2 heads
|
||||
have no payload.
|
||||
|
||||
## Host lease discipline
|
||||
|
||||
Hosts use `LeaseAuthority` with `LeasePolicy` defaults: 20 s lease, 5 s renewal
|
||||
interval, 2 s safety margin, 2 s renewal deadline and 5 s commit deadline. The
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
#pragma once
|
||||
#include "evo/persistence/object_store.hpp"
|
||||
#include "evo/persistence/region_checkpoint.hpp"
|
||||
#include "evo/persistence/region_lease_store.hpp"
|
||||
#include <chrono>
|
||||
#include <expected>
|
||||
#include <memory>
|
||||
#include <span>
|
||||
|
||||
// Linux publication and recovery of environmental-v1 checkpoints (ADR-0061).
|
||||
// Order: validate candidate -> immutable upload -> verified read-back -> fenced
|
||||
// manifest transaction. Nothing before the transaction has authority; a failure
|
||||
// there leaves the previous head and at most an unreferenced orphan object.
|
||||
namespace evo::persistence {
|
||||
enum class PublicationError : std::uint8_t {
|
||||
candidate, // State/manifest not a valid successor; nothing uploaded.
|
||||
digest, // Digest provider failure; nothing uploaded.
|
||||
upload, // Object write failed; no manifest was sent.
|
||||
verify, // Read-back differs from the candidate; no manifest was sent.
|
||||
rejected, // Database refused (see store); the upload is an orphan.
|
||||
unknown, // Outcome unknown (see store); stay paused and reconcile the head.
|
||||
receipt, // Committed receipt disagrees with the local manifest: fail closed.
|
||||
};
|
||||
struct PublicationFailure {
|
||||
PublicationError kind{};
|
||||
LeaseStoreFailure store{}; // Meaningful for rejected/unknown.
|
||||
ObjectError object{}; // Meaningful for upload/verify.
|
||||
};
|
||||
enum class RecoveryError : std::uint8_t {
|
||||
store, // Database unavailable/failed (see store).
|
||||
not_provisioned, // No genesis: recovery never creates one.
|
||||
manifest, // Manifest bytes disagree with the committed digest/head.
|
||||
unsupported, // Profile/versions/build not in this host's retained allowlist.
|
||||
missing_payload, // Referenced object absent: quarantine, no older head.
|
||||
corrupt_payload, // Length/digest mismatch: quarantine, no older head.
|
||||
payload, // Verified bytes fail full state validation: quarantine.
|
||||
};
|
||||
struct RecoveryFailure {
|
||||
RecoveryError kind{};
|
||||
LeaseStoreFailure store{};
|
||||
};
|
||||
struct RecoveredCheckpoint {
|
||||
CommittedCheckpoint committed;
|
||||
CheckpointManifest manifest;
|
||||
std::unique_ptr<processes::EnvironmentalCommunity> state;
|
||||
};
|
||||
|
||||
// Owner/operator connection only: uploads a tick-0 state and records genesis.
|
||||
[[nodiscard]] std::expected<CommittedCheckpoint, PublicationFailure> provision_environmental_region(
|
||||
Postgres& owner, const FilesystemObjectStore& objects, RegionKey key, std::uint64_t frame,
|
||||
const processes::EnvironmentalCommunity& opening, Uuid commit, const Sha256Digest& build,
|
||||
std::size_t restore_budget, std::chrono::milliseconds timeout);
|
||||
// Publishes `state` as the successor of `predecessor` under `lease`. The caller
|
||||
// must hold the committed predecessor it simulated from and a lease that may
|
||||
// begin a commit (LeaseAuthority::may_begin_commit).
|
||||
[[nodiscard]] std::expected<CommittedCheckpoint, PublicationFailure>
|
||||
publish_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
|
||||
const processes::EnvironmentalCommunity& state, Uuid commit,
|
||||
const Sha256Digest& build, std::size_t restore_budget,
|
||||
std::chrono::milliseconds timeout);
|
||||
// Loads and fully verifies the committed head. Readiness additionally requires a
|
||||
// fresh valid lease acquired before this call (M7.7); this does not grant one.
|
||||
[[nodiscard]] std::expected<RecoveredCheckpoint, RecoveryFailure>
|
||||
recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::span<const Sha256Digest> retained_builds,
|
||||
std::size_t restore_budget, std::chrono::milliseconds timeout);
|
||||
} // namespace evo::persistence
|
||||
@@ -0,0 +1,57 @@
|
||||
#pragma once
|
||||
#include "evo/persistence/region_lease.hpp"
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <expected>
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
// Immutable content-addressed payload store on a trusted local/mounted volume
|
||||
// (ADR-0061). Linux only. One namespace directory per world/region; the object
|
||||
// key is the lowercase hex SHA-256 of its bytes. There is no delete or overwrite:
|
||||
// unreferenced uploads are orphans and remain until a reachability-proven GC exists.
|
||||
namespace evo::persistence {
|
||||
enum class ObjectError : std::uint8_t {
|
||||
invalid, // Bad root, namespace or object reference; nothing done.
|
||||
missing, // No object with that key.
|
||||
corrupt, // Length or digest mismatch, or not a regular file: quarantine.
|
||||
bounds, // Larger than the configured maximum; rejected before allocation.
|
||||
io, // Write/sync/link failure; the object is absent or unverified.
|
||||
};
|
||||
struct VerifiedObject {
|
||||
Sha256Digest sha256{};
|
||||
std::uint64_t bytes{};
|
||||
friend bool operator==(const VerifiedObject&, const VerifiedObject&) = default;
|
||||
};
|
||||
class FilesystemObjectStore {
|
||||
public:
|
||||
// trusted_root must be an existing real directory (not a symlink). Namespace
|
||||
// directories w<world>/r<region> are created 0700 and their parents synced.
|
||||
[[nodiscard]] static std::expected<std::unique_ptr<FilesystemObjectStore>, ObjectError>
|
||||
open(const std::filesystem::path& trusted_root, RegionKey key, std::size_t max_object_bytes);
|
||||
FilesystemObjectStore(const FilesystemObjectStore&) = delete;
|
||||
FilesystemObjectStore& operator=(const FilesystemObjectStore&) = delete;
|
||||
FilesystemObjectStore(FilesystemObjectStore&&) = delete;
|
||||
FilesystemObjectStore& operator=(FilesystemObjectStore&&) = delete;
|
||||
~FilesystemObjectStore();
|
||||
// Writes a private temporary file, fsyncs it, links it to its key without
|
||||
// replacing anything, then fsyncs the directory. Identical bytes already stored
|
||||
// are verified and reused; a differing existing object is corrupt, never replaced.
|
||||
[[nodiscard]] std::expected<VerifiedObject, ObjectError>
|
||||
put(std::span<const std::uint8_t> bytes) const;
|
||||
// Reads only an exact-length regular file and verifies its SHA-256.
|
||||
[[nodiscard]] std::expected<std::vector<std::uint8_t>, ObjectError>
|
||||
get(const VerifiedObject& object) const;
|
||||
[[nodiscard]] const std::filesystem::path& directory() const noexcept {
|
||||
return directory_;
|
||||
}
|
||||
|
||||
private:
|
||||
FilesystemObjectStore(std::filesystem::path directory, int descriptor, std::size_t maximum);
|
||||
std::filesystem::path directory_;
|
||||
int descriptor_;
|
||||
std::size_t maximum_;
|
||||
};
|
||||
} // namespace evo::persistence
|
||||
@@ -0,0 +1,89 @@
|
||||
#pragma once
|
||||
#include "evo/persistence/region_lease.hpp"
|
||||
#include "evo/processes/environmental_community.hpp"
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <expected>
|
||||
#include <memory>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
// Region checkpoint manifest v1 and environmental-v1 payload validation (ADR-0061,
|
||||
// schemas/region-commit.md). Portable and storage-free: digests are computed and
|
||||
// objects are stored by the Linux adapters; this only encodes and validates.
|
||||
namespace evo::persistence {
|
||||
// Encoded as u32 on the wire; values above 255 decode as unknown (0), never truncated.
|
||||
enum class CheckpointProfile : std::uint8_t { unknown = 0, environmental_v1 = 1 };
|
||||
// Declared durability, verified at deployment. Development: one volume, no host-loss claim.
|
||||
enum class DurabilityPolicy : std::uint8_t { unknown = 0, single_volume_development = 1 };
|
||||
|
||||
// Exact versions of every nested format/rule the payload depends on. The interval
|
||||
// adapter carries no payload state but its version is still pinned.
|
||||
struct VersionBundle {
|
||||
std::uint32_t rules{2}, envelope{2}, resources{1}, fields{1}, terrain{1}, coordinate{1},
|
||||
generator{1}, material{1}, interval_adapter{1};
|
||||
friend bool operator==(const VersionBundle&, const VersionBundle&) = default;
|
||||
};
|
||||
inline constexpr VersionBundle environmental_v1_versions{};
|
||||
|
||||
struct CheckpointManifest {
|
||||
CheckpointProfile profile{CheckpointProfile::environmental_v1};
|
||||
RegionKey key{};
|
||||
std::uint64_t frame{};
|
||||
Uuid commit{};
|
||||
std::uint64_t sequence{};
|
||||
Uuid predecessor_commit{}; // Zero only at genesis.
|
||||
Sha256Digest predecessor_digest{}; // SHA-256 of the predecessor manifest bytes.
|
||||
std::uint64_t begin_tick{}, end_tick{};
|
||||
Uuid holder{}; // Zero only at genesis (provisioning, not a lease).
|
||||
std::uint64_t epoch{};
|
||||
std::uint64_t tick_microseconds{};
|
||||
VersionBundle versions{};
|
||||
Sha256Digest build{}; // Retained immutable build artifact identity.
|
||||
std::uint32_t rng_streams{}; // environmental-v1 rules use no randomness: 0.
|
||||
std::uint32_t history_profile{}; // No lineage/genome state in environmental-v1: 0.
|
||||
DurabilityPolicy durability{DurabilityPolicy::single_volume_development};
|
||||
std::uint64_t payload_bytes{};
|
||||
Sha256Digest payload_sha256{};
|
||||
friend bool operator==(const CheckpointManifest&, const CheckpointManifest&) = default;
|
||||
};
|
||||
|
||||
inline constexpr std::size_t checkpoint_manifest_bytes = 280;
|
||||
inline constexpr std::size_t max_environmental_payload_bytes = std::size_t{1024} * 1024;
|
||||
inline constexpr std::uint64_t max_checkpoint_ticks = 64;
|
||||
|
||||
enum class CheckpointError : std::uint8_t {
|
||||
format, // Malformed or internally inconsistent manifest.
|
||||
unsupported, // Unknown profile, version, RNG/history or durability declaration.
|
||||
chain, // Not a valid successor of the committed predecessor.
|
||||
payload, // Payload bytes fail full envelope validation or identity/tick checks.
|
||||
};
|
||||
|
||||
// Structural validation of every field; versions/profile are checked separately
|
||||
// so recovery can distinguish corruption from an unsupported (retained) format.
|
||||
[[nodiscard]] std::expected<std::vector<std::uint8_t>, CheckpointError>
|
||||
encode_manifest(const CheckpointManifest& manifest);
|
||||
[[nodiscard]] std::expected<CheckpointManifest, CheckpointError>
|
||||
decode_manifest(std::span<const std::uint8_t> bytes);
|
||||
// Profile, exact versions, zero RNG/history declarations and durability policy.
|
||||
[[nodiscard]] bool supported(const CheckpointManifest& manifest) noexcept;
|
||||
|
||||
// Genesis for operator provisioning only; recovery never creates one.
|
||||
[[nodiscard]] std::expected<CheckpointManifest, CheckpointError>
|
||||
genesis_manifest(RegionKey key, std::uint64_t frame, Uuid commit, std::uint64_t tick_microseconds,
|
||||
const Sha256Digest& build, std::uint64_t payload_bytes,
|
||||
const Sha256Digest& payload_sha256);
|
||||
// Next link from the committed predecessor and its manifest digest under a lease.
|
||||
[[nodiscard]] std::expected<CheckpointManifest, CheckpointError>
|
||||
successor_manifest(const CheckpointManifest& predecessor, const Sha256Digest& predecessor_digest,
|
||||
Uuid commit, const LeaseGrant& lease, std::uint64_t end_tick,
|
||||
const Sha256Digest& build, std::uint64_t payload_bytes,
|
||||
const Sha256Digest& payload_sha256);
|
||||
|
||||
// Fully restores payload bytes (envelope checksum, bounds and replay audit), then
|
||||
// requires manifest identity, end tick, tick quantum, envelope format 2 and a
|
||||
// byte-identical canonical re-encoding. The caller verifies length and SHA-256.
|
||||
[[nodiscard]] std::expected<std::unique_ptr<processes::EnvironmentalCommunity>, CheckpointError>
|
||||
restore_environmental_payload(std::span<const std::uint8_t> payload,
|
||||
const CheckpointManifest& manifest, std::size_t budget);
|
||||
} // namespace evo::persistence
|
||||
@@ -13,6 +13,7 @@ using Sha256Digest = std::array<std::uint8_t, 32>;
|
||||
struct RegionKey {
|
||||
std::uint64_t world{};
|
||||
std::uint64_t region{};
|
||||
friend bool operator==(const RegionKey&, const RegionKey&) = default;
|
||||
};
|
||||
struct RegionHead {
|
||||
std::uint64_t frame{};
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <expected>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
// Linux-only calls to the fenced database functions of migration v2 (ADR-0060).
|
||||
// Each call is one statement on a caller-owned connection; outside an explicit
|
||||
@@ -15,6 +17,8 @@ enum class LeaseStoreError : std::uint8_t {
|
||||
head_mismatch, // Expected predecessor is not the committed head.
|
||||
not_provisioned, // No genesis; runtime can never create one.
|
||||
exhausted, // Epoch or commit sequence would wrap: fail closed.
|
||||
unsupported, // Profile/version/build not admitted, or rules/quantum changed.
|
||||
commit_conflict, // Commit ID already used by another manifest (resolve at M7.5).
|
||||
invalid, // Rejected arguments; nothing changed.
|
||||
store, // Other SQL/configuration failure; nothing committed.
|
||||
unknown, // May have executed; reconnect and reconcile before reuse.
|
||||
@@ -23,10 +27,10 @@ struct LeaseStoreFailure {
|
||||
LeaseStoreError kind{};
|
||||
PgFailure cause{};
|
||||
};
|
||||
struct HeadAdvance {
|
||||
Uuid commit{};
|
||||
Sha256Digest digest{};
|
||||
std::uint64_t end_tick{};
|
||||
// Committed head and the exact manifest bytes its digest covers.
|
||||
struct CommittedCheckpoint {
|
||||
RegionHead head{};
|
||||
std::vector<std::uint8_t> manifest;
|
||||
};
|
||||
|
||||
[[nodiscard]] std::expected<LeaseGrant, LeaseStoreFailure>
|
||||
@@ -41,10 +45,13 @@ release_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease,
|
||||
std::chrono::milliseconds timeout);
|
||||
[[nodiscard]] std::expected<RegionHead, LeaseStoreFailure>
|
||||
load_region_head(Postgres& db, RegionKey key, std::chrono::milliseconds timeout);
|
||||
// Locks the lease then the head; requires unexpired holder/epoch and the exact
|
||||
// expected predecessor. head_mismatch after unknown: load the head and compare.
|
||||
[[nodiscard]] std::expected<CommittedCheckpoint, LeaseStoreFailure>
|
||||
load_region_checkpoint(Postgres& db, RegionKey key, std::chrono::milliseconds timeout);
|
||||
// Canonical manifest v1 bytes; the database parses identity, lease and chain from
|
||||
// them, locks the lease then the head and requires the exact committed predecessor.
|
||||
// The payload must already be verified in the object store. head_mismatch after
|
||||
// unknown: load the head and compare its commit ID (full resolution is M7.5).
|
||||
[[nodiscard]] std::expected<RegionHead, LeaseStoreFailure>
|
||||
advance_region_head(Postgres& db, RegionKey key, const LeaseGrant& lease,
|
||||
const RegionHead& expected, const HeadAdvance& next,
|
||||
std::chrono::milliseconds timeout);
|
||||
publish_region_checkpoint(Postgres& db, std::span<const std::uint8_t> manifest,
|
||||
std::chrono::milliseconds timeout);
|
||||
} // namespace evo::persistence
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
#pragma once
|
||||
#include "evo/persistence/region_lease.hpp"
|
||||
#include <expected>
|
||||
#include <span>
|
||||
|
||||
// SHA-256 from the system OpenSSL libcrypto EVP interface (ADR-0061). Linux
|
||||
// persistence adapters only; no project cryptography. Content identity, not
|
||||
// authentication.
|
||||
namespace evo::persistence {
|
||||
enum class DigestError : std::uint8_t { provider };
|
||||
[[nodiscard]] std::expected<Sha256Digest, DigestError>
|
||||
sha256(std::span<const std::uint8_t> bytes) noexcept;
|
||||
} // namespace evo::persistence
|
||||
@@ -0,0 +1,236 @@
|
||||
-- M7.4 committed checkpoint publication (ADR-0061). The database parses the
|
||||
-- canonical manifest v1 bytes itself, so no separately supplied field can disagree
|
||||
-- with the digested record. Payload bytes live in the object store; the database
|
||||
-- records their length and SHA-256 and never parses them.
|
||||
DO $$
|
||||
BEGIN
|
||||
-- Schema v2 heads carry no payload binding and cannot be migrated into a chain.
|
||||
IF EXISTS (SELECT 1 FROM evolution.region_heads) THEN
|
||||
RAISE EXCEPTION 'v2 region heads have no checkpoint payload; re-provision under v3';
|
||||
END IF;
|
||||
END $$;
|
||||
DROP FUNCTION evolution.advance_region_head(evolution.u64, evolution.u64, uuid, evolution.u64,
|
||||
evolution.u64, uuid, bytea, uuid, bytea, evolution.u64);
|
||||
DROP FUNCTION evolution.provision_region(evolution.u64, evolution.u64, evolution.u64, uuid, bytea);
|
||||
|
||||
-- Immutable build artifacts retained for recovery; owner-managed on promotion.
|
||||
CREATE TABLE evolution.retained_builds (
|
||||
build_identity bytea PRIMARY KEY CHECK (octet_length(build_identity) = 32),
|
||||
retained_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE evolution.region_commits (
|
||||
world_id evolution.u64 NOT NULL,
|
||||
region_id evolution.u64 NOT NULL,
|
||||
commit_sequence evolution.u64 NOT NULL,
|
||||
commit_id uuid NOT NULL UNIQUE,
|
||||
commit_digest bytea NOT NULL CHECK (octet_length(commit_digest) = 32),
|
||||
manifest bytea NOT NULL CHECK (octet_length(manifest) = 280),
|
||||
begin_tick evolution.u64 NOT NULL,
|
||||
end_tick evolution.u64 NOT NULL,
|
||||
published_epoch evolution.u64 NOT NULL,
|
||||
tick_microseconds evolution.u64 NOT NULL,
|
||||
versions bytea NOT NULL CHECK (octet_length(versions) = 36),
|
||||
build_identity bytea NOT NULL REFERENCES evolution.retained_builds,
|
||||
payload_bytes evolution.u64 NOT NULL,
|
||||
payload_sha256 bytea NOT NULL CHECK (octet_length(payload_sha256) = 32),
|
||||
committed_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (world_id, region_id, commit_sequence),
|
||||
UNIQUE (world_id, region_id, commit_sequence, commit_id, commit_digest),
|
||||
FOREIGN KEY (world_id, region_id) REFERENCES evolution.region_leases,
|
||||
CONSTRAINT region_commits_manifest_digest CHECK (commit_digest = sha256(manifest))
|
||||
);
|
||||
-- The head can only name an existing commit with the same identity and digest.
|
||||
ALTER TABLE evolution.region_heads
|
||||
ADD FOREIGN KEY (world_id, region_id, commit_sequence, commit_id, digest)
|
||||
REFERENCES evolution.region_commits (world_id, region_id, commit_sequence, commit_id,
|
||||
commit_digest);
|
||||
REVOKE ALL ON evolution.retained_builds, evolution.region_commits FROM PUBLIC;
|
||||
|
||||
CREATE TYPE evolution.checkpoint_manifest AS (
|
||||
world_id evolution.u64, region_id evolution.u64, frame_id evolution.u64, commit_id uuid,
|
||||
commit_sequence evolution.u64, predecessor_commit uuid, predecessor_digest bytea,
|
||||
begin_tick evolution.u64, end_tick evolution.u64, holder uuid, epoch evolution.u64,
|
||||
tick_microseconds evolution.u64, versions bytea, build_identity bytea,
|
||||
payload_bytes evolution.u64, payload_sha256 bytea
|
||||
);
|
||||
|
||||
CREATE FUNCTION evolution.manifest_uint(m bytea, off integer, width integer)
|
||||
RETURNS numeric LANGUAGE plpgsql IMMUTABLE STRICT SET search_path = pg_catalog, pg_temp AS $$
|
||||
DECLARE
|
||||
v numeric := 0;
|
||||
BEGIN
|
||||
FOR i IN REVERSE width - 1 .. 0 LOOP
|
||||
v := v * 256 + get_byte(m, off + i);
|
||||
END LOOP;
|
||||
RETURN v;
|
||||
END $$;
|
||||
|
||||
-- Mirrors decode_manifest + supported() in src/persistence/region_checkpoint.cpp.
|
||||
-- 22023: malformed; EVL06: parsable but unsupported by this schema.
|
||||
CREATE FUNCTION evolution.parse_checkpoint_manifest(m bytea)
|
||||
RETURNS evolution.checkpoint_manifest LANGUAGE plpgsql IMMUTABLE
|
||||
SET search_path = pg_catalog, pg_temp AS $$
|
||||
DECLARE
|
||||
r evolution.checkpoint_manifest;
|
||||
zero16 constant bytea := '\x00000000000000000000000000000000';
|
||||
zero32 constant bytea := decode(repeat('00', 32), 'hex');
|
||||
BEGIN
|
||||
IF m IS NULL OR octet_length(m) <> 280 OR substring(m FROM 1 FOR 8) <> '\x4145564f52434d31' THEN
|
||||
RAISE EXCEPTION 'malformed checkpoint manifest' USING ERRCODE = '22023';
|
||||
END IF;
|
||||
IF evolution.manifest_uint(m, 8, 4) <> 1 OR evolution.manifest_uint(m, 12, 4) <> 1
|
||||
OR substring(m FROM 161 FOR 36) <> '\x020000000200000001000000010000000100000001000000010000000100000001000000'
|
||||
OR evolution.manifest_uint(m, 228, 4) <> 0 OR evolution.manifest_uint(m, 232, 4) <> 0
|
||||
OR evolution.manifest_uint(m, 236, 4) <> 1 THEN
|
||||
RAISE EXCEPTION 'unsupported checkpoint manifest' USING ERRCODE = 'EVL06';
|
||||
END IF;
|
||||
r.world_id := evolution.manifest_uint(m, 16, 8);
|
||||
r.region_id := evolution.manifest_uint(m, 24, 8);
|
||||
r.frame_id := evolution.manifest_uint(m, 32, 8);
|
||||
r.commit_id := encode(substring(m FROM 41 FOR 16), 'hex')::uuid;
|
||||
r.commit_sequence := evolution.manifest_uint(m, 56, 8);
|
||||
r.predecessor_commit := encode(substring(m FROM 65 FOR 16), 'hex')::uuid;
|
||||
r.predecessor_digest := substring(m FROM 81 FOR 32);
|
||||
r.begin_tick := evolution.manifest_uint(m, 112, 8);
|
||||
r.end_tick := evolution.manifest_uint(m, 120, 8);
|
||||
r.holder := encode(substring(m FROM 129 FOR 16), 'hex')::uuid;
|
||||
r.epoch := evolution.manifest_uint(m, 144, 8);
|
||||
r.tick_microseconds := evolution.manifest_uint(m, 152, 8);
|
||||
r.versions := substring(m FROM 161 FOR 36);
|
||||
r.build_identity := substring(m FROM 197 FOR 32);
|
||||
r.payload_bytes := evolution.manifest_uint(m, 240, 8);
|
||||
r.payload_sha256 := substring(m FROM 249 FOR 32);
|
||||
IF r.world_id = 0 OR r.region_id = 0 OR r.frame_id = 0
|
||||
OR substring(m FROM 41 FOR 16) = zero16 OR r.tick_microseconds = 0
|
||||
OR r.build_identity = zero32 OR r.payload_bytes NOT BETWEEN 1 AND 1048576
|
||||
OR r.payload_sha256 = zero32 THEN
|
||||
RAISE EXCEPTION 'malformed checkpoint manifest' USING ERRCODE = '22023';
|
||||
END IF;
|
||||
IF r.commit_sequence = 0 THEN
|
||||
IF substring(m FROM 65 FOR 16) <> zero16 OR r.predecessor_digest <> zero32
|
||||
OR r.begin_tick <> 0 OR r.end_tick <> 0 OR substring(m FROM 129 FOR 16) <> zero16
|
||||
OR r.epoch <> 0 THEN
|
||||
RAISE EXCEPTION 'malformed checkpoint manifest' USING ERRCODE = '22023';
|
||||
END IF;
|
||||
r.predecessor_commit := NULL;
|
||||
r.holder := NULL;
|
||||
ELSIF substring(m FROM 65 FOR 16) = zero16 OR r.predecessor_commit = r.commit_id
|
||||
OR r.predecessor_digest = zero32 OR substring(m FROM 129 FOR 16) = zero16
|
||||
OR r.epoch = 0 OR r.end_tick <= r.begin_tick OR r.end_tick - r.begin_tick > 64 THEN
|
||||
RAISE EXCEPTION 'malformed checkpoint manifest' USING ERRCODE = '22023';
|
||||
END IF;
|
||||
RETURN r;
|
||||
END $$;
|
||||
|
||||
-- Owner-only: genesis is an explicit authorized opening payload, never recovery.
|
||||
CREATE FUNCTION evolution.provision_region_checkpoint(p_manifest bytea)
|
||||
RETURNS void LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog, pg_temp AS $$
|
||||
DECLARE
|
||||
g evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(p_manifest);
|
||||
d bytea := sha256(p_manifest);
|
||||
BEGIN
|
||||
IF g.commit_sequence <> 0 THEN
|
||||
RAISE EXCEPTION 'genesis manifest required' USING ERRCODE = '22023';
|
||||
END IF;
|
||||
INSERT INTO evolution.region_leases(world_id, region_id, frame_id)
|
||||
VALUES (g.world_id, g.region_id, g.frame_id);
|
||||
INSERT INTO evolution.region_commits(world_id, region_id, commit_sequence, commit_id,
|
||||
commit_digest, manifest, begin_tick, end_tick, published_epoch, tick_microseconds,
|
||||
versions, build_identity, payload_bytes, payload_sha256)
|
||||
VALUES (g.world_id, g.region_id, 0, g.commit_id, d, p_manifest, 0, 0, 0,
|
||||
g.tick_microseconds, g.versions, g.build_identity, g.payload_bytes,
|
||||
g.payload_sha256);
|
||||
INSERT INTO evolution.region_heads(world_id, region_id, commit_sequence, commit_id, digest,
|
||||
end_tick, published_epoch)
|
||||
VALUES (g.world_id, g.region_id, 0, g.commit_id, d, 0, 0);
|
||||
END $$;
|
||||
|
||||
-- Fenced publication: lease row lock and database-time check, then the head lock;
|
||||
-- the manifest must be the exact successor of the committed head. One statement:
|
||||
-- a rejection inserts nothing. A verified orphan payload stays unreferenced.
|
||||
CREATE FUNCTION evolution.publish_region_checkpoint(p_manifest bytea)
|
||||
RETURNS TABLE (region_frame evolution.u64, head_sequence evolution.u64, head_commit uuid,
|
||||
head_digest text, head_tick evolution.u64)
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
|
||||
DECLARE
|
||||
c evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(p_manifest);
|
||||
h evolution.region_heads%ROWTYPE;
|
||||
p evolution.region_commits%ROWTYPE;
|
||||
frame evolution.u64;
|
||||
d bytea := sha256(p_manifest);
|
||||
BEGIN
|
||||
IF c.commit_sequence = 0 THEN
|
||||
RAISE EXCEPTION 'genesis is provisioned, not published' USING ERRCODE = '22023';
|
||||
END IF;
|
||||
PERFORM evolution.check_region_lease(c.world_id, c.region_id, c.holder, c.epoch);
|
||||
SELECT * INTO h FROM evolution.region_heads r
|
||||
WHERE r.world_id = c.world_id AND r.region_id = c.region_id FOR UPDATE;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04';
|
||||
END IF;
|
||||
IF h.commit_sequence = 18446744073709551615 THEN
|
||||
RAISE EXCEPTION 'commit sequences exhausted' USING ERRCODE = 'EVL05';
|
||||
END IF;
|
||||
IF c.commit_sequence <> h.commit_sequence + 1 OR c.predecessor_commit <> h.commit_id
|
||||
OR c.predecessor_digest <> h.digest OR c.begin_tick <> h.end_tick THEN
|
||||
RAISE EXCEPTION 'region head moved' USING ERRCODE = 'EVL03';
|
||||
END IF;
|
||||
SELECT l.frame_id INTO frame FROM evolution.region_leases l
|
||||
WHERE l.world_id = c.world_id AND l.region_id = c.region_id;
|
||||
IF c.frame_id <> frame THEN
|
||||
RAISE EXCEPTION 'foreign frame' USING ERRCODE = '22023';
|
||||
END IF;
|
||||
SELECT * INTO p FROM evolution.region_commits r
|
||||
WHERE r.world_id = c.world_id AND r.region_id = c.region_id
|
||||
AND r.commit_sequence = h.commit_sequence;
|
||||
-- Rules, formats and quantum are fixed for a region; changing them is a migration.
|
||||
IF c.versions <> p.versions OR c.tick_microseconds <> p.tick_microseconds
|
||||
OR NOT EXISTS (SELECT 1 FROM evolution.retained_builds b
|
||||
WHERE b.build_identity = c.build_identity) THEN
|
||||
RAISE EXCEPTION 'unsupported checkpoint' USING ERRCODE = 'EVL06';
|
||||
END IF;
|
||||
IF EXISTS (SELECT 1 FROM evolution.region_commits r WHERE r.commit_id = c.commit_id) THEN
|
||||
RAISE EXCEPTION 'commit identity reused' USING ERRCODE = 'EVL07';
|
||||
END IF;
|
||||
INSERT INTO evolution.region_commits(world_id, region_id, commit_sequence, commit_id,
|
||||
commit_digest, manifest, begin_tick, end_tick, published_epoch, tick_microseconds,
|
||||
versions, build_identity, payload_bytes, payload_sha256)
|
||||
VALUES (c.world_id, c.region_id, c.commit_sequence, c.commit_id, d, p_manifest,
|
||||
c.begin_tick, c.end_tick, c.epoch, c.tick_microseconds, c.versions,
|
||||
c.build_identity, c.payload_bytes, c.payload_sha256);
|
||||
UPDATE evolution.region_heads r
|
||||
SET commit_sequence = c.commit_sequence, commit_id = c.commit_id, digest = d,
|
||||
end_tick = c.end_tick, published_epoch = c.epoch
|
||||
WHERE r.world_id = c.world_id AND r.region_id = c.region_id;
|
||||
RETURN QUERY SELECT frame, c.commit_sequence, c.commit_id, encode(d, 'hex'), c.end_tick;
|
||||
END $$;
|
||||
|
||||
-- Committed head plus its exact manifest bytes, for recovery verification.
|
||||
CREATE FUNCTION evolution.load_region_checkpoint(p_world evolution.u64, p_region evolution.u64)
|
||||
RETURNS TABLE (region_frame evolution.u64, head_sequence evolution.u64, head_commit uuid,
|
||||
head_digest text, head_tick evolution.u64, head_manifest text)
|
||||
LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
|
||||
BEGIN
|
||||
RETURN QUERY SELECT l.frame_id, h.commit_sequence, h.commit_id, encode(h.digest, 'hex'),
|
||||
h.end_tick, encode(c.manifest, 'hex')
|
||||
FROM evolution.region_heads h
|
||||
JOIN evolution.region_leases l USING (world_id, region_id)
|
||||
JOIN evolution.region_commits c USING (world_id, region_id, commit_sequence)
|
||||
WHERE h.world_id = p_world AND h.region_id = p_region;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
REVOKE ALL ON FUNCTION
|
||||
evolution.manifest_uint(bytea, integer, integer),
|
||||
evolution.parse_checkpoint_manifest(bytea),
|
||||
evolution.provision_region_checkpoint(bytea),
|
||||
evolution.publish_region_checkpoint(bytea),
|
||||
evolution.load_region_checkpoint(evolution.u64, evolution.u64)
|
||||
FROM PUBLIC;
|
||||
GRANT EXECUTE ON FUNCTION
|
||||
evolution.publish_region_checkpoint(bytea),
|
||||
evolution.load_region_checkpoint(evolution.u64, evolution.u64)
|
||||
TO :"runtime_role";
|
||||
INSERT INTO evolution.schema_migrations(version, sha256) VALUES (3, :'migration_sha_3');
|
||||
+11
-4
@@ -17,10 +17,15 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution')
|
||||
SELECT coalesce(string_agg(version || ':' || sha256, ',' ORDER BY version), '')
|
||||
AS history FROM evolution.schema_migrations \gset
|
||||
SELECT :'history' = '1:' || :'migration_sha' AS at_v1,
|
||||
:'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' AS at_v2 \gset
|
||||
\if :at_v2
|
||||
:'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' AS at_v2,
|
||||
:'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2'
|
||||
|| ',3:' || :'migration_sha_3' AS at_v3 \gset
|
||||
\if :at_v3
|
||||
\elif :at_v2
|
||||
\ir 0003_region_checkpoints.sql
|
||||
\elif :at_v1
|
||||
\ir 0002_region_fencing.sql
|
||||
\ir 0003_region_checkpoints.sql
|
||||
\else
|
||||
\echo 'Unknown or changed migration history'
|
||||
DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$;
|
||||
@@ -28,6 +33,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution')
|
||||
\else
|
||||
\ir 0001_region_bootstrap.sql
|
||||
\ir 0002_region_fencing.sql
|
||||
\ir 0003_region_checkpoints.sql
|
||||
\endif
|
||||
-- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes
|
||||
-- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC
|
||||
@@ -52,7 +58,8 @@ BEGIN
|
||||
('evolution.renew_region_lease(evolution.u64,evolution.u64,uuid,evolution.u64,integer)'::regprocedure::oid),
|
||||
('evolution.release_region_lease(evolution.u64,evolution.u64,uuid,evolution.u64)'::regprocedure::oid),
|
||||
('evolution.load_region_head(evolution.u64,evolution.u64)'::regprocedure::oid),
|
||||
('evolution.advance_region_head(evolution.u64,evolution.u64,uuid,evolution.u64,evolution.u64,uuid,bytea,uuid,bytea,evolution.u64)'::regprocedure::oid)
|
||||
('evolution.publish_region_checkpoint(bytea)'::regprocedure::oid),
|
||||
('evolution.load_region_checkpoint(evolution.u64,evolution.u64)'::regprocedure::oid)
|
||||
), violations(role_oid, reason) AS (
|
||||
SELECT r.oid, 'privileged attribute or owner identity' FROM pg_catalog.pg_roles r
|
||||
JOIN closure c ON c.oid = r.oid
|
||||
@@ -76,7 +83,7 @@ BEGIN
|
||||
OR EXISTS (SELECT 1 FROM pg_catalog.pg_namespace n WHERE n.nspname = 'public'
|
||||
AND pg_catalog.has_schema_privilege(c.oid, n.oid, 'CREATE'))
|
||||
UNION ALL
|
||||
-- Only the five guarded functions are executable; provisioning/internal are not.
|
||||
-- Only the six guarded functions are executable; provisioning/internal are not.
|
||||
SELECT c.oid, 'EXECUTE outside the fenced function set' FROM closure c
|
||||
CROSS JOIN pg_catalog.pg_proc p
|
||||
WHERE p.pronamespace = 'evolution'::regnamespace
|
||||
|
||||
@@ -1,4 +1,41 @@
|
||||
# Region commit manifest contract v1 (M7.1 design)
|
||||
# Region commit manifest contract v1 (M7.1 design, M7.4a encoding)
|
||||
|
||||
## Canonical manifest encoding v1 (ADR-0061)
|
||||
|
||||
The encoder, decoder and `supported()` check live in
|
||||
`src/persistence/region_checkpoint.cpp`; `evolution.parse_checkpoint_manifest`
|
||||
mirrors them in SQL. The record is exactly 280 bytes, all integers little-endian,
|
||||
with no padding or trailing checksum. The commit digest is SHA-256 of the whole
|
||||
record, and each commit row stores it.
|
||||
|
||||
| Offset | Size | Field |
|
||||
|---|---|---|
|
||||
| 0 | 8 | magic `AEVORCM1` |
|
||||
| 8 | 4 | manifest_version = 1 (other values are unsupported) |
|
||||
| 12 | 4 | profile: 1 = environmental-v1 |
|
||||
| 16 | 24 | world, region, frame (nonzero u64 each) |
|
||||
| 40 | 16 | commit_id (nonzero; globally unique in the database) |
|
||||
| 56 | 8 | commit_sequence |
|
||||
| 64 | 16 | predecessor_commit_id (zero only at genesis; never equal to commit_id) |
|
||||
| 80 | 32 | predecessor manifest SHA-256 (zero only at genesis) |
|
||||
| 112 | 16 | begin_tick, end_tick (genesis 0/0; otherwise 1..64 tick delta) |
|
||||
| 128 | 16 | lease holder (zero only at genesis) |
|
||||
| 144 | 8 | lease epoch (0 only at genesis) |
|
||||
| 152 | 8 | tick_microseconds (nonzero; fixed for the region) |
|
||||
| 160 | 36 | nine u32 versions: rules 2, envelope 2, resources/fields/terrain/coordinate/generator/material/interval-adapter 1 |
|
||||
| 196 | 32 | retained build identity (nonzero; `retained_builds` row required) |
|
||||
| 228 | 4 | rng_streams = 0 (these rules use no randomness) |
|
||||
| 232 | 4 | history_profile = 0 (no genome/lineage state) |
|
||||
| 236 | 4 | durability_policy = 1 (single-volume development) |
|
||||
| 240 | 8 | payload_bytes (1..1 MiB) |
|
||||
| 248 | 32 | payload SHA-256 (nonzero) |
|
||||
|
||||
A structurally invalid record is malformed (SQL 22023). A record that parses but
|
||||
has an unsupported version, profile, RNG, history or durability declaration is
|
||||
unsupported (EVL06). Values above 255 are never truncated onto a supported enum.
|
||||
The payload is the exact environmental envelope format 2; its object key is the
|
||||
hex payload SHA-256 in the region's namespace. Decisions, trace and outbox are not
|
||||
encoded here: M7.5 adds them under a new manifest version.
|
||||
|
||||
Authority: ADR-0058 and docs/architecture/REGION_DURABILITY.md. Logical schema,
|
||||
not an implemented wire codec, SQL migration or permission to serialize structs.
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
#include "evo/persistence/checkpoint_store.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <string>
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
using std::chrono::milliseconds;
|
||||
PublicationFailure fail(PublicationError kind) {
|
||||
return {kind, {}, {}};
|
||||
}
|
||||
// Encodes the candidate, validates the exact payload bytes as recovery would,
|
||||
// uploads and reads back. Returns the manifest bytes ready for the database.
|
||||
std::expected<std::vector<std::uint8_t>, PublicationFailure>
|
||||
stage(const FilesystemObjectStore& objects, const std::vector<std::uint8_t>& payload,
|
||||
const CheckpointManifest& manifest, std::size_t budget) {
|
||||
auto bytes = encode_manifest(manifest);
|
||||
if (!bytes || !restore_environmental_payload(payload, manifest, budget))
|
||||
return std::unexpected{fail(PublicationError::candidate)};
|
||||
auto stored = objects.put(payload);
|
||||
if (!stored)
|
||||
return std::unexpected{PublicationFailure{PublicationError::upload, {}, stored.error()}};
|
||||
auto back = objects.get(*stored);
|
||||
if (!back || *stored != VerifiedObject{manifest.payload_sha256, manifest.payload_bytes} ||
|
||||
!std::ranges::equal(*back, payload))
|
||||
return std::unexpected{PublicationFailure{
|
||||
PublicationError::verify, {}, back ? ObjectError::corrupt : back.error()}};
|
||||
return std::move(*bytes);
|
||||
}
|
||||
std::expected<CommittedCheckpoint, PublicationFailure>
|
||||
confirm(const std::expected<RegionHead, LeaseStoreFailure>& published,
|
||||
std::vector<std::uint8_t> manifest_bytes, const CheckpointManifest& manifest) {
|
||||
if (!published) {
|
||||
const auto kind = published.error().kind == LeaseStoreError::unknown
|
||||
? PublicationError::unknown
|
||||
: PublicationError::rejected;
|
||||
return std::unexpected{PublicationFailure{kind, published.error(), {}}};
|
||||
}
|
||||
// The database digested the bytes independently; the receipt must agree.
|
||||
const auto digest = sha256(manifest_bytes);
|
||||
if (!digest || published->digest != *digest || published->sequence != manifest.sequence ||
|
||||
published->commit != manifest.commit || published->end_tick != manifest.end_tick ||
|
||||
published->frame != manifest.frame)
|
||||
return std::unexpected{fail(PublicationError::receipt)};
|
||||
return CommittedCheckpoint{*published, std::move(manifest_bytes)};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<CommittedCheckpoint, PublicationFailure> provision_environmental_region(
|
||||
Postgres& owner, const FilesystemObjectStore& objects, RegionKey key, std::uint64_t frame,
|
||||
const processes::EnvironmentalCommunity& opening, Uuid commit, const Sha256Digest& build,
|
||||
std::size_t restore_budget, milliseconds timeout) {
|
||||
auto payload = opening.snapshot();
|
||||
if (!payload || opening.progress().tick != 0)
|
||||
return std::unexpected{fail(PublicationError::candidate)};
|
||||
const auto digest = sha256(*payload);
|
||||
if (!digest)
|
||||
return std::unexpected{fail(PublicationError::digest)};
|
||||
auto manifest = genesis_manifest(key, frame, commit, opening.tick_microseconds(), build,
|
||||
payload->size(), *digest);
|
||||
if (!manifest)
|
||||
return std::unexpected{fail(PublicationError::candidate)};
|
||||
auto bytes = stage(objects, *payload, *manifest, restore_budget);
|
||||
if (!bytes)
|
||||
return std::unexpected{bytes.error()};
|
||||
constexpr char digits[] = "0123456789abcdef";
|
||||
std::string parameter = "\\x";
|
||||
for (const auto b : *bytes) {
|
||||
parameter.push_back(digits[b >> 4U]);
|
||||
parameter.push_back(digits[b & 0x0FU]);
|
||||
}
|
||||
const std::array<std::string, 1> p{parameter};
|
||||
auto done = owner.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, timeout);
|
||||
if (!done) {
|
||||
const auto kind = done.error().kind == PgError::unknown ? PublicationError::unknown
|
||||
: PublicationError::rejected;
|
||||
return std::unexpected{
|
||||
PublicationFailure{kind,
|
||||
{done.error().kind == PgError::unknown ? LeaseStoreError::unknown
|
||||
: LeaseStoreError::store,
|
||||
done.error()},
|
||||
{}}};
|
||||
}
|
||||
auto committed = load_region_checkpoint(owner, key, timeout);
|
||||
if (!committed)
|
||||
return std::unexpected{
|
||||
PublicationFailure{PublicationError::unknown, committed.error(), {}}};
|
||||
return confirm(committed->head, std::move(*bytes), *manifest);
|
||||
}
|
||||
|
||||
std::expected<CommittedCheckpoint, PublicationFailure> publish_environmental_checkpoint(
|
||||
Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
||||
const CommittedCheckpoint& predecessor, const processes::EnvironmentalCommunity& state,
|
||||
Uuid commit, const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout) {
|
||||
const auto previous = decode_manifest(predecessor.manifest);
|
||||
auto payload = state.snapshot();
|
||||
if (!previous || !payload)
|
||||
return std::unexpected{fail(PublicationError::candidate)};
|
||||
const auto digest = sha256(*payload);
|
||||
if (!digest)
|
||||
return std::unexpected{fail(PublicationError::digest)};
|
||||
auto manifest = successor_manifest(*previous, predecessor.head.digest, commit, lease,
|
||||
state.progress().tick, build, payload->size(), *digest);
|
||||
if (!manifest)
|
||||
return std::unexpected{fail(PublicationError::candidate)};
|
||||
auto bytes = stage(objects, *payload, *manifest, restore_budget);
|
||||
if (!bytes)
|
||||
return std::unexpected{bytes.error()};
|
||||
auto published = publish_region_checkpoint(db, *bytes, timeout);
|
||||
return confirm(published, std::move(*bytes), *manifest);
|
||||
}
|
||||
|
||||
std::expected<RecoveredCheckpoint, RecoveryFailure>
|
||||
recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::span<const Sha256Digest> retained_builds,
|
||||
std::size_t restore_budget, milliseconds timeout) {
|
||||
auto committed = load_region_checkpoint(db, key, timeout);
|
||||
if (!committed)
|
||||
return std::unexpected{
|
||||
RecoveryFailure{committed.error().kind == LeaseStoreError::not_provisioned
|
||||
? RecoveryError::not_provisioned
|
||||
: RecoveryError::store,
|
||||
committed.error()}};
|
||||
const auto fail_with = [](RecoveryError kind) {
|
||||
return std::unexpected{RecoveryFailure{kind, {}}};
|
||||
};
|
||||
const auto digest = sha256(committed->manifest);
|
||||
auto manifest = decode_manifest(committed->manifest);
|
||||
const auto& head = committed->head;
|
||||
if (!digest || *digest != head.digest)
|
||||
return fail_with(RecoveryError::manifest);
|
||||
if (!manifest)
|
||||
return fail_with(manifest.error() == CheckpointError::unsupported
|
||||
? RecoveryError::unsupported
|
||||
: RecoveryError::manifest);
|
||||
if (manifest->key != key || manifest->frame != head.frame ||
|
||||
manifest->sequence != head.sequence || manifest->commit != head.commit ||
|
||||
manifest->end_tick != head.end_tick)
|
||||
return fail_with(RecoveryError::manifest);
|
||||
if (!supported(*manifest) ||
|
||||
std::ranges::find(retained_builds, manifest->build) == retained_builds.end())
|
||||
return fail_with(RecoveryError::unsupported);
|
||||
auto payload = objects.get({manifest->payload_sha256, manifest->payload_bytes});
|
||||
if (!payload)
|
||||
return fail_with(payload.error() == ObjectError::missing ? RecoveryError::missing_payload
|
||||
: payload.error() == ObjectError::corrupt ? RecoveryError::corrupt_payload
|
||||
: RecoveryError::store);
|
||||
auto state = restore_environmental_payload(*payload, *manifest, restore_budget);
|
||||
if (!state)
|
||||
return fail_with(RecoveryError::payload);
|
||||
return RecoveredCheckpoint{std::move(*committed), *manifest, std::move(*state)};
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
@@ -0,0 +1,200 @@
|
||||
#include "evo/persistence/object_store.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include <atomic>
|
||||
#include <cerrno>
|
||||
#include <fcntl.h>
|
||||
#include <string>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
constexpr std::size_t maximum_object_limit = std::size_t{64} * 1024 * 1024;
|
||||
std::atomic<std::uint64_t> temporary_counter{0};
|
||||
|
||||
class Descriptor {
|
||||
public:
|
||||
explicit Descriptor(int fd) noexcept : fd_(fd) {}
|
||||
Descriptor(const Descriptor&) = delete;
|
||||
Descriptor& operator=(const Descriptor&) = delete;
|
||||
~Descriptor() {
|
||||
if (fd_ >= 0)
|
||||
static_cast<void>(::close(fd_));
|
||||
}
|
||||
[[nodiscard]] int get() const noexcept {
|
||||
return fd_;
|
||||
}
|
||||
[[nodiscard]] int release() noexcept {
|
||||
const int fd = fd_;
|
||||
fd_ = -1;
|
||||
return fd;
|
||||
}
|
||||
// Explicit close so a deferred write error is observed.
|
||||
[[nodiscard]] bool close() noexcept {
|
||||
const int fd = release();
|
||||
return fd < 0 || ::close(fd) == 0;
|
||||
}
|
||||
|
||||
private:
|
||||
int fd_;
|
||||
};
|
||||
std::string key_name(const Sha256Digest& d) {
|
||||
constexpr char digits[] = "0123456789abcdef";
|
||||
std::string out;
|
||||
out.reserve(64);
|
||||
for (const auto b : d) {
|
||||
out.push_back(digits[b >> 4U]);
|
||||
out.push_back(digits[b & 0x0FU]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
// Opens or creates a private subdirectory without following symlinks.
|
||||
int subdirectory(int parent, const std::string& name) {
|
||||
bool created = false;
|
||||
if (::mkdirat(parent, name.c_str(), 0700) == 0)
|
||||
created = true;
|
||||
else if (errno != EEXIST)
|
||||
return -1;
|
||||
const int fd = ::openat(parent, name.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
if (fd >= 0 && created && ::fsync(parent) != 0) {
|
||||
static_cast<void>(::close(fd));
|
||||
return -1;
|
||||
}
|
||||
return fd;
|
||||
}
|
||||
bool write_all(int fd, std::span<const std::uint8_t> bytes) {
|
||||
while (!bytes.empty()) {
|
||||
const auto n = ::write(fd, bytes.data(), bytes.size());
|
||||
if (n < 0 && errno == EINTR)
|
||||
continue;
|
||||
if (n <= 0)
|
||||
return false;
|
||||
bytes = bytes.subspan(static_cast<std::size_t>(n));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
// Exact-length regular file whose content has the expected digest.
|
||||
std::expected<std::vector<std::uint8_t>, ObjectError> read_verified(int directory,
|
||||
const VerifiedObject& object) {
|
||||
const auto name = key_name(object.sha256);
|
||||
const Descriptor fd{::openat(directory, name.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC)};
|
||||
if (fd.get() < 0)
|
||||
return std::unexpected{errno == ENOENT ? ObjectError::missing
|
||||
: errno == ELOOP ? ObjectError::corrupt
|
||||
: ObjectError::io};
|
||||
struct stat info{};
|
||||
if (::fstat(fd.get(), &info) != 0)
|
||||
return std::unexpected{ObjectError::io};
|
||||
// Size is checked before any allocation.
|
||||
if (!S_ISREG(info.st_mode) || info.st_size < 0 ||
|
||||
static_cast<std::uint64_t>(info.st_size) != object.bytes)
|
||||
return std::unexpected{ObjectError::corrupt};
|
||||
std::vector<std::uint8_t> bytes(static_cast<std::size_t>(object.bytes));
|
||||
std::size_t done = 0;
|
||||
while (done < bytes.size()) {
|
||||
const auto n = ::read(fd.get(), bytes.data() + done, bytes.size() - done);
|
||||
if (n < 0 && errno == EINTR)
|
||||
continue;
|
||||
if (n < 0)
|
||||
return std::unexpected{ObjectError::io};
|
||||
if (n == 0)
|
||||
return std::unexpected{ObjectError::corrupt}; // Shrunk after fstat.
|
||||
done += static_cast<std::size_t>(n);
|
||||
}
|
||||
std::uint8_t extra{};
|
||||
if (::read(fd.get(), &extra, 1) != 0)
|
||||
return std::unexpected{ObjectError::corrupt}; // Grew after fstat.
|
||||
const auto digest = sha256(bytes);
|
||||
if (!digest)
|
||||
return std::unexpected{ObjectError::io};
|
||||
if (*digest != object.sha256)
|
||||
return std::unexpected{ObjectError::corrupt};
|
||||
return bytes;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
FilesystemObjectStore::FilesystemObjectStore(std::filesystem::path directory, int descriptor,
|
||||
std::size_t maximum)
|
||||
: directory_(std::move(directory)), descriptor_(descriptor), maximum_(maximum) {}
|
||||
|
||||
FilesystemObjectStore::~FilesystemObjectStore() {
|
||||
static_cast<void>(::close(descriptor_));
|
||||
}
|
||||
|
||||
std::expected<std::unique_ptr<FilesystemObjectStore>, ObjectError>
|
||||
FilesystemObjectStore::open(const std::filesystem::path& trusted_root, RegionKey key,
|
||||
std::size_t max_object_bytes) {
|
||||
if (key.world == 0 || key.region == 0 || max_object_bytes == 0 ||
|
||||
max_object_bytes > maximum_object_limit || trusted_root.empty())
|
||||
return std::unexpected{ObjectError::invalid};
|
||||
const Descriptor root{
|
||||
::open(trusted_root.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC)};
|
||||
if (root.get() < 0)
|
||||
return std::unexpected{ObjectError::invalid};
|
||||
const auto world = "w" + std::to_string(key.world);
|
||||
const auto region = "r" + std::to_string(key.region);
|
||||
const Descriptor world_fd{subdirectory(root.get(), world)};
|
||||
if (world_fd.get() < 0)
|
||||
return std::unexpected{ObjectError::io};
|
||||
Descriptor region_fd{subdirectory(world_fd.get(), region)};
|
||||
if (region_fd.get() < 0)
|
||||
return std::unexpected{ObjectError::io};
|
||||
return std::unique_ptr<FilesystemObjectStore>(new FilesystemObjectStore(
|
||||
trusted_root / world / region, region_fd.release(), max_object_bytes));
|
||||
}
|
||||
|
||||
std::expected<VerifiedObject, ObjectError>
|
||||
FilesystemObjectStore::put(std::span<const std::uint8_t> bytes) const {
|
||||
if (bytes.empty())
|
||||
return std::unexpected{ObjectError::invalid};
|
||||
if (bytes.size() > maximum_)
|
||||
return std::unexpected{ObjectError::bounds};
|
||||
const auto digest = sha256(bytes);
|
||||
if (!digest)
|
||||
return std::unexpected{ObjectError::io};
|
||||
const VerifiedObject object{*digest, bytes.size()};
|
||||
const auto existing = [&]() -> std::expected<VerifiedObject, ObjectError> {
|
||||
auto stored = read_verified(descriptor_, object);
|
||||
if (!stored)
|
||||
return std::unexpected{stored.error()};
|
||||
return object;
|
||||
};
|
||||
// Immutable: identical content is reused; anything else under the key is corrupt.
|
||||
if (auto found = existing(); found || found.error() != ObjectError::missing)
|
||||
return found;
|
||||
|
||||
const auto name = key_name(*digest);
|
||||
const auto temporary = ".tmp-" + name + "-" + std::to_string(::getpid()) + "-" +
|
||||
std::to_string(temporary_counter.fetch_add(1));
|
||||
Descriptor fd{::openat(descriptor_, temporary.c_str(),
|
||||
O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600)};
|
||||
if (fd.get() < 0)
|
||||
return std::unexpected{ObjectError::io};
|
||||
const auto discard = [&] { static_cast<void>(::unlinkat(descriptor_, temporary.c_str(), 0)); };
|
||||
if (!write_all(fd.get(), bytes) || ::fsync(fd.get()) != 0 || !fd.close()) {
|
||||
discard();
|
||||
return std::unexpected{ObjectError::io};
|
||||
}
|
||||
// link() never replaces an existing name, unlike rename().
|
||||
if (::linkat(descriptor_, temporary.c_str(), descriptor_, name.c_str(), 0) != 0) {
|
||||
const bool raced = errno == EEXIST;
|
||||
discard();
|
||||
if (raced)
|
||||
return existing();
|
||||
return std::unexpected{ObjectError::io};
|
||||
}
|
||||
discard();
|
||||
if (::fsync(descriptor_) != 0)
|
||||
return std::unexpected{ObjectError::io};
|
||||
return object;
|
||||
}
|
||||
|
||||
std::expected<std::vector<std::uint8_t>, ObjectError>
|
||||
FilesystemObjectStore::get(const VerifiedObject& object) const {
|
||||
if (object.bytes == 0)
|
||||
return std::unexpected{ObjectError::invalid};
|
||||
if (object.bytes > maximum_)
|
||||
return std::unexpected{ObjectError::bounds};
|
||||
return read_verified(descriptor_, object);
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
@@ -0,0 +1,220 @@
|
||||
#include "evo/persistence/region_checkpoint.hpp"
|
||||
#include <algorithm>
|
||||
#include <ranges>
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
constexpr std::uint64_t manifest_magic = 0x314D43524F564541ULL; // "AEVORCM1"
|
||||
constexpr std::uint32_t manifest_version = 1;
|
||||
constexpr std::uint64_t maximum = ~std::uint64_t{0};
|
||||
|
||||
template <std::size_t N> bool zero(const std::array<std::uint8_t, N>& bytes) {
|
||||
return std::ranges::all_of(bytes, [](std::uint8_t b) { return b == 0; });
|
||||
}
|
||||
class Writer {
|
||||
public:
|
||||
explicit Writer(std::vector<std::uint8_t>& out) : out_(out) {}
|
||||
void u(std::uint64_t value, unsigned width) {
|
||||
for (unsigned i = 0; i < width; ++i) {
|
||||
out_.push_back(static_cast<std::uint8_t>(value & 0xFFU));
|
||||
value >>= 8U;
|
||||
}
|
||||
}
|
||||
template <std::size_t N> void bytes(const std::array<std::uint8_t, N>& value) {
|
||||
out_.insert(out_.end(), value.begin(), value.end());
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<std::uint8_t>& out_;
|
||||
};
|
||||
class Reader {
|
||||
public:
|
||||
explicit Reader(std::span<const std::uint8_t> in) : in_(in) {}
|
||||
std::uint64_t u(unsigned width) {
|
||||
std::uint64_t value{};
|
||||
for (unsigned i = 0; i < width; ++i)
|
||||
value |= static_cast<std::uint64_t>(in_[at_ + i]) << (8U * i);
|
||||
at_ += width;
|
||||
return value;
|
||||
}
|
||||
template <std::size_t N> std::array<std::uint8_t, N> bytes() {
|
||||
std::array<std::uint8_t, N> value{};
|
||||
std::copy_n(in_.begin() + static_cast<std::ptrdiff_t>(at_), N, value.begin());
|
||||
at_ += N;
|
||||
return value;
|
||||
}
|
||||
|
||||
private:
|
||||
std::span<const std::uint8_t> in_;
|
||||
std::size_t at_{};
|
||||
};
|
||||
|
||||
// Every field, independent of which versions this build supports.
|
||||
bool structural(const CheckpointManifest& m) noexcept {
|
||||
if (m.key.world == 0 || m.key.region == 0 || m.frame == 0 || zero(m.commit) ||
|
||||
m.tick_microseconds == 0 || zero(m.build) || m.payload_bytes == 0 ||
|
||||
m.payload_bytes > max_environmental_payload_bytes || zero(m.payload_sha256))
|
||||
return false;
|
||||
if (m.sequence == 0)
|
||||
return zero(m.predecessor_commit) && zero(m.predecessor_digest) && m.begin_tick == 0 &&
|
||||
m.end_tick == 0 && zero(m.holder) && m.epoch == 0;
|
||||
return !zero(m.predecessor_commit) && m.predecessor_commit != m.commit &&
|
||||
!zero(m.predecessor_digest) && !zero(m.holder) && m.epoch != 0 &&
|
||||
m.end_tick > m.begin_tick && m.end_tick - m.begin_tick <= max_checkpoint_ticks;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
bool supported(const CheckpointManifest& m) noexcept {
|
||||
return m.profile == CheckpointProfile::environmental_v1 &&
|
||||
m.versions == environmental_v1_versions && m.rng_streams == 0 &&
|
||||
m.history_profile == 0 && m.durability == DurabilityPolicy::single_volume_development;
|
||||
}
|
||||
|
||||
std::expected<std::vector<std::uint8_t>, CheckpointError>
|
||||
encode_manifest(const CheckpointManifest& m) {
|
||||
if (!structural(m))
|
||||
return std::unexpected{CheckpointError::format};
|
||||
std::vector<std::uint8_t> out;
|
||||
out.reserve(checkpoint_manifest_bytes);
|
||||
Writer w{out};
|
||||
w.u(manifest_magic, 8);
|
||||
w.u(manifest_version, 4);
|
||||
w.u(static_cast<std::uint8_t>(m.profile), 4);
|
||||
w.u(m.key.world, 8);
|
||||
w.u(m.key.region, 8);
|
||||
w.u(m.frame, 8);
|
||||
w.bytes(m.commit);
|
||||
w.u(m.sequence, 8);
|
||||
w.bytes(m.predecessor_commit);
|
||||
w.bytes(m.predecessor_digest);
|
||||
w.u(m.begin_tick, 8);
|
||||
w.u(m.end_tick, 8);
|
||||
w.bytes(m.holder);
|
||||
w.u(m.epoch, 8);
|
||||
w.u(m.tick_microseconds, 8);
|
||||
for (const auto v : {m.versions.rules, m.versions.envelope, m.versions.resources,
|
||||
m.versions.fields, m.versions.terrain, m.versions.coordinate,
|
||||
m.versions.generator, m.versions.material, m.versions.interval_adapter})
|
||||
w.u(v, 4);
|
||||
w.bytes(m.build);
|
||||
w.u(m.rng_streams, 4);
|
||||
w.u(m.history_profile, 4);
|
||||
w.u(static_cast<std::uint8_t>(m.durability), 4);
|
||||
w.u(m.payload_bytes, 8);
|
||||
w.bytes(m.payload_sha256);
|
||||
return out;
|
||||
}
|
||||
|
||||
std::expected<CheckpointManifest, CheckpointError>
|
||||
decode_manifest(std::span<const std::uint8_t> bytes) {
|
||||
if (bytes.size() != checkpoint_manifest_bytes)
|
||||
return std::unexpected{CheckpointError::format};
|
||||
Reader r{bytes};
|
||||
if (r.u(8) != manifest_magic)
|
||||
return std::unexpected{CheckpointError::format};
|
||||
if (r.u(4) != manifest_version)
|
||||
return std::unexpected{CheckpointError::unsupported};
|
||||
CheckpointManifest m;
|
||||
const auto profile = r.u(4);
|
||||
m.profile =
|
||||
profile <= 0xFFU ? static_cast<CheckpointProfile>(profile) : CheckpointProfile::unknown;
|
||||
m.key.world = r.u(8);
|
||||
m.key.region = r.u(8);
|
||||
m.frame = r.u(8);
|
||||
m.commit = r.bytes<16>();
|
||||
m.sequence = r.u(8);
|
||||
m.predecessor_commit = r.bytes<16>();
|
||||
m.predecessor_digest = r.bytes<32>();
|
||||
m.begin_tick = r.u(8);
|
||||
m.end_tick = r.u(8);
|
||||
m.holder = r.bytes<16>();
|
||||
m.epoch = r.u(8);
|
||||
m.tick_microseconds = r.u(8);
|
||||
for (auto* v : {&m.versions.rules, &m.versions.envelope, &m.versions.resources,
|
||||
&m.versions.fields, &m.versions.terrain, &m.versions.coordinate,
|
||||
&m.versions.generator, &m.versions.material, &m.versions.interval_adapter})
|
||||
*v = static_cast<std::uint32_t>(r.u(4));
|
||||
m.build = r.bytes<32>();
|
||||
m.rng_streams = static_cast<std::uint32_t>(r.u(4));
|
||||
m.history_profile = static_cast<std::uint32_t>(r.u(4));
|
||||
const auto durability = r.u(4);
|
||||
m.durability =
|
||||
durability <= 0xFFU ? static_cast<DurabilityPolicy>(durability) : DurabilityPolicy::unknown;
|
||||
m.payload_bytes = r.u(8);
|
||||
m.payload_sha256 = r.bytes<32>();
|
||||
if (!structural(m))
|
||||
return std::unexpected{CheckpointError::format};
|
||||
return m;
|
||||
}
|
||||
|
||||
std::expected<CheckpointManifest, CheckpointError>
|
||||
genesis_manifest(RegionKey key, std::uint64_t frame, Uuid commit, std::uint64_t tick_microseconds,
|
||||
const Sha256Digest& build, std::uint64_t payload_bytes,
|
||||
const Sha256Digest& payload_sha256) {
|
||||
CheckpointManifest m;
|
||||
m.key = key;
|
||||
m.frame = frame;
|
||||
m.commit = commit;
|
||||
m.tick_microseconds = tick_microseconds;
|
||||
m.build = build;
|
||||
m.payload_bytes = payload_bytes;
|
||||
m.payload_sha256 = payload_sha256;
|
||||
if (!structural(m))
|
||||
return std::unexpected{CheckpointError::format};
|
||||
return m;
|
||||
}
|
||||
|
||||
std::expected<CheckpointManifest, CheckpointError>
|
||||
successor_manifest(const CheckpointManifest& p, const Sha256Digest& predecessor_digest, Uuid commit,
|
||||
const LeaseGrant& lease, std::uint64_t end_tick, const Sha256Digest& build,
|
||||
std::uint64_t payload_bytes, const Sha256Digest& payload_sha256) {
|
||||
// This build writes only its own versions; a change requires a migration (M7.9).
|
||||
if (!supported(p))
|
||||
return std::unexpected{CheckpointError::unsupported};
|
||||
if (!structural(p) || p.sequence == maximum || lease.frame != p.frame ||
|
||||
end_tick <= p.end_tick || end_tick - p.end_tick > max_checkpoint_ticks)
|
||||
return std::unexpected{CheckpointError::chain};
|
||||
CheckpointManifest m = p;
|
||||
m.commit = commit;
|
||||
m.sequence = p.sequence + 1;
|
||||
m.predecessor_commit = p.commit;
|
||||
m.predecessor_digest = predecessor_digest;
|
||||
m.begin_tick = p.end_tick;
|
||||
m.end_tick = end_tick;
|
||||
m.holder = lease.holder;
|
||||
m.epoch = lease.epoch;
|
||||
m.build = build;
|
||||
m.payload_bytes = payload_bytes;
|
||||
m.payload_sha256 = payload_sha256;
|
||||
if (!structural(m))
|
||||
return std::unexpected{CheckpointError::format};
|
||||
return m;
|
||||
}
|
||||
|
||||
std::expected<std::unique_ptr<processes::EnvironmentalCommunity>, CheckpointError>
|
||||
restore_environmental_payload(std::span<const std::uint8_t> payload, const CheckpointManifest& m,
|
||||
std::size_t budget) {
|
||||
if (!structural(m))
|
||||
return std::unexpected{CheckpointError::format};
|
||||
if (!supported(m))
|
||||
return std::unexpected{CheckpointError::unsupported};
|
||||
if (payload.size() != m.payload_bytes || payload.size() < 16)
|
||||
return std::unexpected{CheckpointError::payload};
|
||||
Reader header{payload};
|
||||
header.u(8); // Envelope magic is checked by the envelope decoder itself.
|
||||
if (header.u(8) != m.versions.envelope)
|
||||
return std::unexpected{CheckpointError::unsupported};
|
||||
auto restored =
|
||||
processes::EnvironmentalCommunity::restore(payload, {m.key.world, m.frame}, budget);
|
||||
if (!restored)
|
||||
return std::unexpected{CheckpointError::payload};
|
||||
const auto& state = **restored;
|
||||
if (state.progress().tick != m.end_tick || state.tick_microseconds() != m.tick_microseconds)
|
||||
return std::unexpected{CheckpointError::payload};
|
||||
// Canonical: the restored state re-encodes to exactly the committed bytes.
|
||||
const auto again = state.snapshot();
|
||||
if (!again || !std::ranges::equal(*again, payload))
|
||||
return std::unexpected{CheckpointError::payload};
|
||||
return std::move(*restored);
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
@@ -2,6 +2,7 @@
|
||||
#include <array>
|
||||
#include <charconv>
|
||||
#include <optional>
|
||||
#include <span>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
@@ -27,15 +28,19 @@ LeaseStoreFailure classify(const PgFailure& f) {
|
||||
kind = LeaseStoreError::not_provisioned;
|
||||
else if (state == "EVL05")
|
||||
kind = LeaseStoreError::exhausted;
|
||||
else if (state == "EVL06")
|
||||
kind = LeaseStoreError::unsupported;
|
||||
else if (state == "EVL07")
|
||||
kind = LeaseStoreError::commit_conflict;
|
||||
return {kind, f};
|
||||
}
|
||||
LeaseStoreFailure malformed(bool mutation) {
|
||||
// A mutation whose reply cannot be parsed may still have committed.
|
||||
return {mutation ? LeaseStoreError::unknown : LeaseStoreError::store, {}};
|
||||
}
|
||||
template <std::size_t N> std::string hex(const std::array<std::uint8_t, N>& bytes) {
|
||||
std::string hex(std::span<const std::uint8_t> bytes) {
|
||||
std::string out;
|
||||
out.reserve(N * 2);
|
||||
out.reserve(bytes.size() * 2);
|
||||
for (const auto b : bytes) {
|
||||
out.push_back(digits[static_cast<std::size_t>(b >> 4U)]);
|
||||
out.push_back(digits[static_cast<std::size_t>(b & 0x0FU)]);
|
||||
@@ -61,6 +66,19 @@ template <std::size_t N> std::optional<std::array<std::uint8_t, N>> unhex(std::s
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::optional<std::vector<std::uint8_t>> unhex_bytes(std::string_view s) {
|
||||
if (s.size() % 2 != 0)
|
||||
return std::nullopt;
|
||||
std::vector<std::uint8_t> out(s.size() / 2);
|
||||
for (std::size_t i = 0; i < out.size(); ++i) {
|
||||
const auto high = nibble(s[i * 2]);
|
||||
const auto low = nibble(s[(i * 2) + 1]);
|
||||
if (!high || !low)
|
||||
return std::nullopt;
|
||||
out[i] = static_cast<std::uint8_t>((*high << 4U) | *low);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::string uuid_text(const Uuid& id) {
|
||||
const auto h = hex(id);
|
||||
return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) +
|
||||
@@ -89,11 +107,12 @@ template <typename T> std::optional<T> parse_integer(const std::optional<std::st
|
||||
return std::nullopt;
|
||||
return value;
|
||||
}
|
||||
std::string bytea(const Sha256Digest& d) {
|
||||
return "\\x" + hex(d);
|
||||
std::string bytea(std::span<const std::uint8_t> b) {
|
||||
return "\\x" + hex(b);
|
||||
}
|
||||
std::optional<RegionHead> parse_head(const PgRows& rows) {
|
||||
if (rows.size() != 1 || rows[0].size() != 5)
|
||||
// Head columns: frame, sequence, commit, digest hex, end tick (+ optional extras).
|
||||
std::optional<RegionHead> parse_head(const PgRows& rows, std::size_t columns = 5) {
|
||||
if (rows.size() != 1 || rows[0].size() != columns)
|
||||
return std::nullopt;
|
||||
const auto& r = rows[0];
|
||||
const auto frame = parse_integer<std::uint64_t>(r[0]);
|
||||
@@ -177,23 +196,28 @@ std::expected<RegionHead, LeaseStoreFailure> load_region_head(Postgres& db, Regi
|
||||
return *head;
|
||||
}
|
||||
|
||||
std::expected<CommittedCheckpoint, LeaseStoreFailure>
|
||||
load_region_checkpoint(Postgres& db, RegionKey key, milliseconds timeout) {
|
||||
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
|
||||
auto rows = db.query(
|
||||
"SELECT * FROM evolution.load_region_checkpoint($1::evolution.u64, $2::evolution.u64)", p,
|
||||
timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
auto head = parse_head(*rows, 6);
|
||||
const auto& manifest_text = head ? (*rows)[0][5] : std::optional<std::string>{};
|
||||
auto manifest = manifest_text ? unhex_bytes(*manifest_text) : std::nullopt;
|
||||
if (!head || !manifest)
|
||||
return std::unexpected{malformed(false)};
|
||||
return CommittedCheckpoint{*head, std::move(*manifest)};
|
||||
}
|
||||
|
||||
std::expected<RegionHead, LeaseStoreFailure>
|
||||
advance_region_head(Postgres& db, RegionKey key, const LeaseGrant& lease,
|
||||
const RegionHead& expected, const HeadAdvance& next, milliseconds timeout) {
|
||||
const std::array<std::string, 10> p{std::to_string(key.world),
|
||||
std::to_string(key.region),
|
||||
uuid_text(lease.holder),
|
||||
std::to_string(lease.epoch),
|
||||
std::to_string(expected.sequence),
|
||||
uuid_text(expected.commit),
|
||||
bytea(expected.digest),
|
||||
uuid_text(next.commit),
|
||||
bytea(next.digest),
|
||||
std::to_string(next.end_tick)};
|
||||
auto rows = db.query("SELECT * FROM evolution.advance_region_head($1::evolution.u64, "
|
||||
"$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::evolution.u64, "
|
||||
"$6::uuid, $7::bytea, $8::uuid, $9::bytea, $10::evolution.u64)",
|
||||
p, timeout);
|
||||
publish_region_checkpoint(Postgres& db, std::span<const std::uint8_t> manifest,
|
||||
milliseconds timeout) {
|
||||
const std::array<std::string, 1> p{bytea(manifest)};
|
||||
auto rows =
|
||||
db.query("SELECT * FROM evolution.publish_region_checkpoint($1::bytea)", p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
auto head = parse_head(*rows);
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include <openssl/evp.h>
|
||||
|
||||
namespace evo::persistence {
|
||||
std::expected<Sha256Digest, DigestError> sha256(std::span<const std::uint8_t> bytes) noexcept {
|
||||
Sha256Digest out{};
|
||||
unsigned int length = 0;
|
||||
if (EVP_Digest(bytes.data(), bytes.size(), out.data(), &length, EVP_sha256(), nullptr) != 1 ||
|
||||
length != out.size())
|
||||
return std::unexpected{DigestError::provider};
|
||||
return out;
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
#pragma once
|
||||
// Rules-2 environmental community used by region checkpoint tests (M7.4).
|
||||
#include "evo/processes/environmental_community.hpp"
|
||||
#include <array>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace evo::test {
|
||||
inline constexpr world::WorldIdentity region_identity{7, 9};
|
||||
|
||||
inline void require(bool ok, const char* what) {
|
||||
if (!ok)
|
||||
throw std::runtime_error(what);
|
||||
}
|
||||
inline processes::EnvironmentalConfig region_config() {
|
||||
processes::EnvironmentalConfig c;
|
||||
c.rules = 2;
|
||||
c.world = region_identity.world;
|
||||
c.frame = region_identity.frame;
|
||||
c.producer = 1;
|
||||
c.consumer = 2;
|
||||
c.source = 3;
|
||||
c.tick_microseconds = 50000;
|
||||
c.area_um2 = 1;
|
||||
c.irradiance_numerator = 11;
|
||||
c.irradiance_denominator = 3;
|
||||
c.efficiency = 600;
|
||||
c.growth_numerator = 3;
|
||||
c.growth_denominator = 2;
|
||||
c.growth_cost = 1;
|
||||
c.recipe = {1, 2, 1, 1, 1};
|
||||
c.intake.fill(100);
|
||||
c.assimilation = 1000;
|
||||
c.producer_min_mK = c.consumer_min_mK = 270000;
|
||||
c.producer_max_mK = c.consumer_max_mK = 310000;
|
||||
c.decay.fill(1);
|
||||
return c;
|
||||
}
|
||||
inline std::unique_ptr<processes::EnvironmentalCommunity> region_community() {
|
||||
using namespace world;
|
||||
constexpr auto maximum = std::numeric_limits<std::uint64_t>::max();
|
||||
std::array<std::uint64_t, 10> environment{200, 6, 6, 400, 0, 0, 200, 0, 2000, 0};
|
||||
const FieldSite site{{0, 0, -16000}, {{0}, {0}, {0}, {10}, {20}, {3}}, {1}};
|
||||
std::array<FieldOpening, 10> opening{};
|
||||
for (std::size_t k = 0; k < 10; ++k)
|
||||
opening[k] = {static_cast<Field>(k), {&environment[k], 1}};
|
||||
const FieldDefaults defaults{
|
||||
{accounting::Femtograms{0}, accounting::Femtograms{maximum}, accounting::Femtograms{0}}};
|
||||
auto f = FieldStore::create(region_identity, 1, defaults, {&site, 1}, opening, 2000000);
|
||||
require(f.has_value(), "field store");
|
||||
auto t = ChunkStore::create({}, 1, 2000000, {{VolumeUnits{100}, VolumeUnits{100}}});
|
||||
require(t.has_value(), "chunk store");
|
||||
require((*t)->load({0, 0, -1000}).has_value(), "chunk load");
|
||||
std::array<OrganismResources, 2> accounts{};
|
||||
accounts[0].id = {1};
|
||||
accounts[1].id = {2};
|
||||
accounts[0].capacities.fill(100000000);
|
||||
accounts[1].capacities.fill(100000000);
|
||||
auto r = EnvironmentalTransfers::create(region_identity, std::move(*f), std::move(*t), accounts,
|
||||
4000000);
|
||||
require(r.has_value(), "transfers");
|
||||
auto o = processes::EnvironmentalCommunity::create(region_config(), std::move(*r), 8000000);
|
||||
require(o.has_value(), "community");
|
||||
return std::move(*o);
|
||||
}
|
||||
// Deterministic input for tick t; the consumer dies at tick 20 when requested.
|
||||
inline processes::EnvironmentalInput region_input(std::uint64_t tick, bool consumer_dies = false) {
|
||||
processes::EnvironmentalInput in;
|
||||
in.temperature_mK = 290000;
|
||||
in.source_enabled = tick % 7 == 0 ? 0 : 1;
|
||||
if (tick == 16)
|
||||
in.bite = {2, 2, 2, 2, 1};
|
||||
if (consumer_dies && tick == 20)
|
||||
in.terminal[1] = 1;
|
||||
return in;
|
||||
}
|
||||
inline void advance_region(processes::EnvironmentalCommunity& o, std::uint64_t ticks) {
|
||||
for (std::uint64_t i = 0; i < ticks; ++i) {
|
||||
const auto tick = o.progress().tick + 1;
|
||||
require(o.advance(tick, region_input(tick)).has_value(), "advance");
|
||||
}
|
||||
}
|
||||
} // namespace evo::test
|
||||
@@ -4,6 +4,7 @@
|
||||
#include <source_location>
|
||||
#include <stdexcept>
|
||||
void run_region_lease_store_tests();
|
||||
void run_region_checkpoint_store_tests();
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
@@ -72,6 +73,8 @@ int main() {
|
||||
std::puts("postgres adapter integration passed");
|
||||
run_region_lease_store_tests();
|
||||
std::puts("region lease fencing integration passed");
|
||||
run_region_checkpoint_store_tests();
|
||||
std::puts("region checkpoint publication integration passed");
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
std::fprintf(stderr, "%s\n", e.what());
|
||||
|
||||
@@ -0,0 +1,357 @@
|
||||
// Real PostgreSQL + filesystem M7.4 checkpoint publication/recovery (ADR-0061).
|
||||
#include "evo/persistence/checkpoint_store.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include "fixtures/environmental_region.hpp"
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <source_location>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
void run_region_checkpoint_store_tests();
|
||||
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
namespace fs = std::filesystem;
|
||||
void check(bool value, std::source_location at = std::source_location::current()) {
|
||||
if (!value)
|
||||
throw std::runtime_error("region checkpoint store test line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
std::string env(const char* key) {
|
||||
const auto* value = std::getenv(key);
|
||||
if (!value)
|
||||
throw std::runtime_error("missing test environment");
|
||||
return value;
|
||||
}
|
||||
std::unique_ptr<Postgres> connect_as(const std::string& user) {
|
||||
PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
|
||||
auto made = Postgres::connect(options, 5s);
|
||||
CHECK(made);
|
||||
return std::move(*made);
|
||||
}
|
||||
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
|
||||
std::array<std::uint8_t, N> out{};
|
||||
out.fill(value);
|
||||
return out;
|
||||
}
|
||||
std::string hex(std::span<const std::uint8_t> bytes) {
|
||||
constexpr char digits[] = "0123456789abcdef";
|
||||
std::string out;
|
||||
for (const auto b : bytes) {
|
||||
out.push_back(digits[b >> 4U]);
|
||||
out.push_back(digits[b & 0x0FU]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::vector<std::uint8_t> bytes_of(std::string_view s) {
|
||||
return {s.begin(), s.end()};
|
||||
}
|
||||
// Private scratch root; the CI script runs each binary under its own /tmp dir.
|
||||
fs::path scratch(const char* name) {
|
||||
const auto root = fs::path(env("PGHOST")) / name;
|
||||
fs::remove_all(root);
|
||||
fs::create_directory(root);
|
||||
fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace);
|
||||
return root;
|
||||
}
|
||||
const Sha256Digest build = filled<32>(0xB1);
|
||||
constexpr std::size_t budget = 8000000;
|
||||
|
||||
void sha256_vectors_and_database_agreement() {
|
||||
// FIPS 180-2 / NIST example vectors.
|
||||
CHECK(hex(*sha256({})) == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
|
||||
CHECK(hex(*sha256(bytes_of("abc"))) ==
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
|
||||
CHECK(hex(*sha256(bytes_of("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"))) ==
|
||||
"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
|
||||
// Independent implementation: PostgreSQL computes the commit digest.
|
||||
auto db = connect_as(env("PGUSER"));
|
||||
std::vector<std::uint8_t> data(2000); // Hex parameter stays within the 4KiB adapter bound.
|
||||
for (std::size_t i = 0; i < data.size(); ++i)
|
||||
data[i] = static_cast<std::uint8_t>((i * 131U) ^ (i >> 3U));
|
||||
const std::array<std::string, 1> p{"\\x" + hex(data)};
|
||||
auto rows = db->query("SELECT encode(sha256($1::bytea), 'hex')", p, 5s);
|
||||
CHECK(rows && (*rows)[0][0] == hex(*sha256(data)));
|
||||
}
|
||||
|
||||
void object_store_is_immutable_and_verified() {
|
||||
const auto root = scratch("objects");
|
||||
CHECK(!FilesystemObjectStore::open(root / "absent", {1, 1}, 1024));
|
||||
CHECK(!FilesystemObjectStore::open(root, {0, 1}, 1024));
|
||||
fs::create_directory_symlink(root, root.parent_path() / "objects-link");
|
||||
CHECK(!FilesystemObjectStore::open(root.parent_path() / "objects-link", {1, 1}, 1024));
|
||||
fs::remove(root.parent_path() / "objects-link");
|
||||
auto store = FilesystemObjectStore::open(root, {1, 2}, 1024);
|
||||
CHECK(store);
|
||||
const auto dir = (*store)->directory();
|
||||
CHECK(dir == root / "w1" / "r2");
|
||||
CHECK((fs::status(dir).permissions() & fs::perms::all) == fs::perms::owner_all);
|
||||
|
||||
const auto payload = bytes_of("immutable payload bytes");
|
||||
auto put = (*store)->put(payload);
|
||||
CHECK(put && put->bytes == payload.size() && put->sha256 == *sha256(payload));
|
||||
auto again = (*store)->put(payload); // Idempotent for identical bytes.
|
||||
CHECK(again && *again == *put);
|
||||
auto got = (*store)->get(*put);
|
||||
CHECK(got && *got == payload);
|
||||
const auto file = dir / hex(put->sha256);
|
||||
CHECK((fs::status(file).permissions() & fs::perms::all) == fs::perms::owner_read ||
|
||||
(fs::status(file).permissions() & fs::perms::all) ==
|
||||
(fs::perms::owner_read | fs::perms::owner_write));
|
||||
|
||||
CHECK((*store)->put({}).error() == ObjectError::invalid);
|
||||
CHECK((*store)->put(std::vector<std::uint8_t>(1025)).error() == ObjectError::bounds);
|
||||
CHECK((*store)->get({put->sha256, 1025}).error() == ObjectError::bounds);
|
||||
CHECK((*store)->get({filled<32>(7), 5}).error() == ObjectError::missing);
|
||||
CHECK((*store)->get({put->sha256, put->bytes - 1}).error() == ObjectError::corrupt);
|
||||
|
||||
// A crash mid-upload leaves only a temporary name no key can resolve.
|
||||
std::ofstream(dir / (".tmp-" + hex(filled<32>(9)) + "-1-0")) << "partial";
|
||||
CHECK((*store)->get({filled<32>(9), 7}).error() == ObjectError::missing);
|
||||
|
||||
// Corruption is detected, and never repaired by overwriting.
|
||||
{
|
||||
std::fstream f(file, std::ios::in | std::ios::out | std::ios::binary);
|
||||
f.seekp(3);
|
||||
f.put('X');
|
||||
}
|
||||
CHECK((*store)->get(*put).error() == ObjectError::corrupt);
|
||||
CHECK((*store)->put(payload).error() == ObjectError::corrupt);
|
||||
fs::remove(file);
|
||||
fs::create_symlink(dir / "elsewhere", file);
|
||||
CHECK((*store)->get(*put).error() == ObjectError::corrupt);
|
||||
fs::remove(file);
|
||||
CHECK((*store)->put(payload) && (*store)->get(*put));
|
||||
fs::resize_file(file, payload.size() + 1);
|
||||
CHECK((*store)->get(*put).error() == ObjectError::corrupt);
|
||||
fs::remove(file);
|
||||
|
||||
if (::geteuid() != 0) { // Permission faults cannot be simulated as root.
|
||||
fs::permissions(dir, fs::perms::owner_read | fs::perms::owner_exec,
|
||||
fs::perm_options::replace);
|
||||
CHECK((*store)->put(payload).error() == ObjectError::io);
|
||||
fs::permissions(dir, fs::perms::owner_all, fs::perm_options::replace);
|
||||
}
|
||||
}
|
||||
|
||||
struct Region {
|
||||
RegionKey key{7, 3};
|
||||
std::unique_ptr<Postgres> owner, runtime;
|
||||
std::unique_ptr<FilesystemObjectStore> objects;
|
||||
};
|
||||
Region provisioned(const char* name, std::uint64_t region, std::uint8_t genesis_seed) {
|
||||
Region r;
|
||||
r.key.region = region;
|
||||
r.owner = connect_as(env("PGUSER"));
|
||||
r.runtime = connect_as(env("EVO_PG_RUNTIME_USER"));
|
||||
auto objects =
|
||||
FilesystemObjectStore::open(scratch(name), r.key, max_environmental_payload_bytes);
|
||||
CHECK(objects);
|
||||
r.objects = std::move(*objects);
|
||||
CHECK(r.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
||||
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
||||
{}, 5s));
|
||||
auto opening = evo::test::region_community();
|
||||
auto genesis = provision_environmental_region(*r.owner, *r.objects, r.key,
|
||||
evo::test::region_identity.frame, *opening,
|
||||
filled<16>(genesis_seed), build, budget, 5s);
|
||||
CHECK(genesis && genesis->head.sequence == 0);
|
||||
return r;
|
||||
}
|
||||
std::size_t count_objects(const fs::path& dir) {
|
||||
std::size_t n = 0;
|
||||
for (const auto& entry : fs::directory_iterator(dir))
|
||||
n += entry.path().filename().string().starts_with(".tmp-") ? 0 : 1;
|
||||
return n;
|
||||
}
|
||||
|
||||
void publish_recover_and_continue() {
|
||||
auto r = provisioned("publish", 3, 0xC0);
|
||||
const std::array<Sha256Digest, 1> retained{build};
|
||||
auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s);
|
||||
CHECK(lease);
|
||||
// Genesis recovers to the exact opening state.
|
||||
auto recovered =
|
||||
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
||||
CHECK(recovered && recovered->state->progress().tick == 0);
|
||||
auto head = recovered->committed;
|
||||
|
||||
// Genesis cannot be re-provisioned or published over.
|
||||
auto opening = evo::test::region_community();
|
||||
CHECK(!provision_environmental_region(*r.owner, *r.objects, r.key, 9, *opening,
|
||||
filled<16>(0xC9), build, budget, 5s));
|
||||
|
||||
auto original = evo::test::region_community();
|
||||
std::uint8_t seed = 0xC1;
|
||||
for (int batch = 0; batch < 3; ++batch) {
|
||||
evo::test::advance_region(*original, batch == 1 ? 64 : 16);
|
||||
const auto started = std::chrono::steady_clock::now();
|
||||
auto published = publish_environmental_checkpoint(
|
||||
*r.runtime, *r.objects, *lease, head, *original, filled<16>(seed++), build, budget, 5s);
|
||||
const auto us = std::chrono::duration_cast<std::chrono::microseconds>(
|
||||
std::chrono::steady_clock::now() - started)
|
||||
.count();
|
||||
CHECK(published && published->head.sequence == head.head.sequence + 1);
|
||||
CHECK(published->head.end_tick == original->progress().tick);
|
||||
std::printf(
|
||||
"checkpoint_publish_us batch=%d ticks=%llu payload_bytes=%llu us=%lld\n", batch,
|
||||
static_cast<unsigned long long>(published->head.end_tick - head.head.end_tick),
|
||||
static_cast<unsigned long long>(decode_manifest(published->manifest)->payload_bytes),
|
||||
static_cast<long long>(us));
|
||||
head = std::move(*published);
|
||||
}
|
||||
// 65 ticks is more than one batch: refused before upload.
|
||||
const auto before = count_objects(r.objects->directory());
|
||||
evo::test::advance_region(*original, 65);
|
||||
auto oversized = publish_environmental_checkpoint(
|
||||
*r.runtime, *r.objects, *lease, head, *original, filled<16>(0xCA), build, budget, 5s);
|
||||
CHECK(!oversized && oversized.error().kind == PublicationError::candidate);
|
||||
CHECK(count_objects(r.objects->directory()) == before);
|
||||
|
||||
// A new process recovers the committed head; its trajectory matches the original's.
|
||||
auto second = FilesystemObjectStore::open(r.objects->directory().parent_path().parent_path(),
|
||||
r.key, max_environmental_payload_bytes);
|
||||
CHECK(second);
|
||||
auto restored =
|
||||
recover_environmental_checkpoint(*r.runtime, **second, r.key, retained, budget, 5s);
|
||||
CHECK(restored && restored->committed.head == head.head);
|
||||
CHECK(restored->state->progress().tick == 96);
|
||||
auto reference = evo::test::region_community();
|
||||
evo::test::advance_region(*reference, 96);
|
||||
CHECK(reference->snapshot() == restored->state->snapshot());
|
||||
for (std::uint64_t tick = 97; tick <= 140; ++tick) {
|
||||
auto a = reference->advance(tick, evo::test::region_input(tick, true));
|
||||
auto b = restored->state->advance(tick, evo::test::region_input(tick, true));
|
||||
CHECK(a && b && *a == *b);
|
||||
}
|
||||
CHECK(reference->snapshot() == restored->state->snapshot());
|
||||
|
||||
// Retained-build allowlist: a host without this build cannot restore it.
|
||||
const std::array<Sha256Digest, 1> other{filled<32>(0xB2)};
|
||||
auto foreign =
|
||||
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, other, budget, 5s);
|
||||
CHECK(!foreign && foreign.error().kind == RecoveryError::unsupported);
|
||||
// Unretained build is refused by the database; the upload stays an orphan.
|
||||
auto next = evo::test::region_community();
|
||||
evo::test::advance_region(*next, 100);
|
||||
auto refused = publish_environmental_checkpoint(*r.runtime, *r.objects, *lease, head, *next,
|
||||
filled<16>(0xCB), filled<32>(0xB2), budget, 5s);
|
||||
CHECK(!refused && refused.error().kind == PublicationError::rejected &&
|
||||
refused.error().store.kind == LeaseStoreError::unsupported);
|
||||
CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s));
|
||||
}
|
||||
|
||||
void failures_leave_the_previous_head() {
|
||||
auto r = provisioned("failures", 4, 0xD0);
|
||||
const std::array<Sha256Digest, 1> retained{build};
|
||||
auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s);
|
||||
CHECK(lease);
|
||||
auto head = load_region_checkpoint(*r.runtime, r.key, 5s);
|
||||
CHECK(head);
|
||||
auto state = evo::test::region_community();
|
||||
evo::test::advance_region(*state, 8);
|
||||
auto first = publish_environmental_checkpoint(*r.runtime, *r.objects, *lease, *head, *state,
|
||||
filled<16>(0xD1), build, budget, 5s);
|
||||
CHECK(first);
|
||||
const auto manifest = *decode_manifest(first->manifest);
|
||||
const auto object = r.objects->directory() / hex(manifest.payload_sha256);
|
||||
|
||||
// Upload failure: no manifest is sent; the head does not move.
|
||||
evo::test::advance_region(*state, 8);
|
||||
if (::geteuid() != 0) {
|
||||
fs::permissions(r.objects->directory(), fs::perms::owner_read | fs::perms::owner_exec,
|
||||
fs::perm_options::replace);
|
||||
auto upload = publish_environmental_checkpoint(*r.runtime, *r.objects, *lease, *first,
|
||||
*state, filled<16>(0xD2), build, budget, 5s);
|
||||
fs::permissions(r.objects->directory(), fs::perms::owner_all, fs::perm_options::replace);
|
||||
CHECK(!upload && upload.error().kind == PublicationError::upload);
|
||||
CHECK(load_region_head(*r.runtime, r.key, 5s)->sequence == 1);
|
||||
}
|
||||
|
||||
// Manifest transaction failure after a verified upload: fenced writer, orphan retained.
|
||||
auto stale = *lease;
|
||||
stale.holder[0] ^= 1U;
|
||||
auto fenced = publish_environmental_checkpoint(*r.runtime, *r.objects, stale, *first, *state,
|
||||
filled<16>(0xD3), build, budget, 5s);
|
||||
CHECK(!fenced && fenced.error().kind == PublicationError::rejected &&
|
||||
fenced.error().store.kind == LeaseStoreError::fenced);
|
||||
auto orphan_bytes = state->snapshot();
|
||||
auto orphan = r.objects->get({*sha256(*orphan_bytes), orphan_bytes->size()});
|
||||
CHECK(orphan && *orphan == *orphan_bytes); // Uploaded, verified, unreferenced.
|
||||
auto recovered =
|
||||
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
||||
CHECK(recovered && recovered->state->progress().tick == 8); // Never the orphan.
|
||||
|
||||
// Missing and corrupt committed payloads quarantine; no older head is selected.
|
||||
const auto saved = r.objects->get({manifest.payload_sha256, manifest.payload_bytes});
|
||||
CHECK(saved);
|
||||
fs::remove(object);
|
||||
auto missing =
|
||||
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
||||
CHECK(!missing && missing.error().kind == RecoveryError::missing_payload);
|
||||
CHECK(r.objects->put(*saved));
|
||||
fs::permissions(object, fs::perms::owner_all, fs::perm_options::replace);
|
||||
{
|
||||
std::fstream f(object, std::ios::in | std::ios::out | std::ios::binary);
|
||||
f.seekp(100);
|
||||
f.put(static_cast<char>((*saved)[100] ^ 1U));
|
||||
}
|
||||
auto corrupt =
|
||||
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
||||
CHECK(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload);
|
||||
fs::remove(object);
|
||||
CHECK(r.objects->put(*saved));
|
||||
CHECK(recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s));
|
||||
|
||||
// The database refuses tampering with committed manifests even by the owner.
|
||||
auto tamper = r.owner->query("UPDATE evolution.region_commits SET manifest = "
|
||||
"overlay(manifest placing '\\x02'::bytea from 121 for 1) "
|
||||
"WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1",
|
||||
{}, 5s);
|
||||
CHECK(!tamper && std::string(tamper.error().sqlstate.data()) == "23514");
|
||||
auto repoint = r.owner->query("UPDATE evolution.region_heads SET digest = sha256('\\x00') "
|
||||
"WHERE world_id = 7 AND region_id = 4",
|
||||
{}, 5s);
|
||||
CHECK(!repoint && std::string(repoint.error().sqlstate.data()) == "23503");
|
||||
// Defense in depth: even if that constraint were lost, recovery re-digests the
|
||||
// manifest and quarantines. Restores the constraint before continuing.
|
||||
CHECK(r.owner->query("ALTER TABLE evolution.region_commits DROP CONSTRAINT "
|
||||
"region_commits_manifest_digest",
|
||||
{}, 5s));
|
||||
// Flip a payload-digest byte: only the manifest digest check can notice it.
|
||||
CHECK(r.owner->query("UPDATE evolution.region_commits SET manifest = "
|
||||
"set_byte(manifest, 248, get_byte(manifest, 248) # 1) "
|
||||
"WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1",
|
||||
{}, 5s));
|
||||
auto tampered =
|
||||
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
||||
CHECK(!tampered && tampered.error().kind == RecoveryError::manifest);
|
||||
const std::array<std::string, 1> original{"\\x" + hex(first->manifest)};
|
||||
CHECK(r.owner->query("UPDATE evolution.region_commits SET manifest = $1::bytea "
|
||||
"WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1",
|
||||
original, 5s));
|
||||
CHECK(r.owner->query("ALTER TABLE evolution.region_commits ADD CONSTRAINT "
|
||||
"region_commits_manifest_digest CHECK (commit_digest = sha256(manifest))",
|
||||
{}, 5s));
|
||||
CHECK(recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s));
|
||||
|
||||
// Recovery does not create genesis for an unknown region.
|
||||
auto none =
|
||||
recover_environmental_checkpoint(*r.runtime, *r.objects, {7, 99}, retained, budget, 5s);
|
||||
CHECK(!none && none.error().kind == RecoveryError::not_provisioned);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void run_region_checkpoint_store_tests() {
|
||||
sha256_vectors_and_database_agreement();
|
||||
object_store_is_immutable_and_verified();
|
||||
publish_recover_and_continue();
|
||||
failures_leave_the_previous_head();
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
#include "evo/persistence/region_checkpoint.hpp"
|
||||
#include "fixtures/environmental_region.hpp"
|
||||
|
||||
#include <cstdio>
|
||||
#include <source_location>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using evo::test::advance_region;
|
||||
using evo::test::region_community;
|
||||
using evo::test::region_input;
|
||||
void check(bool ok, std::source_location at = std::source_location::current()) {
|
||||
if (!ok)
|
||||
throw std::runtime_error("region checkpoint line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
|
||||
std::array<std::uint8_t, N> out{};
|
||||
out.fill(value);
|
||||
return out;
|
||||
}
|
||||
constexpr RegionKey key{7, 3};
|
||||
const auto build = filled<32>(0xB1);
|
||||
LeaseGrant lease() {
|
||||
return {filled<16>(0x11), 4, 9, std::chrono::milliseconds{20000}};
|
||||
}
|
||||
CheckpointManifest genesis(std::uint64_t payload_bytes = 100) {
|
||||
auto g =
|
||||
genesis_manifest(key, 9, filled<16>(0x01), 50000, build, payload_bytes, filled<32>(0xA0));
|
||||
CHECK(g);
|
||||
return *g;
|
||||
}
|
||||
CheckpointManifest next(std::uint64_t end_tick = 16) {
|
||||
auto n = successor_manifest(genesis(), filled<32>(0xD0), filled<16>(0x02), lease(), end_tick,
|
||||
build, 200, filled<32>(0xA1));
|
||||
CHECK(n);
|
||||
return *n;
|
||||
}
|
||||
std::uint64_t at(const std::vector<std::uint8_t>& b, std::size_t offset, unsigned width) {
|
||||
std::uint64_t value{};
|
||||
for (unsigned i = 0; i < width; ++i)
|
||||
value |= static_cast<std::uint64_t>(b[offset + i]) << (8U * i);
|
||||
return value;
|
||||
}
|
||||
|
||||
void layout_and_round_trip() {
|
||||
const auto m = next();
|
||||
auto bytes = encode_manifest(m);
|
||||
CHECK(bytes && bytes->size() == checkpoint_manifest_bytes);
|
||||
// Golden offsets from schemas/region-commit.md.
|
||||
CHECK(std::string_view(reinterpret_cast<const char*>(bytes->data()), 8) == "AEVORCM1");
|
||||
CHECK(at(*bytes, 8, 4) == 1 && at(*bytes, 12, 4) == 1);
|
||||
CHECK(at(*bytes, 16, 8) == 7 && at(*bytes, 24, 8) == 3 && at(*bytes, 32, 8) == 9);
|
||||
CHECK((*bytes)[40] == 0x02 && at(*bytes, 56, 8) == 1 && (*bytes)[64] == 0x01);
|
||||
CHECK((*bytes)[80] == 0xD0 && at(*bytes, 112, 8) == 0 && at(*bytes, 120, 8) == 16);
|
||||
CHECK((*bytes)[128] == 0x11 && at(*bytes, 144, 8) == 4 && at(*bytes, 152, 8) == 50000);
|
||||
CHECK(at(*bytes, 160, 4) == 2 && at(*bytes, 164, 4) == 2 && at(*bytes, 192, 4) == 1);
|
||||
CHECK((*bytes)[196] == 0xB1 && at(*bytes, 228, 4) == 0 && at(*bytes, 232, 4) == 0);
|
||||
CHECK(at(*bytes, 236, 4) == 1 && at(*bytes, 240, 8) == 200 && (*bytes)[248] == 0xA1);
|
||||
auto decoded = decode_manifest(*bytes);
|
||||
CHECK(decoded && *decoded == m && supported(*decoded));
|
||||
auto g = encode_manifest(genesis());
|
||||
CHECK(g && decode_manifest(*g) == genesis());
|
||||
}
|
||||
|
||||
void malformed_and_unsupported() {
|
||||
auto bytes = *encode_manifest(next());
|
||||
CHECK(decode_manifest(std::span{bytes}.first(279)).error() == CheckpointError::format);
|
||||
auto grown = bytes;
|
||||
grown.push_back(0);
|
||||
CHECK(decode_manifest(grown).error() == CheckpointError::format);
|
||||
auto magic = bytes;
|
||||
magic[0] ^= 1U;
|
||||
CHECK(decode_manifest(magic).error() == CheckpointError::format);
|
||||
auto version = bytes;
|
||||
version[8] = 2;
|
||||
CHECK(decode_manifest(version).error() == CheckpointError::unsupported);
|
||||
// Structurally invalid: zero world, 65-tick batch, oversized payload, zero digest.
|
||||
for (const auto& [offset, value] :
|
||||
{std::pair{16, 0}, std::pair{120, 65}, std::pair{242, 0x11}, std::pair{144, 0}}) {
|
||||
auto b = bytes;
|
||||
b[static_cast<std::size_t>(offset)] = static_cast<std::uint8_t>(value);
|
||||
if (offset == 144)
|
||||
std::fill_n(b.begin() + 144, 8, std::uint8_t{0}); // Epoch zero off genesis.
|
||||
CHECK(decode_manifest(b).error() == CheckpointError::format);
|
||||
}
|
||||
auto sha = bytes;
|
||||
std::fill_n(sha.begin() + 248, 32, std::uint8_t{0});
|
||||
CHECK(decode_manifest(sha).error() == CheckpointError::format);
|
||||
// Parsable but not admitted by this build: decode succeeds, supported() refuses.
|
||||
// 13/237 make profile/durability 257: never truncated onto a supported value.
|
||||
for (const std::size_t offset : {12U, 13U, 160U, 164U, 192U, 228U, 232U, 236U, 237U}) {
|
||||
auto b = bytes;
|
||||
b[offset] = static_cast<std::uint8_t>(b[offset] + 1);
|
||||
auto d = decode_manifest(b);
|
||||
CHECK(d && !supported(*d));
|
||||
}
|
||||
auto g = genesis();
|
||||
g.holder = filled<16>(1);
|
||||
CHECK(encode_manifest(g).error() == CheckpointError::format);
|
||||
g = genesis();
|
||||
g.end_tick = 1;
|
||||
CHECK(encode_manifest(g).error() == CheckpointError::format);
|
||||
CHECK(
|
||||
!genesis_manifest(key, 9, filled<16>(1), 50000, build, 1024 * 1024 + 1, filled<32>(0xA0)));
|
||||
CHECK(!genesis_manifest(key, 0, filled<16>(1), 50000, build, 1, filled<32>(0xA0)));
|
||||
}
|
||||
|
||||
void successor_chain() {
|
||||
const auto g = genesis();
|
||||
const auto d = filled<32>(0xD0);
|
||||
const auto payload = filled<32>(0xA1);
|
||||
CHECK(successor_manifest(g, d, filled<16>(2), lease(), 0, build, 1, payload).error() ==
|
||||
CheckpointError::chain);
|
||||
CHECK(successor_manifest(g, d, filled<16>(2), lease(), 65, build, 1, payload).error() ==
|
||||
CheckpointError::chain);
|
||||
auto foreign = lease();
|
||||
foreign.frame = 8;
|
||||
CHECK(successor_manifest(g, d, filled<16>(2), foreign, 1, build, 1, payload).error() ==
|
||||
CheckpointError::chain);
|
||||
CHECK(successor_manifest(g, d, g.commit, lease(), 1, build, 1, payload).error() ==
|
||||
CheckpointError::format); // Reused commit identity.
|
||||
auto old = g;
|
||||
old.versions.rules = 1;
|
||||
CHECK(successor_manifest(old, d, filled<16>(2), lease(), 1, build, 1, payload).error() ==
|
||||
CheckpointError::unsupported);
|
||||
auto last = next();
|
||||
last.sequence = ~std::uint64_t{0};
|
||||
CHECK(successor_manifest(last, d, filled<16>(3), lease(), 17, build, 1, payload).error() ==
|
||||
CheckpointError::chain);
|
||||
const auto n = next(64);
|
||||
CHECK(n.sequence == 1 && n.predecessor_commit == g.commit && n.predecessor_digest == d);
|
||||
CHECK(n.begin_tick == 0 && n.holder == lease().holder && n.epoch == 4 && n.frame == 9);
|
||||
}
|
||||
|
||||
void payload_validation_and_restored_trajectory() {
|
||||
const RegionKey region{evo::test::region_identity.world, 3};
|
||||
auto original = region_community();
|
||||
auto opening = original->snapshot();
|
||||
CHECK(opening);
|
||||
auto g = genesis_manifest(region, evo::test::region_identity.frame, filled<16>(1), 50000, build,
|
||||
opening->size(), filled<32>(0xA0));
|
||||
CHECK(g && restore_environmental_payload(*opening, *g, 8000000));
|
||||
|
||||
advance_region(*original, 16);
|
||||
auto payload = original->snapshot();
|
||||
CHECK(payload);
|
||||
auto lease_at_frame = lease();
|
||||
lease_at_frame.frame = evo::test::region_identity.frame;
|
||||
auto m = successor_manifest(*g, filled<32>(0xD0), filled<16>(2), lease_at_frame, 16, build,
|
||||
payload->size(), filled<32>(0xA1));
|
||||
CHECK(m);
|
||||
auto restored = restore_environmental_payload(*payload, *m, 8000000);
|
||||
CHECK(restored);
|
||||
// The restored checkpoint continues exactly like the uninterrupted original.
|
||||
for (std::uint64_t tick = 17; tick <= 40; ++tick) {
|
||||
auto a = original->advance(tick, region_input(tick, true));
|
||||
auto b = (*restored)->advance(tick, region_input(tick, true));
|
||||
CHECK(a && b && *a == *b);
|
||||
}
|
||||
CHECK(original->snapshot() == (*restored)->snapshot());
|
||||
|
||||
const auto refused = [&](auto change, CheckpointError error) {
|
||||
auto bytes = *payload;
|
||||
auto manifest = *m;
|
||||
change(bytes, manifest);
|
||||
auto r = restore_environmental_payload(bytes, manifest, 8000000);
|
||||
return !r && r.error() == error;
|
||||
};
|
||||
CHECK(refused([](auto&, auto& x) { x.end_tick = 15; }, CheckpointError::payload));
|
||||
CHECK(refused([](auto&, auto& x) { x.key.world = 8; }, CheckpointError::payload));
|
||||
CHECK(refused([](auto&, auto& x) { x.frame = 10; }, CheckpointError::payload));
|
||||
CHECK(refused([](auto&, auto& x) { x.tick_microseconds = 25000; }, CheckpointError::payload));
|
||||
CHECK(refused([](auto&, auto& x) { --x.payload_bytes; }, CheckpointError::payload));
|
||||
CHECK(refused([](auto& b, auto&) { b[b.size() / 2] ^= 1U; }, CheckpointError::payload));
|
||||
CHECK(refused([](auto&, auto& x) { x.versions.rules = 1; }, CheckpointError::unsupported));
|
||||
CHECK(refused([](auto&, auto& x) { x.rng_streams = 1; }, CheckpointError::unsupported));
|
||||
CHECK(refused([](auto&, auto& x) { x.history_profile = 1; }, CheckpointError::unsupported));
|
||||
CHECK(refused([](auto& b, auto&) { b[8] = 1; }, CheckpointError::unsupported));
|
||||
CHECK(!restore_environmental_payload(*payload, *m, 1000)); // Budget still charged.
|
||||
}
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
try {
|
||||
layout_and_round_trip();
|
||||
malformed_and_unsupported();
|
||||
successor_chain();
|
||||
payload_validation_and_restored_trajectory();
|
||||
std::puts("region checkpoint contracts passed");
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
std::fprintf(stderr, "%s\n", e.what());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
// Real PostgreSQL M7.3 fencing checks (ADR-0060). Run by tools/ci/postgres-check.sh.
|
||||
// Real PostgreSQL M7.3 fencing checks (ADR-0060), publishing M7.4 manifests
|
||||
// (ADR-0061) with synthetic payload digests. Run by tools/ci/postgres-check.sh.
|
||||
#include "evo/persistence/region_checkpoint.hpp"
|
||||
#include "evo/persistence/region_lease_store.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
@@ -8,6 +10,7 @@
|
||||
#include <latch>
|
||||
#include <memory>
|
||||
#include <source_location>
|
||||
#include <span>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
@@ -52,29 +55,76 @@ Sha256Digest digest(std::uint8_t seed) {
|
||||
out.fill(seed);
|
||||
return out;
|
||||
}
|
||||
std::string hex_text(std::uint8_t seed, std::size_t bytes) {
|
||||
const Sha256Digest build = digest(0xB1);
|
||||
void retain_build(Postgres& db) {
|
||||
CHECK(db.query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
||||
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
||||
{}, 5s));
|
||||
}
|
||||
std::string bytea(std::span<const std::uint8_t> bytes) {
|
||||
constexpr char digits[] = "0123456789abcdef";
|
||||
std::string out;
|
||||
for (std::size_t i = 0; i < bytes; ++i) {
|
||||
out.push_back(digits[seed >> 4U]);
|
||||
out.push_back(digits[seed & 0x0FU]);
|
||||
std::string out = "\\x";
|
||||
for (const auto b : bytes) {
|
||||
out.push_back(digits[b >> 4U]);
|
||||
out.push_back(digits[b & 0x0FU]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::string uuid_literal(std::uint8_t seed) {
|
||||
const auto h = hex_text(seed, 16);
|
||||
return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) +
|
||||
'-' + h.substr(20);
|
||||
}
|
||||
// Genesis commit/digest bytes are the region number; frame is 7.
|
||||
// Genesis commit ID bytes are the region number; frame is 7. Payload digests are
|
||||
// synthetic: the database records them and never reads the object store.
|
||||
RegionKey provision(Postgres& db, std::uint64_t region) {
|
||||
const auto seed = static_cast<std::uint8_t>(region);
|
||||
const std::array<std::string, 4> p{std::to_string(region), uuid_literal(seed),
|
||||
"\\x" + hex_text(seed, 32), "7"};
|
||||
CHECK(db.query("SELECT evolution.provision_region(1, $1::evolution.u64, $4::evolution.u64, "
|
||||
"$2::uuid, $3::bytea)",
|
||||
p, 5s));
|
||||
return {1, region};
|
||||
retain_build(db);
|
||||
const RegionKey key{1, region};
|
||||
auto g = genesis_manifest(key, 7, id(static_cast<std::uint8_t>(region)), 50000, build, 100,
|
||||
digest(0xA0));
|
||||
CHECK(g);
|
||||
auto bytes = encode_manifest(*g);
|
||||
CHECK(bytes);
|
||||
const std::array<std::string, 1> p{bytea(*bytes)};
|
||||
CHECK(db.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s));
|
||||
return key;
|
||||
}
|
||||
CommittedCheckpoint committed(Postgres& db, RegionKey key) {
|
||||
auto c = load_region_checkpoint(db, key, 5s);
|
||||
CHECK(c);
|
||||
return std::move(*c);
|
||||
}
|
||||
// Successor fields set directly, so tests can send links the local builder refuses.
|
||||
CheckpointManifest link(const CommittedCheckpoint& from, const LeaseGrant& lease,
|
||||
std::uint8_t commit, std::uint64_t end_tick) {
|
||||
auto m = decode_manifest(from.manifest);
|
||||
CHECK(m);
|
||||
m->commit = id(commit);
|
||||
m->sequence = from.head.sequence + 1;
|
||||
m->predecessor_commit = from.head.commit;
|
||||
m->predecessor_digest = from.head.digest;
|
||||
m->begin_tick = from.head.end_tick;
|
||||
m->end_tick = end_tick;
|
||||
m->holder = lease.holder;
|
||||
m->epoch = lease.epoch;
|
||||
m->payload_sha256 = digest(0xA1);
|
||||
return *m;
|
||||
}
|
||||
std::expected<CommittedCheckpoint, LeaseStoreFailure> publish(Postgres& db,
|
||||
std::vector<std::uint8_t> bytes) {
|
||||
auto head = publish_region_checkpoint(db, bytes, 5s);
|
||||
if (!head)
|
||||
return std::unexpected{head.error()};
|
||||
return CommittedCheckpoint{*head, std::move(bytes)};
|
||||
}
|
||||
std::expected<CommittedCheckpoint, LeaseStoreFailure> publish(Postgres& db,
|
||||
const CheckpointManifest& m) {
|
||||
auto bytes = encode_manifest(m);
|
||||
CHECK(bytes);
|
||||
return publish(db, std::move(*bytes));
|
||||
}
|
||||
// Encodes a valid link, then overwrites raw bytes the encoder would refuse.
|
||||
std::vector<std::uint8_t> patched(const CheckpointManifest& m, std::size_t offset,
|
||||
std::span<const std::uint8_t> value) {
|
||||
auto bytes = encode_manifest(m);
|
||||
CHECK(bytes);
|
||||
std::ranges::copy(value, bytes->begin() + static_cast<std::ptrdiff_t>(offset));
|
||||
return std::move(*bytes);
|
||||
}
|
||||
bool failed(const auto& result, LeaseStoreError kind) {
|
||||
return !result && result.error().kind == kind;
|
||||
@@ -94,7 +144,10 @@ void runtime_cannot_bypass_the_guard() {
|
||||
"INSERT INTO evolution.region_leases(world_id,region_id,frame_id) VALUES (9,9,9)",
|
||||
"SELECT * FROM evolution.region_leases",
|
||||
"SELECT * FROM evolution.region_heads",
|
||||
"SELECT evolution.provision_region(1, 11, 7, gen_random_uuid(), '\\x00')",
|
||||
"SELECT evolution.provision_region_checkpoint('\\x00')",
|
||||
"SELECT * FROM evolution.region_commits",
|
||||
"INSERT INTO evolution.retained_builds VALUES ('\\x00')",
|
||||
"SELECT evolution.parse_checkpoint_manifest('\\x00')",
|
||||
"SELECT evolution.check_region_lease(1, 10, gen_random_uuid(), 1)",
|
||||
"CREATE FUNCTION evolution.bypass() RETURNS int LANGUAGE sql AS 'SELECT 1'",
|
||||
"CREATE TABLE evolution.bypass(x int)",
|
||||
@@ -110,7 +163,7 @@ void runtime_cannot_bypass_the_guard() {
|
||||
CHECK(release_region_lease(*db, key, *lease, 5s));
|
||||
}
|
||||
|
||||
void acquire_renew_release_and_advance() {
|
||||
void acquire_renew_release_and_publish() {
|
||||
auto admin = owner();
|
||||
const auto key = provision(*admin, 20);
|
||||
auto db = runtime();
|
||||
@@ -130,39 +183,50 @@ void acquire_renew_release_and_advance() {
|
||||
stale_epoch.epoch = 2;
|
||||
CHECK(failed(renew_region_lease(*db, key, stale_epoch, 20000ms, 5s), LeaseStoreError::fenced));
|
||||
|
||||
auto genesis = load_region_head(*db, key, 5s);
|
||||
CHECK(genesis && genesis->sequence == 0 && genesis->end_tick == 0 && genesis->frame == 7);
|
||||
CHECK(genesis->commit == id(20) && genesis->digest == digest(20));
|
||||
CHECK(failed(advance_region_head(*db, key, *lease, *genesis, {id(21), digest(21), 65}, 5s),
|
||||
const auto genesis = committed(*db, key);
|
||||
CHECK(genesis.head.sequence == 0 && genesis.head.end_tick == 0 && genesis.head.frame == 7);
|
||||
CHECK(genesis.head.commit == id(20) && genesis.head.digest != Sha256Digest{});
|
||||
const std::array<std::uint8_t, 1> tick65{65};
|
||||
CHECK(failed(publish(*db, patched(link(genesis, *lease, 21, 64), 120, tick65)),
|
||||
LeaseStoreError::invalid));
|
||||
CHECK(failed(advance_region_head(*db, key, *lease, *genesis, {id(20), digest(21), 1}, 5s),
|
||||
const auto reused = id(20);
|
||||
CHECK(failed(publish(*db, patched(link(genesis, *lease, 21, 1), 40, reused)),
|
||||
LeaseStoreError::invalid));
|
||||
const HeadAdvance first{id(21), digest(21), 64};
|
||||
auto head = advance_region_head(*db, key, *lease, *genesis, first, 5s);
|
||||
CHECK(head && head->sequence == 1 && head->end_tick == 64 && head->commit == id(21));
|
||||
CHECK(failed(advance_region_head(*db, key, forged, *head, {id(22), digest(22), 65}, 5s),
|
||||
LeaseStoreError::fenced));
|
||||
// Exact retry after an applied advance: mismatch, then the head proves it committed.
|
||||
CHECK(failed(advance_region_head(*db, key, *lease, *genesis, first, 5s),
|
||||
LeaseStoreError::head_mismatch));
|
||||
auto current = load_region_head(*db, key, 5s);
|
||||
CHECK(current && *current == *head);
|
||||
auto changed = *head;
|
||||
changed.digest[31] ^= 1U;
|
||||
CHECK(failed(advance_region_head(*db, key, *lease, changed, {id(22), digest(22), 65}, 5s),
|
||||
LeaseStoreError::head_mismatch));
|
||||
CHECK(failed(advance_region_head(*db, key, *lease, *head, {id(22), digest(22), 64}, 5s),
|
||||
const auto first = link(genesis, *lease, 21, 64);
|
||||
auto head = publish(*db, first);
|
||||
CHECK(head && head->head.sequence == 1 && head->head.end_tick == 64 &&
|
||||
head->head.commit == id(21));
|
||||
CHECK(failed(publish(*db, link(*head, forged, 22, 65)), LeaseStoreError::fenced));
|
||||
// Exact retry after an applied publication: mismatch, then the head proves it.
|
||||
CHECK(failed(publish(*db, first), LeaseStoreError::head_mismatch));
|
||||
auto current = committed(*db, key);
|
||||
CHECK(current.head == head->head && current.manifest == head->manifest);
|
||||
auto changed = link(*head, *lease, 22, 65);
|
||||
changed.predecessor_digest[31] ^= 1U;
|
||||
CHECK(failed(publish(*db, changed), LeaseStoreError::head_mismatch));
|
||||
const std::array<std::uint8_t, 1> tick64{64};
|
||||
CHECK(failed(publish(*db, patched(link(*head, *lease, 22, 65), 120, tick64)),
|
||||
LeaseStoreError::invalid));
|
||||
auto other_quantum = link(*head, *lease, 22, 65);
|
||||
other_quantum.tick_microseconds = 25000; // Fixed per region; changing it is a migration.
|
||||
CHECK(failed(publish(*db, other_quantum), LeaseStoreError::unsupported));
|
||||
auto unretained = link(*head, *lease, 22, 65);
|
||||
unretained.build = digest(0xB2);
|
||||
CHECK(failed(publish(*db, unretained), LeaseStoreError::unsupported));
|
||||
auto future = link(*head, *lease, 22, 65);
|
||||
future.versions.rules = 3;
|
||||
CHECK(failed(publish(*db, future), LeaseStoreError::unsupported));
|
||||
// Commit IDs are globally unique: region 10's genesis ID cannot be reused here.
|
||||
CHECK(failed(publish(*db, link(*head, *lease, 10, 65)), LeaseStoreError::commit_conflict));
|
||||
|
||||
CHECK(release_region_lease(*db, key, *lease, 5s));
|
||||
CHECK(failed(release_region_lease(*db, key, *lease, 5s), LeaseStoreError::fenced));
|
||||
CHECK(failed(renew_region_lease(*db, key, *lease, 20000ms, 5s), LeaseStoreError::fenced));
|
||||
CHECK(failed(advance_region_head(*db, key, *lease, *head, {id(22), digest(22), 65}, 5s),
|
||||
LeaseStoreError::fenced));
|
||||
CHECK(failed(publish(*db, link(*head, *lease, 22, 65)), LeaseStoreError::fenced));
|
||||
auto next = acquire_region_lease(*db, key, 20000ms, 5s);
|
||||
CHECK(next && next->epoch == 2 && next->holder != lease->holder);
|
||||
auto final_head = advance_region_head(*db, key, *next, *head, {id(22), digest(22), 65}, 5s);
|
||||
CHECK(final_head && final_head->sequence == 2);
|
||||
auto final_head = publish(*db, link(*head, *next, 22, 65));
|
||||
CHECK(final_head && final_head->head.sequence == 2);
|
||||
}
|
||||
|
||||
void database_time_expiry_and_paused_writer() {
|
||||
@@ -176,29 +240,23 @@ void database_time_expiry_and_paused_writer() {
|
||||
auto lease_a = acquire_region_lease(*worker_a, key, fast.duration, 5s);
|
||||
CHECK(lease_a && a.acquired(*lease_a, sent, LeaseClock::now()));
|
||||
CHECK(failed(acquire_region_lease(*worker_b, key, 20000ms, 5s), LeaseStoreError::busy));
|
||||
auto genesis = load_region_head(*worker_a, key, 5s);
|
||||
CHECK(genesis);
|
||||
const auto genesis = committed(*worker_a, key);
|
||||
|
||||
std::this_thread::sleep_for(1200ms); // Worker A is paused past its lease.
|
||||
CHECK(a.state(LeaseClock::now()) == LeaseState::fenced);
|
||||
// Expiry is decided by database time; the successor acquires a newer epoch.
|
||||
auto lease_b = acquire_region_lease(*worker_b, key, 20000ms, 5s);
|
||||
CHECK(lease_b && lease_b->epoch == lease_a->epoch + 1);
|
||||
auto head_b =
|
||||
advance_region_head(*worker_b, key, *lease_b, *genesis, {id(31), digest(31), 3}, 5s);
|
||||
auto head_b = publish(*worker_b, link(genesis, *lease_b, 31, 3));
|
||||
CHECK(head_b);
|
||||
// A resumes and ignores its local fence: every database path rejects it.
|
||||
CHECK(failed(renew_region_lease(*worker_a, key, *lease_a, fast.duration, 5s),
|
||||
LeaseStoreError::fenced));
|
||||
CHECK(
|
||||
failed(advance_region_head(*worker_a, key, *lease_a, *genesis, {id(32), digest(32), 1}, 5s),
|
||||
LeaseStoreError::fenced));
|
||||
CHECK(
|
||||
failed(advance_region_head(*worker_a, key, *lease_a, *head_b, {id(32), digest(32), 4}, 5s),
|
||||
LeaseStoreError::fenced));
|
||||
CHECK(failed(publish(*worker_a, link(genesis, *lease_a, 32, 1)), LeaseStoreError::fenced));
|
||||
CHECK(failed(publish(*worker_a, link(*head_b, *lease_a, 32, 4)), LeaseStoreError::fenced));
|
||||
CHECK(failed(release_region_lease(*worker_a, key, *lease_a, 5s), LeaseStoreError::fenced));
|
||||
auto head = load_region_head(*worker_a, key, 5s);
|
||||
CHECK(head && *head == *head_b);
|
||||
CHECK(head && *head == head_b->head);
|
||||
|
||||
// Expired renewal cannot revive a lease even without a successor.
|
||||
CHECK(release_region_lease(*worker_b, key, *lease_b, 5s));
|
||||
@@ -216,24 +274,23 @@ void queued_writer_uses_post_lock_database_time() {
|
||||
const auto key = provision(*admin, 40);
|
||||
auto worker = runtime();
|
||||
auto lease = acquire_region_lease(*worker, key, 1000ms, 5s);
|
||||
auto genesis = load_region_head(*worker, key, 5s);
|
||||
CHECK(lease && genesis);
|
||||
CHECK(lease);
|
||||
const auto genesis = committed(*worker, key);
|
||||
auto blocker = owner();
|
||||
const std::array<std::string, 1> region{"40"};
|
||||
CHECK(blocker->query("BEGIN", {}, 5s));
|
||||
CHECK(blocker->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 1 AND "
|
||||
"region_id = $1::evolution.u64 FOR UPDATE",
|
||||
region, 5s));
|
||||
std::expected<RegionHead, LeaseStoreFailure> queued = std::unexpected{LeaseStoreFailure{}};
|
||||
std::thread writer([&] {
|
||||
queued = advance_region_head(*worker, key, *lease, *genesis, {id(41), digest(41), 1}, 5s);
|
||||
});
|
||||
std::expected<CommittedCheckpoint, LeaseStoreFailure> queued =
|
||||
std::unexpected{LeaseStoreFailure{}};
|
||||
std::thread writer([&] { queued = publish(*worker, link(genesis, *lease, 41, 1)); });
|
||||
std::this_thread::sleep_for(1300ms);
|
||||
CHECK(blocker->query("ROLLBACK", {}, 5s));
|
||||
writer.join();
|
||||
CHECK(failed(queued, LeaseStoreError::fenced));
|
||||
auto head = load_region_head(*worker, key, 5s);
|
||||
CHECK(head && *head == *genesis);
|
||||
CHECK(head && *head == genesis.head);
|
||||
}
|
||||
|
||||
// Partition: the renewal's outcome is unknown and the connection is gone. The
|
||||
@@ -300,10 +357,9 @@ void contenders_serialize_on_the_database() {
|
||||
// The same holder racing two commits from the same predecessor: exactly one wins.
|
||||
auto lease = acquire_region_lease(*workers[0], key, 20000ms, 5s);
|
||||
CHECK(lease);
|
||||
auto head = load_region_head(*workers[0], key, 5s);
|
||||
CHECK(head);
|
||||
auto head = committed(*workers[0], key);
|
||||
for (std::uint8_t round = 0; round < 32; ++round) {
|
||||
std::array<std::expected<RegionHead, LeaseStoreFailure>, 2> results{
|
||||
std::array<std::expected<CommittedCheckpoint, LeaseStoreFailure>, 2> results{
|
||||
std::unexpected{LeaseStoreFailure{}}, std::unexpected{LeaseStoreFailure{}}};
|
||||
std::latch start{2};
|
||||
std::vector<std::thread> threads;
|
||||
@@ -312,18 +368,17 @@ void contenders_serialize_on_the_database() {
|
||||
threads.emplace_back([&, i] {
|
||||
const auto seed = static_cast<std::uint8_t>(100 + (round * 2) + i);
|
||||
start.arrive_and_wait();
|
||||
results[i] = advance_region_head(*workers[i], key, *lease, *head,
|
||||
{id(seed), digest(seed), head->end_tick + 1}, 5s);
|
||||
results[i] = publish(*workers[i], link(head, *lease, seed, head.head.end_tick + 1));
|
||||
});
|
||||
for (auto& t : threads)
|
||||
t.join();
|
||||
const std::size_t w = results[0] ? 0 : 1;
|
||||
CHECK(results[w] && failed(results[1 - w], LeaseStoreError::head_mismatch));
|
||||
CHECK(results[w]->sequence == head->sequence + 1);
|
||||
CHECK(results[w]->head.sequence == head.head.sequence + 1);
|
||||
head = *results[w];
|
||||
}
|
||||
auto committed = load_region_head(*workers[1], key, 5s);
|
||||
CHECK(committed && *committed == *head && committed->sequence == 32);
|
||||
auto last = load_region_head(*workers[1], key, 5s);
|
||||
CHECK(last && *last == head.head && last->sequence == 32);
|
||||
}
|
||||
|
||||
void exhaustion_fails_closed() {
|
||||
@@ -337,13 +392,24 @@ void exhaustion_fails_closed() {
|
||||
const auto other = provision(*admin, 71);
|
||||
auto lease = acquire_region_lease(*db, other, 20000ms, 5s);
|
||||
CHECK(lease);
|
||||
// Owner-only fixture: a chain link at the last sequence (the head FK requires it).
|
||||
CHECK(admin->query(
|
||||
"INSERT INTO evolution.region_commits(world_id, region_id, commit_sequence, commit_id, "
|
||||
"commit_digest, manifest, begin_tick, end_tick, published_epoch, tick_microseconds, "
|
||||
"versions, build_identity, payload_bytes, payload_sha256) SELECT world_id, region_id, "
|
||||
"18446744073709551615, '000000ff-0000-0000-0000-000000000000', commit_digest, manifest, 0, "
|
||||
"5, 1, tick_microseconds, versions, build_identity, payload_bytes, payload_sha256 FROM "
|
||||
"evolution.region_commits WHERE world_id = 1 AND region_id = 71",
|
||||
{}, 5s));
|
||||
CHECK(admin->query("UPDATE evolution.region_heads SET commit_sequence = 18446744073709551615, "
|
||||
"end_tick = 5, published_epoch = 1 WHERE world_id = 1 AND region_id = 71",
|
||||
"commit_id = '000000ff-0000-0000-0000-000000000000', end_tick = 5, "
|
||||
"published_epoch = 1 WHERE world_id = 1 AND region_id = 71",
|
||||
{}, 5s));
|
||||
auto head = load_region_head(*db, other, 5s);
|
||||
CHECK(head && head->sequence == 18446744073709551615ULL);
|
||||
CHECK(failed(advance_region_head(*db, other, *lease, *head, {id(72), digest(72), 6}, 5s),
|
||||
LeaseStoreError::exhausted));
|
||||
const auto last = committed(*db, other);
|
||||
CHECK(last.head.sequence == 18446744073709551615ULL);
|
||||
auto beyond = link(last, *lease, 72, 6);
|
||||
beyond.sequence = 18446744073709551615ULL; // Any value: exhaustion is checked first.
|
||||
CHECK(failed(publish(*db, beyond), LeaseStoreError::exhausted));
|
||||
}
|
||||
|
||||
struct Summary {
|
||||
@@ -372,8 +438,7 @@ void measure_round_trips() {
|
||||
auto db = runtime();
|
||||
constexpr int n = 200;
|
||||
Summary acquire, renew, advance, release;
|
||||
auto head = load_region_head(*db, key, 5s);
|
||||
CHECK(head);
|
||||
auto head = committed(*db, key);
|
||||
for (int i = 0; i < n; ++i) {
|
||||
std::expected<LeaseGrant, LeaseStoreFailure> lease = std::unexpected{LeaseStoreFailure{}};
|
||||
acquire.samples.push_back(
|
||||
@@ -381,29 +446,27 @@ void measure_round_trips() {
|
||||
CHECK(lease);
|
||||
renew.samples.push_back(
|
||||
timed([&] { CHECK(renew_region_lease(*db, key, *lease, 20000ms, 5s)); }));
|
||||
const auto seed = static_cast<std::uint8_t>(i);
|
||||
Uuid commit = id(seed);
|
||||
commit[0] = 0xAA; // Distinct from genesis 0x50 bytes.
|
||||
commit[1] = static_cast<std::uint8_t>(i >> 8);
|
||||
auto m = link(head, *lease, 0, head.head.end_tick + 64);
|
||||
m.commit[0] = 0xAA; // Unique across all tests: 0xAA, index, zeros.
|
||||
m.commit[1] = static_cast<std::uint8_t>(i);
|
||||
advance.samples.push_back(timed([&] {
|
||||
auto next = advance_region_head(*db, key, *lease, *head,
|
||||
{commit, digest(seed), head->end_tick + 64}, 5s);
|
||||
auto next = publish(*db, m);
|
||||
CHECK(next);
|
||||
head = *next;
|
||||
head = std::move(*next);
|
||||
}));
|
||||
release.samples.push_back(
|
||||
timed([&] { CHECK(release_region_lease(*db, key, *lease, 5s)); }));
|
||||
}
|
||||
acquire.print("acquire");
|
||||
renew.print("renew");
|
||||
advance.print("advance_head");
|
||||
advance.print("publish_manifest");
|
||||
release.print("release");
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void run_region_lease_store_tests() {
|
||||
runtime_cannot_bypass_the_guard();
|
||||
acquire_renew_release_and_advance();
|
||||
acquire_renew_release_and_publish();
|
||||
database_time_expiry_and_paused_writer();
|
||||
queued_writer_uses_post_lock_database_time();
|
||||
partitioned_renewal_fences_locally();
|
||||
|
||||
@@ -37,13 +37,23 @@ if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE=no_such_evo_role; \
|
||||
echo 'Expected missing runtime role rejection' >&2; exit 1
|
||||
fi
|
||||
test "$(PGDATABASE=migration_rollback psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname='evolution'")" = 0
|
||||
# Supported upgrade path: an existing exact v1 database migrates to v2.
|
||||
# Supported upgrade path: an existing exact v1 database migrates to the latest.
|
||||
createdb upgrade_path
|
||||
PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" \
|
||||
-f "$root/migrations/0001_region_bootstrap.sql" > /dev/null
|
||||
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1
|
||||
PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh"
|
||||
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2
|
||||
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3
|
||||
# v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2.
|
||||
v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")"
|
||||
v2_sha="${v2_sha%% *}"
|
||||
createdb v2_heads
|
||||
PGDATABASE=v2_heads psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -v "migration_sha_2=$v2_sha" -v runtime_role=evo_runtime_test -f "$root/migrations/0001_region_bootstrap.sql" -f "$root/migrations/0002_region_fencing.sql" > /dev/null
|
||||
PGDATABASE=v2_heads psql -X -v ON_ERROR_STOP=1 -c "SELECT evolution.provision_region(1, 1, 1, gen_random_uuid(), decode(repeat('00', 32), 'hex'))" > /dev/null
|
||||
if PGDATABASE=v2_heads bash "$root/tools/migrate-postgres.sh"; then
|
||||
echo 'Expected v2 heads to block v3' >&2; exit 1
|
||||
fi
|
||||
test "$(PGDATABASE=v2_heads psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2
|
||||
# Inherited privileges bypass fencing, so every migrate run audits effective rights.
|
||||
psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_bad_runtime NOLOGIN IN ROLE pg_write_all_data'
|
||||
if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE=evo_bad_runtime; \
|
||||
@@ -72,13 +82,17 @@ expect_audit_failure 'GRANT UPDATE ON evolution.region_heads TO PUBLIC' \
|
||||
expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.check_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64) TO evo_runtime_test' \
|
||||
'REVOKE EXECUTE ON FUNCTION evolution.check_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64) FROM evo_runtime_test'
|
||||
expect_audit_failure 'ALTER ROLE evo_runtime_login CREATEDB' 'ALTER ROLE evo_runtime_login NOCREATEDB'
|
||||
expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.provision_region_checkpoint(bytea) TO evo_runtime_test' 'REVOKE EXECUTE ON FUNCTION evolution.provision_region_checkpoint(bytea) FROM evo_runtime_test'
|
||||
expect_audit_failure 'GRANT SELECT ON evolution.region_commits TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commits FROM evo_runtime_test'
|
||||
bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again.
|
||||
# Unknown future versions and checksum drift must not be repaired silently.
|
||||
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=3 WHERE version=2'
|
||||
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=4 WHERE version=3'
|
||||
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=3 WHERE version=4"
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=3"
|
||||
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=2 WHERE version=3"
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=2"
|
||||
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64)"
|
||||
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
||||
echo 'postgres migrations, rollback, constraints, permissions and fencing passed'
|
||||
echo 'postgres migrations, rollback, constraints, permissions, fencing and checkpoints passed'
|
||||
|
||||
@@ -16,6 +16,9 @@ digest="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
|
||||
digest="${digest%% *}"
|
||||
digest_2="$(sha256sum "$root/migrations/0002_region_fencing.sql")"
|
||||
digest_2="${digest_2%% *}"
|
||||
digest_3="$(sha256sum "$root/migrations/0003_region_checkpoints.sql")"
|
||||
digest_3="${digest_3%% *}"
|
||||
psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \
|
||||
--set="migration_sha_2=$digest_2" --set="runtime_role=$runtime_role" \
|
||||
--set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \
|
||||
--set="runtime_role=$runtime_role" \
|
||||
--file="$root/migrations/apply.sql"
|
||||
|
||||
Reference in New Issue
Block a user