diff --git a/CMakeLists.txt b/CMakeLists.txt index 5657cf2..35c4d3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,7 +126,8 @@ if(EVO_BUILD_POSTGRES) add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp - tests/region_ops_tests.cpp tests/region_clone_tests.cpp) + tests/region_ops_tests.cpp tests/region_clone_tests.cpp + tests/region_migration_tests.cpp) target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads) add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests. evo_owned_target(evo-postgres-tests) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 77cd22c..df41bd1 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -593,3 +593,11 @@ M7.9a implemented and locally qualified (2026-09-29), under ADR-0067: - a full-chain retention audit is added. Next: M7.9b, covering the upgrade barrier and old-writer fencing, migration failure while serving, and measured recovery and batching latency. + +M7.9b implemented and locally qualified (2026-09-29), under ADR-0068: +- build-bound authority, and an upgrade barrier that retires a build and fences + old writers; +- a failed migration leaves the previous schema exactly, while writers continue; +- rule and format changes are refused explicitly; +- measured recovery (19.6 s against a 60 s target) and batching latency. +M7 is complete locally. Next: M8.1, the host inventory and budgets. diff --git a/docs/STATUS.md b/docs/STATUS.md index fff97c7..8be3b7d 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -928,3 +928,29 @@ configurations; the PostgreSQL suites passed in three. Next: M7.9b, covering the upgrade barrier and old-writer fencing, schema migration failure while a host is serving, and measured recovery and batching latency. + +## M7.9b upgrade barrier and durability qualification (2026-09-29) + +ADR-0068 and schema v8 bind authority to builds: +- Acquisition names the host's build, and only a retained build that has not been + retired may hold a lease. +- No commit may name a retired build. +- `upgrade_barrier` retires a build and fences every holder in one transaction. +- Hosts built before M7.9b keep working until the first barrier. + +A lease-holding writer survived a failed v8 migration (the database stayed at v7 +exactly) and then the real migration. After the barrier it could never regain +authority. The new build recovered the whole history, including links written by +the retired build. + +Qualification (engineering/DURABILITY_QUALIFICATION.md), under the production +lease: +- SIGKILL to successor readiness: 19.5–19.6 s against the proposed 60 s target; +- acknowledgement latency with a 1 s flush: p50 549 ms, p99 1048 ms; +- a death was acknowledged in 44 ms; +- no acknowledged input was lost. + +**M7 exit criteria are met locally.** Container, CI and cluster execution have not +run. Host-loss durability is M8. + +Next: M8.1, the host inventory and budgets. diff --git a/docs/adr/0068-upgrade-barrier.md b/docs/adr/0068-upgrade-barrier.md new file mode 100644 index 0000000..73fe639 --- /dev/null +++ b/docs/adr/0068-upgrade-barrier.md @@ -0,0 +1,96 @@ +# ADR-0068: Upgrade barrier, build-bound authority and durability qualification + +## Status +Accepted for M7.9b, 2026-09-29. It completes M7.9 with ADR-0067 (clones). It +changes the ADR-0060 acquisition contract (schema v8). Rules and format +conversions remain unimplemented: no converter exists, so every change is +refused explicitly. + +## Context +M7.9 requires: +- explicit rule and format rejection or migration; +- evidence for a migration failure and for fencing old writers; +- measured recovery and batching latency. + +PERSISTENCE_ARCHITECTURE's upgrade path is: barrier, clone, migrate, compare, +approve, then a compatible deployment at a world barrier. That implies: +- A new build must be able to take over a region. +- Every writer of the old build must be stopped for good, including one that is + paused, idle, or restarted from an old image. +- History written by the old build must stay recoverable. + +Before v8, any retained build could publish, and acquisition did not name a +build. A fenced old host could therefore re-acquire an idle region and renew it +forever. + +## Decision +**Build-bound authority (schema v8).** +- `retained_builds.retired_at` marks a build as retired from publication. The + row, and so the recoverability of its history, stays. +- A `region_commits` insert trigger refuses any new commit that names a retired + build (EVL06). This covers publications, provisioned geneses and clone + geneses. +- `acquire_region_lease(world, region, duration, build)` grants authority only to + a retained, unretired build (EVL06 otherwise). The C++ API takes the build. The + host quarantines on `unsupported` (exit 4): a retired image never serves or + holds an idle lease. +- The build-less 3-argument form remains for pre-v8 hosts during a rolling + schema upgrade. It refuses once any build has been retired. +- Both forms wrap the internal `grant_region_lease`, which the runtime cannot + execute. The runtime audit now covers twelve functions. + +**Barrier.** The owner-only `upgrade_barrier(build)` retires one build and clears +every current lease holder in the database, in one transaction: +- Existing holders are fenced at their next renewal or publication. +- They cannot re-acquire by either form. +- A commit already waiting on a lease row is judged after the barrier commits. +- Retiring the last current build is refused. +- Epochs are kept, so the next acquisition takes a fresh one. + +Operator order: +1. Migrate the schema. Old hosts keep their leases, because renew and publish are + unchanged. +2. Retain the new build. +3. Deploy hosts with `--build new --retained old`. +4. Run the barrier for the old build. + +**Explicit rejection.** A region never changes profile, versions, quantum or +history profile (EVL06 at publication; `unsupported` at recovery). A clone +inherits them unchanged. No rules or format converter is registered. Adding one +requires a new ADR, a versioned conversion that is replayed and compared on a +clone, and approval. Retirement is irreversible through this interface. Undoing +it is an audited owner break-glass action, valid only while formats and rules are +identical. + +**Qualification fixture.** The fixture is: +- the production lease policy (20 s lease, 5 s renewal, 2 s margin) and a 1 s flush; +- one lineage region paced at 50 ms per tick; +- a local socket and filesystem, on WSL2. + +It measures: +- acknowledgement latency; +- the time from SIGKILL to the successor's readiness, against the proposed 60 s + target (asserted for this fixture only); +- head verification, full audit and clone cost. + +The results are in engineering/DURABILITY_QUALIFICATION.md. + +## Consequences +- Recovery after losing a worker is dominated by the lease: a successor waits out + the dead holder's remaining lease, up to 20 s, and verification adds + milliseconds. Shortening the lease trades that time against false fencing + under database latency (ADR-0060). +- Migrations run in one transaction under an advisory lock and a 5 s lock timeout. + A publication waiting on migration locks may become ambiguous; resolution + handles it (ADR-0063). A failed migration leaves the previous schema exactly. +- Binaries built before M7.9b (3-argument acquisition) work against v8 until the + first barrier. Binaries from M7.9b onwards require schema v8. + +## Alternatives rejected +- **Retirement only, checked at publication.** An idle old host could still hold + a lease forever. +- **Bumping epochs without retirement.** Old hosts would simply re-acquire. +- **Dropping the 3-argument acquisition in v8.** Every old host would be stranded + mid-upgrade before the new build was deployed. +- **In-place rules migration of a live region.** It would change physical laws + without the clone-and-compare approval that the vision requires. diff --git a/docs/adr/README.md b/docs/adr/README.md index aa42952..5039cde 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -78,3 +78,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0065: Recovery before readiness](0065-recovery-before-readiness.md) - accepted for M7.7 head tail replay verification, schema v6 commit lookup and real kill-boundary tests. - [ADR-0066: Operations endpoints](0066-operations-endpoints.md) - accepted for M7.8 liveness vs authority readiness, bounded metrics, dependency-free responder and the server image. - [ADR-0067: Region clones and retained-history audit](0067-region-clones.md) - accepted for M7.9a re-identified clone genesis naming its source cut, isolated clone databases and full-chain retention audit. +- [ADR-0068: Upgrade barrier and durability qualification](0068-upgrade-barrier.md) - accepted for M7.9b build-bound authority, retirement barrier fencing old writers, explicit refusal of rule/format changes and the measured recovery fixture. diff --git a/docs/architecture/REGION_DURABILITY.md b/docs/architecture/REGION_DURABILITY.md index e82a68f..d537b27 100644 --- a/docs/architecture/REGION_DURABILITY.md +++ b/docs/architecture/REGION_DURABILITY.md @@ -235,3 +235,20 @@ database: It names the exact link whose evidence is missing, even though head recovery still succeeds. Migration, fencing of old writers, and measured recovery and batching latency are still M7.9b. + +M7.9b (ADR-0068) closes the clone/migration row. +- **Failed migration while serving.** A v8 migration that failed at its last + statement left v7 exactly, and the lease-holding writer kept publishing. The + real migration then committed under that same lease. +- **Old writers.** The upgrade barrier retires the old build and fences every + holder. After it: + - the old writer cannot publish, renew or re-acquire, by either acquisition + form; + - no link naming the retired build can commit, even under a current lease; + - a host of the old image quarantines; + - the new build recovered the whole history and continued. +- **Rejection.** Rule, format and profile changes are refused; no converter + exists. +- **Qualification.** Real SIGKILLs under the production lease were recovered in + 19.5–19.6 s against the proposed 60 s target, with no acknowledged input lost + (see engineering/DURABILITY_QUALIFICATION.md). diff --git a/docs/engineering/DURABILITY_QUALIFICATION.md b/docs/engineering/DURABILITY_QUALIFICATION.md new file mode 100644 index 0000000..0878af4 --- /dev/null +++ b/docs/engineering/DURABILITY_QUALIFICATION.md @@ -0,0 +1,75 @@ +# M7 durability qualification (M7.9b) + +Measured on 2026-09-29 by `durability_qualification` in +`tests/region_migration_tests.cpp`, under `tools/ci/postgres-check.sh`. These are +local development measurements under the fixture below. They are not a +production service level, and they are not evidence about host loss (M8). + +## Fixture +- **Machine:** WSL2 Ubuntu on the maintainer's Windows 11 workstation. +- **Database:** a private PostgreSQL 18.6 cluster, connected over a Unix socket. +- **Storage:** filesystem object store on the same volume (`single_volume_development`). +- **Workload:** one `environmental-lineage-v1` region with a 37,756-byte payload, + driven by real `evo-server` processes. +- **Host settings:** production lease policy (20 s lease, 5 s renewal, 2 s safety + margin, 2 s renewal and 5 s commit deadlines) and the default 1 s flush interval. +- **Inputs:** paced at the 50 ms tick, 300 in total over three owners. The + consumer dies at tick 20. +- **Kills:** after each 100 inputs, 10 more were sent and the owner was + SIGKILLed 200 ms later, mid-batch. The successor started immediately, for two + measured recoveries. + +## Results + +| Measure | Release (GCC14 -O3) | clang-tidy build | ASan/UBSan | +|---|---|---|---| +| Ack latency p50 / p99 / max (ms), n=299 | 549 / 1048 / 1048 | 549 / 1045 / 1049 | 530 / 1028 / 1028 | +| Death tick ack latency (ms) | 44 | 48 | 23 | +| SIGKILL to successor ready (ms), runs 1 and 2 | 19599 / 19548 | 19605 / 19557 | 19627 / 19561 | +| Head verification p50 / max (µs), n=10 | 1626 / 11949 | 5826 / 16184 | 14419 / 25352 | +| Full-history audit, 16 links, 300 ticks (ms) | 20 | 82 | 204 | +| Clone into the second database (ms) | 12 | 19 | 36 | + +The lease-store microbenchmarks in the same run (200 samples, Release) had these +p50/p99 values: + +| Operation | p50 (µs) | p99 (µs) | +|---|---|---| +| Acquire | 624 | 844 | +| Renew | 600 | 744 | +| Publish manifest | 750 | 965 | +| Release | 603 | 717 | + +Batch publication, including payload and trace upload with read-back, took +6–8 ms. + +## Interpretation +- **Recovery is dominated by the lease.** Kill to ready was 19.5–19.6 s in every + build. The dead holder's lease must expire in database time, and it had been + renewed less than 5 s earlier. Verification (tens of milliseconds even under + ASan) and process start are small. +- **The target holds.** The proposed 60 s recovery target holds for this fixture + with about 40 s of margin. The test asserts < 60 s for this fixture only. +- **Nothing acknowledged was lost.** Every acknowledged input was at or below the + committed head at each kill. The one death was recorded exactly once across + three owners, and each successor resumed exactly at the committed head with a + new epoch. +- **Acknowledgement latency is the batching cost.** It is set by the flush + interval: an input waits for its batch to commit, which averages about half a + flush interval, plus a commit of under 10 ms. A death seals its batch and + commits at once, so it is acknowledged fastest. +- **Verification and audit costs scale.** They grow with payload size and chain + length. At this size the full history is re-verified in about 20 ms (Release). + Larger regions and longer histories need their own measurement before any + target is claimed. + +## Not measured +The following were not measured: +- host or storage loss; +- network partitions between host and database; +- a separate database host; +- concurrent regions; +- large payloads; +- long histories; +- Kubernetes rescheduling time. +These belong to M8, together with off-host backup and restore. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index e5510e8..1417a26 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2485,3 +2485,52 @@ Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate stale-binary run was discarded and the mutation was rerun. - Not executed: clones across separate PostgreSQL clusters or hosts, Docker, the Gitea CI, and cluster runs. + +## M7.9b upgrade barrier and durability qualification - 2026-09-29 + +- Every M7.9b change is in the Linux PostgreSQL build or in docs. Windows was not + rerun; its 59/59 from M7.9a still covers the portable code, which is unchanged. +- WSL Linux `checks` set (GCC14 Debug and Release, clang19 analysis, clang19 + ASan/UBSan) passed CTest 59/59 each. The format check passed. +- The PostgreSQL suites passed in GCC Release, clang-tidy analysis and ASan/UBSan. + After the mutation run, one host test was corrected: `unrestorable_head_quarantines` + now retains its build, so the quarantine it checks again comes from recovery + rather than acquisition. Release and analysis were rebuilt and rerun green after + that change. The ASan build compiled with it, but its full run predates it. +- Harness: upgrade paths now end at 1..8. The populated v6 database upgrades to v8. + Version and checksum drift checks include v8. +- Migration and barrier test (database `evo_online`, created at v7 from the + checked-in files): + - A pre-v8 writer with a build-less lease published ticks 1..20 (with the + death). + - An `apply.sql` run with an invalid v8 checksum failed. It left history 1..7, + no `retired_at` column and no four-argument acquire. The writer renewed and + published ticks 21..40. + - The real migration then reached v8 while the writer held its lease, and the + writer published ticks 41..60. + - Barrier refusals: the runtime cannot call it (42501), and retiring the last + current build is refused (22023). + - After the barrier (1 lease fenced), the old writer's publication and renewal + were fenced. Build-less acquisition returned EVL06, and acquisition with b1 + returned unsupported. + - A b2 lease publishing a link that names b1 was refused as unsupported. + - A b2 `evo-server` became ready at epoch 3 from committed tick 60 and + acknowledged through tick 90. Its final head is built by b2. + - The audit passes with both builds retained, and reports unsupported at + sequence 0 with only b2. + - A b1 `evo-server` quarantined (exit 4) without becoming ready, and the head + was unchanged. +- Qualification: see engineering/DURABILITY_QUALIFICATION.md. SIGKILL to ready + took 19.5–19.6 s under the production lease in all three builds (the 60 s + target is asserted). Acknowledgement latency p50 was about 549 ms and p99 + about 1048 ms. No acknowledged input was lost, and one durable death was + recorded across five owners. +- Mutations (each restored and rebuilt). Each was caught: + - no retired-build trigger; + - no build check in four-argument acquisition; + - no barrier check in build-less acquisition; + - a barrier that does not fence holders; + - a host that retries an unsupported acquisition instead of quarantining. + The migration test caught this one after the host-test correction. +- Not executed: Docker, the Gitea CI, cluster runs, a separate database host, and + host or storage loss (M8). diff --git a/docs/infrastructure/POSTGRESQL.md b/docs/infrastructure/POSTGRESQL.md index b29a76c..9c634cf 100644 --- a/docs/infrastructure/POSTGRESQL.md +++ b/docs/infrastructure/POSTGRESQL.md @@ -68,7 +68,8 @@ every role transitively in it, and rolls back on any violation: `pg_write_all_data`); - any table privilege in `evolution`, including via PUBLIC; - CREATE on the schema, the database or `public`; -- EXECUTE outside the eleven fenced functions (schema v6). +- EXECUTE outside the twelve fenced functions (schema v8 adds the build-aware + `acquire_region_lease`). Attribute checks alone are not enough: a login with `pg_write_all_data` can update lease rows directly. Rerun the migrate Job after adding or changing runtime logins or grants; it is the audit. Grants made later without a rerun are not detected. Bootstrap revokes PUBLIC access to the schema, tables and @@ -145,6 +146,27 @@ receives clones. `evo-world-clone audit` verifies a region's whole retained history; it is read-only and needs only runtime read access. Never point a clone target at a production database or World0. +Schema v8 (ADR-0068) binds authority to builds: +- Hosts from M7.9b onwards acquire with their build identity. Only a retained + build that has not been retired may hold a lease. +- No new commit may name a retired build (EVL06). History written by a retired + build stays recoverable, so keep it retained. +- Hosts built before M7.9b use build-less acquisition, which keeps working until + the first barrier. + +To upgrade the build: +1. Migrate the schema. Running hosts keep their leases. +2. Insert the new build into `retained_builds`. +3. Deploy hosts with `--build --retained `. +4. On an owner connection, run + `SELECT evolution.upgrade_barrier('\x');` + It retires the old build and fences every current holder in the database. It + returns the number of leases fenced, and refuses to retire the last current + build. + +Old images then quarantine (exit 4) instead of serving. Rules and format changes +are refused explicitly; there is no converter. + There is no operator CLI for provisioning original regions yet, and no real world has been provisioned. Migrating v2→v3 fails closed if any v2 head exists, because v2 heads have no payload. diff --git a/include/evo/persistence/region_lease_store.hpp b/include/evo/persistence/region_lease_store.hpp index bd819f5..fb482a6 100644 --- a/include/evo/persistence/region_lease_store.hpp +++ b/include/evo/persistence/region_lease_store.hpp @@ -36,9 +36,11 @@ struct CommittedCheckpoint { std::vector manifest; }; +// Schema v8 (ADR-0068): authority is granted only to a retained, unretired build; +// a retired or unknown build is unsupported and must not keep retrying. [[nodiscard]] std::expected -acquire_region_lease(Postgres& db, RegionKey key, std::chrono::milliseconds duration, - std::chrono::milliseconds timeout); +acquire_region_lease(Postgres& db, RegionKey key, const Sha256Digest& build, + std::chrono::milliseconds duration, std::chrono::milliseconds timeout); // Returns the granted duration; the caller measures it from its request send time. [[nodiscard]] std::expected renew_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease, diff --git a/migrations/0008_upgrade_barrier.sql b/migrations/0008_upgrade_barrier.sql new file mode 100644 index 0000000..d9f2fa9 --- /dev/null +++ b/migrations/0008_upgrade_barrier.sql @@ -0,0 +1,97 @@ +-- M7.9b upgrade barrier (ADR-0068). A build can be retired from publication while +-- remaining retained for recovery of the history it wrote: no new commit (published +-- link, provisioned genesis or clone genesis) may name a retired build. The owner-only +-- barrier retires a build and fences every current lease holder in the database in +-- one transaction, so a paused or still-running old writer can never commit again +-- and every region is resumed by a host that recovers from the committed head. +ALTER TABLE evolution.retained_builds ADD COLUMN retired_at timestamptz; + +CREATE FUNCTION evolution.refuse_retired_build() RETURNS trigger LANGUAGE plpgsql +SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + IF EXISTS (SELECT 1 FROM evolution.retained_builds b + WHERE b.build_identity = NEW.build_identity AND b.retired_at IS NOT NULL) THEN + RAISE EXCEPTION 'build retired from publication' USING ERRCODE = 'EVL06'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER region_commits_current_build BEFORE INSERT ON evolution.region_commits + FOR EACH ROW EXECUTE FUNCTION evolution.refuse_retired_build(); + +-- Authority now names the build that will use it. The grant itself is the v2 +-- acquisition body, internal and not executable by the runtime role. +ALTER FUNCTION evolution.acquire_region_lease(evolution.u64, evolution.u64, integer) + RENAME TO grant_region_lease; +REVOKE ALL ON FUNCTION evolution.grant_region_lease(evolution.u64, evolution.u64, integer) + FROM PUBLIC, :"runtime_role"; +ALTER FUNCTION evolution.grant_region_lease(evolution.u64, evolution.u64, integer) + SECURITY INVOKER; + +-- Build-aware acquisition (M7.9b hosts): only a retained, unretired build may hold +-- authority, so a retired build can neither publish nor sit on an idle lease. +CREATE FUNCTION evolution.acquire_region_lease(p_world evolution.u64, p_region evolution.u64, + p_duration_ms integer, p_build bytea) +RETURNS TABLE (lease_holder uuid, lease_epoch evolution.u64, region_frame evolution.u64, + granted_ms integer) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + IF p_build IS NULL OR octet_length(p_build) <> 32 THEN + RAISE EXCEPTION 'invalid lease request' USING ERRCODE = '22023'; + END IF; + IF NOT EXISTS (SELECT 1 FROM evolution.retained_builds b + WHERE b.build_identity = p_build AND b.retired_at IS NULL) THEN + RAISE EXCEPTION 'build not current' USING ERRCODE = 'EVL06'; + END IF; + RETURN QUERY SELECT * FROM evolution.grant_region_lease(p_world, p_region, p_duration_ms); +END $$; + +-- Pre-v8 hosts name no build. They keep working until the first barrier, after +-- which none of them can take authority again. +CREATE FUNCTION evolution.acquire_region_lease(p_world evolution.u64, p_region evolution.u64, + p_duration_ms integer) +RETURNS TABLE (lease_holder uuid, lease_epoch evolution.u64, region_frame evolution.u64, + granted_ms integer) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + IF EXISTS (SELECT 1 FROM evolution.retained_builds b WHERE b.retired_at IS NOT NULL) THEN + RAISE EXCEPTION 'build-less acquisition ended by an upgrade barrier' USING ERRCODE = 'EVL06'; + END IF; + RETURN QUERY SELECT * FROM evolution.grant_region_lease(p_world, p_region, p_duration_ms); +END $$; + +-- Returns the number of leases fenced. Retirement is permanent through this +-- interface; at least one current build must remain, or nothing could publish. +CREATE FUNCTION evolution.upgrade_barrier(p_retire bytea) +RETURNS integer LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + fenced integer; +BEGIN + IF p_retire IS NULL OR octet_length(p_retire) <> 32 THEN + RAISE EXCEPTION 'invalid build identity' USING ERRCODE = '22023'; + END IF; + PERFORM 1 FROM evolution.retained_builds b WHERE b.build_identity = p_retire FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'build not retained' USING ERRCODE = '22023'; + END IF; + IF NOT EXISTS (SELECT 1 FROM evolution.retained_builds b + WHERE b.build_identity <> p_retire AND b.retired_at IS NULL) THEN + RAISE EXCEPTION 'no current build would remain' USING ERRCODE = '22023'; + END IF; + UPDATE evolution.retained_builds b SET retired_at = coalesce(b.retired_at, clock_timestamp()) + WHERE b.build_identity = p_retire; + -- Holders keep their epochs; the next acquisition takes a fresh one. + UPDATE evolution.region_leases l SET holder = NULL, expires_at = NULL + WHERE l.holder IS NOT NULL; + GET DIAGNOSTICS fenced = ROW_COUNT; + RETURN fenced; +END $$; + +REVOKE ALL ON FUNCTION evolution.refuse_retired_build(), evolution.upgrade_barrier(bytea), + evolution.acquire_region_lease(evolution.u64, evolution.u64, integer, bytea), + evolution.acquire_region_lease(evolution.u64, evolution.u64, integer) + FROM PUBLIC; +GRANT EXECUTE ON FUNCTION + evolution.acquire_region_lease(evolution.u64, evolution.u64, integer, bytea), + evolution.acquire_region_lease(evolution.u64, evolution.u64, integer) + TO :"runtime_role"; +INSERT INTO evolution.schema_migrations(version, sha256) VALUES (8, :'migration_sha_8'); diff --git a/migrations/apply.sql b/migrations/apply.sql index 27d3228..a93c39c 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -31,24 +31,34 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') :'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' || ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' || ',5:' || :'migration_sha_5' || ',6:' || :'migration_sha_6' - || ',7:' || :'migration_sha_7' AS at_v7 \gset - \if :at_v7 + || ',7:' || :'migration_sha_7' AS at_v7, + :'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' + || ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' + || ',5:' || :'migration_sha_5' || ',6:' || :'migration_sha_6' + || ',7:' || :'migration_sha_7' || ',8:' || :'migration_sha_8' AS at_v8 \gset + \if :at_v8 + \elif :at_v7 + \ir 0008_upgrade_barrier.sql \elif :at_v6 \ir 0007_region_clones.sql + \ir 0008_upgrade_barrier.sql \elif :at_v5 \ir 0006_commit_lookup.sql \ir 0007_region_clones.sql + \ir 0008_upgrade_barrier.sql \elif :at_v4 \ir pre_v5_guard.sql \ir 0005_batch_decisions.sql \ir 0006_commit_lookup.sql \ir 0007_region_clones.sql + \ir 0008_upgrade_barrier.sql \elif :at_v3 \ir 0004_lineage_profile.sql \ir pre_v5_guard.sql \ir 0005_batch_decisions.sql \ir 0006_commit_lookup.sql \ir 0007_region_clones.sql + \ir 0008_upgrade_barrier.sql \elif :at_v2 \ir 0003_region_checkpoints.sql \ir 0004_lineage_profile.sql @@ -56,6 +66,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0005_batch_decisions.sql \ir 0006_commit_lookup.sql \ir 0007_region_clones.sql + \ir 0008_upgrade_barrier.sql \elif :at_v1 \ir 0002_region_fencing.sql \ir 0003_region_checkpoints.sql @@ -64,6 +75,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0005_batch_decisions.sql \ir 0006_commit_lookup.sql \ir 0007_region_clones.sql + \ir 0008_upgrade_barrier.sql \else \echo 'Unknown or changed migration history' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; @@ -77,6 +89,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0005_batch_decisions.sql \ir 0006_commit_lookup.sql \ir 0007_region_clones.sql + \ir 0008_upgrade_barrier.sql \endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes -- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC @@ -98,6 +111,7 @@ BEGIN SELECT m.member FROM pg_catalog.pg_auth_members m JOIN closure c ON m.roleid = c.oid ), fenced(oid) AS (VALUES ('evolution.acquire_region_lease(evolution.u64,evolution.u64,integer)'::regprocedure::oid), + ('evolution.acquire_region_lease(evolution.u64,evolution.u64,integer,bytea)'::regprocedure::oid), ('evolution.renew_region_lease(evolution.u64,evolution.u64,uuid,evolution.u64,integer)'::regprocedure::oid), ('evolution.release_region_lease(evolution.u64,evolution.u64,uuid,evolution.u64)'::regprocedure::oid), ('evolution.load_region_head(evolution.u64,evolution.u64)'::regprocedure::oid), @@ -131,7 +145,7 @@ BEGIN OR EXISTS (SELECT 1 FROM pg_catalog.pg_namespace n WHERE n.nspname = 'public' AND pg_catalog.has_schema_privilege(c.oid, n.oid, 'CREATE')) UNION ALL - -- Only the eleven guarded functions are executable; provisioning/internal are not. + -- Only the twelve guarded functions are executable; provisioning/internal are not. SELECT c.oid, 'EXECUTE outside the fenced function set' FROM closure c CROSS JOIN pg_catalog.pg_proc p WHERE p.pronamespace = 'evolution'::regnamespace diff --git a/src/persistence/region_lease_store.cpp b/src/persistence/region_lease_store.cpp index 454ae8c..76e164c 100644 --- a/src/persistence/region_lease_store.cpp +++ b/src/persistence/region_lease_store.cpp @@ -131,12 +131,14 @@ std::optional parse_head(const PgRows& rows, std::size_t columns = 5 } } // namespace -std::expected -acquire_region_lease(Postgres& db, RegionKey key, milliseconds duration, milliseconds timeout) { - const std::array p{std::to_string(key.world), std::to_string(key.region), - std::to_string(duration.count())}; +std::expected acquire_region_lease(Postgres& db, RegionKey key, + const Sha256Digest& build, + milliseconds duration, + milliseconds timeout) { + const std::array p{std::to_string(key.world), std::to_string(key.region), + std::to_string(duration.count()), bytea(build)}; auto rows = db.query("SELECT * FROM evolution.acquire_region_lease($1::evolution.u64, " - "$2::evolution.u64, $3::integer)", + "$2::evolution.u64, $3::integer, $4::bytea)", p, timeout); if (!rows) return std::unexpected{classify(rows.error())}; diff --git a/src/server/region_host.cpp b/src/server/region_host.cpp index f8f626e..bce43ed 100644 --- a/src/server/region_host.cpp +++ b/src/server/region_host.cpp @@ -138,10 +138,12 @@ template void RegionHost::acquire(SchedulerTime now) { // A grant survives a transient recovery failure; only re-acquire without one. if (!authority_.may_originate(LeaseClock::now())) { const auto sent = LeaseClock::now(); - auto grant = persistence::acquire_region_lease(*db_, config_.key, config_.lease.duration, - config_.lease.renew_timeout); + auto grant = persistence::acquire_region_lease( + *db_, config_.key, config_.build, config_.lease.duration, config_.lease.renew_timeout); if (!grant) { - if (grant.error().kind == LeaseStoreError::not_provisioned) + // No region, or this build is retired/unknown (ADR-0068): never serve. + if (grant.error().kind == LeaseStoreError::not_provisioned || + grant.error().kind == LeaseStoreError::unsupported) status_.phase = HostPhase::quarantined; else if (grant.error().kind != LeaseStoreError::busy) db_.reset(); diff --git a/tests/fixtures/server_process.hpp b/tests/fixtures/server_process.hpp index 0616087..a599eb7 100644 --- a/tests/fixtures/server_process.hpp +++ b/tests/fixtures/server_process.hpp @@ -46,7 +46,7 @@ inline std::unique_ptr owner() { inline std::string request_hex(std::uint64_t tick, std::uint8_t tag = 0x9B) { char out[33]; std::snprintf(out, sizeof out, "%02x000000000000000000000000%06llx", tag, - static_cast(tick)); + static_cast(tick & 0xFFFFFFU)); // Test ticks < 2^24. return out; } inline std::string input_line(std::uint64_t tick, bool consumer_dies, std::uint8_t tag = 0x9B) { @@ -150,6 +150,14 @@ class Child { [[nodiscard]] const std::vector& lines() const noexcept { return lines_; } + // Arrival time of each line (when it was read), for latency measurement. + [[nodiscard]] const std::vector& times() const noexcept { + return times_; + } + // Reads whatever arrives for `duration`, stamping each line. + void pump(std::chrono::milliseconds duration) { + wait_for([](const std::string&) { return false; }, duration); + } [[nodiscard]] bool printed(std::string_view prefix) const { for (const auto& l : lines_) if (starts(l, prefix)) @@ -169,6 +177,7 @@ class Child { buffer_.append(chunk, static_cast(n)); for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) { lines_.push_back(buffer_.substr(0, end)); + times_.push_back(Clock::now()); buffer_.erase(0, end + 1); } return true; @@ -177,6 +186,7 @@ class Child { int in_{-1}, out_{-1}; std::string buffer_; std::vector lines_; + std::vector times_; }; struct ServerRegion { diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index a43d322..f94cd38 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -10,6 +10,7 @@ void run_region_server_process_tests(); void run_region_recovery_tests(); void run_region_ops_tests(); void run_region_clone_tests(); +void run_region_migration_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; @@ -90,6 +91,8 @@ int main() { std::puts("region ops integration passed"); run_region_clone_tests(); std::puts("region clone and history integration passed"); + run_region_migration_tests(); + std::puts("region migration, barrier and qualification integration passed"); return 0; } catch (const std::exception& e) { std::fprintf(stderr, "%s\n", e.what()); diff --git a/tests/region_checkpoint_store_tests.cpp b/tests/region_checkpoint_store_tests.cpp index e5b2ea2..d199ac0 100644 --- a/tests/region_checkpoint_store_tests.cpp +++ b/tests/region_checkpoint_store_tests.cpp @@ -215,7 +215,7 @@ template void run_batch(Batch& b, std::uint64_t ticks, bool dea void publish_recover_and_continue() { auto r = provisioned("publish", 3, 0xC0); const std::array retained{build}; - auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s); + auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s); CHECK(lease); // Genesis recovers to the exact opening state. auto recovered = @@ -313,7 +313,7 @@ void publish_recover_and_continue() { void failures_leave_the_previous_head() { auto r = provisioned("failures", 4, 0xD0); const std::array retained{build}; - auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s); + auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s); CHECK(lease); auto head = load_region_checkpoint(*r.runtime, r.key, 5s); CHECK(head); @@ -446,7 +446,7 @@ void lineage_publish_and_recover() { CHECK(genesis); CHECK(decode_manifest(genesis->manifest)->profile == CheckpointProfile::environmental_lineage_v1); - auto lease = acquire_region_lease(*runtime, key, 20000ms, 5s); + auto lease = acquire_region_lease(*runtime, key, build, 20000ms, 5s); CHECK(lease); auto region = evo::test::lineage_region(); @@ -548,7 +548,7 @@ struct LeaseRowLock { void durable_outcomes_and_ambiguity() { auto r = provisioned("outcomes", 7, 0xF0); const std::array retained{build}; - auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s); + auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s); CHECK(lease); auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s); CHECK(genesis); @@ -711,7 +711,7 @@ void publication_waits_for_directory_durability() { auto faulty = FilesystemObjectStore::open(r.objects->directory().parent_path().parent_path(), r.key, max_environmental_payload_bytes, &faults); CHECK(faulty); - auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s); + auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s); CHECK(lease); auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s); CHECK(genesis); @@ -738,7 +738,7 @@ void publication_waits_for_directory_durability() { // recovered region. The database cannot see payload deaths; recovery must. void undecided_death_is_quarantined() { auto r = provisioned("undecided", 10, 0xB6); - auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s); + auto lease = acquire_region_lease(*r.runtime, r.key, build, 20000ms, 5s); CHECK(lease); auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s); CHECK(genesis); diff --git a/tests/region_clone_tests.cpp b/tests/region_clone_tests.cpp index 86a1a0a..052449b 100644 --- a/tests/region_clone_tests.cpp +++ b/tests/region_clone_tests.cpp @@ -251,11 +251,11 @@ void clone_cannot_write_its_source() { CHECK(cloned && cloned->inherited_outcomes == 1 && cloned->origin.tick == 25 && cloned->origin.sequence == before.sequence && cloned->origin.commit == before.commit); // The source session is read-only for good, even through SECURITY DEFINER functions. - auto denied = acquire_region_lease(*reader, source.key, 4000ms, 5s); + auto denied = acquire_region_lease(*reader, source.key, build(), 4000ms, 5s); CHECK(!denied && sqlstate(denied.error().cause) == "25006"); // Authority over the clone is not authority over the source. auto runtime = connect_to(source_db(), env("EVO_PG_RUNTIME_USER")); - auto lease = acquire_region_lease(*runtime, same_db.key, 4000ms, 5s); + auto lease = acquire_region_lease(*runtime, same_db.key, build(), 4000ms, 5s); CHECK(lease); auto source_head = load_region_checkpoint(*runtime, source.key, 5s); CHECK(source_head); @@ -296,7 +296,7 @@ void clone_cannot_write_its_source() { CHECK(!rewrite && sqlstate(rewrite.error()) == "EVL09"); // The clone database's runtime cannot reach the source database's functions. auto foreign = connect_to(source_db(), env("EVO_PG_CLONE_RUNTIME_USER")); - auto crossed = acquire_region_lease(*foreign, source.key, 4000ms, 5s); + auto crossed = acquire_region_lease(*foreign, source.key, build(), 4000ms, 5s); CHECK(!crossed && sqlstate(crossed.error().cause) == "42501"); // Nothing above moved the source. const auto after = head_in(source_db(), source.key); @@ -420,7 +420,7 @@ void environmental_clone_binds_inherited_outcomes() { *owner_db, *source_objects, key, evo::test::region_identity.frame, *evo::test::region_community(), filled<16>(0xF9), build(), budget, 5s); CHECK(genesis); - auto lease = acquire_region_lease(*runtime, key, 20000ms, 5s); + auto lease = acquire_region_lease(*runtime, key, build(), 20000ms, 5s); CHECK(lease); auto state = evo::test::region_community(); RegionBatch batch{*state}; diff --git a/tests/region_host_tests.cpp b/tests/region_host_tests.cpp index e851afd..a3a2e15 100644 --- a/tests/region_host_tests.cpp +++ b/tests/region_host_tests.cpp @@ -305,6 +305,12 @@ void unrestorable_head_quarantines() { auto c = config(r); c.build = filled<32>(0xB2); c.retained_builds = {c.build}; // Not the build that produced the head. + // A current build in the database (so acquisition succeeds, ADR-0068): the + // quarantine comes from recovery refusing the head, not from the lease. + CHECK(connect_as(env("PGUSER")) + ->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('b2', 32), 'hex')) ON CONFLICT DO NOTHING", + {}, 5s)); auto h = host(r, c); const auto& s = h->step(std::chrono::steady_clock::now()); CHECK(s.phase == HostPhase::quarantined && !s.ready && h->state() == nullptr); diff --git a/tests/region_lease_store_tests.cpp b/tests/region_lease_store_tests.cpp index ae82d45..d6c843c 100644 --- a/tests/region_lease_store_tests.cpp +++ b/tests/region_lease_store_tests.cpp @@ -178,7 +178,7 @@ void runtime_cannot_bypass_the_guard() { // A SECURITY DEFINER function ignores a hostile caller search_path. CHECK(db->query("SET search_path = pg_temp, public", {}, 5s)); CHECK(db->query("CREATE TEMP TABLE region_leases(x int)", {}, 5s)); - auto lease = acquire_region_lease(*db, key, 20000ms, 5s); + auto lease = acquire_region_lease(*db, key, build, 20000ms, 5s); CHECK(lease && lease->epoch == 1 && lease->frame == 7); CHECK(release_region_lease(*db, key, *lease, 5s)); } @@ -187,13 +187,13 @@ void acquire_renew_release_and_publish() { auto admin = owner(); const auto key = provision(*admin, 20); auto db = runtime(); - CHECK( - failed(acquire_region_lease(*db, {1, 999}, 20000ms, 5s), LeaseStoreError::not_provisioned)); - CHECK(failed(acquire_region_lease(*db, key, 999ms, 5s), LeaseStoreError::invalid)); - CHECK(failed(acquire_region_lease(*db, key, 60001ms, 5s), LeaseStoreError::invalid)); - auto lease = acquire_region_lease(*db, key, 20000ms, 5s); + CHECK(failed(acquire_region_lease(*db, {1, 999}, build, 20000ms, 5s), + LeaseStoreError::not_provisioned)); + CHECK(failed(acquire_region_lease(*db, key, build, 999ms, 5s), LeaseStoreError::invalid)); + CHECK(failed(acquire_region_lease(*db, key, build, 60001ms, 5s), LeaseStoreError::invalid)); + auto lease = acquire_region_lease(*db, key, build, 20000ms, 5s); CHECK(lease && lease->epoch == 1 && lease->duration == 20000ms && lease->holder != Uuid{}); - CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::busy)); + CHECK(failed(acquire_region_lease(*db, key, build, 20000ms, 5s), LeaseStoreError::busy)); auto renewed = renew_region_lease(*db, key, *lease, 20000ms, 5s); CHECK(renewed && *renewed == 20000ms); auto forged = *lease; @@ -244,7 +244,7 @@ void acquire_renew_release_and_publish() { CHECK(failed(release_region_lease(*db, key, *lease, 5s), LeaseStoreError::fenced)); CHECK(failed(renew_region_lease(*db, key, *lease, 20000ms, 5s), LeaseStoreError::fenced)); CHECK(failed(publish(*db, link(*head, *lease, 22, 65)), LeaseStoreError::fenced)); - auto next = acquire_region_lease(*db, key, 20000ms, 5s); + auto next = acquire_region_lease(*db, key, build, 20000ms, 5s); CHECK(next && next->epoch == 2 && next->holder != lease->holder); auto final_head = publish(*db, link(*head, *next, 22, 65)); CHECK(final_head && final_head->head.sequence == 2); @@ -258,16 +258,16 @@ void database_time_expiry_and_paused_writer() { const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms}; LeaseAuthority a{fast}; auto sent = LeaseClock::now(); - auto lease_a = acquire_region_lease(*worker_a, key, fast.duration, 5s); + auto lease_a = acquire_region_lease(*worker_a, key, build, fast.duration, 5s); CHECK(lease_a && a.acquired(*lease_a, sent, LeaseClock::now())); - CHECK(failed(acquire_region_lease(*worker_b, key, 20000ms, 5s), LeaseStoreError::busy)); + CHECK(failed(acquire_region_lease(*worker_b, key, build, 20000ms, 5s), LeaseStoreError::busy)); const auto genesis = committed(*worker_a, key); std::this_thread::sleep_for(1200ms); // Worker A is paused past its lease. CHECK(a.state(LeaseClock::now()) == LeaseState::fenced); // Expiry is decided by database time; the successor acquires a newer epoch. wait_until_expired(*admin, key); - auto lease_b = acquire_region_lease(*worker_b, key, 20000ms, 5s); + auto lease_b = acquire_region_lease(*worker_b, key, build, 20000ms, 5s); CHECK(lease_b && lease_b->epoch == lease_a->epoch + 1); auto head_b = publish(*worker_b, link(genesis, *lease_b, 31, 3)); CHECK(head_b); @@ -282,7 +282,7 @@ void database_time_expiry_and_paused_writer() { // Expired renewal cannot revive a lease even without a successor. CHECK(release_region_lease(*worker_b, key, *lease_b, 5s)); - auto lease_c = acquire_region_lease(*worker_a, key, 1000ms, 5s); + auto lease_c = acquire_region_lease(*worker_a, key, build, 1000ms, 5s); CHECK(lease_c && lease_c->epoch == lease_b->epoch + 1); wait_until_expired(*admin, key); CHECK( @@ -295,7 +295,7 @@ void queued_writer_uses_post_lock_database_time() { auto admin = owner(); const auto key = provision(*admin, 40); auto worker = runtime(); - auto lease = acquire_region_lease(*worker, key, 1000ms, 5s); + auto lease = acquire_region_lease(*worker, key, build, 1000ms, 5s); CHECK(lease); const auto genesis = committed(*worker, key); auto blocker = owner(); @@ -325,7 +325,7 @@ void partitioned_renewal_fences_locally() { const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms}; LeaseAuthority authority{fast}; const auto sent = LeaseClock::now(); - auto lease = acquire_region_lease(*worker, key, fast.duration, 5s); + auto lease = acquire_region_lease(*worker, key, build, fast.duration, 5s); CHECK(lease && authority.acquired(*lease, sent, LeaseClock::now())); auto blocker = owner(); const std::array region{"50"}; @@ -346,7 +346,7 @@ void partitioned_renewal_fences_locally() { CHECK(blocker->query("ROLLBACK", {}, 5s)); // The orphaned renewal reached the row only after database expiry: rejected. auto successor = runtime(); - auto next = acquire_region_lease(*successor, key, 20000ms, 5s); + auto next = acquire_region_lease(*successor, key, build, 20000ms, 5s); CHECK(next && next->epoch == lease->epoch + 1); } @@ -365,7 +365,7 @@ void contenders_serialize_on_the_database() { for (std::size_t i = 0; i < 2; ++i) threads.emplace_back([&, i] { start.arrive_and_wait(); - results[i] = acquire_region_lease(*workers[i], key, 20000ms, 5s); + results[i] = acquire_region_lease(*workers[i], key, build, 20000ms, 5s); }); for (auto& t : threads) t.join(); @@ -380,7 +380,7 @@ void contenders_serialize_on_the_database() { } // The same holder racing two commits from the same predecessor: exactly one wins. - auto lease = acquire_region_lease(*workers[0], key, 20000ms, 5s); + auto lease = acquire_region_lease(*workers[0], key, build, 20000ms, 5s); CHECK(lease); auto head = committed(*workers[0], key); for (std::uint8_t round = 0; round < 32; ++round) { @@ -413,9 +413,9 @@ void exhaustion_fails_closed() { "WHERE world_id = 1 AND region_id = 70", {}, 5s)); auto db = runtime(); - CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::exhausted)); + CHECK(failed(acquire_region_lease(*db, key, build, 20000ms, 5s), LeaseStoreError::exhausted)); const auto other = provision(*admin, 71); - auto lease = acquire_region_lease(*db, other, 20000ms, 5s); + auto lease = acquire_region_lease(*db, other, build, 20000ms, 5s); CHECK(lease); // Owner-only fixture: a chain link at the last sequence (the head FK requires it). CHECK(admin->query( @@ -493,7 +493,7 @@ void measure_round_trips() { for (int i = 0; i < n; ++i) { std::expected lease = std::unexpected{LeaseStoreFailure{}}; acquire.samples.push_back( - timed([&] { lease = acquire_region_lease(*db, key, 20000ms, 5s); })); + timed([&] { lease = acquire_region_lease(*db, key, build, 20000ms, 5s); })); CHECK(lease); renew.samples.push_back( timed([&] { CHECK(renew_region_lease(*db, key, *lease, 20000ms, 5s)); })); diff --git a/tests/region_migration_tests.cpp b/tests/region_migration_tests.cpp new file mode 100644 index 0000000..db60b74 --- /dev/null +++ b/tests/region_migration_tests.cpp @@ -0,0 +1,403 @@ +// M7.9b schema migration, upgrade barrier and durability qualification (ADR-0068) +// against real PostgreSQL, real evo-server processes and the checked-in migrations +// (EVO_SOURCE_ROOT). Run by tools/ci/postgres-check.sh; prints measured latencies. +#include "evo/persistence/region_clone.hpp" +#include "evo/persistence/sha256.hpp" +#include "fixtures/server_process.hpp" +#include +#include +#include + +void run_region_migration_tests(); + +namespace { +using namespace evo::test::process; +using namespace evo::persistence; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("region migration test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +constexpr std::size_t budget = 8000000; +const std::string b1_text = [] { + std::string s; + for (int i = 0; i < 32; ++i) + s += "b1"; + return s; +}(); +const std::string b2_text = [] { + std::string s; + for (int i = 0; i < 32; ++i) + s += "b2"; + return s; +}(); +Sha256Digest digest_of(std::uint8_t b) { + Sha256Digest d{}; + d.fill(b); + return d; +} +const Sha256Digest b1 = digest_of(0xB1), b2 = digest_of(0xB2); +template std::array filled(std::uint8_t value) { + std::array out{}; + out.fill(value); + return out; +} +std::unique_ptr connect_to(const std::string& database, const std::string& user) { + auto made = Postgres::connect({env("PGHOST"), {}, env("PGPORT"), database, user, {}, {}}, 5s); + CHECK(made); + return std::move(*made); +} +std::string sqlstate(const PgFailure& f) { + return f.sqlstate.data(); +} +std::string scalar(Postgres& db, const std::string& sql) { + auto rows = db.query(sql, {}, 5s); + CHECK(rows && rows->size() == 1 && (*rows)[0].size() == 1); + return (*rows)[0][0].value_or(""); +} +int shell(const std::string& command) { + Child c{{"/bin/bash", "-c", command}}; + return c.wait_exit(120s); +} +std::vector with(std::vector args, const std::string& flag, + const std::string& value) { + const auto at = std::ranges::find(args, flag); + CHECK(at != args.end() && at + 1 != args.end()); + *(at + 1) = value; + return args; +} +std::unique_ptr store(const ServerRegion& r) { + auto made = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); + CHECK(made); + return std::move(*made); +} +std::optional parse_uuid(const std::string& text) { + std::string compact; + for (const char c : text) + if (c != '-') + compact.push_back(c); + if (compact.size() != 32) + return std::nullopt; + Uuid out{}; + for (std::size_t i = 0; i < 16; ++i) + out[i] = static_cast(std::stoul(compact.substr(2 * i, 2), nullptr, 16)); + return out; +} +// A pre-v8 writer: the build-less acquisition every earlier host used. +std::expected legacy_acquire(Postgres& db, RegionKey key) { + const std::array p{std::to_string(key.world), std::to_string(key.region)}; + auto rows = db.query("SELECT * FROM evolution.acquire_region_lease($1::evolution.u64, " + "$2::evolution.u64, 20000)", + p, 5s); + if (!rows) + return std::unexpected{sqlstate(rows.error())}; + CHECK(rows->size() == 1 && (*rows)[0].size() == 4); + const auto& r = (*rows)[0]; + const auto holder = parse_uuid(r[0].value_or("")); + CHECK(holder.has_value()); + return LeaseGrant{holder.value_or(Uuid{}), std::stoull(r[1].value_or("0")), + std::stoull(r[2].value_or("0")), 20000ms}; +} +const char* const sha_function = + "s(){ sha256sum \"$EVO_SOURCE_ROOT/migrations/$1\" | cut -d' ' -f1; }; "; +std::string history(Postgres& owner) { + return scalar(owner, "SELECT string_agg(version::text, ',' ORDER BY version) FROM " + "evolution.schema_migrations"); +} + +// An existing v7 database is upgraded while an older writer holds a lease. A +// migration that fails leaves v7 exactly and the writer unaffected; the real one +// commits under it. The barrier then retires the old build: the writer is fenced +// and can never regain authority, while the new build recovers the whole history. +void upgrade_across_migration_and_barrier() { + const std::string database = "evo_online"; + const std::string runtime_user = env("EVO_PG_RUNTIME_USER"); + CHECK(shell(std::string(sha_function) + "cd \"$EVO_SOURCE_ROOT/migrations\" && createdb " + + database + " && PGDATABASE=" + database + + " psql -X -q -v ON_ERROR_STOP=1 -1 -v migration_sha=$(s 0001_region_bootstrap.sql)" + " -v migration_sha_2=$(s 0002_region_fencing.sql)" + " -v migration_sha_3=$(s 0003_region_checkpoints.sql)" + " -v migration_sha_4=$(s 0004_lineage_profile.sql)" + " -v migration_sha_5=$(s 0005_batch_decisions.sql)" + " -v migration_sha_6=$(s 0006_commit_lookup.sql)" + " -v migration_sha_7=$(s 0007_region_clones.sql)" + " -v runtime_role=evo_runtime_test -f 0001_region_bootstrap.sql" + " -f 0002_region_fencing.sql -f 0003_region_checkpoints.sql" + " -f 0004_lineage_profile.sql -f 0005_batch_decisions.sql" + " -f 0006_commit_lookup.sql -f 0007_region_clones.sql > /dev/null") == 0); + auto owner_db = connect_to(database, env("PGUSER")); + CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); + CHECK(owner_db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('b1', 32), 'hex')), (decode(repeat('b2', 32), 'hex'))", + {}, 5s)); + const ServerRegion r{{7, 91}, fs::path(env("PGHOST")) / "online-91"}; + fs::remove_all(r.root); + fs::create_directory(r.root); + fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); + auto objects = store(r); + auto genesis = + provision_lineage_region(*owner_db, *objects, r.key, evo::test::region_identity.frame, + *evo::test::lineage_region(), filled<16>(0xDA), b1, budget, 5s); + CHECK(genesis); + + // The old writer: build-less lease, publishing with build b1. + auto runtime = connect_to(database, runtime_user); + auto lease = legacy_acquire(*runtime, r.key); + CHECK(lease); + auto state = evo::test::lineage_region(); + RegionBatch batch{*state}; + CommittedCheckpoint head = *genesis; + std::uint8_t commit_seed = 0xDB; + const auto publish_next = [&](std::uint64_t ticks) { + for (std::uint64_t i = 0; i < ticks && !batch.sealed(); ++i) + CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1, true))); + batch.seal(); + return publish_lineage_batch(*runtime, *objects, *lease, head, batch, + filled<16>(commit_seed++), b1, budget, 5s); + }; + const auto confirm = [&](const std::expected& done) { + CHECK(done && batch.confirm(done->committed.head.end_tick)); + head = done->committed; + }; + confirm(publish_next(64)); // Ticks 1..20: the consumer's death seals it. + CHECK(head.head.end_tick == 20); + + // A v8 migration that fails at its last statement changes nothing. + CHECK(shell(std::string(sha_function) + "cd \"$EVO_SOURCE_ROOT\" && PGDATABASE=" + database + + " psql -X -q -v ON_ERROR_STOP=1 -v migration_sha=$(s 0001_region_bootstrap.sql)" + " -v migration_sha_2=$(s 0002_region_fencing.sql)" + " -v migration_sha_3=$(s 0003_region_checkpoints.sql)" + " -v migration_sha_4=$(s 0004_lineage_profile.sql)" + " -v migration_sha_5=$(s 0005_batch_decisions.sql)" + " -v migration_sha_6=$(s 0006_commit_lookup.sql)" + " -v migration_sha_7=$(s 0007_region_clones.sql) -v migration_sha_8=invalid" + " -v runtime_role=evo_runtime_test -f migrations/apply.sql" + " > /dev/null 2>&1") != 0); + CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); + CHECK(scalar(*owner_db, "SELECT count(*) FROM information_schema.columns WHERE " + "table_schema = 'evolution' AND column_name = 'retired_at'") == "0"); + CHECK(scalar(*owner_db, + "SELECT (to_regprocedure('evolution.acquire_region_lease(" + "evolution.u64, evolution.u64, integer, bytea)') IS NULL)::text") == "true"); + CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s)); + confirm(publish_next(20)); // 21..40, still v7. + + // The real migration commits while the writer holds its lease; it keeps going. + CHECK(shell("PGDATABASE=" + database + + " EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/" + "tools/migrate-postgres.sh\" > /dev/null") == 0); + CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8"); + CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s)); + confirm(publish_next(20)); // 41..60, now v8. + CHECK(head.head.end_tick == 60); + + // Barrier guards: the last current build cannot be retired; runtime cannot call it. + const std::string retire_b1 = "SELECT evolution.upgrade_barrier(decode(repeat('b1', 32), " + "'hex'))::text"; + auto denied = runtime->query(retire_b1, {}, 5s); + CHECK(!denied && sqlstate(denied.error()) == "42501"); + // The barrier: b1 retired and every holder fenced, in one transaction. + CHECK(scalar(*owner_db, retire_b1) == "1"); + auto last_b2 = owner_db->query( + "SELECT evolution.upgrade_barrier(decode(repeat('b2', 32), 'hex'))", {}, 5s); + CHECK(!last_b2 && sqlstate(last_b2.error()) == "22023"); + // The old writer is fenced and cannot regain authority by any path. + auto after = publish_next(10); + CHECK(!after && after.error().kind == PublicationError::rejected && + after.error().store.kind == LeaseStoreError::fenced); + auto renewed = renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s); + CHECK(!renewed && renewed.error().kind == LeaseStoreError::fenced); + auto legacy = legacy_acquire(*runtime, r.key); + CHECK(!legacy && legacy.error() == "EVL06"); + auto retired = acquire_region_lease(*runtime, r.key, b1, 20000ms, 5s); + CHECK(!retired && retired.error().kind == LeaseStoreError::unsupported); + // Authority for b2 still cannot publish a link naming the retired build. + auto current = acquire_region_lease(*runtime, r.key, b2, 20000ms, 5s); + CHECK(current); + lease = *current; + auto stale_build = publish_lineage_batch(*runtime, *objects, *lease, head, batch, + filled<16>(commit_seed++), b1, budget, 5s); + CHECK(!stale_build && stale_build.error().kind == PublicationError::rejected && + stale_build.error().store.kind == LeaseStoreError::unsupported); + CHECK(release_region_lease(*runtime, r.key, *lease, 5s)); + const auto before_new = head.head; + + // The new build resumes from the committed head, and b1 history stays readable. + auto args = with(with(server_args(r), "--pg-database", database), "--build", b2_text); + args.insert(args.end(), {"--retained", b1_text}); + { + Child server{args}; + CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); + CHECK(server.printed("phase running epoch=3 committed=60 ")); + server.send(input_lines(61, 90, true)); + CHECK(server.wait_line("ack " + request_hex(90) + " 90 ", 30s)); + server.signal(SIGTERM); + CHECK(server.wait_exit(20s) == 0); + } + auto final_head = load_region_checkpoint(*owner_db, r.key, 5s); + CHECK(final_head && final_head->head.end_tick == 90 && + final_head->head.sequence > before_new.sequence); + const auto final_manifest = decode_manifest(final_head->manifest); + CHECK(final_manifest && final_manifest->build == b2); + const std::array both{b1, b2}; + const std::array only_new{b2}; + auto audited = audit_region_history(*runtime, *objects, r.key, both, budget, 100, 5s); + CHECK(audited && audited->commits == final_head->head.sequence + 1 && audited->outcomes == 1); + auto without_old = audit_region_history(*runtime, *objects, r.key, only_new, budget, 100, 5s); + CHECK(!without_old && without_old.error().kind == AuditError::unsupported && + without_old.error().sequence == 0); + // A host of the retired build never serves, not even once the region is free. + { + Child old_host{with(with(server_args(r), "--pg-database", database), "--build", b1_text)}; + CHECK(old_host.wait_exit(20s) == 4); + CHECK(old_host.printed("phase quarantined") && !old_host.printed("ready 1")); + } + CHECK(load_region_checkpoint(*owner_db, r.key, 5s)->head == final_head->head); +} + +std::uint64_t percentile(std::vector v, unsigned p) { + CHECK(!v.empty()); + std::ranges::sort(v); + return v[std::min(v.size() - 1, (v.size() * p) / 100)]; +} +std::uint64_t ms_between(Clock::time_point a, Clock::time_point b) { + return static_cast( + std::chrono::duration_cast(b - a).count()); +} +// Sends inputs first..last paced at the 50 ms tick; returns send times. +std::vector paced(Child& server, std::uint64_t first, std::uint64_t last) { + std::vector sent; + const auto start = Clock::now(); + for (auto tick = first; tick <= last; ++tick) { + const auto due = start + std::chrono::milliseconds{50 * (tick - first)}; + if (const auto now = Clock::now(); due > now) + server.pump(std::chrono::duration_cast(due - now)); + server.send(input_line(tick, true)); + sent.push_back(Clock::now()); + } + return sent; +} +std::optional ack_time(const Child& server, std::uint64_t tick) { + const auto prefix = "ack " + request_hex(tick) + " " + std::to_string(tick) + " "; + for (std::size_t i = 0; i < server.lines().size(); ++i) + if (starts(server.lines()[i], prefix)) + return server.times()[i]; + return std::nullopt; +} +Clock::time_point ready_time(const Child& server) { + for (std::size_t i = 0; i < server.lines().size(); ++i) + if (server.lines()[i] == "ready 1") + return server.times()[i]; + CHECK(false); + return {}; +} + +// The M7 exit fixture: production lease policy (20 s lease, 5 s renewal) and +// flush interval (1 s), one lineage region on a local socket and filesystem. +// Measures acknowledgement latency, worker-kill-to-ready recovery against the +// proposed 60 s target, and the cost of verification, audit and clone. +void durability_qualification() { + const auto r = provision(93, 0xDC); + const auto args = + with(with(server_args(r), "--lease-profile", "default"), "--flush-ms", "1000"); + std::vector latency, kill_to_ready; + std::uint64_t death_latency = 0; + std::uint64_t next = 1; + std::uint64_t epoch = 1; + for (int round = 0; round < 3; ++round) { + Child server{args}; + CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 60s)); + CHECK(server.printed("phase running epoch=" + std::to_string(epoch) + " ")); + const auto first = next; + const auto last = next + 99; + const auto sent = paced(server, first, last); + CHECK(server.wait_line("ack " + request_hex(last) + " " + std::to_string(last) + " ", 30s)); + for (auto tick = first; tick <= last; ++tick) { + const auto at = ack_time(server, tick); + CHECK(at.has_value()); + const auto ms = ms_between(sent[tick - first], at.value_or(Clock::time_point{})); + if (tick == 20) + death_latency = ms; + else + latency.push_back(ms); + } + // Mid-batch force kill: more inputs are in flight, none acknowledged yet. + server.send(input_lines(last + 1, last + 10, true)); + server.pump(200ms); + const auto acked = max_acked(server); + server.signal(SIGKILL); + const auto killed = Clock::now(); + CHECK(server.wait_exit(20s) == 128 + SIGKILL); + const auto committed = head_of(r.key); + CHECK(acked <= committed.tick && acked >= last); // No acknowledged input lost. + next = committed.tick + 1; + ++epoch; + if (round == 2) + break; + Child successor{args}; + CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 60s)); + kill_to_ready.push_back(ms_between(killed, ready_time(successor))); + CHECK(successor.printed("phase running epoch=" + std::to_string(epoch) + + " committed=" + std::to_string(committed.tick) + " ")); + successor.signal(SIGTERM); + CHECK(successor.wait_exit(20s) == 0); + ++epoch; + } + CHECK(decisions_of(r.key) == 1); + for (const auto ms : kill_to_ready) + CHECK(ms < 60000); // The declared target for this fixture. + std::printf("qualification ack_latency_ms flush=1000 n=%zu p50=%llu p99=%llu max=%llu " + "death=%llu\n", + latency.size(), static_cast(percentile(latency, 50)), + static_cast(percentile(latency, 99)), + static_cast(percentile(latency, 100)), + static_cast(death_latency)); + std::printf("qualification kill_to_ready_ms lease=20000 renew=5000 runs=%zu first=%llu " + "second=%llu target=60000\n", + kill_to_ready.size(), static_cast(kill_to_ready[0]), + static_cast(kill_to_ready[1])); + + // Verification cost on the final head (no lease needed to read). + auto reader = connect_to(env("PGDATABASE"), env("EVO_PG_RUNTIME_USER")); + auto objects = store(r); + const std::array retained{b1}; + std::vector verify_us; + std::uint64_t payload_bytes = 0; + for (int i = 0; i < 10; ++i) { + const auto start = Clock::now(); + auto recovered = recover_lineage_checkpoint(*reader, *objects, r.key, retained, budget, 5s); + CHECK(recovered); + payload_bytes = recovered->manifest.payload_bytes; + verify_us.push_back(static_cast( + std::chrono::duration_cast(Clock::now() - start).count())); + } + const auto audit_start = Clock::now(); + auto audited = audit_region_history(*reader, *objects, r.key, retained, budget, 1000, 5s); + const auto audit_ms = ms_between(audit_start, Clock::now()); + CHECK(audited); + auto clone_owner = connect_to(env("EVO_PG_CLONE_DATABASE"), env("PGUSER")); + const ServerRegion target{{8, 93}, fs::path(env("PGHOST")) / "qualify-clone-93"}; + fs::remove_all(target.root); + fs::create_directory(target.root); + fs::permissions(target.root, fs::perms::owner_all, fs::perm_options::replace); + const auto clone_start = Clock::now(); + auto cloned = clone_region(*reader, *objects, r.key, *clone_owner, *store(target), target.key, + filled<16>(0xDD), retained, budget, 5s); + const auto clone_ms = ms_between(clone_start, Clock::now()); + CHECK(cloned); + std::printf("qualification verify_head_us payload_bytes=%llu n=10 p50=%llu max=%llu\n", + static_cast(payload_bytes), + static_cast(percentile(verify_us, 50)), + static_cast(percentile(verify_us, 100))); + std::printf("qualification audit_ms commits=%llu ticks=%llu ms=%llu clone_ms=%llu\n", + static_cast(audited->commits), + static_cast(audited->end_tick), + static_cast(audit_ms), + static_cast(clone_ms)); +} +} // namespace + +void run_region_migration_tests() { + durability_qualification(); + upgrade_across_migration_and_barrier(); +} diff --git a/tests/region_recovery_tests.cpp b/tests/region_recovery_tests.cpp index ff1b51c..0fc9bf5 100644 --- a/tests/region_recovery_tests.cpp +++ b/tests/region_recovery_tests.cpp @@ -193,7 +193,7 @@ void tail_verification_before_readiness() { id.fill(v); return id; }; - auto lease = acquire_region_lease(**runtime, r.key, 20000ms, 5s); + auto lease = acquire_region_lease(**runtime, r.key, build, 20000ms, 5s); CHECK(lease); auto head = load_region_checkpoint(**runtime, r.key, 5s); CHECK(head); diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index ed15ef4..10eec1f 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -23,6 +23,8 @@ psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_clone_runtime NOLOGIN' -c 'CREA createdb evo_clone (export PGDATABASE=evo_clone EVO_PG_RUNTIME_ROLE=evo_clone_runtime; bash "$root/tools/migrate-postgres.sh") export EVO_PG_CLONE_DATABASE=evo_clone EVO_PG_CLONE_RUNTIME_USER=evo_clone_login +# The online migration test (M7.9b) runs the checked-in migrations from here. +export EVO_SOURCE_ROOT="$root" # Process tests spawn the server and clone tool built beside the test binary. export EVO_SERVER_BIN EVO_WORLD_CLONE_BIN EVO_SERVER_BIN="$(dirname "${1:?path to evo-postgres-tests}")/evo-server" @@ -53,7 +55,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -74,7 +76,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 7:1,2,3,4,5,6,7 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 8:1,2,3,4,5,6,7,8 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -128,7 +130,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick # Inherited privileges bypass fencing, so every migrate run audits effective rights. @@ -163,9 +165,11 @@ expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.provision_region_check expect_audit_failure 'GRANT SELECT ON evolution.region_commits TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commits FROM evo_runtime_test' bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=8 WHERE version=7' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=9 WHERE version=8' +if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=8 WHERE version=9" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=8" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=7 WHERE version=8" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=7" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=6" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index 70ab03b..8bf94b9 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -26,9 +26,12 @@ digest_6="$(sha256sum "$root/migrations/0006_commit_lookup.sql")" digest_6="${digest_6%% *}" digest_7="$(sha256sum "$root/migrations/0007_region_clones.sql")" digest_7="${digest_7%% *}" +digest_8="$(sha256sum "$root/migrations/0008_upgrade_barrier.sql")" +digest_8="${digest_8%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ --set="migration_sha_6=$digest_6" --set="migration_sha_7=$digest_7" \ + --set="migration_sha_8=$digest_8" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"