Complete M7.7 recovery verification before readiness

ADR-0065: before any readiness, recovery now verifies the head's tail.
The trace object must exist with its manifest digest, its header must
bind profile, region, predecessor and range; the predecessor (schema v6
load_region_commit) must hash to the digest the head names; replaying
the trace from the predecessor's verified payload must reproduce every
report and the exact head bytes; and trace-derived decisions must equal
the head's durable decisions. Missing, corrupt or non-reproducing tails
quarantine (RecoveryError::tail); nothing older or empty is served.

Test-only crash seams (HostConfig::crash_hook, PublishHooks) and
evo-server --crash-at deliver real SIGKILL at before-upload,
before-transaction, after-commit and after-ack. The kill matrix,
a SIGSTOP/SIGCONT stale owner and damaged-head startups all behave as
the durability contract requires; successors verify before readiness.

Validated: PostgreSQL suites under GCC Release, clang-tidy and
ASan/UBSan; tail-skip and replay-equality mutations each caught. No
Docker/Gitea CI, production-lease kill test or deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-28 22:31:40 -04:00
co-authored by Claude Opus 5.5
parent d6d0408aae
commit 30411a83f9
24 changed files with 990 additions and 284 deletions
+1 -1
View File
@@ -115,7 +115,7 @@ if(EVO_BUILD_POSTGRES)
# Contention tests use one thread and one connection per contender.
add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp
tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp
tests/region_server_process_tests.cpp)
tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp)
target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads)
add_dependencies(evo-postgres-tests evo-server) # Spawned by the process tests.
evo_owned_target(evo-postgres-tests)
+42 -2
View File
@@ -10,6 +10,10 @@
// outcome <tick> <actor> <organism> <commit_sequence>
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
//
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
// after a step's acknowledgements are written and flushed).
#include "evo/server/region_host.hpp"
#include <algorithm>
@@ -22,6 +26,7 @@
#include <fstream>
#include <optional>
#include <poll.h>
#include <signal.h>
#include <string>
#include <string_view>
#include <unistd.h>
@@ -41,8 +46,18 @@ struct Options {
std::filesystem::path objects, ready_file;
server::HostConfig host;
std::chrono::milliseconds drain{10000};
std::string crash_point;
std::uint64_t crash_ordinal{};
};
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
[[noreturn]] void crash_now() {
std::fflush(stdout);
::kill(::getpid(), SIGKILL);
for (;;)
::pause();
}
std::optional<std::uint64_t> number(std::string_view text) {
std::uint64_t value{};
const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value);
@@ -125,7 +140,18 @@ std::optional<Options> parse(int argc, char** argv) {
o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms};
else if (flag == "--lease-profile" && value == "default")
o.host.lease = {};
else
else if (flag == "--crash-at") {
const auto colon = value.find(':');
const auto ordinal =
colon == std::string_view::npos ? std::nullopt : number(value.substr(colon + 1));
const auto point = value.substr(0, colon);
if (!ordinal || *ordinal == 0 ||
(point != "before-upload" && point != "before-transaction" &&
point != "after-commit" && point != "after-ack"))
return std::nullopt;
o.crash_point = point;
o.crash_ordinal = *ordinal;
} else
return std::nullopt;
}
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
@@ -243,6 +269,17 @@ class LineReader {
};
template <typename State> int serve(Options& o) {
if (!o.crash_point.empty() && o.crash_point != "after-ack") {
o.host.crash_hook = [point = o.crash_point, ordinal = o.crash_ordinal,
seen = std::uint64_t{0}](server::CrashPoint at) mutable {
const char* name = at == server::CrashPoint::before_upload ? "before-upload"
: at == server::CrashPoint::before_transaction ? "before-transaction"
: "after-commit";
if (point == name && ++seen == ordinal)
crash_now();
};
}
std::uint64_t ack_steps = 0;
auto objects = persistence::FilesystemObjectStore::open(o.objects, o.host.key,
persistence::max_lineage_payload_bytes);
if (!objects) {
@@ -317,7 +354,8 @@ template <typename State> int serve(Options& o) {
static_cast<unsigned long long>(s.simulated_tick));
}
set_ready(s.ready);
for (const auto& a : host.take_acks())
const auto acknowledged = host.take_acks();
for (const auto& a : acknowledged)
std::printf("ack %s %llu %llu\n", text(a.request).c_str(),
static_cast<unsigned long long>(a.tick),
static_cast<unsigned long long>(a.commit_sequence));
@@ -330,6 +368,8 @@ template <typename State> int serve(Options& o) {
for (const auto& d : host.take_dropped())
std::printf("dropped %s %s\n", text(d.input.request).c_str(), dropped_name(d.reason));
std::fflush(stdout);
if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal)
crash_now();
const auto finish = [&](int code) {
set_ready(false);
std::printf(
@@ -3,7 +3,7 @@
apiVersion: batch/v1
kind: Job
metadata:
name: evolution-db-migrate-v5
name: evolution-db-migrate-v6
namespace: evolution-dev
spec:
backoffLimit: 0
+9
View File
@@ -567,3 +567,12 @@ M7.6 implemented and locally qualified (2026-09-28), under ADR-0064.
- `evo-server` passed real SIGTERM drain and SIGKILL tests, with successor
recovery.
Next: M7.7 recovery verification at every kill boundary.
M7.7 implemented and locally qualified (2026-09-28), under ADR-0065.
- Before readiness, recovery replays the head's trace from its predecessor's
verified payload, requires the exact head bytes and decisions, and
quarantines on any missing or corrupt tail. Schema v6 adds the read-only
commit lookup.
- Real SIGKILL at five boundaries, a resumed SIGSTOPped owner, and damaged heads
all behave as the crash matrix requires.
Next: M7.8 operations endpoints and dev image.
+19
View File
@@ -861,3 +861,22 @@ and ASan/UBSan.
Next: M7.7 recovery verification at every kill boundary. There is no gateway,
metrics or deployment yet.
## M7.7 recovery before readiness (2026-09-28)
ADR-0065: before readiness, recovery now replays the head's validation trace
from the predecessor's verified payload. It must reproduce the head bytes
exactly and derive exactly the durable decisions. A missing, corrupt or
non-reproducing tail, or a missing predecessor payload, quarantines the region,
and nothing older or empty is ever served. Schema v6 adds the read-only
`load_region_commit`.
Real `evo-server` processes were SIGKILLed at five publication and ACK
boundaries using a test-only `--crash-at` injector. Every result matched the
crash matrix, and each successor verified the head before becoming ready. A
SIGSTOPped owner resumed fenced, and damaged heads exited quarantined.
Tail-skip and replay-equality mutations were each caught. The PostgreSQL suites
passed under Release, analysis and ASan/UBSan.
Next: M7.8 operations endpoints and a dev image.
@@ -0,0 +1,78 @@
# ADR-0065: Recovery verification before readiness
## Status
Accepted for M7.7, 2026-09-28. It extends ADR-0061 (recovery), ADR-0063 (batch
evidence) and ADR-0064 (host). Clone, migration and durability qualification are
M7.9.
## Context
The M7.7 criterion is: load a verified committed checkpoint, verify the tail,
hash, inventories and history, and hold valid current authority before serving.
Kill tests must cover every publication and acknowledgement boundary; a corrupt
or missing tail must quarantine; there must be no empty fallback world.
The full-checkpoint profile (ADR-0058) has no replay tail beyond the head. The
head's own batch evidence (trace, decisions) was published in M7.5, but recovery
did not re-verify it. The crash matrix rows were exercised in process, but not by
killing a real process at each boundary.
## Decision
**Tail verification.** Recovery, which the host completes before readiness under
a freshly acquired lease, now also verifies the head's batch evidence after the
manifest, build, payload and decision checks. For any head after genesis:
1. The manifest must be v2.
2. The trace object must exist with its manifest length and SHA-256, and its
header must bind the profile, region, predecessor commit and digest, and the
tick range.
3. The predecessor is loaded by sequence through the new read-only
`evolution.load_region_commit` (schema v6; runtime function set is eleven).
Its manifest must hash to the digest the head names.
4. The predecessor's payload is fetched, verified and restored. Replaying the
trace must reproduce every report and the head payload bytes exactly.
5. The decisions derived from the trace must equal the head commit's durable
decisions.
A missing, corrupt or non-reproducing trace, or a missing predecessor payload, is
`RecoveryError::tail`; a decision mismatch is `history`. Either quarantines the
region: no older head, genesis or empty world is ever served.
The replay equality is the inventory check. It proves the head state follows,
under the pinned rules, from a predecessor that was itself verified when it was
published. The envelope's own restore audit covers internal totals. Recovery
cost grows by one predecessor restore and at most 64 replayed ticks.
**Kill boundaries.** `HostConfig::crash_hook` and `PublishHooks` are test-only
seams at three points: before upload, before the transaction (after verified
uploads), and after commit (before any acknowledgement). `evo-server --crash-at
<point>:<n>` delivers a real SIGKILL there, or right after a step's
acknowledgements are flushed (`after-ack`).
## Alternatives Considered
- **Verifying the whole chain on every recovery**: unbounded cost. Earlier links
were verified when published, and the chain digests bind them.
- **Skipping trace verification**: leaves a published but unreproducible head
servable.
- **Kill points chosen by sleeps and signals**: nondeterministic; hooks fire
exactly at the boundary.
## Benefits
- Real killed processes at every boundary show the contract end to end.
- A resumed stale owner cannot serve.
- Damaged heads and tails never become ready.
## Costs
- Recovery reads two payloads and one trace, and replays up to 64 ticks.
- Predecessor objects must be retained (there is no GC).
- The crash seams exist in production binaries but are inert unless configured.
## Risks
- Tail verification covers the head link only; a damaged older link is detected
only when it becomes a predecessor, or by a future audit.
- The kill tests use the development 4 s lease on a single host with a local
database.
- VM freezes and host loss remain M8.
## Consequences
- M7.8 exposes readiness and quarantine to probes and metrics.
- M7.9 measures recovery latency with the production lease, and qualifies clones
and migrations against this verification.
+1
View File
@@ -75,3 +75,4 @@ design. No unadopted candidate is a link dependency of the proof.
- [ADR-0062: Lineage checkpoint profile](0062-lineage-checkpoint-profile.md) - accepted for M7.4b archive codec, LineageRegion closure and history-capable profile 2.
- [ADR-0063: Batch outcomes](0063-batch-outcomes.md) - accepted for M7.5 manifest v2, validation trace replay, durable decisions/outbox and commit resolution.
- [ADR-0064: Region host](0064-region-host.md) - accepted for M7.6 single-owner host state machine, bounded admission, final-only-after-commit and evo-server process.
- [ADR-0065: Recovery before readiness](0065-recovery-before-readiness.md) - accepted for M7.7 head tail replay verification, schema v6 commit lookup and real kill-boundary tests.
+10
View File
@@ -210,3 +210,13 @@ M7.6 (ADR-0064) executed its host rows:
- Shutdown drains, commits and releases.
- A real SIGKILLed `evo-server` left no acknowledged tick uncommitted, and its
successor became ready only after lease expiry, from the committed head.
M7.7 (ADR-0065) executed the matrix with real `evo-server` processes SIGKILLed at
each boundary: before the transaction (first and second batch), before upload,
after commit but before acknowledgement, and after acknowledgement.
- In every case acknowledgements stayed at or below the committed head.
- Deaths were final only once committed, and verified orphans remained.
- Each successor verified the head and its tail before readiness.
A SIGSTOPped owner resumed fenced: it was not ready, refused admission and
acknowledged nothing new. Corrupt payloads and missing traces quarantined with
no fallback. Host loss and clone or migration remain M8 and M7.9.
+42
View File
@@ -2339,3 +2339,45 @@ Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate
- Not executed: Docker/Gitea CI, TSan (the host is single-threaded; libpq is
uninstrumented), remote PostgreSQL/TLS, a production 20 s lease kill test,
durable client deduplication, and a network gateway.
## M7.7 recovery before readiness - 2026-09-28
- WSL Linux: GCC14 Release, clang19 analysis (clang-tidy, clean on the first
build) and clang19 ASan/UBSan builds of `evo-postgres-tests` and `evo-server`
passed the full PostgreSQL suite, including the M7.7 recovery tests. Runs:
Release 2, analysis 1, ASan 2. The repository format check passed after
reordering one fixture include.
- Windows was not rerun. This change touches only Linux persistence and host
code, SQL, tests and docs; M7.6's Windows run (58/58) covers the portable
targets, which are unchanged.
- Kill matrix (real SIGKILL via `--crash-at`, inputs 1..40 with a death at tick
20, 5 s flush):
| Crash point | Head (seq/tick) | Acked up to | Decisions | Objects |
|---|---|---|---|---|
| before-transaction:1 | 0/0 | none | 0 | 3 (verified orphans) |
| before-upload:2 | 1/20 | 20 | 1 | 3 |
| before-transaction:2 | 1/20 | 20 | 1 | 5 (orphans) |
| after-commit:2 | 2/40 | 20 (lost acks are the safe direction) | 1 | 5 |
| after-ack:2 | 2/40 | 40 | 1 | 5 |
The lease stayed held after every kill. Each successor became ready at epoch 2
from the committed head (with tail verification), then drained and released.
- Resumed stale owner: the owner was SIGSTOPped for 4.5 s (4 s lease). The
successor became ready at epoch 2 with committed=10 and processed ticks 11–20.
After SIGCONT the old owner printed `ready 0`, rejected new inputs as
`not_ready`, acknowledged nothing beyond tick 10, and exited 0 on SIGTERM
without touching the successor's chain.
- Damaged heads: a flipped head payload byte and a missing head trace each made
`evo-server` exit 4 (quarantined) without ever printing `ready 1`, and the head
was unchanged. After repair a successor became ready at epoch 4.
- Tail verification (in-process): a missing or corrupt head trace and a missing
predecessor payload are `tail`, and recovery succeeds again after repair. A
published trace of different inputs over the same range (a host bypass the
database cannot detect) is `tail`.
- Mutations (restored and rerun green): skipping tail verification made the
quarantine test time out because the server became ready. Dropping replay
byte-equality failed the forged-trace test (line 262).
- Not executed: Docker/Gitea CI, the production 20 s lease, host loss or VM
freeze, multi-host network partitions, or verification of links older than
the head.
+5 -2
View File
@@ -68,7 +68,7 @@ every role transitively in it, and rolls back on any violation:
`pg_write_all_data`);
- any table privilege in `evolution`, including via PUBLIC;
- CREATE on the schema, the database or `public`;
- EXECUTE outside the ten fenced functions (schema v5).
- EXECUTE outside the eleven fenced functions (schema v6).
Attribute checks alone are not enough: a login with `pg_write_all_data` can update
lease rows directly. Rerun the migrate Job after adding or changing runtime logins
or grants; it is the audit. Grants made later without a rerun are not detected. Bootstrap revokes PUBLIC access to the schema, tables and
@@ -185,6 +185,9 @@ Exit codes:
- 4: quarantined;
- 2: usage error.
Use the default 20 s lease in any shared environment. `--lease-profile fast` exists
`--crash-at <point>:<n>` (before-upload, before-transaction, after-commit or
after-ack) is a test-only fault injector that SIGKILLs the process at that
boundary; never set it in shared environments. Use the default 20 s lease in any
shared environment. `--lease-profile fast` exists
only for local tests. The runtime credentials come from `--pg-*` flags and a
passfile; never place passwords on the command line.
+17 -4
View File
@@ -5,6 +5,7 @@
#include "evo/persistence/region_lease_store.hpp"
#include <chrono>
#include <expected>
#include <functional>
#include <memory>
#include <span>
#include <vector>
@@ -41,6 +42,11 @@ struct PublicationFailure {
PreparedCommit prepared; // Present for rejected/unknown.
};
// Durably confirmed batch: its decisions are now final and may be acknowledged.
// Test/fault-injection seam: called after verified uploads, immediately before the
// fenced transaction is sent. Production callers pass nothing.
struct PublishHooks {
std::function<void()> before_transaction;
};
struct PublishedBatch {
CommittedCheckpoint committed;
std::vector<BatchDecision> decisions;
@@ -54,6 +60,7 @@ enum class RecoveryError : std::uint8_t {
corrupt_payload, // Length/digest mismatch: quarantine, no older head.
payload, // Verified bytes fail full state validation: quarantine.
history, // Durable decisions disagree with the restored lifecycle: quarantine.
tail, // Head batch evidence missing, corrupt or not reproducible: quarantine.
};
struct RecoveryFailure {
RecoveryError kind{};
@@ -85,9 +92,15 @@ publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects,
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
const RegionBatch<processes::EnvironmentalCommunity>& batch,
Uuid commit, const Sha256Digest& build, std::size_t restore_budget,
std::chrono::milliseconds timeout);
// Loads and fully verifies the committed head and reconciles durable decisions.
// Readiness additionally requires a fresh valid lease acquired before this call.
std::chrono::milliseconds timeout, const PublishHooks& hooks = {});
// Loads and fully verifies the committed head before any readiness (ADR-0065):
// manifest digest and chain fields, retained build, payload length/SHA-256 and full
// restore; then the head's tail: its trace object binds to the committed
// predecessor, replaying it from the predecessor's verified payload reproduces the
// head bytes, and its derived decisions equal the durable ones; finally every
// durable decision agrees with the restored lifecycle. Any failure quarantines;
// nothing older or empty is ever substituted. Readiness also requires a fresh
// valid lease acquired before this call.
[[nodiscard]] std::expected<RecoveredCheckpoint, RecoveryFailure>
recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
std::span<const Sha256Digest> retained_builds,
@@ -106,7 +119,7 @@ publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const
const CommittedCheckpoint& predecessor,
const RegionBatch<processes::LineageRegion>& batch, Uuid commit,
const Sha256Digest& build, std::size_t restore_budget,
std::chrono::milliseconds timeout);
std::chrono::milliseconds timeout, const PublishHooks& hooks = {});
[[nodiscard]] std::expected<RecoveredLineageCheckpoint, RecoveryFailure>
recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
std::span<const Sha256Digest> retained_builds,
@@ -50,6 +50,11 @@ release_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease,
load_region_head(Postgres& db, RegionKey key, std::chrono::milliseconds timeout);
[[nodiscard]] std::expected<CommittedCheckpoint, LeaseStoreFailure>
load_region_checkpoint(Postgres& db, RegionKey key, std::chrono::milliseconds timeout);
// Any committed link by sequence (schema v6): its identity, digest, end tick and
// exact manifest bytes. not_provisioned when the region has no such commit.
[[nodiscard]] std::expected<CommittedCheckpoint, LeaseStoreFailure>
load_region_commit(Postgres& db, RegionKey key, std::uint64_t sequence,
std::chrono::milliseconds timeout);
// Canonical manifest v2 bytes plus the decision block they digest (ADR-0063). The
// database parses identity, lease and chain, locks the lease then the head, requires
// the exact committed predecessor and records decisions and outbox rows in the same
+9
View File
@@ -75,6 +75,12 @@ struct HostStatus {
std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{};
};
// Commit boundaries for fault injection (M7.7 kill tests).
enum class CrashPoint : std::uint8_t {
before_upload, // Batch sealed; nothing uploaded or sent.
before_transaction, // Payload and trace uploaded and verified; manifest not sent.
after_commit, // Commit confirmed; no acknowledgement or outcome emitted yet.
};
using Connect =
std::function<std::expected<std::unique_ptr<persistence::Postgres>, persistence::PgFailure>()>;
struct HostConfig {
@@ -90,6 +96,9 @@ struct HostConfig {
persistence::Sha256Digest build{};
std::vector<persistence::Sha256Digest> retained_builds;
std::size_t restore_budget{8000000};
// Test-only: called at each commit boundary (e.g. to SIGKILL the process).
// Production configurations leave it empty.
std::function<void(CrashPoint)> crash_hook;
};
enum class HostConfigError : std::uint8_t { invalid };
+23
View File
@@ -0,0 +1,23 @@
-- M7.7 recovery tail verification (ADR-0065): read any committed link of a region
-- by sequence, so recovery can replay the head's trace from its predecessor.
-- Read-only; returns nothing a runtime could not already derive from the chain.
CREATE FUNCTION evolution.load_region_commit(p_world evolution.u64, p_region evolution.u64,
p_sequence evolution.u64)
RETURNS TABLE (region_frame evolution.u64, commit_sequence evolution.u64, commit_id uuid,
commit_digest text, end_tick evolution.u64, manifest text)
LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
BEGIN
RETURN QUERY SELECT l.frame_id, c.commit_sequence, c.commit_id, encode(c.commit_digest, 'hex'),
c.end_tick, encode(c.manifest, 'hex')
FROM evolution.region_commits c
JOIN evolution.region_leases l USING (world_id, region_id)
WHERE c.world_id = p_world AND c.region_id = p_region AND c.commit_sequence = p_sequence;
IF NOT FOUND THEN
RAISE EXCEPTION 'no such commit' USING ERRCODE = 'EVL04';
END IF;
END $$;
REVOKE ALL ON FUNCTION evolution.load_region_commit(evolution.u64, evolution.u64, evolution.u64)
FROM PUBLIC;
GRANT EXECUTE ON FUNCTION evolution.load_region_commit(evolution.u64, evolution.u64, evolution.u64)
TO :"runtime_role";
INSERT INTO evolution.schema_migrations(version, sha256) VALUES (6, :'migration_sha_6');
+15 -4
View File
@@ -24,26 +24,35 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution')
|| ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' AS at_v4,
:'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2'
|| ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4'
|| ',5:' || :'migration_sha_5' AS at_v5 \gset
\if :at_v5
|| ',5:' || :'migration_sha_5' AS at_v5,
:'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2'
|| ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4'
|| ',5:' || :'migration_sha_5' || ',6:' || :'migration_sha_6' AS at_v6 \gset
\if :at_v6
\elif :at_v5
\ir 0006_commit_lookup.sql
\elif :at_v4
\ir pre_v5_guard.sql
\ir 0005_batch_decisions.sql
\ir 0006_commit_lookup.sql
\elif :at_v3
\ir 0004_lineage_profile.sql
\ir pre_v5_guard.sql
\ir 0005_batch_decisions.sql
\ir 0006_commit_lookup.sql
\elif :at_v2
\ir 0003_region_checkpoints.sql
\ir 0004_lineage_profile.sql
\ir pre_v5_guard.sql
\ir 0005_batch_decisions.sql
\ir 0006_commit_lookup.sql
\elif :at_v1
\ir 0002_region_fencing.sql
\ir 0003_region_checkpoints.sql
\ir 0004_lineage_profile.sql
\ir pre_v5_guard.sql
\ir 0005_batch_decisions.sql
\ir 0006_commit_lookup.sql
\else
\echo 'Unknown or changed migration history'
DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$;
@@ -55,6 +64,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution')
\ir 0004_lineage_profile.sql
\ir pre_v5_guard.sql
\ir 0005_batch_decisions.sql
\ir 0006_commit_lookup.sql
\endif
-- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes
-- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC
@@ -84,7 +94,8 @@ BEGIN
('evolution.resolve_region_commit(evolution.u64,evolution.u64,uuid,bytea,bytea)'::regprocedure::oid),
('evolution.load_region_decisions(evolution.u64,evolution.u64)'::regprocedure::oid),
('evolution.pending_region_outbox(evolution.u64,evolution.u64,integer)'::regprocedure::oid),
('evolution.acknowledge_region_outbox(evolution.u64,evolution.u64,integer,integer,evolution.u64)'::regprocedure::oid)
('evolution.acknowledge_region_outbox(evolution.u64,evolution.u64,integer,integer,evolution.u64)'::regprocedure::oid),
('evolution.load_region_commit(evolution.u64,evolution.u64,evolution.u64)'::regprocedure::oid)
), violations(role_oid, reason) AS (
SELECT r.oid, 'privileged attribute or owner identity' FROM pg_catalog.pg_roles r
JOIN closure c ON c.oid = r.oid
@@ -108,7 +119,7 @@ BEGIN
OR EXISTS (SELECT 1 FROM pg_catalog.pg_namespace n WHERE n.nspname = 'public'
AND pg_catalog.has_schema_privilege(c.oid, n.oid, 'CREATE'))
UNION ALL
-- Only the ten guarded functions are executable; provisioning/internal are not.
-- Only the eleven guarded functions are executable; provisioning/internal are not.
SELECT c.oid, 'EXECUTE outside the fenced function set' FROM closure c
CROSS JOIN pg_catalog.pg_proc p
WHERE p.pronamespace = 'evolution'::regnamespace
+79 -5
View File
@@ -175,7 +175,8 @@ std::expected<PublishedBatch, PublicationFailure>
publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
const CommittedCheckpoint& predecessor,
const RegionBatch<typename Codec::State>& batch, Uuid commit,
const Sha256Digest& build, std::size_t budget, milliseconds timeout) {
const Sha256Digest& build, std::size_t budget, milliseconds timeout,
const PublishHooks& hooks) {
const auto previous = decode_manifest(predecessor.manifest);
if (!previous || previous->profile != Codec::profile || !batch.sealed() ||
batch.steps().empty() || batch.begin_tick() != previous->end_tick)
@@ -231,6 +232,8 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra
return std::unexpected{stored.error()};
PreparedCommit prepared{previous->key, commit, *bytes, *decision_bytes};
if (hooks.before_transaction)
hooks.before_transaction();
auto published = publish_region_checkpoint(db, *bytes, *decision_bytes, timeout);
auto committed = confirm(published, std::move(*bytes), *manifest, std::move(prepared));
if (!committed)
@@ -238,6 +241,72 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra
return PublishedBatch{std::move(*committed), decisions};
}
// Replays the head's recorded batch from its committed predecessor (ADR-0065).
template <typename Codec>
std::expected<void, RecoveryFailure>
verify_tail(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const CheckpointManifest& m, std::span<const std::uint8_t> head_payload,
const typename Codec::State& head_state, std::span<const CommittedDecision> durable,
std::size_t budget, milliseconds timeout) {
const auto fail_with = [](RecoveryError kind) {
return std::unexpected{RecoveryFailure{kind, {}}};
};
if (m.sequence == 0)
return {}; // Genesis has no batch.
if (m.manifest_version != 2)
return fail_with(RecoveryError::unsupported);
auto trace_bytes = objects.get({m.batch.trace_sha256, m.batch.trace_bytes});
if (!trace_bytes)
return fail_with(trace_bytes.error() == ObjectError::io ? RecoveryError::store
: RecoveryError::tail);
auto trace = decode_trace(*trace_bytes);
const TraceHeader expected{static_cast<std::uint32_t>(Codec::profile),
key,
m.predecessor_commit,
m.predecessor_digest,
m.begin_tick,
m.end_tick};
if (!trace || trace->header != expected)
return fail_with(RecoveryError::tail);
auto predecessor = load_region_commit(db, key, m.sequence - 1, timeout);
if (!predecessor)
return predecessor.error().kind == LeaseStoreError::not_provisioned
? fail_with(RecoveryError::tail)
: std::unexpected{RecoveryFailure{RecoveryError::store, predecessor.error()}};
const auto digest = sha256(predecessor->manifest);
auto previous = decode_manifest(predecessor->manifest);
if (!digest || *digest != predecessor->head.digest ||
predecessor->head.digest != m.predecessor_digest ||
predecessor->head.commit != m.predecessor_commit || !previous ||
previous->end_tick != m.begin_tick)
return fail_with(RecoveryError::tail);
auto base = objects.get({previous->payload_sha256, previous->payload_bytes});
if (!base)
return fail_with(base.error() == ObjectError::io ? RecoveryError::store
: RecoveryError::tail);
auto replay = Codec::restore(*base, *previous, budget);
if (!replay)
return fail_with(RecoveryError::tail);
for (const auto& step : trace->steps) {
auto report = (*replay)->advance(step.report.tick, step.input);
if (!report || *report != step.report)
return fail_with(RecoveryError::tail);
}
const auto replayed = Codec::snapshot(**replay);
if (!replayed || !std::ranges::equal(*replayed, head_payload))
return fail_with(RecoveryError::tail);
// The head's durable outcomes are exactly those its trace derives.
const auto organisms = Codec::organisms(head_state);
const auto derived = derive_decisions(trace->steps, organisms.producer, organisms.consumer);
std::vector<BatchDecision> recorded;
for (const auto& d : durable)
if (d.commit_sequence == m.sequence)
recorded.push_back(d.decision);
if (derived != recorded || derived.size() != m.batch.decision_count)
return fail_with(RecoveryError::history);
return {};
}
template <typename Codec>
std::expected<typename Codec::Recovered, RecoveryFailure>
recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
@@ -284,6 +353,10 @@ recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
return store_failure(decisions.error());
if (!Codec::reconcile(**state, *decisions, head))
return fail_with(RecoveryError::history);
if (auto tail = verify_tail<Codec>(db, objects, key, *manifest, *payload, **state, *decisions,
budget, timeout);
!tail)
return std::unexpected{tail.error()};
return typename Codec::Recovered{std::move(*committed), *manifest, std::move(*state),
std::move(*decisions)};
}
@@ -301,9 +374,9 @@ publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects,
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
const RegionBatch<processes::EnvironmentalCommunity>& batch,
Uuid commit, const Sha256Digest& build, std::size_t restore_budget,
milliseconds timeout) {
milliseconds timeout, const PublishHooks& hooks) {
return publish_batch<EnvironmentalCodec>(db, objects, lease, predecessor, batch, commit, build,
restore_budget, timeout);
restore_budget, timeout, hooks);
}
std::expected<RecoveredCheckpoint, RecoveryFailure>
recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
@@ -324,9 +397,10 @@ std::expected<PublishedBatch, PublicationFailure>
publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
const CommittedCheckpoint& predecessor,
const RegionBatch<processes::LineageRegion>& batch, Uuid commit,
const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout) {
const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout,
const PublishHooks& hooks) {
return publish_batch<LineageCodec>(db, objects, lease, predecessor, batch, commit, build,
restore_budget, timeout);
restore_budget, timeout, hooks);
}
std::expected<RecoveredLineageCheckpoint, RecoveryFailure>
recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
+19
View File
@@ -216,6 +216,25 @@ load_region_checkpoint(Postgres& db, RegionKey key, milliseconds timeout) {
return CommittedCheckpoint{*head, std::move(*manifest)};
}
std::expected<CommittedCheckpoint, LeaseStoreFailure>
load_region_commit(Postgres& db, RegionKey key, std::uint64_t sequence, milliseconds timeout) {
const std::array<std::string, 3> p{std::to_string(key.world), std::to_string(key.region),
std::to_string(sequence)};
auto rows = db.query("SELECT * FROM evolution.load_region_commit($1::evolution.u64, "
"$2::evolution.u64, $3::evolution.u64)",
p, timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
auto head = parse_head(*rows, 6);
if (!head || rows->size() != 1)
return std::unexpected{malformed(false)};
const auto& manifest_text = (*rows)[0][5];
auto manifest = manifest_text ? unhex_bytes(*manifest_text) : std::nullopt;
if (!manifest)
return std::unexpected{malformed(false)};
return CommittedCheckpoint{*head, std::move(*manifest)};
}
std::expected<RegionHead, LeaseStoreFailure>
publish_region_checkpoint(Postgres& db, std::span<const std::uint8_t> manifest,
std::span<const std::uint8_t> decisions, milliseconds timeout) {
+9 -1
View File
@@ -228,6 +228,8 @@ template <typename State> persistence::Uuid RegionHost<State>::next_commit_id()
template <typename State>
void RegionHost<State>::confirmed(const persistence::PublishedBatch& published, SchedulerTime now) {
if (config_.crash_hook)
config_.crash_hook(CrashPoint::after_commit);
const auto& head = published.committed.head;
if (!batch_ || !batch_->confirm(head.end_tick)) {
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
@@ -255,9 +257,15 @@ template <typename State> void RegionHost<State>::commit(SchedulerTime now) {
return;
const auto commit_id = next_commit_id();
batch_->seal();
const auto& crash = config_.crash_hook;
if (crash)
crash(CrashPoint::before_upload);
persistence::PublishHooks hooks;
if (crash)
hooks.before_transaction = [&crash] { crash(CrashPoint::before_transaction); };
auto published =
Ops<State>::publish(*db_, *objects_, *grant, *committed_, *batch_, commit_id, config_.build,
config_.restore_budget, config_.lease.commit_timeout);
config_.restore_budget, config_.lease.commit_timeout, hooks);
if (published) {
confirmed(*published, now);
return;
+297
View File
@@ -0,0 +1,297 @@
#pragma once
// Shared real-process helpers for evo-server tests (M7.6/M7.7). Linux only.
#include "environmental_region.hpp"
#include "evo/persistence/checkpoint_store.hpp"
#include <array>
#include <cerrno>
#include <chrono>
#include <csignal>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <memory>
#include <poll.h>
#include <spawn.h>
#include <stdexcept>
#include <string>
#include <string_view>
#include <sys/wait.h>
#include <unistd.h>
#include <utility>
#include <vector>
extern char** environ;
namespace evo::test::process {
using namespace std::chrono_literals;
namespace fs = std::filesystem;
using Clock = std::chrono::steady_clock;
inline void expect(bool ok, const char* what) {
if (!ok)
throw std::runtime_error(std::string("server fixture: ") + what);
}
inline std::string env(const char* key) {
const auto* value = std::getenv(key);
expect(value != nullptr, "missing test environment");
return value;
}
inline std::unique_ptr<persistence::Postgres> owner() {
auto made = persistence::Postgres::connect(
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s);
expect(made.has_value(), "owner connection");
return std::move(*made);
}
// Request identity: a tag byte (one per writer) and the tick.
inline std::string request_hex(std::uint64_t tick, std::uint8_t tag = 0x9B) {
char out[33];
std::snprintf(out, sizeof out, "%02x000000000000000000000000%06llx", tag,
static_cast<unsigned long long>(tick));
return out;
}
inline std::string input_line(std::uint64_t tick, bool consumer_dies, std::uint8_t tag = 0x9B) {
const auto in = region_input(tick, consumer_dies);
std::string line = "input " + request_hex(tick, tag) + " " + std::to_string(in.temperature_mK) +
" " + std::to_string(in.source_enabled);
for (const auto b : in.bite)
line += " " + std::to_string(b);
for (const auto t : in.terminal)
line += " " + std::to_string(t);
return line + "\n";
}
inline std::string input_lines(std::uint64_t first, std::uint64_t last, bool deaths,
std::uint8_t tag = 0x9B) {
std::string out;
for (auto tick = first; tick <= last; ++tick)
out += input_line(tick, deaths, tag);
return out;
}
inline bool starts(const std::string& line, std::string_view prefix) {
return line.starts_with(prefix);
}
class Child {
public:
explicit Child(const std::vector<std::string>& args) {
int to_child[2], from_child[2];
expect(::pipe(to_child) == 0 && ::pipe(from_child) == 0, "pipes");
posix_spawn_file_actions_t actions;
posix_spawn_file_actions_init(&actions);
posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO);
posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO);
posix_spawn_file_actions_addclose(&actions, to_child[1]);
posix_spawn_file_actions_addclose(&actions, from_child[0]);
std::vector<char*> argv;
argv.reserve(args.size() + 1);
for (const auto& a : args)
argv.push_back(const_cast<char*>(a.c_str()));
argv.push_back(nullptr);
expect(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0, "spawn");
posix_spawn_file_actions_destroy(&actions);
::close(to_child[0]);
::close(from_child[1]);
in_ = to_child[1];
out_ = from_child[0];
}
Child(const Child&) = delete;
Child& operator=(const Child&) = delete;
~Child() {
if (pid_ > 0) {
::kill(pid_, SIGKILL);
::waitpid(pid_, nullptr, 0);
}
::close(in_);
::close(out_);
}
void send(const std::string& text) {
std::size_t done = 0;
while (done < text.size()) {
const auto n = ::write(in_, text.data() + done, text.size() - done);
expect(n > 0 || errno == EINTR, "write to child");
if (n > 0)
done += static_cast<std::size_t>(n);
}
}
// Reads lines until one satisfies `want`; every line is kept in lines().
template <typename Want> bool wait_for(Want want, std::chrono::milliseconds timeout) {
const auto deadline = Clock::now() + timeout;
for (std::size_t seen = 0;;) {
for (; seen < lines_.size(); ++seen)
if (want(lines_[seen]))
return true;
const auto left =
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
if (left.count() <= 0 || !read_some(static_cast<int>(left.count())))
return false;
}
}
bool wait_line(std::string_view prefix, std::chrono::milliseconds timeout) {
return wait_for([prefix](const std::string& l) { return starts(l, prefix); }, timeout);
}
void signal(int sig) {
expect(::kill(pid_, sig) == 0, "signal");
}
// Exit code, or 128+signal; drains remaining output.
int wait_exit(std::chrono::milliseconds timeout) {
const auto deadline = Clock::now() + timeout;
int status = 0;
for (;;) {
read_some(20);
const auto r = ::waitpid(pid_, &status, WNOHANG);
if (r == pid_)
break;
expect(Clock::now() < deadline, "child exit deadline");
}
while (read_some(20)) {
}
pid_ = -1;
return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
}
[[nodiscard]] const std::vector<std::string>& lines() const noexcept {
return lines_;
}
[[nodiscard]] bool printed(std::string_view prefix) const {
for (const auto& l : lines_)
if (starts(l, prefix))
return true;
return false;
}
private:
bool read_some(int timeout_ms) {
pollfd fd{out_, POLLIN, 0};
if (::poll(&fd, 1, timeout_ms) <= 0)
return false;
char chunk[4096];
const auto n = ::read(out_, chunk, sizeof chunk);
if (n <= 0)
return false;
buffer_.append(chunk, static_cast<std::size_t>(n));
for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) {
lines_.push_back(buffer_.substr(0, end));
buffer_.erase(0, end + 1);
}
return true;
}
pid_t pid_{-1};
int in_{-1}, out_{-1};
std::string buffer_;
std::vector<std::string> lines_;
};
struct ServerRegion {
persistence::RegionKey key;
fs::path root;
// The region's object namespace directory.
[[nodiscard]] fs::path objects() const {
return root / ("w" + std::to_string(key.world)) / ("r" + std::to_string(key.region));
}
};
// A fresh lineage region (world 7) provisioned by the owner.
inline ServerRegion provision(std::uint64_t region, std::uint8_t genesis) {
ServerRegion r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))};
fs::remove_all(r.root);
fs::create_directory(r.root);
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
auto db = owner();
expect(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s)
.has_value(),
"retain build");
auto objects = persistence::FilesystemObjectStore::open(r.root, r.key,
persistence::max_lineage_payload_bytes);
expect(objects.has_value(), "object store");
persistence::Sha256Digest build{};
build.fill(0xB1);
persistence::Uuid commit{};
commit.fill(genesis);
expect(persistence::provision_lineage_region(*db, **objects, r.key, region_identity.frame,
*lineage_region(), commit, build, 8000000, 5s)
.has_value(),
"provision");
return r;
}
inline std::vector<std::string> server_args(const ServerRegion& r,
std::vector<std::string> extra = {}) {
std::vector<std::string> args{
env("EVO_SERVER_BIN"),
"--profile",
"lineage",
"--world",
std::to_string(r.key.world),
"--region",
std::to_string(r.key.region),
"--frame",
std::to_string(region_identity.frame),
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
env("EVO_PG_RUNTIME_USER"),
"--objects",
r.root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--flush-ms",
"500",
"--lease-profile",
"fast",
"--ready-file",
(r.root / "ready").string()};
args.insert(args.end(), extra.begin(), extra.end());
return args;
}
// (tick, commit_sequence) of every ack line.
inline std::vector<std::pair<std::uint64_t, std::uint64_t>> acks(const Child& c) {
std::vector<std::pair<std::uint64_t, std::uint64_t>> out;
for (const auto& line : c.lines()) {
unsigned long long tick = 0, sequence = 0;
char request[40];
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
out.emplace_back(tick, sequence);
}
return out;
}
inline std::uint64_t max_acked(const Child& c) {
std::uint64_t most = 0;
for (const auto& [tick, sequence] : acks(c))
most = std::max(most, tick);
return most;
}
struct Head {
std::uint64_t tick{}, sequence{};
bool released{};
};
inline Head head_of(persistence::RegionKey key) {
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
auto rows = owner()->query(
"SELECT h.end_tick::text, h.commit_sequence::text, (l.holder IS NULL)::text FROM "
"evolution.region_heads h JOIN evolution.region_leases l USING (world_id, region_id) "
"WHERE world_id = $1::evolution.u64 AND region_id = $2::evolution.u64",
p, 5s);
expect(rows && rows->size() == 1 && (*rows)[0].size() == 3, "head row");
const auto& row = (*rows)[0];
expect(row[0] && row[1] && row[2], "head values");
return {std::stoull(row[0].value_or("0")), std::stoull(row[1].value_or("0")),
row[2].value_or("") == "true"};
}
inline std::size_t decisions_of(persistence::RegionKey key) {
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
auto rows = owner()->query("SELECT count(*)::text FROM evolution.region_decisions WHERE "
"world_id = $1::evolution.u64 AND region_id = $2::evolution.u64",
p, 5s);
expect(rows && rows->size() == 1, "decision count");
return std::stoull((*rows)[0][0].value_or("0"));
}
// Object files in the region namespace (temporaries excluded).
inline std::size_t object_count(const ServerRegion& r) {
std::size_t n = 0;
for (const auto& entry : fs::directory_iterator(r.objects()))
n += entry.path().filename().string().starts_with(".tmp-") ? 0 : 1;
return n;
}
} // namespace evo::test::process
+3
View File
@@ -7,6 +7,7 @@ void run_region_lease_store_tests();
void run_region_checkpoint_store_tests();
void run_region_host_tests();
void run_region_server_process_tests();
void run_region_recovery_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
@@ -81,6 +82,8 @@ int main() {
std::puts("region host integration passed");
run_region_server_process_tests();
std::puts("region server process integration passed");
run_region_recovery_tests();
std::puts("region recovery integration passed");
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "%s\n", e.what());
+276
View File
@@ -0,0 +1,276 @@
// M7.7 recovery before readiness (ADR-0065): real evo-server processes killed at
// every publication/acknowledgement boundary, a resumed stale owner, damaged heads
// that must never serve, and tail verification. Run by tools/ci/postgres-check.sh.
#include "evo/persistence/sha256.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <fstream>
#include <source_location>
#include <thread>
void run_region_recovery_tests();
namespace {
using namespace evo::test::process;
using namespace evo::persistence;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region recovery test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string hex(std::span<const std::uint8_t> bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out;
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
std::vector<char> read_file(const fs::path& p) {
std::ifstream in(p, std::ios::binary);
return {std::istreambuf_iterator<char>(in), {}};
}
void write_file(const fs::path& p, const std::vector<char>& bytes) {
fs::remove(p);
std::ofstream(p, std::ios::binary)
.write(bytes.data(), static_cast<std::streamsize>(bytes.size()));
fs::permissions(p, fs::perms::owner_read | fs::perms::owner_write, fs::perm_options::replace);
}
CheckpointManifest head_manifest(RegionKey key) {
auto db = owner();
auto committed = load_region_checkpoint(*db, key, 5s);
CHECK(committed);
auto m = decode_manifest(committed->manifest);
CHECK(m);
return *m;
}
// Successor: waits for readiness, requires it resumed from the committed head, drains.
void successor_resumes(const ServerRegion& r, std::uint64_t epoch) {
const auto head = head_of(r.key);
Child next{server_args(r)};
CHECK(next.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
CHECK(next.printed("phase running epoch=" + std::to_string(epoch) +
" committed=" + std::to_string(head.tick) + " "));
next.signal(SIGTERM);
CHECK(next.wait_exit(20s) == 0);
const auto after = head_of(r.key);
CHECK(after.tick == head.tick && after.sequence == head.sequence && after.released);
}
struct Boundary {
const char* point;
int ordinal;
std::uint64_t region;
std::uint8_t genesis;
};
// Inputs 1..40 with the consumer's death at tick 20: commit 1 is the death batch
// (ticks 1..20), commit 2 the next interval flush. The process is SIGKILLed at the
// chosen boundary; durable state, acknowledgements and orphans must match it.
void kill_at(const Boundary& b) {
const auto r = provision(b.region, b.genesis);
std::uint64_t acked = 0;
bool outcome = false;
{
// A long flush interval makes commit 1 exactly the death batch.
Child server{server_args(r, {"--flush-ms", "5000", "--crash-at",
std::string(b.point) + ":" + std::to_string(b.ordinal)})};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
server.send(input_lines(1, 40, true));
CHECK(server.wait_exit(30s) == 128 + SIGKILL);
acked = max_acked(server);
outcome = server.printed("outcome 20 1 11 1");
}
const auto head = head_of(r.key);
const auto decisions = decisions_of(r.key);
const auto objects = object_count(r); // Genesis payload plus 2 per uploaded batch.
const std::string point = b.point;
CHECK(!head.released); // Killed holders never release; successors wait for expiry.
CHECK(acked <= head.tick); // Never an acknowledgement beyond the committed head.
if (b.ordinal == 1) {
// The death batch never committed: nothing final, the death is not a fact.
CHECK(point == "before-transaction");
CHECK(head.sequence == 0 && head.tick == 0 && acked == 0 && decisions == 0 && !outcome);
CHECK(objects == 3); // Verified orphans only.
} else if (point == "before-upload" || point == "before-transaction") {
CHECK(head.sequence == 1 && head.tick == 20 && acked == 20 && decisions == 1 && outcome);
CHECK(objects == (point == "before-upload" ? 3U : 5U));
} else if (point == "after-commit") {
// Durable but never acknowledged: the lost acks are the safe direction.
CHECK(head.sequence == 2 && head.tick > 20 && acked == 20 && decisions == 1 && outcome);
} else {
CHECK(point == "after-ack");
CHECK(head.sequence == 2 && acked == head.tick && decisions == 1 && outcome);
}
std::printf("kill_at %s:%d head=%llu/%llu acked=%llu objects=%zu\n", b.point, b.ordinal,
static_cast<unsigned long long>(head.sequence),
static_cast<unsigned long long>(head.tick), static_cast<unsigned long long>(acked),
objects);
successor_resumes(r, 2); // Verified head, tail and history before readiness.
}
// A paused (SIGSTOP) owner loses its lease; after SIGCONT it fences itself, refuses
// admission and never acknowledges past its own commits, while the successor's
// chain continues untouched.
void resumed_stale_owner_is_fenced() {
const auto r = provision(46, 0x62);
Child stale{server_args(r)};
CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
stale.send(input_lines(1, 10, false, 0xA1));
CHECK(stale.wait_line("ack " + request_hex(10, 0xA1) + " 10 ", 15s));
stale.signal(SIGSTOP);
std::this_thread::sleep_for(4500ms); // Longer than the 4 s development lease.
Child owner_now{server_args(r)};
CHECK(owner_now.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
CHECK(owner_now.printed("phase running epoch=2 committed=10 "));
owner_now.send(input_lines(11, 20, false, 0xB1));
CHECK(owner_now.wait_line("ack " + request_hex(20, 0xB1) + " 20 ", 15s));
stale.signal(SIGCONT);
CHECK(stale.wait_line("ready 0", 10s));
stale.send(input_lines(11, 12, false, 0xA2));
CHECK(stale.wait_line("rejected " + request_hex(11, 0xA2) + " not_ready", 10s));
CHECK(max_acked(stale) == 10);
const auto head = head_of(r.key);
CHECK(head.tick >= 20 && head.sequence >= 2 && !head.released);
stale.signal(SIGTERM);
CHECK(stale.wait_exit(20s) == 0); // It never regained authority: nothing to release.
owner_now.signal(SIGTERM);
CHECK(owner_now.wait_exit(20s) == 0);
CHECK(head_of(r.key).released && head_of(r.key).tick == head.tick);
}
// A committed head whose payload or tail cannot be verified is never served, and
// no older or empty world replaces it.
void damaged_heads_never_become_ready() {
const auto r = provision(47, 0x63);
{
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
server.send(input_lines(1, 10, false));
CHECK(server.wait_line("ack " + request_hex(10) + " 10 ", 15s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
}
const auto head = head_of(r.key);
const auto m = head_manifest(r.key);
const auto payload = r.objects() / hex(m.payload_sha256);
const auto trace = r.objects() / hex(m.batch.trace_sha256);
const auto quarantined = [&] {
Child server{server_args(r)};
const auto code = server.wait_exit(20s);
return code == 4 && server.printed("phase quarantined") && !server.printed("ready 1");
};
const auto payload_bytes = read_file(payload);
auto flipped = payload_bytes;
flipped[flipped.size() / 2] = static_cast<char>(flipped[flipped.size() / 2] ^ 1);
write_file(payload, flipped);
CHECK(quarantined());
write_file(payload, payload_bytes);
const auto trace_bytes = read_file(trace);
fs::remove(trace);
CHECK(quarantined());
write_file(trace, trace_bytes);
const auto after = head_of(r.key);
CHECK(after.tick == head.tick && after.sequence == head.sequence); // Nothing replaced it.
successor_resumes(r, 4); // Epochs 2 and 3 were taken by the quarantined attempts.
}
// In-process: recovery replays the head's trace from its predecessor before any
// readiness; missing, damaged or non-reproducing evidence quarantines (tail).
void tail_verification_before_readiness() {
const auto r = provision(48, 0x01);
auto runtime = Postgres::connect(
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}},
5s);
CHECK(runtime);
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(objects);
Sha256Digest build{};
build.fill(0xB1);
const std::array<Sha256Digest, 1> retained{build};
const auto commit = [](std::uint8_t v) {
Uuid id{};
id.fill(v);
return id;
};
auto lease = acquire_region_lease(**runtime, r.key, 20000ms, 5s);
CHECK(lease);
auto head = load_region_checkpoint(**runtime, r.key, 5s);
CHECK(head);
auto state = evo::test::lineage_region();
evo::persistence::RegionBatch<evo::processes::LineageRegion> batch{*state};
for (int n = 0; n < 2; ++n) {
for (int i = 0; i < 10; ++i)
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
batch.seal();
auto published =
publish_lineage_batch(**runtime, **objects, *lease, *head, batch,
commit(static_cast<std::uint8_t>(2 + n)), build, 8000000, 5s);
CHECK(published && batch.confirm(published->committed.head.end_tick));
head = published->committed;
}
const auto recover = [&] {
return recover_lineage_checkpoint(**runtime, **objects, r.key, retained, 8000000, 5s);
};
CHECK(recover());
const auto m = *decode_manifest(head->manifest);
const auto trace = r.objects() / hex(m.batch.trace_sha256);
const auto trace_bytes = read_file(trace);
const auto refused = [&](RecoveryError kind) {
auto rec = recover();
return !rec && rec.error().kind == kind;
};
fs::remove(trace);
CHECK(refused(RecoveryError::tail));
auto damaged = trace_bytes;
damaged[200] = static_cast<char>(damaged[200] ^ 1);
write_file(trace, damaged);
CHECK(refused(RecoveryError::tail));
write_file(trace, trace_bytes);
// The predecessor's payload is needed to replay the head: its loss quarantines.
auto first = load_region_commit(**runtime, r.key, 1, 5s);
CHECK(first);
const auto previous = *decode_manifest(first->manifest);
const auto base = r.objects() / hex(previous.payload_sha256);
const auto base_bytes = read_file(base);
fs::remove(base);
CHECK(refused(RecoveryError::tail));
write_file(base, base_bytes);
CHECK(recover());
// Evidence that does not reproduce the payload: a trace of different inputs over
// the same range, published by bypassing the host (the database cannot tell).
for (int i = 0; i < 10; ++i)
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
auto alternate = recover();
CHECK(alternate);
evo::persistence::RegionBatch<evo::processes::LineageRegion> other{*alternate->state};
for (int i = 0; i < 10; ++i) {
auto in = evo::test::region_input(other.end_tick() + 1);
in.source_enabled = 1 - in.source_enabled; // Not what actually happened.
CHECK(other.advance(in));
}
auto payload = state->snapshot();
auto forged = encode_trace({2, r.key, m.commit, head->head.digest, m.end_tick, m.end_tick + 10},
other.steps());
CHECK(payload && forged && (*objects)->put(*payload) && (*objects)->put(*forged));
auto manifest = successor_manifest(m, head->head.digest, commit(5), *lease, m.end_tick + 10,
build, payload->size(), *sha256(*payload),
{forged->size(), *sha256(*forged), 0, {}});
CHECK(manifest);
CHECK(publish_region_checkpoint(**runtime, *encode_manifest(*manifest), {}, 5s));
CHECK(refused(RecoveryError::tail));
CHECK(release_region_lease(**runtime, r.key, *lease, 5s));
}
} // namespace
void run_region_recovery_tests() {
for (const auto& b :
{Boundary{"before-transaction", 1, 41, 0x5D}, Boundary{"before-upload", 2, 42, 0x5E},
Boundary{"before-transaction", 2, 43, 0x5F}, Boundary{"after-commit", 2, 44, 0x60},
Boundary{"after-ack", 2, 45, 0x61}})
kill_at(b);
resumed_stale_owner_is_fenced();
damaged_heads_never_become_ready();
tail_verification_before_readiness();
}
+21 -260
View File
@@ -1,249 +1,20 @@
// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL
// with recovery by a successor process. Run by tools/ci/postgres-check.sh, which
// sets EVO_SERVER_BIN.
#include "evo/persistence/checkpoint_store.hpp"
#include "fixtures/environmental_region.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <cerrno>
#include <chrono>
#include <csignal>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <poll.h>
#include <source_location>
#include <spawn.h>
#include <stdexcept>
#include <string>
#include <sys/wait.h>
#include <thread>
#include <unistd.h>
#include <vector>
extern char** environ;
void run_region_server_process_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
namespace fs = std::filesystem;
using Clock = std::chrono::steady_clock;
using namespace evo::test::process;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region server process test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
std::unique_ptr<Postgres> owner() {
auto made = Postgres::connect(
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s);
CHECK(made);
return std::move(*made);
}
std::string hex32(std::uint64_t tick) {
char out[33];
std::snprintf(out, sizeof out, "9b000000000000000000000000%06llx",
static_cast<unsigned long long>(tick));
return out;
}
std::string input_line(std::uint64_t tick, bool consumer_dies) {
const auto in = evo::test::region_input(tick, consumer_dies);
std::string line = "input " + hex32(tick) + " " + std::to_string(in.temperature_mK) + " " +
std::to_string(in.source_enabled);
for (const auto b : in.bite)
line += " " + std::to_string(b);
for (const auto t : in.terminal)
line += " " + std::to_string(t);
return line + "\n";
}
class Child {
public:
explicit Child(const std::vector<std::string>& args) {
int to_child[2], from_child[2];
CHECK(::pipe(to_child) == 0 && ::pipe(from_child) == 0);
posix_spawn_file_actions_t actions;
posix_spawn_file_actions_init(&actions);
posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO);
posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO);
posix_spawn_file_actions_addclose(&actions, to_child[1]);
posix_spawn_file_actions_addclose(&actions, from_child[0]);
std::vector<char*> argv;
argv.reserve(args.size() + 1);
for (const auto& a : args)
argv.push_back(const_cast<char*>(a.c_str()));
argv.push_back(nullptr);
CHECK(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0);
posix_spawn_file_actions_destroy(&actions);
::close(to_child[0]);
::close(from_child[1]);
in_ = to_child[1];
out_ = from_child[0];
}
Child(const Child&) = delete;
Child& operator=(const Child&) = delete;
~Child() {
if (pid_ > 0) {
::kill(pid_, SIGKILL);
::waitpid(pid_, nullptr, 0);
}
::close(in_);
::close(out_);
}
void send(const std::string& text) {
std::size_t done = 0;
while (done < text.size()) {
const auto n = ::write(in_, text.data() + done, text.size() - done);
CHECK(n > 0 || errno == EINTR);
if (n > 0)
done += static_cast<std::size_t>(n);
}
}
// Reads lines until one satisfies `want`; every line is kept in lines().
template <typename Want> bool wait_for(Want want, std::chrono::milliseconds timeout) {
const auto deadline = Clock::now() + timeout;
for (std::size_t seen = 0;;) {
for (; seen < lines_.size(); ++seen)
if (want(lines_[seen]))
return true;
const auto left =
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
if (left.count() <= 0 || !read_some(static_cast<int>(left.count())))
return false;
}
}
void signal(int sig) {
CHECK(::kill(pid_, sig) == 0);
}
// Exit code, or 128+signal; drains remaining output.
int wait_exit(std::chrono::milliseconds timeout) {
const auto deadline = Clock::now() + timeout;
int status = 0;
for (;;) {
read_some(20);
const auto r = ::waitpid(pid_, &status, WNOHANG);
if (r == pid_)
break;
CHECK(Clock::now() < deadline);
}
while (read_some(20)) {
}
pid_ = -1;
return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
}
[[nodiscard]] const std::vector<std::string>& lines() const noexcept {
return lines_;
}
private:
bool read_some(int timeout_ms) {
pollfd fd{out_, POLLIN, 0};
if (::poll(&fd, 1, timeout_ms) <= 0)
return false;
char chunk[4096];
const auto n = ::read(out_, chunk, sizeof chunk);
if (n <= 0)
return false;
buffer_.append(chunk, static_cast<std::size_t>(n));
for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) {
lines_.push_back(buffer_.substr(0, end));
buffer_.erase(0, end + 1);
}
return true;
}
pid_t pid_{-1};
int in_{-1}, out_{-1};
std::string buffer_;
std::vector<std::string> lines_;
};
struct Region {
RegionKey key;
fs::path root;
};
Region provision(std::uint64_t region, std::uint8_t genesis) {
Region r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))};
fs::remove_all(r.root);
fs::create_directory(r.root);
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
auto db = owner();
CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(objects);
Sha256Digest build{};
build.fill(0xB1);
Uuid commit{};
commit.fill(genesis);
CHECK(provision_lineage_region(*db, **objects, r.key, evo::test::region_identity.frame,
*evo::test::lineage_region(), commit, build, 8000000, 5s));
return r;
}
std::vector<std::string> server_args(const Region& r) {
return {env("EVO_SERVER_BIN"),
"--profile",
"lineage",
"--world",
"7",
"--region",
std::to_string(r.key.region),
"--frame",
std::to_string(evo::test::region_identity.frame),
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
env("EVO_PG_RUNTIME_USER"),
"--objects",
r.root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--flush-ms",
"500",
"--lease-profile",
"fast",
"--ready-file",
(r.root / "ready").string()};
}
// (tick, commit_sequence) of every ack line.
std::vector<std::pair<std::uint64_t, std::uint64_t>> acks(const Child& c) {
std::vector<std::pair<std::uint64_t, std::uint64_t>> out;
for (const auto& line : c.lines()) {
unsigned long long tick = 0, sequence = 0;
char request[40];
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
out.emplace_back(tick, sequence);
}
return out;
}
std::string head_of(RegionKey key) {
const std::array<std::string, 1> p{std::to_string(key.region)};
auto rows = owner()->query("SELECT h.end_tick::text || ':' || h.commit_sequence::text || ':' "
"|| coalesce(l.holder::text, 'released') FROM "
"evolution.region_heads h JOIN evolution.region_leases l USING "
"(world_id, region_id) WHERE world_id = 7 AND region_id = "
"$1::evolution.u64",
p, 5s);
CHECK(rows && rows->size() == 1);
const auto& value = (*rows)[0][0];
CHECK(value.has_value());
return value.value_or("");
}
std::uint64_t head_tick(RegionKey key) {
return std::stoull(head_of(key).substr(0, head_of(key).find(':')));
}
bool starts(const std::string& line, std::string_view prefix) {
return line.starts_with(prefix);
}
// SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every
// acknowledged tick is committed, the backlog never exceeds its bound, and the
@@ -253,19 +24,12 @@ void sigterm_drains_and_releases() {
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
CHECK(fs::exists(r.root / "ready"));
std::string burst;
for (std::uint64_t tick = 1; tick <= 80; ++tick)
burst += input_line(tick, true);
server.send(burst); // More than the 64-input backlog: stdin backpressures.
CHECK(server.wait_for(
[](const std::string& l) { return starts(l, "ack " + hex32(80) + " 80 "); }, 30s));
server.send(input_lines(1, 80, true)); // More than the 64-input backlog: stdin backpressures.
CHECK(server.wait_line("ack " + request_hex(80) + " 80 ", 30s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
const auto& lines = server.lines();
CHECK(std::ranges::find(lines, "exit 0") != lines.end());
CHECK(std::ranges::find_if(lines, [](const std::string& l) {
return starts(l, "phase stopped");
}) != lines.end());
CHECK(std::ranges::find(lines, "exit 0") != lines.end() && server.printed("phase stopped"));
unsigned long long max_queued = 999;
for (const auto& l : lines)
std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued);
@@ -277,12 +41,14 @@ void sigterm_drains_and_releases() {
const auto outcome = std::ranges::find_if(
lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); });
const auto ack20 = std::ranges::find_if(
lines, [](const std::string& l) { return starts(l, "ack " + hex32(20) + " 20 "); });
lines, [](const std::string& l) { return starts(l, "ack " + request_hex(20) + " 20 "); });
CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome);
CHECK(head_of(r.key).starts_with("80:") && head_of(r.key).ends_with(":released"));
const auto head = head_of(r.key);
CHECK(head.tick == 80 && head.released);
CHECK(!fs::exists(r.root / "ready"));
std::printf("server_sigterm commits=%s acks=%zu max_queued=%llu\n", head_of(r.key).c_str(),
all.size(), max_queued);
std::printf("server_sigterm head=%llu sequence=%llu acks=%zu max_queued=%llu\n",
static_cast<unsigned long long>(head.tick),
static_cast<unsigned long long>(head.sequence), all.size(), max_queued);
}
// SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was
@@ -293,22 +59,16 @@ void sigkill_recovers_through_a_successor() {
{
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
std::string burst;
for (std::uint64_t tick = 1; tick <= 60; ++tick)
burst += input_line(tick, false);
server.send(burst);
CHECK(server.wait_for([](const std::string& l) { return starts(l, "ack "); }, 15s));
server.send(input_lines(1, 60, false));
CHECK(server.wait_line("ack ", 15s));
std::this_thread::sleep_for(300ms); // Ticks run past the last commit.
server.signal(SIGKILL);
CHECK(server.wait_exit(10s) == 128 + SIGKILL);
for (const auto& [tick, sequence] : acks(server)) {
static_cast<void>(sequence);
acked_max = std::max(acked_max, tick);
}
acked_max = max_acked(server);
}
const auto committed = head_tick(r.key);
CHECK(acked_max > 0 && committed >= acked_max); // No acknowledged tick was lost.
CHECK(!head_of(r.key).ends_with(":released")); // The killed holder never released.
const auto committed = head_of(r.key);
CHECK(acked_max > 0 && committed.tick >= acked_max); // No acknowledged tick was lost.
CHECK(!committed.released); // The killed holder never released.
const auto started = Clock::now();
Child successor{server_args(r)};
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
@@ -317,14 +77,15 @@ void sigkill_recovers_through_a_successor() {
const auto running = std::ranges::find_if(
successor.lines(), [](const std::string& l) { return starts(l, "phase running"); });
CHECK(running != successor.lines().end());
CHECK(running->find("committed=" + std::to_string(committed) + " ") != std::string::npos);
CHECK(running->find("committed=" + std::to_string(committed.tick) + " ") != std::string::npos);
CHECK(running->find("epoch=2 ") != std::string::npos);
successor.signal(SIGTERM);
CHECK(successor.wait_exit(20s) == 0);
CHECK(head_tick(r.key) == committed && head_of(r.key).ends_with(":released"));
const auto after = head_of(r.key);
CHECK(after.tick == committed.tick && after.released);
std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n",
static_cast<unsigned long long>(acked_max),
static_cast<unsigned long long>(committed), static_cast<long long>(to_ready));
static_cast<unsigned long long>(committed.tick), static_cast<long long>(to_ready));
}
} // namespace
+6 -4
View File
@@ -46,7 +46,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha"
-f "$root/migrations/0001_region_bootstrap.sql" > /dev/null
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1
PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh"
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6
# v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2.
v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")"
v2_sha="${v2_sha%% *}"
@@ -67,7 +67,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \
-c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \
-c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null
PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh"
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 5:1,2,3,4,5
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 6:1,2,3,4,5,6
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1
# Review P1: a v4 database holding a published (possibly terminal) checkpoint has
# no durable decisions for it. The v5 upgrade must refuse before changing anything.
@@ -136,9 +136,11 @@ expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.provision_region_check
expect_audit_failure 'GRANT SELECT ON evolution.region_commits TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commits FROM evo_runtime_test'
bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again.
# Unknown future versions and checksum drift must not be repaired silently.
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=6 WHERE version=5'
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=7 WHERE version=6'
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=6 WHERE version=7"
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=6"
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=5 WHERE version=6"
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=5"
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=4"
+3
View File
@@ -22,8 +22,11 @@ digest_4="$(sha256sum "$root/migrations/0004_lineage_profile.sql")"
digest_4="${digest_4%% *}"
digest_5="$(sha256sum "$root/migrations/0005_batch_decisions.sql")"
digest_5="${digest_5%% *}"
digest_6="$(sha256sum "$root/migrations/0006_commit_lookup.sql")"
digest_6="${digest_6%% *}"
psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \
--set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \
--set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \
--set="migration_sha_6=$digest_6" \
--set="runtime_role=$runtime_role" \
--file="$root/migrations/apply.sql"