From 30411a83f90901bc171483cddc3f9548a756dbe4 Mon Sep 17 00:00:00 2001 From: dingo Date: Mon, 28 Sep 2026 22:31:40 -0400 Subject: [PATCH] Complete M7.7 recovery verification before readiness ADR-0065: before any readiness, recovery now verifies the head's tail. The trace object must exist with its manifest digest, its header must bind profile, region, predecessor and range; the predecessor (schema v6 load_region_commit) must hash to the digest the head names; replaying the trace from the predecessor's verified payload must reproduce every report and the exact head bytes; and trace-derived decisions must equal the head's durable decisions. Missing, corrupt or non-reproducing tails quarantine (RecoveryError::tail); nothing older or empty is served. Test-only crash seams (HostConfig::crash_hook, PublishHooks) and evo-server --crash-at deliver real SIGKILL at before-upload, before-transaction, after-commit and after-ack. The kill matrix, a SIGSTOP/SIGCONT stale owner and damaged-head startups all behave as the durability contract requires; successors verify before readiness. Validated: PostgreSQL suites under GCC Release, clang-tidy and ASan/UBSan; tail-skip and replay-equality mutations each caught. No Docker/Gitea CI, production-lease kill test or deployment. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 2 +- apps/evo-server/main.cpp | 44 ++- .../postgres-migration/job.yaml.example | 2 +- docs/ROADMAP.md | 9 + docs/STATUS.md | 19 ++ docs/adr/0065-recovery-before-readiness.md | 78 +++++ docs/adr/README.md | 1 + docs/architecture/REGION_DURABILITY.md | 10 + docs/engineering/VALIDATION.md | 42 +++ docs/infrastructure/POSTGRESQL.md | 7 +- include/evo/persistence/checkpoint_store.hpp | 21 +- .../evo/persistence/region_lease_store.hpp | 5 + include/evo/server/region_host.hpp | 9 + migrations/0006_commit_lookup.sql | 23 ++ migrations/apply.sql | 19 +- src/persistence/checkpoint_store.cpp | 84 ++++- src/persistence/region_lease_store.cpp | 19 ++ src/server/region_host.cpp | 10 +- tests/fixtures/server_process.hpp | 297 ++++++++++++++++++ tests/postgres_tests.cpp | 3 + tests/region_recovery_tests.cpp | 276 ++++++++++++++++ tests/region_server_process_tests.cpp | 281 ++--------------- tools/ci/postgres-check.sh | 10 +- tools/migrate-postgres.sh | 3 + 24 files changed, 990 insertions(+), 284 deletions(-) create mode 100644 docs/adr/0065-recovery-before-readiness.md create mode 100644 migrations/0006_commit_lookup.sql create mode 100644 tests/fixtures/server_process.hpp create mode 100644 tests/region_recovery_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 73e402e..901fb99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,7 +115,7 @@ if(EVO_BUILD_POSTGRES) # Contention tests use one thread and one connection per contender. add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp - tests/region_server_process_tests.cpp) + tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp) target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads) add_dependencies(evo-postgres-tests evo-server) # Spawned by the process tests. evo_owned_target(evo-postgres-tests) diff --git a/apps/evo-server/main.cpp b/apps/evo-server/main.cpp index f980e81..7caf485 100644 --- a/apps/evo-server/main.cpp +++ b/apps/evo-server/main.cpp @@ -10,6 +10,10 @@ // outcome // stats max_queued= max_tick_debt= commits= store_failures= // Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined. +// +// --crash-at : (tests only, M7.7) SIGKILLs this process at the n-th +// occurrence of before-upload, before-transaction, after-commit or after-ack (right +// after a step's acknowledgements are written and flushed). #include "evo/server/region_host.hpp" #include @@ -22,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -41,8 +46,18 @@ struct Options { std::filesystem::path objects, ready_file; server::HostConfig host; std::chrono::milliseconds drain{10000}; + std::string crash_point; + std::uint64_t crash_ordinal{}; }; +// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary. +[[noreturn]] void crash_now() { + std::fflush(stdout); + ::kill(::getpid(), SIGKILL); + for (;;) + ::pause(); +} + std::optional number(std::string_view text) { std::uint64_t value{}; const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value); @@ -125,7 +140,18 @@ std::optional parse(int argc, char** argv) { o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms}; else if (flag == "--lease-profile" && value == "default") o.host.lease = {}; - else + else if (flag == "--crash-at") { + const auto colon = value.find(':'); + const auto ordinal = + colon == std::string_view::npos ? std::nullopt : number(value.substr(colon + 1)); + const auto point = value.substr(0, colon); + if (!ordinal || *ordinal == 0 || + (point != "before-upload" && point != "before-transaction" && + point != "after-commit" && point != "after-ack")) + return std::nullopt; + o.crash_point = point; + o.crash_ordinal = *ordinal; + } else return std::nullopt; } if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end()) @@ -243,6 +269,17 @@ class LineReader { }; template int serve(Options& o) { + if (!o.crash_point.empty() && o.crash_point != "after-ack") { + o.host.crash_hook = [point = o.crash_point, ordinal = o.crash_ordinal, + seen = std::uint64_t{0}](server::CrashPoint at) mutable { + const char* name = at == server::CrashPoint::before_upload ? "before-upload" + : at == server::CrashPoint::before_transaction ? "before-transaction" + : "after-commit"; + if (point == name && ++seen == ordinal) + crash_now(); + }; + } + std::uint64_t ack_steps = 0; auto objects = persistence::FilesystemObjectStore::open(o.objects, o.host.key, persistence::max_lineage_payload_bytes); if (!objects) { @@ -317,7 +354,8 @@ template int serve(Options& o) { static_cast(s.simulated_tick)); } set_ready(s.ready); - for (const auto& a : host.take_acks()) + const auto acknowledged = host.take_acks(); + for (const auto& a : acknowledged) std::printf("ack %s %llu %llu\n", text(a.request).c_str(), static_cast(a.tick), static_cast(a.commit_sequence)); @@ -330,6 +368,8 @@ template int serve(Options& o) { for (const auto& d : host.take_dropped()) std::printf("dropped %s %s\n", text(d.input.request).c_str(), dropped_name(d.reason)); std::fflush(stdout); + if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal) + crash_now(); const auto finish = [&](int code) { set_ready(false); std::printf( diff --git a/deploy/kubernetes/postgres-migration/job.yaml.example b/deploy/kubernetes/postgres-migration/job.yaml.example index c1221ef..7bbeb3d 100644 --- a/deploy/kubernetes/postgres-migration/job.yaml.example +++ b/deploy/kubernetes/postgres-migration/job.yaml.example @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: evolution-db-migrate-v5 + name: evolution-db-migrate-v6 namespace: evolution-dev spec: backoffLimit: 0 diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index ffa63c7..bf8fad4 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -567,3 +567,12 @@ M7.6 implemented and locally qualified (2026-09-28), under ADR-0064. - `evo-server` passed real SIGTERM drain and SIGKILL tests, with successor recovery. Next: M7.7 recovery verification at every kill boundary. + +M7.7 implemented and locally qualified (2026-09-28), under ADR-0065. +- Before readiness, recovery replays the head's trace from its predecessor's + verified payload, requires the exact head bytes and decisions, and + quarantines on any missing or corrupt tail. Schema v6 adds the read-only + commit lookup. +- Real SIGKILL at five boundaries, a resumed SIGSTOPped owner, and damaged heads + all behave as the crash matrix requires. +Next: M7.8 operations endpoints and dev image. diff --git a/docs/STATUS.md b/docs/STATUS.md index e1b5e2a..7b6306f 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -861,3 +861,22 @@ and ASan/UBSan. Next: M7.7 recovery verification at every kill boundary. There is no gateway, metrics or deployment yet. + +## M7.7 recovery before readiness (2026-09-28) + +ADR-0065: before readiness, recovery now replays the head's validation trace +from the predecessor's verified payload. It must reproduce the head bytes +exactly and derive exactly the durable decisions. A missing, corrupt or +non-reproducing tail, or a missing predecessor payload, quarantines the region, +and nothing older or empty is ever served. Schema v6 adds the read-only +`load_region_commit`. + +Real `evo-server` processes were SIGKILLed at five publication and ACK +boundaries using a test-only `--crash-at` injector. Every result matched the +crash matrix, and each successor verified the head before becoming ready. A +SIGSTOPped owner resumed fenced, and damaged heads exited quarantined. + +Tail-skip and replay-equality mutations were each caught. The PostgreSQL suites +passed under Release, analysis and ASan/UBSan. + +Next: M7.8 operations endpoints and a dev image. diff --git a/docs/adr/0065-recovery-before-readiness.md b/docs/adr/0065-recovery-before-readiness.md new file mode 100644 index 0000000..e048daa --- /dev/null +++ b/docs/adr/0065-recovery-before-readiness.md @@ -0,0 +1,78 @@ +# ADR-0065: Recovery verification before readiness + +## Status +Accepted for M7.7, 2026-09-28. It extends ADR-0061 (recovery), ADR-0063 (batch +evidence) and ADR-0064 (host). Clone, migration and durability qualification are +M7.9. + +## Context +The M7.7 criterion is: load a verified committed checkpoint, verify the tail, +hash, inventories and history, and hold valid current authority before serving. +Kill tests must cover every publication and acknowledgement boundary; a corrupt +or missing tail must quarantine; there must be no empty fallback world. + +The full-checkpoint profile (ADR-0058) has no replay tail beyond the head. The +head's own batch evidence (trace, decisions) was published in M7.5, but recovery +did not re-verify it. The crash matrix rows were exercised in process, but not by +killing a real process at each boundary. + +## Decision +**Tail verification.** Recovery, which the host completes before readiness under +a freshly acquired lease, now also verifies the head's batch evidence after the +manifest, build, payload and decision checks. For any head after genesis: +1. The manifest must be v2. +2. The trace object must exist with its manifest length and SHA-256, and its + header must bind the profile, region, predecessor commit and digest, and the + tick range. +3. The predecessor is loaded by sequence through the new read-only + `evolution.load_region_commit` (schema v6; runtime function set is eleven). + Its manifest must hash to the digest the head names. +4. The predecessor's payload is fetched, verified and restored. Replaying the + trace must reproduce every report and the head payload bytes exactly. +5. The decisions derived from the trace must equal the head commit's durable + decisions. + +A missing, corrupt or non-reproducing trace, or a missing predecessor payload, is +`RecoveryError::tail`; a decision mismatch is `history`. Either quarantines the +region: no older head, genesis or empty world is ever served. + +The replay equality is the inventory check. It proves the head state follows, +under the pinned rules, from a predecessor that was itself verified when it was +published. The envelope's own restore audit covers internal totals. Recovery +cost grows by one predecessor restore and at most 64 replayed ticks. + +**Kill boundaries.** `HostConfig::crash_hook` and `PublishHooks` are test-only +seams at three points: before upload, before the transaction (after verified +uploads), and after commit (before any acknowledgement). `evo-server --crash-at +:` delivers a real SIGKILL there, or right after a step's +acknowledgements are flushed (`after-ack`). + +## Alternatives Considered +- **Verifying the whole chain on every recovery**: unbounded cost. Earlier links + were verified when published, and the chain digests bind them. +- **Skipping trace verification**: leaves a published but unreproducible head + servable. +- **Kill points chosen by sleeps and signals**: nondeterministic; hooks fire + exactly at the boundary. + +## Benefits +- Real killed processes at every boundary show the contract end to end. +- A resumed stale owner cannot serve. +- Damaged heads and tails never become ready. + +## Costs +- Recovery reads two payloads and one trace, and replays up to 64 ticks. +- Predecessor objects must be retained (there is no GC). +- The crash seams exist in production binaries but are inert unless configured. + +## Risks +- Tail verification covers the head link only; a damaged older link is detected + only when it becomes a predecessor, or by a future audit. +- The kill tests use the development 4 s lease on a single host with a local + database. +- VM freezes and host loss remain M8. + +## Consequences +- M7.8 exposes readiness and quarantine to probes and metrics. +- M7.9 measures recovery latency with the production lease, and qualifies clones + and migrations against this verification. diff --git a/docs/adr/README.md b/docs/adr/README.md index 14f36f8..a30fbe0 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -75,3 +75,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0062: Lineage checkpoint profile](0062-lineage-checkpoint-profile.md) - accepted for M7.4b archive codec, LineageRegion closure and history-capable profile 2. - [ADR-0063: Batch outcomes](0063-batch-outcomes.md) - accepted for M7.5 manifest v2, validation trace replay, durable decisions/outbox and commit resolution. - [ADR-0064: Region host](0064-region-host.md) - accepted for M7.6 single-owner host state machine, bounded admission, final-only-after-commit and evo-server process. +- [ADR-0065: Recovery before readiness](0065-recovery-before-readiness.md) - accepted for M7.7 head tail replay verification, schema v6 commit lookup and real kill-boundary tests. diff --git a/docs/architecture/REGION_DURABILITY.md b/docs/architecture/REGION_DURABILITY.md index 210b601..98aab3f 100644 --- a/docs/architecture/REGION_DURABILITY.md +++ b/docs/architecture/REGION_DURABILITY.md @@ -210,3 +210,13 @@ M7.6 (ADR-0064) executed its host rows: - Shutdown drains, commits and releases. - A real SIGKILLed `evo-server` left no acknowledged tick uncommitted, and its successor became ready only after lease expiry, from the committed head. + +M7.7 (ADR-0065) executed the matrix with real `evo-server` processes SIGKILLed at +each boundary: before the transaction (first and second batch), before upload, +after commit but before acknowledgement, and after acknowledgement. +- In every case acknowledgements stayed at or below the committed head. +- Deaths were final only once committed, and verified orphans remained. +- Each successor verified the head and its tail before readiness. +A SIGSTOPped owner resumed fenced: it was not ready, refused admission and +acknowledged nothing new. Corrupt payloads and missing traces quarantined with +no fallback. Host loss and clone or migration remain M8 and M7.9. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 55f54c3..343a32c 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2339,3 +2339,45 @@ Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate - Not executed: Docker/Gitea CI, TSan (the host is single-threaded; libpq is uninstrumented), remote PostgreSQL/TLS, a production 20 s lease kill test, durable client deduplication, and a network gateway. + +## M7.7 recovery before readiness - 2026-09-28 + +- WSL Linux: GCC14 Release, clang19 analysis (clang-tidy, clean on the first + build) and clang19 ASan/UBSan builds of `evo-postgres-tests` and `evo-server` + passed the full PostgreSQL suite, including the M7.7 recovery tests. Runs: + Release 2, analysis 1, ASan 2. The repository format check passed after + reordering one fixture include. +- Windows was not rerun. This change touches only Linux persistence and host + code, SQL, tests and docs; M7.6's Windows run (58/58) covers the portable + targets, which are unchanged. +- Kill matrix (real SIGKILL via `--crash-at`, inputs 1..40 with a death at tick + 20, 5 s flush): + + | Crash point | Head (seq/tick) | Acked up to | Decisions | Objects | + |---|---|---|---|---| + | before-transaction:1 | 0/0 | none | 0 | 3 (verified orphans) | + | before-upload:2 | 1/20 | 20 | 1 | 3 | + | before-transaction:2 | 1/20 | 20 | 1 | 5 (orphans) | + | after-commit:2 | 2/40 | 20 (lost acks are the safe direction) | 1 | 5 | + | after-ack:2 | 2/40 | 40 | 1 | 5 | + + The lease stayed held after every kill. Each successor became ready at epoch 2 + from the committed head (with tail verification), then drained and released. +- Resumed stale owner: the owner was SIGSTOPped for 4.5 s (4 s lease). The + successor became ready at epoch 2 with committed=10 and processed ticks 11–20. + After SIGCONT the old owner printed `ready 0`, rejected new inputs as + `not_ready`, acknowledged nothing beyond tick 10, and exited 0 on SIGTERM + without touching the successor's chain. +- Damaged heads: a flipped head payload byte and a missing head trace each made + `evo-server` exit 4 (quarantined) without ever printing `ready 1`, and the head + was unchanged. After repair a successor became ready at epoch 4. +- Tail verification (in-process): a missing or corrupt head trace and a missing + predecessor payload are `tail`, and recovery succeeds again after repair. A + published trace of different inputs over the same range (a host bypass the + database cannot detect) is `tail`. +- Mutations (restored and rerun green): skipping tail verification made the + quarantine test time out because the server became ready. Dropping replay + byte-equality failed the forged-trace test (line 262). +- Not executed: Docker/Gitea CI, the production 20 s lease, host loss or VM + freeze, multi-host network partitions, or verification of links older than + the head. diff --git a/docs/infrastructure/POSTGRESQL.md b/docs/infrastructure/POSTGRESQL.md index 0516c85..7759105 100644 --- a/docs/infrastructure/POSTGRESQL.md +++ b/docs/infrastructure/POSTGRESQL.md @@ -68,7 +68,7 @@ every role transitively in it, and rolls back on any violation: `pg_write_all_data`); - any table privilege in `evolution`, including via PUBLIC; - CREATE on the schema, the database or `public`; -- EXECUTE outside the ten fenced functions (schema v5). +- EXECUTE outside the eleven fenced functions (schema v6). Attribute checks alone are not enough: a login with `pg_write_all_data` can update lease rows directly. Rerun the migrate Job after adding or changing runtime logins or grants; it is the audit. Grants made later without a rerun are not detected. Bootstrap revokes PUBLIC access to the schema, tables and @@ -185,6 +185,9 @@ Exit codes: - 4: quarantined; - 2: usage error. -Use the default 20 s lease in any shared environment. `--lease-profile fast` exists +`--crash-at :` (before-upload, before-transaction, after-commit or +after-ack) is a test-only fault injector that SIGKILLs the process at that +boundary; never set it in shared environments. Use the default 20 s lease in any +shared environment. `--lease-profile fast` exists only for local tests. The runtime credentials come from `--pg-*` flags and a passfile; never place passwords on the command line. diff --git a/include/evo/persistence/checkpoint_store.hpp b/include/evo/persistence/checkpoint_store.hpp index 7469088..042cce0 100644 --- a/include/evo/persistence/checkpoint_store.hpp +++ b/include/evo/persistence/checkpoint_store.hpp @@ -5,6 +5,7 @@ #include "evo/persistence/region_lease_store.hpp" #include #include +#include #include #include #include @@ -41,6 +42,11 @@ struct PublicationFailure { PreparedCommit prepared; // Present for rejected/unknown. }; // Durably confirmed batch: its decisions are now final and may be acknowledged. +// Test/fault-injection seam: called after verified uploads, immediately before the +// fenced transaction is sent. Production callers pass nothing. +struct PublishHooks { + std::function before_transaction; +}; struct PublishedBatch { CommittedCheckpoint committed; std::vector decisions; @@ -54,6 +60,7 @@ enum class RecoveryError : std::uint8_t { corrupt_payload, // Length/digest mismatch: quarantine, no older head. payload, // Verified bytes fail full state validation: quarantine. history, // Durable decisions disagree with the restored lifecycle: quarantine. + tail, // Head batch evidence missing, corrupt or not reproducible: quarantine. }; struct RecoveryFailure { RecoveryError kind{}; @@ -85,9 +92,15 @@ publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, const CommittedCheckpoint& predecessor, const RegionBatch& batch, Uuid commit, const Sha256Digest& build, std::size_t restore_budget, - std::chrono::milliseconds timeout); -// Loads and fully verifies the committed head and reconciles durable decisions. -// Readiness additionally requires a fresh valid lease acquired before this call. + std::chrono::milliseconds timeout, const PublishHooks& hooks = {}); +// Loads and fully verifies the committed head before any readiness (ADR-0065): +// manifest digest and chain fields, retained build, payload length/SHA-256 and full +// restore; then the head's tail: its trace object binds to the committed +// predecessor, replaying it from the predecessor's verified payload reproduces the +// head bytes, and its derived decisions equal the durable ones; finally every +// durable decision agrees with the restored lifecycle. Any failure quarantines; +// nothing older or empty is ever substituted. Readiness also requires a fresh +// valid lease acquired before this call. [[nodiscard]] std::expected recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, std::span retained_builds, @@ -106,7 +119,7 @@ publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const const CommittedCheckpoint& predecessor, const RegionBatch& batch, Uuid commit, const Sha256Digest& build, std::size_t restore_budget, - std::chrono::milliseconds timeout); + std::chrono::milliseconds timeout, const PublishHooks& hooks = {}); [[nodiscard]] std::expected recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, std::span retained_builds, diff --git a/include/evo/persistence/region_lease_store.hpp b/include/evo/persistence/region_lease_store.hpp index d02fb74..bd819f5 100644 --- a/include/evo/persistence/region_lease_store.hpp +++ b/include/evo/persistence/region_lease_store.hpp @@ -50,6 +50,11 @@ release_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease, load_region_head(Postgres& db, RegionKey key, std::chrono::milliseconds timeout); [[nodiscard]] std::expected load_region_checkpoint(Postgres& db, RegionKey key, std::chrono::milliseconds timeout); +// Any committed link by sequence (schema v6): its identity, digest, end tick and +// exact manifest bytes. not_provisioned when the region has no such commit. +[[nodiscard]] std::expected +load_region_commit(Postgres& db, RegionKey key, std::uint64_t sequence, + std::chrono::milliseconds timeout); // Canonical manifest v2 bytes plus the decision block they digest (ADR-0063). The // database parses identity, lease and chain, locks the lease then the head, requires // the exact committed predecessor and records decisions and outbox rows in the same diff --git a/include/evo/server/region_host.hpp b/include/evo/server/region_host.hpp index 9b994b2..4818c92 100644 --- a/include/evo/server/region_host.hpp +++ b/include/evo/server/region_host.hpp @@ -75,6 +75,12 @@ struct HostStatus { std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{}; }; +// Commit boundaries for fault injection (M7.7 kill tests). +enum class CrashPoint : std::uint8_t { + before_upload, // Batch sealed; nothing uploaded or sent. + before_transaction, // Payload and trace uploaded and verified; manifest not sent. + after_commit, // Commit confirmed; no acknowledgement or outcome emitted yet. +}; using Connect = std::function, persistence::PgFailure>()>; struct HostConfig { @@ -90,6 +96,9 @@ struct HostConfig { persistence::Sha256Digest build{}; std::vector retained_builds; std::size_t restore_budget{8000000}; + // Test-only: called at each commit boundary (e.g. to SIGKILL the process). + // Production configurations leave it empty. + std::function crash_hook; }; enum class HostConfigError : std::uint8_t { invalid }; diff --git a/migrations/0006_commit_lookup.sql b/migrations/0006_commit_lookup.sql new file mode 100644 index 0000000..27d2669 --- /dev/null +++ b/migrations/0006_commit_lookup.sql @@ -0,0 +1,23 @@ +-- M7.7 recovery tail verification (ADR-0065): read any committed link of a region +-- by sequence, so recovery can replay the head's trace from its predecessor. +-- Read-only; returns nothing a runtime could not already derive from the chain. +CREATE FUNCTION evolution.load_region_commit(p_world evolution.u64, p_region evolution.u64, + p_sequence evolution.u64) +RETURNS TABLE (region_frame evolution.u64, commit_sequence evolution.u64, commit_id uuid, + commit_digest text, end_tick evolution.u64, manifest text) +LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + RETURN QUERY SELECT l.frame_id, c.commit_sequence, c.commit_id, encode(c.commit_digest, 'hex'), + c.end_tick, encode(c.manifest, 'hex') + FROM evolution.region_commits c + JOIN evolution.region_leases l USING (world_id, region_id) + WHERE c.world_id = p_world AND c.region_id = p_region AND c.commit_sequence = p_sequence; + IF NOT FOUND THEN + RAISE EXCEPTION 'no such commit' USING ERRCODE = 'EVL04'; + END IF; +END $$; +REVOKE ALL ON FUNCTION evolution.load_region_commit(evolution.u64, evolution.u64, evolution.u64) + FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.load_region_commit(evolution.u64, evolution.u64, evolution.u64) + TO :"runtime_role"; +INSERT INTO evolution.schema_migrations(version, sha256) VALUES (6, :'migration_sha_6'); diff --git a/migrations/apply.sql b/migrations/apply.sql index 4b7eade..49f5f12 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -24,26 +24,35 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') || ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' AS at_v4, :'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' || ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' - || ',5:' || :'migration_sha_5' AS at_v5 \gset - \if :at_v5 + || ',5:' || :'migration_sha_5' AS at_v5, + :'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' + || ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' + || ',5:' || :'migration_sha_5' || ',6:' || :'migration_sha_6' AS at_v6 \gset + \if :at_v6 + \elif :at_v5 + \ir 0006_commit_lookup.sql \elif :at_v4 \ir pre_v5_guard.sql \ir 0005_batch_decisions.sql + \ir 0006_commit_lookup.sql \elif :at_v3 \ir 0004_lineage_profile.sql \ir pre_v5_guard.sql \ir 0005_batch_decisions.sql + \ir 0006_commit_lookup.sql \elif :at_v2 \ir 0003_region_checkpoints.sql \ir 0004_lineage_profile.sql \ir pre_v5_guard.sql \ir 0005_batch_decisions.sql + \ir 0006_commit_lookup.sql \elif :at_v1 \ir 0002_region_fencing.sql \ir 0003_region_checkpoints.sql \ir 0004_lineage_profile.sql \ir pre_v5_guard.sql \ir 0005_batch_decisions.sql + \ir 0006_commit_lookup.sql \else \echo 'Unknown or changed migration history' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; @@ -55,6 +64,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0004_lineage_profile.sql \ir pre_v5_guard.sql \ir 0005_batch_decisions.sql + \ir 0006_commit_lookup.sql \endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes -- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC @@ -84,7 +94,8 @@ BEGIN ('evolution.resolve_region_commit(evolution.u64,evolution.u64,uuid,bytea,bytea)'::regprocedure::oid), ('evolution.load_region_decisions(evolution.u64,evolution.u64)'::regprocedure::oid), ('evolution.pending_region_outbox(evolution.u64,evolution.u64,integer)'::regprocedure::oid), - ('evolution.acknowledge_region_outbox(evolution.u64,evolution.u64,integer,integer,evolution.u64)'::regprocedure::oid) + ('evolution.acknowledge_region_outbox(evolution.u64,evolution.u64,integer,integer,evolution.u64)'::regprocedure::oid), + ('evolution.load_region_commit(evolution.u64,evolution.u64,evolution.u64)'::regprocedure::oid) ), violations(role_oid, reason) AS ( SELECT r.oid, 'privileged attribute or owner identity' FROM pg_catalog.pg_roles r JOIN closure c ON c.oid = r.oid @@ -108,7 +119,7 @@ BEGIN OR EXISTS (SELECT 1 FROM pg_catalog.pg_namespace n WHERE n.nspname = 'public' AND pg_catalog.has_schema_privilege(c.oid, n.oid, 'CREATE')) UNION ALL - -- Only the ten guarded functions are executable; provisioning/internal are not. + -- Only the eleven guarded functions are executable; provisioning/internal are not. SELECT c.oid, 'EXECUTE outside the fenced function set' FROM closure c CROSS JOIN pg_catalog.pg_proc p WHERE p.pronamespace = 'evolution'::regnamespace diff --git a/src/persistence/checkpoint_store.cpp b/src/persistence/checkpoint_store.cpp index 07260a5..11543c2 100644 --- a/src/persistence/checkpoint_store.cpp +++ b/src/persistence/checkpoint_store.cpp @@ -175,7 +175,8 @@ std::expected publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, const CommittedCheckpoint& predecessor, const RegionBatch& batch, Uuid commit, - const Sha256Digest& build, std::size_t budget, milliseconds timeout) { + const Sha256Digest& build, std::size_t budget, milliseconds timeout, + const PublishHooks& hooks) { const auto previous = decode_manifest(predecessor.manifest); if (!previous || previous->profile != Codec::profile || !batch.sealed() || batch.steps().empty() || batch.begin_tick() != previous->end_tick) @@ -231,6 +232,8 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra return std::unexpected{stored.error()}; PreparedCommit prepared{previous->key, commit, *bytes, *decision_bytes}; + if (hooks.before_transaction) + hooks.before_transaction(); auto published = publish_region_checkpoint(db, *bytes, *decision_bytes, timeout); auto committed = confirm(published, std::move(*bytes), *manifest, std::move(prepared)); if (!committed) @@ -238,6 +241,72 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra return PublishedBatch{std::move(*committed), decisions}; } +// Replays the head's recorded batch from its committed predecessor (ADR-0065). +template +std::expected +verify_tail(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, + const CheckpointManifest& m, std::span head_payload, + const typename Codec::State& head_state, std::span durable, + std::size_t budget, milliseconds timeout) { + const auto fail_with = [](RecoveryError kind) { + return std::unexpected{RecoveryFailure{kind, {}}}; + }; + if (m.sequence == 0) + return {}; // Genesis has no batch. + if (m.manifest_version != 2) + return fail_with(RecoveryError::unsupported); + auto trace_bytes = objects.get({m.batch.trace_sha256, m.batch.trace_bytes}); + if (!trace_bytes) + return fail_with(trace_bytes.error() == ObjectError::io ? RecoveryError::store + : RecoveryError::tail); + auto trace = decode_trace(*trace_bytes); + const TraceHeader expected{static_cast(Codec::profile), + key, + m.predecessor_commit, + m.predecessor_digest, + m.begin_tick, + m.end_tick}; + if (!trace || trace->header != expected) + return fail_with(RecoveryError::tail); + auto predecessor = load_region_commit(db, key, m.sequence - 1, timeout); + if (!predecessor) + return predecessor.error().kind == LeaseStoreError::not_provisioned + ? fail_with(RecoveryError::tail) + : std::unexpected{RecoveryFailure{RecoveryError::store, predecessor.error()}}; + const auto digest = sha256(predecessor->manifest); + auto previous = decode_manifest(predecessor->manifest); + if (!digest || *digest != predecessor->head.digest || + predecessor->head.digest != m.predecessor_digest || + predecessor->head.commit != m.predecessor_commit || !previous || + previous->end_tick != m.begin_tick) + return fail_with(RecoveryError::tail); + auto base = objects.get({previous->payload_sha256, previous->payload_bytes}); + if (!base) + return fail_with(base.error() == ObjectError::io ? RecoveryError::store + : RecoveryError::tail); + auto replay = Codec::restore(*base, *previous, budget); + if (!replay) + return fail_with(RecoveryError::tail); + for (const auto& step : trace->steps) { + auto report = (*replay)->advance(step.report.tick, step.input); + if (!report || *report != step.report) + return fail_with(RecoveryError::tail); + } + const auto replayed = Codec::snapshot(**replay); + if (!replayed || !std::ranges::equal(*replayed, head_payload)) + return fail_with(RecoveryError::tail); + // The head's durable outcomes are exactly those its trace derives. + const auto organisms = Codec::organisms(head_state); + const auto derived = derive_decisions(trace->steps, organisms.producer, organisms.consumer); + std::vector recorded; + for (const auto& d : durable) + if (d.commit_sequence == m.sequence) + recorded.push_back(d.decision); + if (derived != recorded || derived.size() != m.batch.decision_count) + return fail_with(RecoveryError::history); + return {}; +} + template std::expected recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, @@ -284,6 +353,10 @@ recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, return store_failure(decisions.error()); if (!Codec::reconcile(**state, *decisions, head)) return fail_with(RecoveryError::history); + if (auto tail = verify_tail(db, objects, key, *manifest, *payload, **state, *decisions, + budget, timeout); + !tail) + return std::unexpected{tail.error()}; return typename Codec::Recovered{std::move(*committed), *manifest, std::move(*state), std::move(*decisions)}; } @@ -301,9 +374,9 @@ publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, const CommittedCheckpoint& predecessor, const RegionBatch& batch, Uuid commit, const Sha256Digest& build, std::size_t restore_budget, - milliseconds timeout) { + milliseconds timeout, const PublishHooks& hooks) { return publish_batch(db, objects, lease, predecessor, batch, commit, build, - restore_budget, timeout); + restore_budget, timeout, hooks); } std::expected recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, @@ -324,9 +397,10 @@ std::expected publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, const CommittedCheckpoint& predecessor, const RegionBatch& batch, Uuid commit, - const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout) { + const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout, + const PublishHooks& hooks) { return publish_batch(db, objects, lease, predecessor, batch, commit, build, - restore_budget, timeout); + restore_budget, timeout, hooks); } std::expected recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, diff --git a/src/persistence/region_lease_store.cpp b/src/persistence/region_lease_store.cpp index 31924a8..454ae8c 100644 --- a/src/persistence/region_lease_store.cpp +++ b/src/persistence/region_lease_store.cpp @@ -216,6 +216,25 @@ load_region_checkpoint(Postgres& db, RegionKey key, milliseconds timeout) { return CommittedCheckpoint{*head, std::move(*manifest)}; } +std::expected +load_region_commit(Postgres& db, RegionKey key, std::uint64_t sequence, milliseconds timeout) { + const std::array p{std::to_string(key.world), std::to_string(key.region), + std::to_string(sequence)}; + auto rows = db.query("SELECT * FROM evolution.load_region_commit($1::evolution.u64, " + "$2::evolution.u64, $3::evolution.u64)", + p, timeout); + if (!rows) + return std::unexpected{classify(rows.error())}; + auto head = parse_head(*rows, 6); + if (!head || rows->size() != 1) + return std::unexpected{malformed(false)}; + const auto& manifest_text = (*rows)[0][5]; + auto manifest = manifest_text ? unhex_bytes(*manifest_text) : std::nullopt; + if (!manifest) + return std::unexpected{malformed(false)}; + return CommittedCheckpoint{*head, std::move(*manifest)}; +} + std::expected publish_region_checkpoint(Postgres& db, std::span manifest, std::span decisions, milliseconds timeout) { diff --git a/src/server/region_host.cpp b/src/server/region_host.cpp index 71f4585..8398680 100644 --- a/src/server/region_host.cpp +++ b/src/server/region_host.cpp @@ -228,6 +228,8 @@ template persistence::Uuid RegionHost::next_commit_id() template void RegionHost::confirmed(const persistence::PublishedBatch& published, SchedulerTime now) { + if (config_.crash_hook) + config_.crash_hook(CrashPoint::after_commit); const auto& head = published.committed.head; if (!batch_ || !batch_->confirm(head.end_tick)) { lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); @@ -255,9 +257,15 @@ template void RegionHost::commit(SchedulerTime now) { return; const auto commit_id = next_commit_id(); batch_->seal(); + const auto& crash = config_.crash_hook; + if (crash) + crash(CrashPoint::before_upload); + persistence::PublishHooks hooks; + if (crash) + hooks.before_transaction = [&crash] { crash(CrashPoint::before_transaction); }; auto published = Ops::publish(*db_, *objects_, *grant, *committed_, *batch_, commit_id, config_.build, - config_.restore_budget, config_.lease.commit_timeout); + config_.restore_budget, config_.lease.commit_timeout, hooks); if (published) { confirmed(*published, now); return; diff --git a/tests/fixtures/server_process.hpp b/tests/fixtures/server_process.hpp new file mode 100644 index 0000000..0616087 --- /dev/null +++ b/tests/fixtures/server_process.hpp @@ -0,0 +1,297 @@ +#pragma once +// Shared real-process helpers for evo-server tests (M7.6/M7.7). Linux only. +#include "environmental_region.hpp" +#include "evo/persistence/checkpoint_store.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char** environ; + +namespace evo::test::process { +using namespace std::chrono_literals; +namespace fs = std::filesystem; +using Clock = std::chrono::steady_clock; + +inline void expect(bool ok, const char* what) { + if (!ok) + throw std::runtime_error(std::string("server fixture: ") + what); +} +inline std::string env(const char* key) { + const auto* value = std::getenv(key); + expect(value != nullptr, "missing test environment"); + return value; +} +inline std::unique_ptr owner() { + auto made = persistence::Postgres::connect( + {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s); + expect(made.has_value(), "owner connection"); + return std::move(*made); +} +// Request identity: a tag byte (one per writer) and the tick. +inline std::string request_hex(std::uint64_t tick, std::uint8_t tag = 0x9B) { + char out[33]; + std::snprintf(out, sizeof out, "%02x000000000000000000000000%06llx", tag, + static_cast(tick)); + return out; +} +inline std::string input_line(std::uint64_t tick, bool consumer_dies, std::uint8_t tag = 0x9B) { + const auto in = region_input(tick, consumer_dies); + std::string line = "input " + request_hex(tick, tag) + " " + std::to_string(in.temperature_mK) + + " " + std::to_string(in.source_enabled); + for (const auto b : in.bite) + line += " " + std::to_string(b); + for (const auto t : in.terminal) + line += " " + std::to_string(t); + return line + "\n"; +} +inline std::string input_lines(std::uint64_t first, std::uint64_t last, bool deaths, + std::uint8_t tag = 0x9B) { + std::string out; + for (auto tick = first; tick <= last; ++tick) + out += input_line(tick, deaths, tag); + return out; +} +inline bool starts(const std::string& line, std::string_view prefix) { + return line.starts_with(prefix); +} + +class Child { + public: + explicit Child(const std::vector& args) { + int to_child[2], from_child[2]; + expect(::pipe(to_child) == 0 && ::pipe(from_child) == 0, "pipes"); + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_init(&actions); + posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO); + posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO); + posix_spawn_file_actions_addclose(&actions, to_child[1]); + posix_spawn_file_actions_addclose(&actions, from_child[0]); + std::vector argv; + argv.reserve(args.size() + 1); + for (const auto& a : args) + argv.push_back(const_cast(a.c_str())); + argv.push_back(nullptr); + expect(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0, "spawn"); + posix_spawn_file_actions_destroy(&actions); + ::close(to_child[0]); + ::close(from_child[1]); + in_ = to_child[1]; + out_ = from_child[0]; + } + Child(const Child&) = delete; + Child& operator=(const Child&) = delete; + ~Child() { + if (pid_ > 0) { + ::kill(pid_, SIGKILL); + ::waitpid(pid_, nullptr, 0); + } + ::close(in_); + ::close(out_); + } + void send(const std::string& text) { + std::size_t done = 0; + while (done < text.size()) { + const auto n = ::write(in_, text.data() + done, text.size() - done); + expect(n > 0 || errno == EINTR, "write to child"); + if (n > 0) + done += static_cast(n); + } + } + // Reads lines until one satisfies `want`; every line is kept in lines(). + template bool wait_for(Want want, std::chrono::milliseconds timeout) { + const auto deadline = Clock::now() + timeout; + for (std::size_t seen = 0;;) { + for (; seen < lines_.size(); ++seen) + if (want(lines_[seen])) + return true; + const auto left = + std::chrono::duration_cast(deadline - Clock::now()); + if (left.count() <= 0 || !read_some(static_cast(left.count()))) + return false; + } + } + bool wait_line(std::string_view prefix, std::chrono::milliseconds timeout) { + return wait_for([prefix](const std::string& l) { return starts(l, prefix); }, timeout); + } + void signal(int sig) { + expect(::kill(pid_, sig) == 0, "signal"); + } + // Exit code, or 128+signal; drains remaining output. + int wait_exit(std::chrono::milliseconds timeout) { + const auto deadline = Clock::now() + timeout; + int status = 0; + for (;;) { + read_some(20); + const auto r = ::waitpid(pid_, &status, WNOHANG); + if (r == pid_) + break; + expect(Clock::now() < deadline, "child exit deadline"); + } + while (read_some(20)) { + } + pid_ = -1; + return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status); + } + [[nodiscard]] const std::vector& lines() const noexcept { + return lines_; + } + [[nodiscard]] bool printed(std::string_view prefix) const { + for (const auto& l : lines_) + if (starts(l, prefix)) + return true; + return false; + } + + private: + bool read_some(int timeout_ms) { + pollfd fd{out_, POLLIN, 0}; + if (::poll(&fd, 1, timeout_ms) <= 0) + return false; + char chunk[4096]; + const auto n = ::read(out_, chunk, sizeof chunk); + if (n <= 0) + return false; + buffer_.append(chunk, static_cast(n)); + for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) { + lines_.push_back(buffer_.substr(0, end)); + buffer_.erase(0, end + 1); + } + return true; + } + pid_t pid_{-1}; + int in_{-1}, out_{-1}; + std::string buffer_; + std::vector lines_; +}; + +struct ServerRegion { + persistence::RegionKey key; + fs::path root; + // The region's object namespace directory. + [[nodiscard]] fs::path objects() const { + return root / ("w" + std::to_string(key.world)) / ("r" + std::to_string(key.region)); + } +}; +// A fresh lineage region (world 7) provisioned by the owner. +inline ServerRegion provision(std::uint64_t region, std::uint8_t genesis) { + ServerRegion r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))}; + fs::remove_all(r.root); + fs::create_directory(r.root); + fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); + auto db = owner(); + expect(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", + {}, 5s) + .has_value(), + "retain build"); + auto objects = persistence::FilesystemObjectStore::open(r.root, r.key, + persistence::max_lineage_payload_bytes); + expect(objects.has_value(), "object store"); + persistence::Sha256Digest build{}; + build.fill(0xB1); + persistence::Uuid commit{}; + commit.fill(genesis); + expect(persistence::provision_lineage_region(*db, **objects, r.key, region_identity.frame, + *lineage_region(), commit, build, 8000000, 5s) + .has_value(), + "provision"); + return r; +} +inline std::vector server_args(const ServerRegion& r, + std::vector extra = {}) { + std::vector args{ + env("EVO_SERVER_BIN"), + "--profile", + "lineage", + "--world", + std::to_string(r.key.world), + "--region", + std::to_string(r.key.region), + "--frame", + std::to_string(region_identity.frame), + "--pg-host", + env("PGHOST"), + "--pg-port", + env("PGPORT"), + "--pg-database", + env("PGDATABASE"), + "--pg-user", + env("EVO_PG_RUNTIME_USER"), + "--objects", + r.root.string(), + "--build", + "b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1", + "--flush-ms", + "500", + "--lease-profile", + "fast", + "--ready-file", + (r.root / "ready").string()}; + args.insert(args.end(), extra.begin(), extra.end()); + return args; +} +// (tick, commit_sequence) of every ack line. +inline std::vector> acks(const Child& c) { + std::vector> out; + for (const auto& line : c.lines()) { + unsigned long long tick = 0, sequence = 0; + char request[40]; + if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3) + out.emplace_back(tick, sequence); + } + return out; +} +inline std::uint64_t max_acked(const Child& c) { + std::uint64_t most = 0; + for (const auto& [tick, sequence] : acks(c)) + most = std::max(most, tick); + return most; +} +struct Head { + std::uint64_t tick{}, sequence{}; + bool released{}; +}; +inline Head head_of(persistence::RegionKey key) { + const std::array p{std::to_string(key.world), std::to_string(key.region)}; + auto rows = owner()->query( + "SELECT h.end_tick::text, h.commit_sequence::text, (l.holder IS NULL)::text FROM " + "evolution.region_heads h JOIN evolution.region_leases l USING (world_id, region_id) " + "WHERE world_id = $1::evolution.u64 AND region_id = $2::evolution.u64", + p, 5s); + expect(rows && rows->size() == 1 && (*rows)[0].size() == 3, "head row"); + const auto& row = (*rows)[0]; + expect(row[0] && row[1] && row[2], "head values"); + return {std::stoull(row[0].value_or("0")), std::stoull(row[1].value_or("0")), + row[2].value_or("") == "true"}; +} +inline std::size_t decisions_of(persistence::RegionKey key) { + const std::array p{std::to_string(key.world), std::to_string(key.region)}; + auto rows = owner()->query("SELECT count(*)::text FROM evolution.region_decisions WHERE " + "world_id = $1::evolution.u64 AND region_id = $2::evolution.u64", + p, 5s); + expect(rows && rows->size() == 1, "decision count"); + return std::stoull((*rows)[0][0].value_or("0")); +} +// Object files in the region namespace (temporaries excluded). +inline std::size_t object_count(const ServerRegion& r) { + std::size_t n = 0; + for (const auto& entry : fs::directory_iterator(r.objects())) + n += entry.path().filename().string().starts_with(".tmp-") ? 0 : 1; + return n; +} +} // namespace evo::test::process diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index a21ab46..fa41d77 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -7,6 +7,7 @@ void run_region_lease_store_tests(); void run_region_checkpoint_store_tests(); void run_region_host_tests(); void run_region_server_process_tests(); +void run_region_recovery_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; @@ -81,6 +82,8 @@ int main() { std::puts("region host integration passed"); run_region_server_process_tests(); std::puts("region server process integration passed"); + run_region_recovery_tests(); + std::puts("region recovery integration passed"); return 0; } catch (const std::exception& e) { std::fprintf(stderr, "%s\n", e.what()); diff --git a/tests/region_recovery_tests.cpp b/tests/region_recovery_tests.cpp new file mode 100644 index 0000000..ff1b51c --- /dev/null +++ b/tests/region_recovery_tests.cpp @@ -0,0 +1,276 @@ +// M7.7 recovery before readiness (ADR-0065): real evo-server processes killed at +// every publication/acknowledgement boundary, a resumed stale owner, damaged heads +// that must never serve, and tail verification. Run by tools/ci/postgres-check.sh. +#include "evo/persistence/sha256.hpp" +#include "fixtures/server_process.hpp" +#include +#include +#include +#include + +void run_region_recovery_tests(); + +namespace { +using namespace evo::test::process; +using namespace evo::persistence; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("region recovery test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +std::string hex(std::span bytes) { + constexpr char digits[] = "0123456789abcdef"; + std::string out; + for (const auto b : bytes) { + out.push_back(digits[b >> 4U]); + out.push_back(digits[b & 0x0FU]); + } + return out; +} +std::vector read_file(const fs::path& p) { + std::ifstream in(p, std::ios::binary); + return {std::istreambuf_iterator(in), {}}; +} +void write_file(const fs::path& p, const std::vector& bytes) { + fs::remove(p); + std::ofstream(p, std::ios::binary) + .write(bytes.data(), static_cast(bytes.size())); + fs::permissions(p, fs::perms::owner_read | fs::perms::owner_write, fs::perm_options::replace); +} +CheckpointManifest head_manifest(RegionKey key) { + auto db = owner(); + auto committed = load_region_checkpoint(*db, key, 5s); + CHECK(committed); + auto m = decode_manifest(committed->manifest); + CHECK(m); + return *m; +} +// Successor: waits for readiness, requires it resumed from the committed head, drains. +void successor_resumes(const ServerRegion& r, std::uint64_t epoch) { + const auto head = head_of(r.key); + Child next{server_args(r)}; + CHECK(next.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); + CHECK(next.printed("phase running epoch=" + std::to_string(epoch) + + " committed=" + std::to_string(head.tick) + " ")); + next.signal(SIGTERM); + CHECK(next.wait_exit(20s) == 0); + const auto after = head_of(r.key); + CHECK(after.tick == head.tick && after.sequence == head.sequence && after.released); +} + +struct Boundary { + const char* point; + int ordinal; + std::uint64_t region; + std::uint8_t genesis; +}; +// Inputs 1..40 with the consumer's death at tick 20: commit 1 is the death batch +// (ticks 1..20), commit 2 the next interval flush. The process is SIGKILLed at the +// chosen boundary; durable state, acknowledgements and orphans must match it. +void kill_at(const Boundary& b) { + const auto r = provision(b.region, b.genesis); + std::uint64_t acked = 0; + bool outcome = false; + { + // A long flush interval makes commit 1 exactly the death batch. + Child server{server_args(r, {"--flush-ms", "5000", "--crash-at", + std::string(b.point) + ":" + std::to_string(b.ordinal)})}; + CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); + server.send(input_lines(1, 40, true)); + CHECK(server.wait_exit(30s) == 128 + SIGKILL); + acked = max_acked(server); + outcome = server.printed("outcome 20 1 11 1"); + } + const auto head = head_of(r.key); + const auto decisions = decisions_of(r.key); + const auto objects = object_count(r); // Genesis payload plus 2 per uploaded batch. + const std::string point = b.point; + CHECK(!head.released); // Killed holders never release; successors wait for expiry. + CHECK(acked <= head.tick); // Never an acknowledgement beyond the committed head. + if (b.ordinal == 1) { + // The death batch never committed: nothing final, the death is not a fact. + CHECK(point == "before-transaction"); + CHECK(head.sequence == 0 && head.tick == 0 && acked == 0 && decisions == 0 && !outcome); + CHECK(objects == 3); // Verified orphans only. + } else if (point == "before-upload" || point == "before-transaction") { + CHECK(head.sequence == 1 && head.tick == 20 && acked == 20 && decisions == 1 && outcome); + CHECK(objects == (point == "before-upload" ? 3U : 5U)); + } else if (point == "after-commit") { + // Durable but never acknowledged: the lost acks are the safe direction. + CHECK(head.sequence == 2 && head.tick > 20 && acked == 20 && decisions == 1 && outcome); + } else { + CHECK(point == "after-ack"); + CHECK(head.sequence == 2 && acked == head.tick && decisions == 1 && outcome); + } + std::printf("kill_at %s:%d head=%llu/%llu acked=%llu objects=%zu\n", b.point, b.ordinal, + static_cast(head.sequence), + static_cast(head.tick), static_cast(acked), + objects); + successor_resumes(r, 2); // Verified head, tail and history before readiness. +} + +// A paused (SIGSTOP) owner loses its lease; after SIGCONT it fences itself, refuses +// admission and never acknowledges past its own commits, while the successor's +// chain continues untouched. +void resumed_stale_owner_is_fenced() { + const auto r = provision(46, 0x62); + Child stale{server_args(r)}; + CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); + stale.send(input_lines(1, 10, false, 0xA1)); + CHECK(stale.wait_line("ack " + request_hex(10, 0xA1) + " 10 ", 15s)); + stale.signal(SIGSTOP); + std::this_thread::sleep_for(4500ms); // Longer than the 4 s development lease. + Child owner_now{server_args(r)}; + CHECK(owner_now.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); + CHECK(owner_now.printed("phase running epoch=2 committed=10 ")); + owner_now.send(input_lines(11, 20, false, 0xB1)); + CHECK(owner_now.wait_line("ack " + request_hex(20, 0xB1) + " 20 ", 15s)); + stale.signal(SIGCONT); + CHECK(stale.wait_line("ready 0", 10s)); + stale.send(input_lines(11, 12, false, 0xA2)); + CHECK(stale.wait_line("rejected " + request_hex(11, 0xA2) + " not_ready", 10s)); + CHECK(max_acked(stale) == 10); + const auto head = head_of(r.key); + CHECK(head.tick >= 20 && head.sequence >= 2 && !head.released); + stale.signal(SIGTERM); + CHECK(stale.wait_exit(20s) == 0); // It never regained authority: nothing to release. + owner_now.signal(SIGTERM); + CHECK(owner_now.wait_exit(20s) == 0); + CHECK(head_of(r.key).released && head_of(r.key).tick == head.tick); +} + +// A committed head whose payload or tail cannot be verified is never served, and +// no older or empty world replaces it. +void damaged_heads_never_become_ready() { + const auto r = provision(47, 0x63); + { + Child server{server_args(r)}; + CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); + server.send(input_lines(1, 10, false)); + CHECK(server.wait_line("ack " + request_hex(10) + " 10 ", 15s)); + server.signal(SIGTERM); + CHECK(server.wait_exit(20s) == 0); + } + const auto head = head_of(r.key); + const auto m = head_manifest(r.key); + const auto payload = r.objects() / hex(m.payload_sha256); + const auto trace = r.objects() / hex(m.batch.trace_sha256); + const auto quarantined = [&] { + Child server{server_args(r)}; + const auto code = server.wait_exit(20s); + return code == 4 && server.printed("phase quarantined") && !server.printed("ready 1"); + }; + const auto payload_bytes = read_file(payload); + auto flipped = payload_bytes; + flipped[flipped.size() / 2] = static_cast(flipped[flipped.size() / 2] ^ 1); + write_file(payload, flipped); + CHECK(quarantined()); + write_file(payload, payload_bytes); + const auto trace_bytes = read_file(trace); + fs::remove(trace); + CHECK(quarantined()); + write_file(trace, trace_bytes); + const auto after = head_of(r.key); + CHECK(after.tick == head.tick && after.sequence == head.sequence); // Nothing replaced it. + successor_resumes(r, 4); // Epochs 2 and 3 were taken by the quarantined attempts. +} + +// In-process: recovery replays the head's trace from its predecessor before any +// readiness; missing, damaged or non-reproducing evidence quarantines (tail). +void tail_verification_before_readiness() { + const auto r = provision(48, 0x01); + auto runtime = Postgres::connect( + {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}}, + 5s); + CHECK(runtime); + auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); + CHECK(objects); + Sha256Digest build{}; + build.fill(0xB1); + const std::array retained{build}; + const auto commit = [](std::uint8_t v) { + Uuid id{}; + id.fill(v); + return id; + }; + auto lease = acquire_region_lease(**runtime, r.key, 20000ms, 5s); + CHECK(lease); + auto head = load_region_checkpoint(**runtime, r.key, 5s); + CHECK(head); + auto state = evo::test::lineage_region(); + evo::persistence::RegionBatch batch{*state}; + for (int n = 0; n < 2; ++n) { + for (int i = 0; i < 10; ++i) + CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1))); + batch.seal(); + auto published = + publish_lineage_batch(**runtime, **objects, *lease, *head, batch, + commit(static_cast(2 + n)), build, 8000000, 5s); + CHECK(published && batch.confirm(published->committed.head.end_tick)); + head = published->committed; + } + const auto recover = [&] { + return recover_lineage_checkpoint(**runtime, **objects, r.key, retained, 8000000, 5s); + }; + CHECK(recover()); + const auto m = *decode_manifest(head->manifest); + const auto trace = r.objects() / hex(m.batch.trace_sha256); + const auto trace_bytes = read_file(trace); + const auto refused = [&](RecoveryError kind) { + auto rec = recover(); + return !rec && rec.error().kind == kind; + }; + fs::remove(trace); + CHECK(refused(RecoveryError::tail)); + auto damaged = trace_bytes; + damaged[200] = static_cast(damaged[200] ^ 1); + write_file(trace, damaged); + CHECK(refused(RecoveryError::tail)); + write_file(trace, trace_bytes); + // The predecessor's payload is needed to replay the head: its loss quarantines. + auto first = load_region_commit(**runtime, r.key, 1, 5s); + CHECK(first); + const auto previous = *decode_manifest(first->manifest); + const auto base = r.objects() / hex(previous.payload_sha256); + const auto base_bytes = read_file(base); + fs::remove(base); + CHECK(refused(RecoveryError::tail)); + write_file(base, base_bytes); + CHECK(recover()); + + // Evidence that does not reproduce the payload: a trace of different inputs over + // the same range, published by bypassing the host (the database cannot tell). + for (int i = 0; i < 10; ++i) + CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1))); + auto alternate = recover(); + CHECK(alternate); + evo::persistence::RegionBatch other{*alternate->state}; + for (int i = 0; i < 10; ++i) { + auto in = evo::test::region_input(other.end_tick() + 1); + in.source_enabled = 1 - in.source_enabled; // Not what actually happened. + CHECK(other.advance(in)); + } + auto payload = state->snapshot(); + auto forged = encode_trace({2, r.key, m.commit, head->head.digest, m.end_tick, m.end_tick + 10}, + other.steps()); + CHECK(payload && forged && (*objects)->put(*payload) && (*objects)->put(*forged)); + auto manifest = successor_manifest(m, head->head.digest, commit(5), *lease, m.end_tick + 10, + build, payload->size(), *sha256(*payload), + {forged->size(), *sha256(*forged), 0, {}}); + CHECK(manifest); + CHECK(publish_region_checkpoint(**runtime, *encode_manifest(*manifest), {}, 5s)); + CHECK(refused(RecoveryError::tail)); + CHECK(release_region_lease(**runtime, r.key, *lease, 5s)); +} +} // namespace + +void run_region_recovery_tests() { + for (const auto& b : + {Boundary{"before-transaction", 1, 41, 0x5D}, Boundary{"before-upload", 2, 42, 0x5E}, + Boundary{"before-transaction", 2, 43, 0x5F}, Boundary{"after-commit", 2, 44, 0x60}, + Boundary{"after-ack", 2, 45, 0x61}}) + kill_at(b); + resumed_stale_owner_is_fenced(); + damaged_heads_never_become_ready(); + tail_verification_before_readiness(); +} diff --git a/tests/region_server_process_tests.cpp b/tests/region_server_process_tests.cpp index 52e5ffd..83d8368 100644 --- a/tests/region_server_process_tests.cpp +++ b/tests/region_server_process_tests.cpp @@ -1,249 +1,20 @@ // Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL // with recovery by a successor process. Run by tools/ci/postgres-check.sh, which // sets EVO_SERVER_BIN. -#include "evo/persistence/checkpoint_store.hpp" -#include "fixtures/environmental_region.hpp" +#include "fixtures/server_process.hpp" #include -#include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include #include -#include -#include -extern char** environ; void run_region_server_process_tests(); namespace { -using namespace evo::persistence; -using namespace std::chrono_literals; -namespace fs = std::filesystem; -using Clock = std::chrono::steady_clock; +using namespace evo::test::process; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region server process test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) -std::string env(const char* key) { - const auto* value = std::getenv(key); - if (!value) - throw std::runtime_error("missing test environment"); - return value; -} -std::unique_ptr owner() { - auto made = Postgres::connect( - {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s); - CHECK(made); - return std::move(*made); -} -std::string hex32(std::uint64_t tick) { - char out[33]; - std::snprintf(out, sizeof out, "9b000000000000000000000000%06llx", - static_cast(tick)); - return out; -} -std::string input_line(std::uint64_t tick, bool consumer_dies) { - const auto in = evo::test::region_input(tick, consumer_dies); - std::string line = "input " + hex32(tick) + " " + std::to_string(in.temperature_mK) + " " + - std::to_string(in.source_enabled); - for (const auto b : in.bite) - line += " " + std::to_string(b); - for (const auto t : in.terminal) - line += " " + std::to_string(t); - return line + "\n"; -} - -class Child { - public: - explicit Child(const std::vector& args) { - int to_child[2], from_child[2]; - CHECK(::pipe(to_child) == 0 && ::pipe(from_child) == 0); - posix_spawn_file_actions_t actions; - posix_spawn_file_actions_init(&actions); - posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO); - posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO); - posix_spawn_file_actions_addclose(&actions, to_child[1]); - posix_spawn_file_actions_addclose(&actions, from_child[0]); - std::vector argv; - argv.reserve(args.size() + 1); - for (const auto& a : args) - argv.push_back(const_cast(a.c_str())); - argv.push_back(nullptr); - CHECK(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0); - posix_spawn_file_actions_destroy(&actions); - ::close(to_child[0]); - ::close(from_child[1]); - in_ = to_child[1]; - out_ = from_child[0]; - } - Child(const Child&) = delete; - Child& operator=(const Child&) = delete; - ~Child() { - if (pid_ > 0) { - ::kill(pid_, SIGKILL); - ::waitpid(pid_, nullptr, 0); - } - ::close(in_); - ::close(out_); - } - void send(const std::string& text) { - std::size_t done = 0; - while (done < text.size()) { - const auto n = ::write(in_, text.data() + done, text.size() - done); - CHECK(n > 0 || errno == EINTR); - if (n > 0) - done += static_cast(n); - } - } - // Reads lines until one satisfies `want`; every line is kept in lines(). - template bool wait_for(Want want, std::chrono::milliseconds timeout) { - const auto deadline = Clock::now() + timeout; - for (std::size_t seen = 0;;) { - for (; seen < lines_.size(); ++seen) - if (want(lines_[seen])) - return true; - const auto left = - std::chrono::duration_cast(deadline - Clock::now()); - if (left.count() <= 0 || !read_some(static_cast(left.count()))) - return false; - } - } - void signal(int sig) { - CHECK(::kill(pid_, sig) == 0); - } - // Exit code, or 128+signal; drains remaining output. - int wait_exit(std::chrono::milliseconds timeout) { - const auto deadline = Clock::now() + timeout; - int status = 0; - for (;;) { - read_some(20); - const auto r = ::waitpid(pid_, &status, WNOHANG); - if (r == pid_) - break; - CHECK(Clock::now() < deadline); - } - while (read_some(20)) { - } - pid_ = -1; - return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status); - } - [[nodiscard]] const std::vector& lines() const noexcept { - return lines_; - } - - private: - bool read_some(int timeout_ms) { - pollfd fd{out_, POLLIN, 0}; - if (::poll(&fd, 1, timeout_ms) <= 0) - return false; - char chunk[4096]; - const auto n = ::read(out_, chunk, sizeof chunk); - if (n <= 0) - return false; - buffer_.append(chunk, static_cast(n)); - for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) { - lines_.push_back(buffer_.substr(0, end)); - buffer_.erase(0, end + 1); - } - return true; - } - pid_t pid_{-1}; - int in_{-1}, out_{-1}; - std::string buffer_; - std::vector lines_; -}; - -struct Region { - RegionKey key; - fs::path root; -}; -Region provision(std::uint64_t region, std::uint8_t genesis) { - Region r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))}; - fs::remove_all(r.root); - fs::create_directory(r.root); - fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); - auto db = owner(); - CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " - "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", - {}, 5s)); - auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); - CHECK(objects); - Sha256Digest build{}; - build.fill(0xB1); - Uuid commit{}; - commit.fill(genesis); - CHECK(provision_lineage_region(*db, **objects, r.key, evo::test::region_identity.frame, - *evo::test::lineage_region(), commit, build, 8000000, 5s)); - return r; -} -std::vector server_args(const Region& r) { - return {env("EVO_SERVER_BIN"), - "--profile", - "lineage", - "--world", - "7", - "--region", - std::to_string(r.key.region), - "--frame", - std::to_string(evo::test::region_identity.frame), - "--pg-host", - env("PGHOST"), - "--pg-port", - env("PGPORT"), - "--pg-database", - env("PGDATABASE"), - "--pg-user", - env("EVO_PG_RUNTIME_USER"), - "--objects", - r.root.string(), - "--build", - "b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1", - "--flush-ms", - "500", - "--lease-profile", - "fast", - "--ready-file", - (r.root / "ready").string()}; -} -// (tick, commit_sequence) of every ack line. -std::vector> acks(const Child& c) { - std::vector> out; - for (const auto& line : c.lines()) { - unsigned long long tick = 0, sequence = 0; - char request[40]; - if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3) - out.emplace_back(tick, sequence); - } - return out; -} -std::string head_of(RegionKey key) { - const std::array p{std::to_string(key.region)}; - auto rows = owner()->query("SELECT h.end_tick::text || ':' || h.commit_sequence::text || ':' " - "|| coalesce(l.holder::text, 'released') FROM " - "evolution.region_heads h JOIN evolution.region_leases l USING " - "(world_id, region_id) WHERE world_id = 7 AND region_id = " - "$1::evolution.u64", - p, 5s); - CHECK(rows && rows->size() == 1); - const auto& value = (*rows)[0][0]; - CHECK(value.has_value()); - return value.value_or(""); -} -std::uint64_t head_tick(RegionKey key) { - return std::stoull(head_of(key).substr(0, head_of(key).find(':'))); -} -bool starts(const std::string& line, std::string_view prefix) { - return line.starts_with(prefix); -} // SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every // acknowledged tick is committed, the backlog never exceeds its bound, and the @@ -253,19 +24,12 @@ void sigterm_drains_and_releases() { Child server{server_args(r)}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); CHECK(fs::exists(r.root / "ready")); - std::string burst; - for (std::uint64_t tick = 1; tick <= 80; ++tick) - burst += input_line(tick, true); - server.send(burst); // More than the 64-input backlog: stdin backpressures. - CHECK(server.wait_for( - [](const std::string& l) { return starts(l, "ack " + hex32(80) + " 80 "); }, 30s)); + server.send(input_lines(1, 80, true)); // More than the 64-input backlog: stdin backpressures. + CHECK(server.wait_line("ack " + request_hex(80) + " 80 ", 30s)); server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); const auto& lines = server.lines(); - CHECK(std::ranges::find(lines, "exit 0") != lines.end()); - CHECK(std::ranges::find_if(lines, [](const std::string& l) { - return starts(l, "phase stopped"); - }) != lines.end()); + CHECK(std::ranges::find(lines, "exit 0") != lines.end() && server.printed("phase stopped")); unsigned long long max_queued = 999; for (const auto& l : lines) std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued); @@ -277,12 +41,14 @@ void sigterm_drains_and_releases() { const auto outcome = std::ranges::find_if( lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); }); const auto ack20 = std::ranges::find_if( - lines, [](const std::string& l) { return starts(l, "ack " + hex32(20) + " 20 "); }); + lines, [](const std::string& l) { return starts(l, "ack " + request_hex(20) + " 20 "); }); CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome); - CHECK(head_of(r.key).starts_with("80:") && head_of(r.key).ends_with(":released")); + const auto head = head_of(r.key); + CHECK(head.tick == 80 && head.released); CHECK(!fs::exists(r.root / "ready")); - std::printf("server_sigterm commits=%s acks=%zu max_queued=%llu\n", head_of(r.key).c_str(), - all.size(), max_queued); + std::printf("server_sigterm head=%llu sequence=%llu acks=%zu max_queued=%llu\n", + static_cast(head.tick), + static_cast(head.sequence), all.size(), max_queued); } // SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was @@ -293,22 +59,16 @@ void sigkill_recovers_through_a_successor() { { Child server{server_args(r)}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); - std::string burst; - for (std::uint64_t tick = 1; tick <= 60; ++tick) - burst += input_line(tick, false); - server.send(burst); - CHECK(server.wait_for([](const std::string& l) { return starts(l, "ack "); }, 15s)); + server.send(input_lines(1, 60, false)); + CHECK(server.wait_line("ack ", 15s)); std::this_thread::sleep_for(300ms); // Ticks run past the last commit. server.signal(SIGKILL); CHECK(server.wait_exit(10s) == 128 + SIGKILL); - for (const auto& [tick, sequence] : acks(server)) { - static_cast(sequence); - acked_max = std::max(acked_max, tick); - } + acked_max = max_acked(server); } - const auto committed = head_tick(r.key); - CHECK(acked_max > 0 && committed >= acked_max); // No acknowledged tick was lost. - CHECK(!head_of(r.key).ends_with(":released")); // The killed holder never released. + const auto committed = head_of(r.key); + CHECK(acked_max > 0 && committed.tick >= acked_max); // No acknowledged tick was lost. + CHECK(!committed.released); // The killed holder never released. const auto started = Clock::now(); Child successor{server_args(r)}; CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); @@ -317,14 +77,15 @@ void sigkill_recovers_through_a_successor() { const auto running = std::ranges::find_if( successor.lines(), [](const std::string& l) { return starts(l, "phase running"); }); CHECK(running != successor.lines().end()); - CHECK(running->find("committed=" + std::to_string(committed) + " ") != std::string::npos); + CHECK(running->find("committed=" + std::to_string(committed.tick) + " ") != std::string::npos); CHECK(running->find("epoch=2 ") != std::string::npos); successor.signal(SIGTERM); CHECK(successor.wait_exit(20s) == 0); - CHECK(head_tick(r.key) == committed && head_of(r.key).ends_with(":released")); + const auto after = head_of(r.key); + CHECK(after.tick == committed.tick && after.released); std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n", static_cast(acked_max), - static_cast(committed), static_cast(to_ready)); + static_cast(committed.tick), static_cast(to_ready)); } } // namespace diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index 5602209..8703d11 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -46,7 +46,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -67,7 +67,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 5:1,2,3,4,5 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 6:1,2,3,4,5,6 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -136,9 +136,11 @@ expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.provision_region_check expect_audit_failure 'GRANT SELECT ON evolution.region_commits TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commits FROM evo_runtime_test' bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=6 WHERE version=5' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=7 WHERE version=6' +if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=6 WHERE version=7" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=6" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=5 WHERE version=6" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=5" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=4" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index 6b1ddea..107adbe 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -22,8 +22,11 @@ digest_4="$(sha256sum "$root/migrations/0004_lineage_profile.sql")" digest_4="${digest_4%% *}" digest_5="$(sha256sum "$root/migrations/0005_batch_decisions.sql")" digest_5="${digest_5%% *}" +digest_6="$(sha256sum "$root/migrations/0006_commit_lookup.sql")" +digest_6="${digest_6%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ + --set="migration_sha_6=$digest_6" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"