Complete M7.7 recovery verification before readiness
ADR-0065: before any readiness, recovery now verifies the head's tail. The trace object must exist with its manifest digest, its header must bind profile, region, predecessor and range; the predecessor (schema v6 load_region_commit) must hash to the digest the head names; replaying the trace from the predecessor's verified payload must reproduce every report and the exact head bytes; and trace-derived decisions must equal the head's durable decisions. Missing, corrupt or non-reproducing tails quarantine (RecoveryError::tail); nothing older or empty is served. Test-only crash seams (HostConfig::crash_hook, PublishHooks) and evo-server --crash-at deliver real SIGKILL at before-upload, before-transaction, after-commit and after-ack. The kill matrix, a SIGSTOP/SIGCONT stale owner and damaged-head startups all behave as the durability contract requires; successors verify before readiness. Validated: PostgreSQL suites under GCC Release, clang-tidy and ASan/UBSan; tail-skip and replay-equality mutations each caught. No Docker/Gitea CI, production-lease kill test or deployment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+1
-1
@@ -115,7 +115,7 @@ if(EVO_BUILD_POSTGRES)
|
||||
# Contention tests use one thread and one connection per contender.
|
||||
add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp
|
||||
tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp
|
||||
tests/region_server_process_tests.cpp)
|
||||
tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp)
|
||||
target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads)
|
||||
add_dependencies(evo-postgres-tests evo-server) # Spawned by the process tests.
|
||||
evo_owned_target(evo-postgres-tests)
|
||||
|
||||
@@ -10,6 +10,10 @@
|
||||
// outcome <tick> <actor> <organism> <commit_sequence>
|
||||
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
|
||||
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
|
||||
//
|
||||
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
|
||||
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
|
||||
// after a step's acknowledgements are written and flushed).
|
||||
#include "evo/server/region_host.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -22,6 +26,7 @@
|
||||
#include <fstream>
|
||||
#include <optional>
|
||||
#include <poll.h>
|
||||
#include <signal.h>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <unistd.h>
|
||||
@@ -41,8 +46,18 @@ struct Options {
|
||||
std::filesystem::path objects, ready_file;
|
||||
server::HostConfig host;
|
||||
std::chrono::milliseconds drain{10000};
|
||||
std::string crash_point;
|
||||
std::uint64_t crash_ordinal{};
|
||||
};
|
||||
|
||||
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
|
||||
[[noreturn]] void crash_now() {
|
||||
std::fflush(stdout);
|
||||
::kill(::getpid(), SIGKILL);
|
||||
for (;;)
|
||||
::pause();
|
||||
}
|
||||
|
||||
std::optional<std::uint64_t> number(std::string_view text) {
|
||||
std::uint64_t value{};
|
||||
const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value);
|
||||
@@ -125,7 +140,18 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms};
|
||||
else if (flag == "--lease-profile" && value == "default")
|
||||
o.host.lease = {};
|
||||
else
|
||||
else if (flag == "--crash-at") {
|
||||
const auto colon = value.find(':');
|
||||
const auto ordinal =
|
||||
colon == std::string_view::npos ? std::nullopt : number(value.substr(colon + 1));
|
||||
const auto point = value.substr(0, colon);
|
||||
if (!ordinal || *ordinal == 0 ||
|
||||
(point != "before-upload" && point != "before-transaction" &&
|
||||
point != "after-commit" && point != "after-ack"))
|
||||
return std::nullopt;
|
||||
o.crash_point = point;
|
||||
o.crash_ordinal = *ordinal;
|
||||
} else
|
||||
return std::nullopt;
|
||||
}
|
||||
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
|
||||
@@ -243,6 +269,17 @@ class LineReader {
|
||||
};
|
||||
|
||||
template <typename State> int serve(Options& o) {
|
||||
if (!o.crash_point.empty() && o.crash_point != "after-ack") {
|
||||
o.host.crash_hook = [point = o.crash_point, ordinal = o.crash_ordinal,
|
||||
seen = std::uint64_t{0}](server::CrashPoint at) mutable {
|
||||
const char* name = at == server::CrashPoint::before_upload ? "before-upload"
|
||||
: at == server::CrashPoint::before_transaction ? "before-transaction"
|
||||
: "after-commit";
|
||||
if (point == name && ++seen == ordinal)
|
||||
crash_now();
|
||||
};
|
||||
}
|
||||
std::uint64_t ack_steps = 0;
|
||||
auto objects = persistence::FilesystemObjectStore::open(o.objects, o.host.key,
|
||||
persistence::max_lineage_payload_bytes);
|
||||
if (!objects) {
|
||||
@@ -317,7 +354,8 @@ template <typename State> int serve(Options& o) {
|
||||
static_cast<unsigned long long>(s.simulated_tick));
|
||||
}
|
||||
set_ready(s.ready);
|
||||
for (const auto& a : host.take_acks())
|
||||
const auto acknowledged = host.take_acks();
|
||||
for (const auto& a : acknowledged)
|
||||
std::printf("ack %s %llu %llu\n", text(a.request).c_str(),
|
||||
static_cast<unsigned long long>(a.tick),
|
||||
static_cast<unsigned long long>(a.commit_sequence));
|
||||
@@ -330,6 +368,8 @@ template <typename State> int serve(Options& o) {
|
||||
for (const auto& d : host.take_dropped())
|
||||
std::printf("dropped %s %s\n", text(d.input.request).c_str(), dropped_name(d.reason));
|
||||
std::fflush(stdout);
|
||||
if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal)
|
||||
crash_now();
|
||||
const auto finish = [&](int code) {
|
||||
set_ready(false);
|
||||
std::printf(
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: evolution-db-migrate-v5
|
||||
name: evolution-db-migrate-v6
|
||||
namespace: evolution-dev
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
|
||||
@@ -567,3 +567,12 @@ M7.6 implemented and locally qualified (2026-09-28), under ADR-0064.
|
||||
- `evo-server` passed real SIGTERM drain and SIGKILL tests, with successor
|
||||
recovery.
|
||||
Next: M7.7 recovery verification at every kill boundary.
|
||||
|
||||
M7.7 implemented and locally qualified (2026-09-28), under ADR-0065.
|
||||
- Before readiness, recovery replays the head's trace from its predecessor's
|
||||
verified payload, requires the exact head bytes and decisions, and
|
||||
quarantines on any missing or corrupt tail. Schema v6 adds the read-only
|
||||
commit lookup.
|
||||
- Real SIGKILL at five boundaries, a resumed SIGSTOPped owner, and damaged heads
|
||||
all behave as the crash matrix requires.
|
||||
Next: M7.8 operations endpoints and dev image.
|
||||
|
||||
@@ -861,3 +861,22 @@ and ASan/UBSan.
|
||||
|
||||
Next: M7.7 recovery verification at every kill boundary. There is no gateway,
|
||||
metrics or deployment yet.
|
||||
|
||||
## M7.7 recovery before readiness (2026-09-28)
|
||||
|
||||
ADR-0065: before readiness, recovery now replays the head's validation trace
|
||||
from the predecessor's verified payload. It must reproduce the head bytes
|
||||
exactly and derive exactly the durable decisions. A missing, corrupt or
|
||||
non-reproducing tail, or a missing predecessor payload, quarantines the region,
|
||||
and nothing older or empty is ever served. Schema v6 adds the read-only
|
||||
`load_region_commit`.
|
||||
|
||||
Real `evo-server` processes were SIGKILLed at five publication and ACK
|
||||
boundaries using a test-only `--crash-at` injector. Every result matched the
|
||||
crash matrix, and each successor verified the head before becoming ready. A
|
||||
SIGSTOPped owner resumed fenced, and damaged heads exited quarantined.
|
||||
|
||||
Tail-skip and replay-equality mutations were each caught. The PostgreSQL suites
|
||||
passed under Release, analysis and ASan/UBSan.
|
||||
|
||||
Next: M7.8 operations endpoints and a dev image.
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
# ADR-0065: Recovery verification before readiness
|
||||
|
||||
## Status
|
||||
Accepted for M7.7, 2026-09-28. It extends ADR-0061 (recovery), ADR-0063 (batch
|
||||
evidence) and ADR-0064 (host). Clone, migration and durability qualification are
|
||||
M7.9.
|
||||
|
||||
## Context
|
||||
The M7.7 criterion is: load a verified committed checkpoint, verify the tail,
|
||||
hash, inventories and history, and hold valid current authority before serving.
|
||||
Kill tests must cover every publication and acknowledgement boundary; a corrupt
|
||||
or missing tail must quarantine; there must be no empty fallback world.
|
||||
|
||||
The full-checkpoint profile (ADR-0058) has no replay tail beyond the head. The
|
||||
head's own batch evidence (trace, decisions) was published in M7.5, but recovery
|
||||
did not re-verify it. The crash matrix rows were exercised in process, but not by
|
||||
killing a real process at each boundary.
|
||||
|
||||
## Decision
|
||||
**Tail verification.** Recovery, which the host completes before readiness under
|
||||
a freshly acquired lease, now also verifies the head's batch evidence after the
|
||||
manifest, build, payload and decision checks. For any head after genesis:
|
||||
1. The manifest must be v2.
|
||||
2. The trace object must exist with its manifest length and SHA-256, and its
|
||||
header must bind the profile, region, predecessor commit and digest, and the
|
||||
tick range.
|
||||
3. The predecessor is loaded by sequence through the new read-only
|
||||
`evolution.load_region_commit` (schema v6; runtime function set is eleven).
|
||||
Its manifest must hash to the digest the head names.
|
||||
4. The predecessor's payload is fetched, verified and restored. Replaying the
|
||||
trace must reproduce every report and the head payload bytes exactly.
|
||||
5. The decisions derived from the trace must equal the head commit's durable
|
||||
decisions.
|
||||
|
||||
A missing, corrupt or non-reproducing trace, or a missing predecessor payload, is
|
||||
`RecoveryError::tail`; a decision mismatch is `history`. Either quarantines the
|
||||
region: no older head, genesis or empty world is ever served.
|
||||
|
||||
The replay equality is the inventory check. It proves the head state follows,
|
||||
under the pinned rules, from a predecessor that was itself verified when it was
|
||||
published. The envelope's own restore audit covers internal totals. Recovery
|
||||
cost grows by one predecessor restore and at most 64 replayed ticks.
|
||||
|
||||
**Kill boundaries.** `HostConfig::crash_hook` and `PublishHooks` are test-only
|
||||
seams at three points: before upload, before the transaction (after verified
|
||||
uploads), and after commit (before any acknowledgement). `evo-server --crash-at
|
||||
<point>:<n>` delivers a real SIGKILL there, or right after a step's
|
||||
acknowledgements are flushed (`after-ack`).
|
||||
|
||||
## Alternatives Considered
|
||||
- **Verifying the whole chain on every recovery**: unbounded cost. Earlier links
|
||||
were verified when published, and the chain digests bind them.
|
||||
- **Skipping trace verification**: leaves a published but unreproducible head
|
||||
servable.
|
||||
- **Kill points chosen by sleeps and signals**: nondeterministic; hooks fire
|
||||
exactly at the boundary.
|
||||
|
||||
## Benefits
|
||||
- Real killed processes at every boundary show the contract end to end.
|
||||
- A resumed stale owner cannot serve.
|
||||
- Damaged heads and tails never become ready.
|
||||
|
||||
## Costs
|
||||
- Recovery reads two payloads and one trace, and replays up to 64 ticks.
|
||||
- Predecessor objects must be retained (there is no GC).
|
||||
- The crash seams exist in production binaries but are inert unless configured.
|
||||
|
||||
## Risks
|
||||
- Tail verification covers the head link only; a damaged older link is detected
|
||||
only when it becomes a predecessor, or by a future audit.
|
||||
- The kill tests use the development 4 s lease on a single host with a local
|
||||
database.
|
||||
- VM freezes and host loss remain M8.
|
||||
|
||||
## Consequences
|
||||
- M7.8 exposes readiness and quarantine to probes and metrics.
|
||||
- M7.9 measures recovery latency with the production lease, and qualifies clones
|
||||
and migrations against this verification.
|
||||
@@ -75,3 +75,4 @@ design. No unadopted candidate is a link dependency of the proof.
|
||||
- [ADR-0062: Lineage checkpoint profile](0062-lineage-checkpoint-profile.md) - accepted for M7.4b archive codec, LineageRegion closure and history-capable profile 2.
|
||||
- [ADR-0063: Batch outcomes](0063-batch-outcomes.md) - accepted for M7.5 manifest v2, validation trace replay, durable decisions/outbox and commit resolution.
|
||||
- [ADR-0064: Region host](0064-region-host.md) - accepted for M7.6 single-owner host state machine, bounded admission, final-only-after-commit and evo-server process.
|
||||
- [ADR-0065: Recovery before readiness](0065-recovery-before-readiness.md) - accepted for M7.7 head tail replay verification, schema v6 commit lookup and real kill-boundary tests.
|
||||
|
||||
@@ -210,3 +210,13 @@ M7.6 (ADR-0064) executed its host rows:
|
||||
- Shutdown drains, commits and releases.
|
||||
- A real SIGKILLed `evo-server` left no acknowledged tick uncommitted, and its
|
||||
successor became ready only after lease expiry, from the committed head.
|
||||
|
||||
M7.7 (ADR-0065) executed the matrix with real `evo-server` processes SIGKILLed at
|
||||
each boundary: before the transaction (first and second batch), before upload,
|
||||
after commit but before acknowledgement, and after acknowledgement.
|
||||
- In every case acknowledgements stayed at or below the committed head.
|
||||
- Deaths were final only once committed, and verified orphans remained.
|
||||
- Each successor verified the head and its tail before readiness.
|
||||
A SIGSTOPped owner resumed fenced: it was not ready, refused admission and
|
||||
acknowledged nothing new. Corrupt payloads and missing traces quarantined with
|
||||
no fallback. Host loss and clone or migration remain M8 and M7.9.
|
||||
|
||||
@@ -2339,3 +2339,45 @@ Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate
|
||||
- Not executed: Docker/Gitea CI, TSan (the host is single-threaded; libpq is
|
||||
uninstrumented), remote PostgreSQL/TLS, a production 20 s lease kill test,
|
||||
durable client deduplication, and a network gateway.
|
||||
|
||||
## M7.7 recovery before readiness - 2026-09-28
|
||||
|
||||
- WSL Linux: GCC14 Release, clang19 analysis (clang-tidy, clean on the first
|
||||
build) and clang19 ASan/UBSan builds of `evo-postgres-tests` and `evo-server`
|
||||
passed the full PostgreSQL suite, including the M7.7 recovery tests. Runs:
|
||||
Release 2, analysis 1, ASan 2. The repository format check passed after
|
||||
reordering one fixture include.
|
||||
- Windows was not rerun. This change touches only Linux persistence and host
|
||||
code, SQL, tests and docs; M7.6's Windows run (58/58) covers the portable
|
||||
targets, which are unchanged.
|
||||
- Kill matrix (real SIGKILL via `--crash-at`, inputs 1..40 with a death at tick
|
||||
20, 5 s flush):
|
||||
|
||||
| Crash point | Head (seq/tick) | Acked up to | Decisions | Objects |
|
||||
|---|---|---|---|---|
|
||||
| before-transaction:1 | 0/0 | none | 0 | 3 (verified orphans) |
|
||||
| before-upload:2 | 1/20 | 20 | 1 | 3 |
|
||||
| before-transaction:2 | 1/20 | 20 | 1 | 5 (orphans) |
|
||||
| after-commit:2 | 2/40 | 20 (lost acks are the safe direction) | 1 | 5 |
|
||||
| after-ack:2 | 2/40 | 40 | 1 | 5 |
|
||||
|
||||
The lease stayed held after every kill. Each successor became ready at epoch 2
|
||||
from the committed head (with tail verification), then drained and released.
|
||||
- Resumed stale owner: the owner was SIGSTOPped for 4.5 s (4 s lease). The
|
||||
successor became ready at epoch 2 with committed=10 and processed ticks 11–20.
|
||||
After SIGCONT the old owner printed `ready 0`, rejected new inputs as
|
||||
`not_ready`, acknowledged nothing beyond tick 10, and exited 0 on SIGTERM
|
||||
without touching the successor's chain.
|
||||
- Damaged heads: a flipped head payload byte and a missing head trace each made
|
||||
`evo-server` exit 4 (quarantined) without ever printing `ready 1`, and the head
|
||||
was unchanged. After repair a successor became ready at epoch 4.
|
||||
- Tail verification (in-process): a missing or corrupt head trace and a missing
|
||||
predecessor payload are `tail`, and recovery succeeds again after repair. A
|
||||
published trace of different inputs over the same range (a host bypass the
|
||||
database cannot detect) is `tail`.
|
||||
- Mutations (restored and rerun green): skipping tail verification made the
|
||||
quarantine test time out because the server became ready. Dropping replay
|
||||
byte-equality failed the forged-trace test (line 262).
|
||||
- Not executed: Docker/Gitea CI, the production 20 s lease, host loss or VM
|
||||
freeze, multi-host network partitions, or verification of links older than
|
||||
the head.
|
||||
|
||||
@@ -68,7 +68,7 @@ every role transitively in it, and rolls back on any violation:
|
||||
`pg_write_all_data`);
|
||||
- any table privilege in `evolution`, including via PUBLIC;
|
||||
- CREATE on the schema, the database or `public`;
|
||||
- EXECUTE outside the ten fenced functions (schema v5).
|
||||
- EXECUTE outside the eleven fenced functions (schema v6).
|
||||
Attribute checks alone are not enough: a login with `pg_write_all_data` can update
|
||||
lease rows directly. Rerun the migrate Job after adding or changing runtime logins
|
||||
or grants; it is the audit. Grants made later without a rerun are not detected. Bootstrap revokes PUBLIC access to the schema, tables and
|
||||
@@ -185,6 +185,9 @@ Exit codes:
|
||||
- 4: quarantined;
|
||||
- 2: usage error.
|
||||
|
||||
Use the default 20 s lease in any shared environment. `--lease-profile fast` exists
|
||||
`--crash-at <point>:<n>` (before-upload, before-transaction, after-commit or
|
||||
after-ack) is a test-only fault injector that SIGKILLs the process at that
|
||||
boundary; never set it in shared environments. Use the default 20 s lease in any
|
||||
shared environment. `--lease-profile fast` exists
|
||||
only for local tests. The runtime credentials come from `--pg-*` flags and a
|
||||
passfile; never place passwords on the command line.
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "evo/persistence/region_lease_store.hpp"
|
||||
#include <chrono>
|
||||
#include <expected>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
@@ -41,6 +42,11 @@ struct PublicationFailure {
|
||||
PreparedCommit prepared; // Present for rejected/unknown.
|
||||
};
|
||||
// Durably confirmed batch: its decisions are now final and may be acknowledged.
|
||||
// Test/fault-injection seam: called after verified uploads, immediately before the
|
||||
// fenced transaction is sent. Production callers pass nothing.
|
||||
struct PublishHooks {
|
||||
std::function<void()> before_transaction;
|
||||
};
|
||||
struct PublishedBatch {
|
||||
CommittedCheckpoint committed;
|
||||
std::vector<BatchDecision> decisions;
|
||||
@@ -54,6 +60,7 @@ enum class RecoveryError : std::uint8_t {
|
||||
corrupt_payload, // Length/digest mismatch: quarantine, no older head.
|
||||
payload, // Verified bytes fail full state validation: quarantine.
|
||||
history, // Durable decisions disagree with the restored lifecycle: quarantine.
|
||||
tail, // Head batch evidence missing, corrupt or not reproducible: quarantine.
|
||||
};
|
||||
struct RecoveryFailure {
|
||||
RecoveryError kind{};
|
||||
@@ -85,9 +92,15 @@ publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
|
||||
const RegionBatch<processes::EnvironmentalCommunity>& batch,
|
||||
Uuid commit, const Sha256Digest& build, std::size_t restore_budget,
|
||||
std::chrono::milliseconds timeout);
|
||||
// Loads and fully verifies the committed head and reconciles durable decisions.
|
||||
// Readiness additionally requires a fresh valid lease acquired before this call.
|
||||
std::chrono::milliseconds timeout, const PublishHooks& hooks = {});
|
||||
// Loads and fully verifies the committed head before any readiness (ADR-0065):
|
||||
// manifest digest and chain fields, retained build, payload length/SHA-256 and full
|
||||
// restore; then the head's tail: its trace object binds to the committed
|
||||
// predecessor, replaying it from the predecessor's verified payload reproduces the
|
||||
// head bytes, and its derived decisions equal the durable ones; finally every
|
||||
// durable decision agrees with the restored lifecycle. Any failure quarantines;
|
||||
// nothing older or empty is ever substituted. Readiness also requires a fresh
|
||||
// valid lease acquired before this call.
|
||||
[[nodiscard]] std::expected<RecoveredCheckpoint, RecoveryFailure>
|
||||
recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::span<const Sha256Digest> retained_builds,
|
||||
@@ -106,7 +119,7 @@ publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const
|
||||
const CommittedCheckpoint& predecessor,
|
||||
const RegionBatch<processes::LineageRegion>& batch, Uuid commit,
|
||||
const Sha256Digest& build, std::size_t restore_budget,
|
||||
std::chrono::milliseconds timeout);
|
||||
std::chrono::milliseconds timeout, const PublishHooks& hooks = {});
|
||||
[[nodiscard]] std::expected<RecoveredLineageCheckpoint, RecoveryFailure>
|
||||
recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::span<const Sha256Digest> retained_builds,
|
||||
|
||||
@@ -50,6 +50,11 @@ release_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease,
|
||||
load_region_head(Postgres& db, RegionKey key, std::chrono::milliseconds timeout);
|
||||
[[nodiscard]] std::expected<CommittedCheckpoint, LeaseStoreFailure>
|
||||
load_region_checkpoint(Postgres& db, RegionKey key, std::chrono::milliseconds timeout);
|
||||
// Any committed link by sequence (schema v6): its identity, digest, end tick and
|
||||
// exact manifest bytes. not_provisioned when the region has no such commit.
|
||||
[[nodiscard]] std::expected<CommittedCheckpoint, LeaseStoreFailure>
|
||||
load_region_commit(Postgres& db, RegionKey key, std::uint64_t sequence,
|
||||
std::chrono::milliseconds timeout);
|
||||
// Canonical manifest v2 bytes plus the decision block they digest (ADR-0063). The
|
||||
// database parses identity, lease and chain, locks the lease then the head, requires
|
||||
// the exact committed predecessor and records decisions and outbox rows in the same
|
||||
|
||||
@@ -75,6 +75,12 @@ struct HostStatus {
|
||||
std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{};
|
||||
};
|
||||
|
||||
// Commit boundaries for fault injection (M7.7 kill tests).
|
||||
enum class CrashPoint : std::uint8_t {
|
||||
before_upload, // Batch sealed; nothing uploaded or sent.
|
||||
before_transaction, // Payload and trace uploaded and verified; manifest not sent.
|
||||
after_commit, // Commit confirmed; no acknowledgement or outcome emitted yet.
|
||||
};
|
||||
using Connect =
|
||||
std::function<std::expected<std::unique_ptr<persistence::Postgres>, persistence::PgFailure>()>;
|
||||
struct HostConfig {
|
||||
@@ -90,6 +96,9 @@ struct HostConfig {
|
||||
persistence::Sha256Digest build{};
|
||||
std::vector<persistence::Sha256Digest> retained_builds;
|
||||
std::size_t restore_budget{8000000};
|
||||
// Test-only: called at each commit boundary (e.g. to SIGKILL the process).
|
||||
// Production configurations leave it empty.
|
||||
std::function<void(CrashPoint)> crash_hook;
|
||||
};
|
||||
enum class HostConfigError : std::uint8_t { invalid };
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
-- M7.7 recovery tail verification (ADR-0065): read any committed link of a region
|
||||
-- by sequence, so recovery can replay the head's trace from its predecessor.
|
||||
-- Read-only; returns nothing a runtime could not already derive from the chain.
|
||||
CREATE FUNCTION evolution.load_region_commit(p_world evolution.u64, p_region evolution.u64,
|
||||
p_sequence evolution.u64)
|
||||
RETURNS TABLE (region_frame evolution.u64, commit_sequence evolution.u64, commit_id uuid,
|
||||
commit_digest text, end_tick evolution.u64, manifest text)
|
||||
LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
|
||||
BEGIN
|
||||
RETURN QUERY SELECT l.frame_id, c.commit_sequence, c.commit_id, encode(c.commit_digest, 'hex'),
|
||||
c.end_tick, encode(c.manifest, 'hex')
|
||||
FROM evolution.region_commits c
|
||||
JOIN evolution.region_leases l USING (world_id, region_id)
|
||||
WHERE c.world_id = p_world AND c.region_id = p_region AND c.commit_sequence = p_sequence;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'no such commit' USING ERRCODE = 'EVL04';
|
||||
END IF;
|
||||
END $$;
|
||||
REVOKE ALL ON FUNCTION evolution.load_region_commit(evolution.u64, evolution.u64, evolution.u64)
|
||||
FROM PUBLIC;
|
||||
GRANT EXECUTE ON FUNCTION evolution.load_region_commit(evolution.u64, evolution.u64, evolution.u64)
|
||||
TO :"runtime_role";
|
||||
INSERT INTO evolution.schema_migrations(version, sha256) VALUES (6, :'migration_sha_6');
|
||||
+15
-4
@@ -24,26 +24,35 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution')
|
||||
|| ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' AS at_v4,
|
||||
:'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2'
|
||||
|| ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4'
|
||||
|| ',5:' || :'migration_sha_5' AS at_v5 \gset
|
||||
\if :at_v5
|
||||
|| ',5:' || :'migration_sha_5' AS at_v5,
|
||||
:'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2'
|
||||
|| ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4'
|
||||
|| ',5:' || :'migration_sha_5' || ',6:' || :'migration_sha_6' AS at_v6 \gset
|
||||
\if :at_v6
|
||||
\elif :at_v5
|
||||
\ir 0006_commit_lookup.sql
|
||||
\elif :at_v4
|
||||
\ir pre_v5_guard.sql
|
||||
\ir 0005_batch_decisions.sql
|
||||
\ir 0006_commit_lookup.sql
|
||||
\elif :at_v3
|
||||
\ir 0004_lineage_profile.sql
|
||||
\ir pre_v5_guard.sql
|
||||
\ir 0005_batch_decisions.sql
|
||||
\ir 0006_commit_lookup.sql
|
||||
\elif :at_v2
|
||||
\ir 0003_region_checkpoints.sql
|
||||
\ir 0004_lineage_profile.sql
|
||||
\ir pre_v5_guard.sql
|
||||
\ir 0005_batch_decisions.sql
|
||||
\ir 0006_commit_lookup.sql
|
||||
\elif :at_v1
|
||||
\ir 0002_region_fencing.sql
|
||||
\ir 0003_region_checkpoints.sql
|
||||
\ir 0004_lineage_profile.sql
|
||||
\ir pre_v5_guard.sql
|
||||
\ir 0005_batch_decisions.sql
|
||||
\ir 0006_commit_lookup.sql
|
||||
\else
|
||||
\echo 'Unknown or changed migration history'
|
||||
DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$;
|
||||
@@ -55,6 +64,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution')
|
||||
\ir 0004_lineage_profile.sql
|
||||
\ir pre_v5_guard.sql
|
||||
\ir 0005_batch_decisions.sql
|
||||
\ir 0006_commit_lookup.sql
|
||||
\endif
|
||||
-- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes
|
||||
-- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC
|
||||
@@ -84,7 +94,8 @@ BEGIN
|
||||
('evolution.resolve_region_commit(evolution.u64,evolution.u64,uuid,bytea,bytea)'::regprocedure::oid),
|
||||
('evolution.load_region_decisions(evolution.u64,evolution.u64)'::regprocedure::oid),
|
||||
('evolution.pending_region_outbox(evolution.u64,evolution.u64,integer)'::regprocedure::oid),
|
||||
('evolution.acknowledge_region_outbox(evolution.u64,evolution.u64,integer,integer,evolution.u64)'::regprocedure::oid)
|
||||
('evolution.acknowledge_region_outbox(evolution.u64,evolution.u64,integer,integer,evolution.u64)'::regprocedure::oid),
|
||||
('evolution.load_region_commit(evolution.u64,evolution.u64,evolution.u64)'::regprocedure::oid)
|
||||
), violations(role_oid, reason) AS (
|
||||
SELECT r.oid, 'privileged attribute or owner identity' FROM pg_catalog.pg_roles r
|
||||
JOIN closure c ON c.oid = r.oid
|
||||
@@ -108,7 +119,7 @@ BEGIN
|
||||
OR EXISTS (SELECT 1 FROM pg_catalog.pg_namespace n WHERE n.nspname = 'public'
|
||||
AND pg_catalog.has_schema_privilege(c.oid, n.oid, 'CREATE'))
|
||||
UNION ALL
|
||||
-- Only the ten guarded functions are executable; provisioning/internal are not.
|
||||
-- Only the eleven guarded functions are executable; provisioning/internal are not.
|
||||
SELECT c.oid, 'EXECUTE outside the fenced function set' FROM closure c
|
||||
CROSS JOIN pg_catalog.pg_proc p
|
||||
WHERE p.pronamespace = 'evolution'::regnamespace
|
||||
|
||||
@@ -175,7 +175,8 @@ std::expected<PublishedBatch, PublicationFailure>
|
||||
publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
||||
const CommittedCheckpoint& predecessor,
|
||||
const RegionBatch<typename Codec::State>& batch, Uuid commit,
|
||||
const Sha256Digest& build, std::size_t budget, milliseconds timeout) {
|
||||
const Sha256Digest& build, std::size_t budget, milliseconds timeout,
|
||||
const PublishHooks& hooks) {
|
||||
const auto previous = decode_manifest(predecessor.manifest);
|
||||
if (!previous || previous->profile != Codec::profile || !batch.sealed() ||
|
||||
batch.steps().empty() || batch.begin_tick() != previous->end_tick)
|
||||
@@ -231,6 +232,8 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra
|
||||
return std::unexpected{stored.error()};
|
||||
|
||||
PreparedCommit prepared{previous->key, commit, *bytes, *decision_bytes};
|
||||
if (hooks.before_transaction)
|
||||
hooks.before_transaction();
|
||||
auto published = publish_region_checkpoint(db, *bytes, *decision_bytes, timeout);
|
||||
auto committed = confirm(published, std::move(*bytes), *manifest, std::move(prepared));
|
||||
if (!committed)
|
||||
@@ -238,6 +241,72 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra
|
||||
return PublishedBatch{std::move(*committed), decisions};
|
||||
}
|
||||
|
||||
// Replays the head's recorded batch from its committed predecessor (ADR-0065).
|
||||
template <typename Codec>
|
||||
std::expected<void, RecoveryFailure>
|
||||
verify_tail(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const CheckpointManifest& m, std::span<const std::uint8_t> head_payload,
|
||||
const typename Codec::State& head_state, std::span<const CommittedDecision> durable,
|
||||
std::size_t budget, milliseconds timeout) {
|
||||
const auto fail_with = [](RecoveryError kind) {
|
||||
return std::unexpected{RecoveryFailure{kind, {}}};
|
||||
};
|
||||
if (m.sequence == 0)
|
||||
return {}; // Genesis has no batch.
|
||||
if (m.manifest_version != 2)
|
||||
return fail_with(RecoveryError::unsupported);
|
||||
auto trace_bytes = objects.get({m.batch.trace_sha256, m.batch.trace_bytes});
|
||||
if (!trace_bytes)
|
||||
return fail_with(trace_bytes.error() == ObjectError::io ? RecoveryError::store
|
||||
: RecoveryError::tail);
|
||||
auto trace = decode_trace(*trace_bytes);
|
||||
const TraceHeader expected{static_cast<std::uint32_t>(Codec::profile),
|
||||
key,
|
||||
m.predecessor_commit,
|
||||
m.predecessor_digest,
|
||||
m.begin_tick,
|
||||
m.end_tick};
|
||||
if (!trace || trace->header != expected)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto predecessor = load_region_commit(db, key, m.sequence - 1, timeout);
|
||||
if (!predecessor)
|
||||
return predecessor.error().kind == LeaseStoreError::not_provisioned
|
||||
? fail_with(RecoveryError::tail)
|
||||
: std::unexpected{RecoveryFailure{RecoveryError::store, predecessor.error()}};
|
||||
const auto digest = sha256(predecessor->manifest);
|
||||
auto previous = decode_manifest(predecessor->manifest);
|
||||
if (!digest || *digest != predecessor->head.digest ||
|
||||
predecessor->head.digest != m.predecessor_digest ||
|
||||
predecessor->head.commit != m.predecessor_commit || !previous ||
|
||||
previous->end_tick != m.begin_tick)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto base = objects.get({previous->payload_sha256, previous->payload_bytes});
|
||||
if (!base)
|
||||
return fail_with(base.error() == ObjectError::io ? RecoveryError::store
|
||||
: RecoveryError::tail);
|
||||
auto replay = Codec::restore(*base, *previous, budget);
|
||||
if (!replay)
|
||||
return fail_with(RecoveryError::tail);
|
||||
for (const auto& step : trace->steps) {
|
||||
auto report = (*replay)->advance(step.report.tick, step.input);
|
||||
if (!report || *report != step.report)
|
||||
return fail_with(RecoveryError::tail);
|
||||
}
|
||||
const auto replayed = Codec::snapshot(**replay);
|
||||
if (!replayed || !std::ranges::equal(*replayed, head_payload))
|
||||
return fail_with(RecoveryError::tail);
|
||||
// The head's durable outcomes are exactly those its trace derives.
|
||||
const auto organisms = Codec::organisms(head_state);
|
||||
const auto derived = derive_decisions(trace->steps, organisms.producer, organisms.consumer);
|
||||
std::vector<BatchDecision> recorded;
|
||||
for (const auto& d : durable)
|
||||
if (d.commit_sequence == m.sequence)
|
||||
recorded.push_back(d.decision);
|
||||
if (derived != recorded || derived.size() != m.batch.decision_count)
|
||||
return fail_with(RecoveryError::history);
|
||||
return {};
|
||||
}
|
||||
|
||||
template <typename Codec>
|
||||
std::expected<typename Codec::Recovered, RecoveryFailure>
|
||||
recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
@@ -284,6 +353,10 @@ recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
return store_failure(decisions.error());
|
||||
if (!Codec::reconcile(**state, *decisions, head))
|
||||
return fail_with(RecoveryError::history);
|
||||
if (auto tail = verify_tail<Codec>(db, objects, key, *manifest, *payload, **state, *decisions,
|
||||
budget, timeout);
|
||||
!tail)
|
||||
return std::unexpected{tail.error()};
|
||||
return typename Codec::Recovered{std::move(*committed), *manifest, std::move(*state),
|
||||
std::move(*decisions)};
|
||||
}
|
||||
@@ -301,9 +374,9 @@ publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
|
||||
const RegionBatch<processes::EnvironmentalCommunity>& batch,
|
||||
Uuid commit, const Sha256Digest& build, std::size_t restore_budget,
|
||||
milliseconds timeout) {
|
||||
milliseconds timeout, const PublishHooks& hooks) {
|
||||
return publish_batch<EnvironmentalCodec>(db, objects, lease, predecessor, batch, commit, build,
|
||||
restore_budget, timeout);
|
||||
restore_budget, timeout, hooks);
|
||||
}
|
||||
std::expected<RecoveredCheckpoint, RecoveryFailure>
|
||||
recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
@@ -324,9 +397,10 @@ std::expected<PublishedBatch, PublicationFailure>
|
||||
publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
||||
const CommittedCheckpoint& predecessor,
|
||||
const RegionBatch<processes::LineageRegion>& batch, Uuid commit,
|
||||
const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout) {
|
||||
const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout,
|
||||
const PublishHooks& hooks) {
|
||||
return publish_batch<LineageCodec>(db, objects, lease, predecessor, batch, commit, build,
|
||||
restore_budget, timeout);
|
||||
restore_budget, timeout, hooks);
|
||||
}
|
||||
std::expected<RecoveredLineageCheckpoint, RecoveryFailure>
|
||||
recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
|
||||
@@ -216,6 +216,25 @@ load_region_checkpoint(Postgres& db, RegionKey key, milliseconds timeout) {
|
||||
return CommittedCheckpoint{*head, std::move(*manifest)};
|
||||
}
|
||||
|
||||
std::expected<CommittedCheckpoint, LeaseStoreFailure>
|
||||
load_region_commit(Postgres& db, RegionKey key, std::uint64_t sequence, milliseconds timeout) {
|
||||
const std::array<std::string, 3> p{std::to_string(key.world), std::to_string(key.region),
|
||||
std::to_string(sequence)};
|
||||
auto rows = db.query("SELECT * FROM evolution.load_region_commit($1::evolution.u64, "
|
||||
"$2::evolution.u64, $3::evolution.u64)",
|
||||
p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
auto head = parse_head(*rows, 6);
|
||||
if (!head || rows->size() != 1)
|
||||
return std::unexpected{malformed(false)};
|
||||
const auto& manifest_text = (*rows)[0][5];
|
||||
auto manifest = manifest_text ? unhex_bytes(*manifest_text) : std::nullopt;
|
||||
if (!manifest)
|
||||
return std::unexpected{malformed(false)};
|
||||
return CommittedCheckpoint{*head, std::move(*manifest)};
|
||||
}
|
||||
|
||||
std::expected<RegionHead, LeaseStoreFailure>
|
||||
publish_region_checkpoint(Postgres& db, std::span<const std::uint8_t> manifest,
|
||||
std::span<const std::uint8_t> decisions, milliseconds timeout) {
|
||||
|
||||
@@ -228,6 +228,8 @@ template <typename State> persistence::Uuid RegionHost<State>::next_commit_id()
|
||||
|
||||
template <typename State>
|
||||
void RegionHost<State>::confirmed(const persistence::PublishedBatch& published, SchedulerTime now) {
|
||||
if (config_.crash_hook)
|
||||
config_.crash_hook(CrashPoint::after_commit);
|
||||
const auto& head = published.committed.head;
|
||||
if (!batch_ || !batch_->confirm(head.end_tick)) {
|
||||
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
||||
@@ -255,9 +257,15 @@ template <typename State> void RegionHost<State>::commit(SchedulerTime now) {
|
||||
return;
|
||||
const auto commit_id = next_commit_id();
|
||||
batch_->seal();
|
||||
const auto& crash = config_.crash_hook;
|
||||
if (crash)
|
||||
crash(CrashPoint::before_upload);
|
||||
persistence::PublishHooks hooks;
|
||||
if (crash)
|
||||
hooks.before_transaction = [&crash] { crash(CrashPoint::before_transaction); };
|
||||
auto published =
|
||||
Ops<State>::publish(*db_, *objects_, *grant, *committed_, *batch_, commit_id, config_.build,
|
||||
config_.restore_budget, config_.lease.commit_timeout);
|
||||
config_.restore_budget, config_.lease.commit_timeout, hooks);
|
||||
if (published) {
|
||||
confirmed(*published, now);
|
||||
return;
|
||||
|
||||
Vendored
+297
@@ -0,0 +1,297 @@
|
||||
#pragma once
|
||||
// Shared real-process helpers for evo-server tests (M7.6/M7.7). Linux only.
|
||||
#include "environmental_region.hpp"
|
||||
#include "evo/persistence/checkpoint_store.hpp"
|
||||
#include <array>
|
||||
#include <cerrno>
|
||||
#include <chrono>
|
||||
#include <csignal>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <poll.h>
|
||||
#include <spawn.h>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
extern char** environ;
|
||||
|
||||
namespace evo::test::process {
|
||||
using namespace std::chrono_literals;
|
||||
namespace fs = std::filesystem;
|
||||
using Clock = std::chrono::steady_clock;
|
||||
|
||||
inline void expect(bool ok, const char* what) {
|
||||
if (!ok)
|
||||
throw std::runtime_error(std::string("server fixture: ") + what);
|
||||
}
|
||||
inline std::string env(const char* key) {
|
||||
const auto* value = std::getenv(key);
|
||||
expect(value != nullptr, "missing test environment");
|
||||
return value;
|
||||
}
|
||||
inline std::unique_ptr<persistence::Postgres> owner() {
|
||||
auto made = persistence::Postgres::connect(
|
||||
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s);
|
||||
expect(made.has_value(), "owner connection");
|
||||
return std::move(*made);
|
||||
}
|
||||
// Request identity: a tag byte (one per writer) and the tick.
|
||||
inline std::string request_hex(std::uint64_t tick, std::uint8_t tag = 0x9B) {
|
||||
char out[33];
|
||||
std::snprintf(out, sizeof out, "%02x000000000000000000000000%06llx", tag,
|
||||
static_cast<unsigned long long>(tick));
|
||||
return out;
|
||||
}
|
||||
inline std::string input_line(std::uint64_t tick, bool consumer_dies, std::uint8_t tag = 0x9B) {
|
||||
const auto in = region_input(tick, consumer_dies);
|
||||
std::string line = "input " + request_hex(tick, tag) + " " + std::to_string(in.temperature_mK) +
|
||||
" " + std::to_string(in.source_enabled);
|
||||
for (const auto b : in.bite)
|
||||
line += " " + std::to_string(b);
|
||||
for (const auto t : in.terminal)
|
||||
line += " " + std::to_string(t);
|
||||
return line + "\n";
|
||||
}
|
||||
inline std::string input_lines(std::uint64_t first, std::uint64_t last, bool deaths,
|
||||
std::uint8_t tag = 0x9B) {
|
||||
std::string out;
|
||||
for (auto tick = first; tick <= last; ++tick)
|
||||
out += input_line(tick, deaths, tag);
|
||||
return out;
|
||||
}
|
||||
inline bool starts(const std::string& line, std::string_view prefix) {
|
||||
return line.starts_with(prefix);
|
||||
}
|
||||
|
||||
class Child {
|
||||
public:
|
||||
explicit Child(const std::vector<std::string>& args) {
|
||||
int to_child[2], from_child[2];
|
||||
expect(::pipe(to_child) == 0 && ::pipe(from_child) == 0, "pipes");
|
||||
posix_spawn_file_actions_t actions;
|
||||
posix_spawn_file_actions_init(&actions);
|
||||
posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO);
|
||||
posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO);
|
||||
posix_spawn_file_actions_addclose(&actions, to_child[1]);
|
||||
posix_spawn_file_actions_addclose(&actions, from_child[0]);
|
||||
std::vector<char*> argv;
|
||||
argv.reserve(args.size() + 1);
|
||||
for (const auto& a : args)
|
||||
argv.push_back(const_cast<char*>(a.c_str()));
|
||||
argv.push_back(nullptr);
|
||||
expect(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0, "spawn");
|
||||
posix_spawn_file_actions_destroy(&actions);
|
||||
::close(to_child[0]);
|
||||
::close(from_child[1]);
|
||||
in_ = to_child[1];
|
||||
out_ = from_child[0];
|
||||
}
|
||||
Child(const Child&) = delete;
|
||||
Child& operator=(const Child&) = delete;
|
||||
~Child() {
|
||||
if (pid_ > 0) {
|
||||
::kill(pid_, SIGKILL);
|
||||
::waitpid(pid_, nullptr, 0);
|
||||
}
|
||||
::close(in_);
|
||||
::close(out_);
|
||||
}
|
||||
void send(const std::string& text) {
|
||||
std::size_t done = 0;
|
||||
while (done < text.size()) {
|
||||
const auto n = ::write(in_, text.data() + done, text.size() - done);
|
||||
expect(n > 0 || errno == EINTR, "write to child");
|
||||
if (n > 0)
|
||||
done += static_cast<std::size_t>(n);
|
||||
}
|
||||
}
|
||||
// Reads lines until one satisfies `want`; every line is kept in lines().
|
||||
template <typename Want> bool wait_for(Want want, std::chrono::milliseconds timeout) {
|
||||
const auto deadline = Clock::now() + timeout;
|
||||
for (std::size_t seen = 0;;) {
|
||||
for (; seen < lines_.size(); ++seen)
|
||||
if (want(lines_[seen]))
|
||||
return true;
|
||||
const auto left =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
|
||||
if (left.count() <= 0 || !read_some(static_cast<int>(left.count())))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
bool wait_line(std::string_view prefix, std::chrono::milliseconds timeout) {
|
||||
return wait_for([prefix](const std::string& l) { return starts(l, prefix); }, timeout);
|
||||
}
|
||||
void signal(int sig) {
|
||||
expect(::kill(pid_, sig) == 0, "signal");
|
||||
}
|
||||
// Exit code, or 128+signal; drains remaining output.
|
||||
int wait_exit(std::chrono::milliseconds timeout) {
|
||||
const auto deadline = Clock::now() + timeout;
|
||||
int status = 0;
|
||||
for (;;) {
|
||||
read_some(20);
|
||||
const auto r = ::waitpid(pid_, &status, WNOHANG);
|
||||
if (r == pid_)
|
||||
break;
|
||||
expect(Clock::now() < deadline, "child exit deadline");
|
||||
}
|
||||
while (read_some(20)) {
|
||||
}
|
||||
pid_ = -1;
|
||||
return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
|
||||
}
|
||||
[[nodiscard]] const std::vector<std::string>& lines() const noexcept {
|
||||
return lines_;
|
||||
}
|
||||
[[nodiscard]] bool printed(std::string_view prefix) const {
|
||||
for (const auto& l : lines_)
|
||||
if (starts(l, prefix))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
private:
|
||||
bool read_some(int timeout_ms) {
|
||||
pollfd fd{out_, POLLIN, 0};
|
||||
if (::poll(&fd, 1, timeout_ms) <= 0)
|
||||
return false;
|
||||
char chunk[4096];
|
||||
const auto n = ::read(out_, chunk, sizeof chunk);
|
||||
if (n <= 0)
|
||||
return false;
|
||||
buffer_.append(chunk, static_cast<std::size_t>(n));
|
||||
for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) {
|
||||
lines_.push_back(buffer_.substr(0, end));
|
||||
buffer_.erase(0, end + 1);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
pid_t pid_{-1};
|
||||
int in_{-1}, out_{-1};
|
||||
std::string buffer_;
|
||||
std::vector<std::string> lines_;
|
||||
};
|
||||
|
||||
struct ServerRegion {
|
||||
persistence::RegionKey key;
|
||||
fs::path root;
|
||||
// The region's object namespace directory.
|
||||
[[nodiscard]] fs::path objects() const {
|
||||
return root / ("w" + std::to_string(key.world)) / ("r" + std::to_string(key.region));
|
||||
}
|
||||
};
|
||||
// A fresh lineage region (world 7) provisioned by the owner.
|
||||
inline ServerRegion provision(std::uint64_t region, std::uint8_t genesis) {
|
||||
ServerRegion r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))};
|
||||
fs::remove_all(r.root);
|
||||
fs::create_directory(r.root);
|
||||
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
|
||||
auto db = owner();
|
||||
expect(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
||||
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
||||
{}, 5s)
|
||||
.has_value(),
|
||||
"retain build");
|
||||
auto objects = persistence::FilesystemObjectStore::open(r.root, r.key,
|
||||
persistence::max_lineage_payload_bytes);
|
||||
expect(objects.has_value(), "object store");
|
||||
persistence::Sha256Digest build{};
|
||||
build.fill(0xB1);
|
||||
persistence::Uuid commit{};
|
||||
commit.fill(genesis);
|
||||
expect(persistence::provision_lineage_region(*db, **objects, r.key, region_identity.frame,
|
||||
*lineage_region(), commit, build, 8000000, 5s)
|
||||
.has_value(),
|
||||
"provision");
|
||||
return r;
|
||||
}
|
||||
inline std::vector<std::string> server_args(const ServerRegion& r,
|
||||
std::vector<std::string> extra = {}) {
|
||||
std::vector<std::string> args{
|
||||
env("EVO_SERVER_BIN"),
|
||||
"--profile",
|
||||
"lineage",
|
||||
"--world",
|
||||
std::to_string(r.key.world),
|
||||
"--region",
|
||||
std::to_string(r.key.region),
|
||||
"--frame",
|
||||
std::to_string(region_identity.frame),
|
||||
"--pg-host",
|
||||
env("PGHOST"),
|
||||
"--pg-port",
|
||||
env("PGPORT"),
|
||||
"--pg-database",
|
||||
env("PGDATABASE"),
|
||||
"--pg-user",
|
||||
env("EVO_PG_RUNTIME_USER"),
|
||||
"--objects",
|
||||
r.root.string(),
|
||||
"--build",
|
||||
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
|
||||
"--flush-ms",
|
||||
"500",
|
||||
"--lease-profile",
|
||||
"fast",
|
||||
"--ready-file",
|
||||
(r.root / "ready").string()};
|
||||
args.insert(args.end(), extra.begin(), extra.end());
|
||||
return args;
|
||||
}
|
||||
// (tick, commit_sequence) of every ack line.
|
||||
inline std::vector<std::pair<std::uint64_t, std::uint64_t>> acks(const Child& c) {
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> out;
|
||||
for (const auto& line : c.lines()) {
|
||||
unsigned long long tick = 0, sequence = 0;
|
||||
char request[40];
|
||||
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
|
||||
out.emplace_back(tick, sequence);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
inline std::uint64_t max_acked(const Child& c) {
|
||||
std::uint64_t most = 0;
|
||||
for (const auto& [tick, sequence] : acks(c))
|
||||
most = std::max(most, tick);
|
||||
return most;
|
||||
}
|
||||
struct Head {
|
||||
std::uint64_t tick{}, sequence{};
|
||||
bool released{};
|
||||
};
|
||||
inline Head head_of(persistence::RegionKey key) {
|
||||
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
|
||||
auto rows = owner()->query(
|
||||
"SELECT h.end_tick::text, h.commit_sequence::text, (l.holder IS NULL)::text FROM "
|
||||
"evolution.region_heads h JOIN evolution.region_leases l USING (world_id, region_id) "
|
||||
"WHERE world_id = $1::evolution.u64 AND region_id = $2::evolution.u64",
|
||||
p, 5s);
|
||||
expect(rows && rows->size() == 1 && (*rows)[0].size() == 3, "head row");
|
||||
const auto& row = (*rows)[0];
|
||||
expect(row[0] && row[1] && row[2], "head values");
|
||||
return {std::stoull(row[0].value_or("0")), std::stoull(row[1].value_or("0")),
|
||||
row[2].value_or("") == "true"};
|
||||
}
|
||||
inline std::size_t decisions_of(persistence::RegionKey key) {
|
||||
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
|
||||
auto rows = owner()->query("SELECT count(*)::text FROM evolution.region_decisions WHERE "
|
||||
"world_id = $1::evolution.u64 AND region_id = $2::evolution.u64",
|
||||
p, 5s);
|
||||
expect(rows && rows->size() == 1, "decision count");
|
||||
return std::stoull((*rows)[0][0].value_or("0"));
|
||||
}
|
||||
// Object files in the region namespace (temporaries excluded).
|
||||
inline std::size_t object_count(const ServerRegion& r) {
|
||||
std::size_t n = 0;
|
||||
for (const auto& entry : fs::directory_iterator(r.objects()))
|
||||
n += entry.path().filename().string().starts_with(".tmp-") ? 0 : 1;
|
||||
return n;
|
||||
}
|
||||
} // namespace evo::test::process
|
||||
@@ -7,6 +7,7 @@ void run_region_lease_store_tests();
|
||||
void run_region_checkpoint_store_tests();
|
||||
void run_region_host_tests();
|
||||
void run_region_server_process_tests();
|
||||
void run_region_recovery_tests();
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
@@ -81,6 +82,8 @@ int main() {
|
||||
std::puts("region host integration passed");
|
||||
run_region_server_process_tests();
|
||||
std::puts("region server process integration passed");
|
||||
run_region_recovery_tests();
|
||||
std::puts("region recovery integration passed");
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
std::fprintf(stderr, "%s\n", e.what());
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
// M7.7 recovery before readiness (ADR-0065): real evo-server processes killed at
|
||||
// every publication/acknowledgement boundary, a resumed stale owner, damaged heads
|
||||
// that must never serve, and tail verification. Run by tools/ci/postgres-check.sh.
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include "fixtures/server_process.hpp"
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
#include <source_location>
|
||||
#include <thread>
|
||||
|
||||
void run_region_recovery_tests();
|
||||
|
||||
namespace {
|
||||
using namespace evo::test::process;
|
||||
using namespace evo::persistence;
|
||||
void check(bool value, std::source_location at = std::source_location::current()) {
|
||||
if (!value)
|
||||
throw std::runtime_error("region recovery test line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
std::string hex(std::span<const std::uint8_t> bytes) {
|
||||
constexpr char digits[] = "0123456789abcdef";
|
||||
std::string out;
|
||||
for (const auto b : bytes) {
|
||||
out.push_back(digits[b >> 4U]);
|
||||
out.push_back(digits[b & 0x0FU]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::vector<char> read_file(const fs::path& p) {
|
||||
std::ifstream in(p, std::ios::binary);
|
||||
return {std::istreambuf_iterator<char>(in), {}};
|
||||
}
|
||||
void write_file(const fs::path& p, const std::vector<char>& bytes) {
|
||||
fs::remove(p);
|
||||
std::ofstream(p, std::ios::binary)
|
||||
.write(bytes.data(), static_cast<std::streamsize>(bytes.size()));
|
||||
fs::permissions(p, fs::perms::owner_read | fs::perms::owner_write, fs::perm_options::replace);
|
||||
}
|
||||
CheckpointManifest head_manifest(RegionKey key) {
|
||||
auto db = owner();
|
||||
auto committed = load_region_checkpoint(*db, key, 5s);
|
||||
CHECK(committed);
|
||||
auto m = decode_manifest(committed->manifest);
|
||||
CHECK(m);
|
||||
return *m;
|
||||
}
|
||||
// Successor: waits for readiness, requires it resumed from the committed head, drains.
|
||||
void successor_resumes(const ServerRegion& r, std::uint64_t epoch) {
|
||||
const auto head = head_of(r.key);
|
||||
Child next{server_args(r)};
|
||||
CHECK(next.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
||||
CHECK(next.printed("phase running epoch=" + std::to_string(epoch) +
|
||||
" committed=" + std::to_string(head.tick) + " "));
|
||||
next.signal(SIGTERM);
|
||||
CHECK(next.wait_exit(20s) == 0);
|
||||
const auto after = head_of(r.key);
|
||||
CHECK(after.tick == head.tick && after.sequence == head.sequence && after.released);
|
||||
}
|
||||
|
||||
struct Boundary {
|
||||
const char* point;
|
||||
int ordinal;
|
||||
std::uint64_t region;
|
||||
std::uint8_t genesis;
|
||||
};
|
||||
// Inputs 1..40 with the consumer's death at tick 20: commit 1 is the death batch
|
||||
// (ticks 1..20), commit 2 the next interval flush. The process is SIGKILLed at the
|
||||
// chosen boundary; durable state, acknowledgements and orphans must match it.
|
||||
void kill_at(const Boundary& b) {
|
||||
const auto r = provision(b.region, b.genesis);
|
||||
std::uint64_t acked = 0;
|
||||
bool outcome = false;
|
||||
{
|
||||
// A long flush interval makes commit 1 exactly the death batch.
|
||||
Child server{server_args(r, {"--flush-ms", "5000", "--crash-at",
|
||||
std::string(b.point) + ":" + std::to_string(b.ordinal)})};
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
||||
server.send(input_lines(1, 40, true));
|
||||
CHECK(server.wait_exit(30s) == 128 + SIGKILL);
|
||||
acked = max_acked(server);
|
||||
outcome = server.printed("outcome 20 1 11 1");
|
||||
}
|
||||
const auto head = head_of(r.key);
|
||||
const auto decisions = decisions_of(r.key);
|
||||
const auto objects = object_count(r); // Genesis payload plus 2 per uploaded batch.
|
||||
const std::string point = b.point;
|
||||
CHECK(!head.released); // Killed holders never release; successors wait for expiry.
|
||||
CHECK(acked <= head.tick); // Never an acknowledgement beyond the committed head.
|
||||
if (b.ordinal == 1) {
|
||||
// The death batch never committed: nothing final, the death is not a fact.
|
||||
CHECK(point == "before-transaction");
|
||||
CHECK(head.sequence == 0 && head.tick == 0 && acked == 0 && decisions == 0 && !outcome);
|
||||
CHECK(objects == 3); // Verified orphans only.
|
||||
} else if (point == "before-upload" || point == "before-transaction") {
|
||||
CHECK(head.sequence == 1 && head.tick == 20 && acked == 20 && decisions == 1 && outcome);
|
||||
CHECK(objects == (point == "before-upload" ? 3U : 5U));
|
||||
} else if (point == "after-commit") {
|
||||
// Durable but never acknowledged: the lost acks are the safe direction.
|
||||
CHECK(head.sequence == 2 && head.tick > 20 && acked == 20 && decisions == 1 && outcome);
|
||||
} else {
|
||||
CHECK(point == "after-ack");
|
||||
CHECK(head.sequence == 2 && acked == head.tick && decisions == 1 && outcome);
|
||||
}
|
||||
std::printf("kill_at %s:%d head=%llu/%llu acked=%llu objects=%zu\n", b.point, b.ordinal,
|
||||
static_cast<unsigned long long>(head.sequence),
|
||||
static_cast<unsigned long long>(head.tick), static_cast<unsigned long long>(acked),
|
||||
objects);
|
||||
successor_resumes(r, 2); // Verified head, tail and history before readiness.
|
||||
}
|
||||
|
||||
// A paused (SIGSTOP) owner loses its lease; after SIGCONT it fences itself, refuses
|
||||
// admission and never acknowledges past its own commits, while the successor's
|
||||
// chain continues untouched.
|
||||
void resumed_stale_owner_is_fenced() {
|
||||
const auto r = provision(46, 0x62);
|
||||
Child stale{server_args(r)};
|
||||
CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
||||
stale.send(input_lines(1, 10, false, 0xA1));
|
||||
CHECK(stale.wait_line("ack " + request_hex(10, 0xA1) + " 10 ", 15s));
|
||||
stale.signal(SIGSTOP);
|
||||
std::this_thread::sleep_for(4500ms); // Longer than the 4 s development lease.
|
||||
Child owner_now{server_args(r)};
|
||||
CHECK(owner_now.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
||||
CHECK(owner_now.printed("phase running epoch=2 committed=10 "));
|
||||
owner_now.send(input_lines(11, 20, false, 0xB1));
|
||||
CHECK(owner_now.wait_line("ack " + request_hex(20, 0xB1) + " 20 ", 15s));
|
||||
stale.signal(SIGCONT);
|
||||
CHECK(stale.wait_line("ready 0", 10s));
|
||||
stale.send(input_lines(11, 12, false, 0xA2));
|
||||
CHECK(stale.wait_line("rejected " + request_hex(11, 0xA2) + " not_ready", 10s));
|
||||
CHECK(max_acked(stale) == 10);
|
||||
const auto head = head_of(r.key);
|
||||
CHECK(head.tick >= 20 && head.sequence >= 2 && !head.released);
|
||||
stale.signal(SIGTERM);
|
||||
CHECK(stale.wait_exit(20s) == 0); // It never regained authority: nothing to release.
|
||||
owner_now.signal(SIGTERM);
|
||||
CHECK(owner_now.wait_exit(20s) == 0);
|
||||
CHECK(head_of(r.key).released && head_of(r.key).tick == head.tick);
|
||||
}
|
||||
|
||||
// A committed head whose payload or tail cannot be verified is never served, and
|
||||
// no older or empty world replaces it.
|
||||
void damaged_heads_never_become_ready() {
|
||||
const auto r = provision(47, 0x63);
|
||||
{
|
||||
Child server{server_args(r)};
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
||||
server.send(input_lines(1, 10, false));
|
||||
CHECK(server.wait_line("ack " + request_hex(10) + " 10 ", 15s));
|
||||
server.signal(SIGTERM);
|
||||
CHECK(server.wait_exit(20s) == 0);
|
||||
}
|
||||
const auto head = head_of(r.key);
|
||||
const auto m = head_manifest(r.key);
|
||||
const auto payload = r.objects() / hex(m.payload_sha256);
|
||||
const auto trace = r.objects() / hex(m.batch.trace_sha256);
|
||||
const auto quarantined = [&] {
|
||||
Child server{server_args(r)};
|
||||
const auto code = server.wait_exit(20s);
|
||||
return code == 4 && server.printed("phase quarantined") && !server.printed("ready 1");
|
||||
};
|
||||
const auto payload_bytes = read_file(payload);
|
||||
auto flipped = payload_bytes;
|
||||
flipped[flipped.size() / 2] = static_cast<char>(flipped[flipped.size() / 2] ^ 1);
|
||||
write_file(payload, flipped);
|
||||
CHECK(quarantined());
|
||||
write_file(payload, payload_bytes);
|
||||
const auto trace_bytes = read_file(trace);
|
||||
fs::remove(trace);
|
||||
CHECK(quarantined());
|
||||
write_file(trace, trace_bytes);
|
||||
const auto after = head_of(r.key);
|
||||
CHECK(after.tick == head.tick && after.sequence == head.sequence); // Nothing replaced it.
|
||||
successor_resumes(r, 4); // Epochs 2 and 3 were taken by the quarantined attempts.
|
||||
}
|
||||
|
||||
// In-process: recovery replays the head's trace from its predecessor before any
|
||||
// readiness; missing, damaged or non-reproducing evidence quarantines (tail).
|
||||
void tail_verification_before_readiness() {
|
||||
const auto r = provision(48, 0x01);
|
||||
auto runtime = Postgres::connect(
|
||||
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}},
|
||||
5s);
|
||||
CHECK(runtime);
|
||||
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
||||
CHECK(objects);
|
||||
Sha256Digest build{};
|
||||
build.fill(0xB1);
|
||||
const std::array<Sha256Digest, 1> retained{build};
|
||||
const auto commit = [](std::uint8_t v) {
|
||||
Uuid id{};
|
||||
id.fill(v);
|
||||
return id;
|
||||
};
|
||||
auto lease = acquire_region_lease(**runtime, r.key, 20000ms, 5s);
|
||||
CHECK(lease);
|
||||
auto head = load_region_checkpoint(**runtime, r.key, 5s);
|
||||
CHECK(head);
|
||||
auto state = evo::test::lineage_region();
|
||||
evo::persistence::RegionBatch<evo::processes::LineageRegion> batch{*state};
|
||||
for (int n = 0; n < 2; ++n) {
|
||||
for (int i = 0; i < 10; ++i)
|
||||
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
|
||||
batch.seal();
|
||||
auto published =
|
||||
publish_lineage_batch(**runtime, **objects, *lease, *head, batch,
|
||||
commit(static_cast<std::uint8_t>(2 + n)), build, 8000000, 5s);
|
||||
CHECK(published && batch.confirm(published->committed.head.end_tick));
|
||||
head = published->committed;
|
||||
}
|
||||
const auto recover = [&] {
|
||||
return recover_lineage_checkpoint(**runtime, **objects, r.key, retained, 8000000, 5s);
|
||||
};
|
||||
CHECK(recover());
|
||||
const auto m = *decode_manifest(head->manifest);
|
||||
const auto trace = r.objects() / hex(m.batch.trace_sha256);
|
||||
const auto trace_bytes = read_file(trace);
|
||||
const auto refused = [&](RecoveryError kind) {
|
||||
auto rec = recover();
|
||||
return !rec && rec.error().kind == kind;
|
||||
};
|
||||
fs::remove(trace);
|
||||
CHECK(refused(RecoveryError::tail));
|
||||
auto damaged = trace_bytes;
|
||||
damaged[200] = static_cast<char>(damaged[200] ^ 1);
|
||||
write_file(trace, damaged);
|
||||
CHECK(refused(RecoveryError::tail));
|
||||
write_file(trace, trace_bytes);
|
||||
// The predecessor's payload is needed to replay the head: its loss quarantines.
|
||||
auto first = load_region_commit(**runtime, r.key, 1, 5s);
|
||||
CHECK(first);
|
||||
const auto previous = *decode_manifest(first->manifest);
|
||||
const auto base = r.objects() / hex(previous.payload_sha256);
|
||||
const auto base_bytes = read_file(base);
|
||||
fs::remove(base);
|
||||
CHECK(refused(RecoveryError::tail));
|
||||
write_file(base, base_bytes);
|
||||
CHECK(recover());
|
||||
|
||||
// Evidence that does not reproduce the payload: a trace of different inputs over
|
||||
// the same range, published by bypassing the host (the database cannot tell).
|
||||
for (int i = 0; i < 10; ++i)
|
||||
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
|
||||
auto alternate = recover();
|
||||
CHECK(alternate);
|
||||
evo::persistence::RegionBatch<evo::processes::LineageRegion> other{*alternate->state};
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
auto in = evo::test::region_input(other.end_tick() + 1);
|
||||
in.source_enabled = 1 - in.source_enabled; // Not what actually happened.
|
||||
CHECK(other.advance(in));
|
||||
}
|
||||
auto payload = state->snapshot();
|
||||
auto forged = encode_trace({2, r.key, m.commit, head->head.digest, m.end_tick, m.end_tick + 10},
|
||||
other.steps());
|
||||
CHECK(payload && forged && (*objects)->put(*payload) && (*objects)->put(*forged));
|
||||
auto manifest = successor_manifest(m, head->head.digest, commit(5), *lease, m.end_tick + 10,
|
||||
build, payload->size(), *sha256(*payload),
|
||||
{forged->size(), *sha256(*forged), 0, {}});
|
||||
CHECK(manifest);
|
||||
CHECK(publish_region_checkpoint(**runtime, *encode_manifest(*manifest), {}, 5s));
|
||||
CHECK(refused(RecoveryError::tail));
|
||||
CHECK(release_region_lease(**runtime, r.key, *lease, 5s));
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void run_region_recovery_tests() {
|
||||
for (const auto& b :
|
||||
{Boundary{"before-transaction", 1, 41, 0x5D}, Boundary{"before-upload", 2, 42, 0x5E},
|
||||
Boundary{"before-transaction", 2, 43, 0x5F}, Boundary{"after-commit", 2, 44, 0x60},
|
||||
Boundary{"after-ack", 2, 45, 0x61}})
|
||||
kill_at(b);
|
||||
resumed_stale_owner_is_fenced();
|
||||
damaged_heads_never_become_ready();
|
||||
tail_verification_before_readiness();
|
||||
}
|
||||
@@ -1,249 +1,20 @@
|
||||
// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL
|
||||
// with recovery by a successor process. Run by tools/ci/postgres-check.sh, which
|
||||
// sets EVO_SERVER_BIN.
|
||||
#include "evo/persistence/checkpoint_store.hpp"
|
||||
#include "fixtures/environmental_region.hpp"
|
||||
#include "fixtures/server_process.hpp"
|
||||
#include <algorithm>
|
||||
#include <cerrno>
|
||||
#include <chrono>
|
||||
#include <csignal>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <poll.h>
|
||||
#include <source_location>
|
||||
#include <spawn.h>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <sys/wait.h>
|
||||
#include <thread>
|
||||
#include <unistd.h>
|
||||
#include <vector>
|
||||
|
||||
extern char** environ;
|
||||
void run_region_server_process_tests();
|
||||
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
namespace fs = std::filesystem;
|
||||
using Clock = std::chrono::steady_clock;
|
||||
using namespace evo::test::process;
|
||||
void check(bool value, std::source_location at = std::source_location::current()) {
|
||||
if (!value)
|
||||
throw std::runtime_error("region server process test line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
std::string env(const char* key) {
|
||||
const auto* value = std::getenv(key);
|
||||
if (!value)
|
||||
throw std::runtime_error("missing test environment");
|
||||
return value;
|
||||
}
|
||||
std::unique_ptr<Postgres> owner() {
|
||||
auto made = Postgres::connect(
|
||||
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s);
|
||||
CHECK(made);
|
||||
return std::move(*made);
|
||||
}
|
||||
std::string hex32(std::uint64_t tick) {
|
||||
char out[33];
|
||||
std::snprintf(out, sizeof out, "9b000000000000000000000000%06llx",
|
||||
static_cast<unsigned long long>(tick));
|
||||
return out;
|
||||
}
|
||||
std::string input_line(std::uint64_t tick, bool consumer_dies) {
|
||||
const auto in = evo::test::region_input(tick, consumer_dies);
|
||||
std::string line = "input " + hex32(tick) + " " + std::to_string(in.temperature_mK) + " " +
|
||||
std::to_string(in.source_enabled);
|
||||
for (const auto b : in.bite)
|
||||
line += " " + std::to_string(b);
|
||||
for (const auto t : in.terminal)
|
||||
line += " " + std::to_string(t);
|
||||
return line + "\n";
|
||||
}
|
||||
|
||||
class Child {
|
||||
public:
|
||||
explicit Child(const std::vector<std::string>& args) {
|
||||
int to_child[2], from_child[2];
|
||||
CHECK(::pipe(to_child) == 0 && ::pipe(from_child) == 0);
|
||||
posix_spawn_file_actions_t actions;
|
||||
posix_spawn_file_actions_init(&actions);
|
||||
posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO);
|
||||
posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO);
|
||||
posix_spawn_file_actions_addclose(&actions, to_child[1]);
|
||||
posix_spawn_file_actions_addclose(&actions, from_child[0]);
|
||||
std::vector<char*> argv;
|
||||
argv.reserve(args.size() + 1);
|
||||
for (const auto& a : args)
|
||||
argv.push_back(const_cast<char*>(a.c_str()));
|
||||
argv.push_back(nullptr);
|
||||
CHECK(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0);
|
||||
posix_spawn_file_actions_destroy(&actions);
|
||||
::close(to_child[0]);
|
||||
::close(from_child[1]);
|
||||
in_ = to_child[1];
|
||||
out_ = from_child[0];
|
||||
}
|
||||
Child(const Child&) = delete;
|
||||
Child& operator=(const Child&) = delete;
|
||||
~Child() {
|
||||
if (pid_ > 0) {
|
||||
::kill(pid_, SIGKILL);
|
||||
::waitpid(pid_, nullptr, 0);
|
||||
}
|
||||
::close(in_);
|
||||
::close(out_);
|
||||
}
|
||||
void send(const std::string& text) {
|
||||
std::size_t done = 0;
|
||||
while (done < text.size()) {
|
||||
const auto n = ::write(in_, text.data() + done, text.size() - done);
|
||||
CHECK(n > 0 || errno == EINTR);
|
||||
if (n > 0)
|
||||
done += static_cast<std::size_t>(n);
|
||||
}
|
||||
}
|
||||
// Reads lines until one satisfies `want`; every line is kept in lines().
|
||||
template <typename Want> bool wait_for(Want want, std::chrono::milliseconds timeout) {
|
||||
const auto deadline = Clock::now() + timeout;
|
||||
for (std::size_t seen = 0;;) {
|
||||
for (; seen < lines_.size(); ++seen)
|
||||
if (want(lines_[seen]))
|
||||
return true;
|
||||
const auto left =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
|
||||
if (left.count() <= 0 || !read_some(static_cast<int>(left.count())))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
void signal(int sig) {
|
||||
CHECK(::kill(pid_, sig) == 0);
|
||||
}
|
||||
// Exit code, or 128+signal; drains remaining output.
|
||||
int wait_exit(std::chrono::milliseconds timeout) {
|
||||
const auto deadline = Clock::now() + timeout;
|
||||
int status = 0;
|
||||
for (;;) {
|
||||
read_some(20);
|
||||
const auto r = ::waitpid(pid_, &status, WNOHANG);
|
||||
if (r == pid_)
|
||||
break;
|
||||
CHECK(Clock::now() < deadline);
|
||||
}
|
||||
while (read_some(20)) {
|
||||
}
|
||||
pid_ = -1;
|
||||
return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
|
||||
}
|
||||
[[nodiscard]] const std::vector<std::string>& lines() const noexcept {
|
||||
return lines_;
|
||||
}
|
||||
|
||||
private:
|
||||
bool read_some(int timeout_ms) {
|
||||
pollfd fd{out_, POLLIN, 0};
|
||||
if (::poll(&fd, 1, timeout_ms) <= 0)
|
||||
return false;
|
||||
char chunk[4096];
|
||||
const auto n = ::read(out_, chunk, sizeof chunk);
|
||||
if (n <= 0)
|
||||
return false;
|
||||
buffer_.append(chunk, static_cast<std::size_t>(n));
|
||||
for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) {
|
||||
lines_.push_back(buffer_.substr(0, end));
|
||||
buffer_.erase(0, end + 1);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
pid_t pid_{-1};
|
||||
int in_{-1}, out_{-1};
|
||||
std::string buffer_;
|
||||
std::vector<std::string> lines_;
|
||||
};
|
||||
|
||||
struct Region {
|
||||
RegionKey key;
|
||||
fs::path root;
|
||||
};
|
||||
Region provision(std::uint64_t region, std::uint8_t genesis) {
|
||||
Region r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))};
|
||||
fs::remove_all(r.root);
|
||||
fs::create_directory(r.root);
|
||||
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
|
||||
auto db = owner();
|
||||
CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
||||
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
||||
{}, 5s));
|
||||
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
||||
CHECK(objects);
|
||||
Sha256Digest build{};
|
||||
build.fill(0xB1);
|
||||
Uuid commit{};
|
||||
commit.fill(genesis);
|
||||
CHECK(provision_lineage_region(*db, **objects, r.key, evo::test::region_identity.frame,
|
||||
*evo::test::lineage_region(), commit, build, 8000000, 5s));
|
||||
return r;
|
||||
}
|
||||
std::vector<std::string> server_args(const Region& r) {
|
||||
return {env("EVO_SERVER_BIN"),
|
||||
"--profile",
|
||||
"lineage",
|
||||
"--world",
|
||||
"7",
|
||||
"--region",
|
||||
std::to_string(r.key.region),
|
||||
"--frame",
|
||||
std::to_string(evo::test::region_identity.frame),
|
||||
"--pg-host",
|
||||
env("PGHOST"),
|
||||
"--pg-port",
|
||||
env("PGPORT"),
|
||||
"--pg-database",
|
||||
env("PGDATABASE"),
|
||||
"--pg-user",
|
||||
env("EVO_PG_RUNTIME_USER"),
|
||||
"--objects",
|
||||
r.root.string(),
|
||||
"--build",
|
||||
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
|
||||
"--flush-ms",
|
||||
"500",
|
||||
"--lease-profile",
|
||||
"fast",
|
||||
"--ready-file",
|
||||
(r.root / "ready").string()};
|
||||
}
|
||||
// (tick, commit_sequence) of every ack line.
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> acks(const Child& c) {
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> out;
|
||||
for (const auto& line : c.lines()) {
|
||||
unsigned long long tick = 0, sequence = 0;
|
||||
char request[40];
|
||||
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
|
||||
out.emplace_back(tick, sequence);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::string head_of(RegionKey key) {
|
||||
const std::array<std::string, 1> p{std::to_string(key.region)};
|
||||
auto rows = owner()->query("SELECT h.end_tick::text || ':' || h.commit_sequence::text || ':' "
|
||||
"|| coalesce(l.holder::text, 'released') FROM "
|
||||
"evolution.region_heads h JOIN evolution.region_leases l USING "
|
||||
"(world_id, region_id) WHERE world_id = 7 AND region_id = "
|
||||
"$1::evolution.u64",
|
||||
p, 5s);
|
||||
CHECK(rows && rows->size() == 1);
|
||||
const auto& value = (*rows)[0][0];
|
||||
CHECK(value.has_value());
|
||||
return value.value_or("");
|
||||
}
|
||||
std::uint64_t head_tick(RegionKey key) {
|
||||
return std::stoull(head_of(key).substr(0, head_of(key).find(':')));
|
||||
}
|
||||
bool starts(const std::string& line, std::string_view prefix) {
|
||||
return line.starts_with(prefix);
|
||||
}
|
||||
|
||||
// SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every
|
||||
// acknowledged tick is committed, the backlog never exceeds its bound, and the
|
||||
@@ -253,19 +24,12 @@ void sigterm_drains_and_releases() {
|
||||
Child server{server_args(r)};
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
||||
CHECK(fs::exists(r.root / "ready"));
|
||||
std::string burst;
|
||||
for (std::uint64_t tick = 1; tick <= 80; ++tick)
|
||||
burst += input_line(tick, true);
|
||||
server.send(burst); // More than the 64-input backlog: stdin backpressures.
|
||||
CHECK(server.wait_for(
|
||||
[](const std::string& l) { return starts(l, "ack " + hex32(80) + " 80 "); }, 30s));
|
||||
server.send(input_lines(1, 80, true)); // More than the 64-input backlog: stdin backpressures.
|
||||
CHECK(server.wait_line("ack " + request_hex(80) + " 80 ", 30s));
|
||||
server.signal(SIGTERM);
|
||||
CHECK(server.wait_exit(20s) == 0);
|
||||
const auto& lines = server.lines();
|
||||
CHECK(std::ranges::find(lines, "exit 0") != lines.end());
|
||||
CHECK(std::ranges::find_if(lines, [](const std::string& l) {
|
||||
return starts(l, "phase stopped");
|
||||
}) != lines.end());
|
||||
CHECK(std::ranges::find(lines, "exit 0") != lines.end() && server.printed("phase stopped"));
|
||||
unsigned long long max_queued = 999;
|
||||
for (const auto& l : lines)
|
||||
std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued);
|
||||
@@ -277,12 +41,14 @@ void sigterm_drains_and_releases() {
|
||||
const auto outcome = std::ranges::find_if(
|
||||
lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); });
|
||||
const auto ack20 = std::ranges::find_if(
|
||||
lines, [](const std::string& l) { return starts(l, "ack " + hex32(20) + " 20 "); });
|
||||
lines, [](const std::string& l) { return starts(l, "ack " + request_hex(20) + " 20 "); });
|
||||
CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome);
|
||||
CHECK(head_of(r.key).starts_with("80:") && head_of(r.key).ends_with(":released"));
|
||||
const auto head = head_of(r.key);
|
||||
CHECK(head.tick == 80 && head.released);
|
||||
CHECK(!fs::exists(r.root / "ready"));
|
||||
std::printf("server_sigterm commits=%s acks=%zu max_queued=%llu\n", head_of(r.key).c_str(),
|
||||
all.size(), max_queued);
|
||||
std::printf("server_sigterm head=%llu sequence=%llu acks=%zu max_queued=%llu\n",
|
||||
static_cast<unsigned long long>(head.tick),
|
||||
static_cast<unsigned long long>(head.sequence), all.size(), max_queued);
|
||||
}
|
||||
|
||||
// SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was
|
||||
@@ -293,22 +59,16 @@ void sigkill_recovers_through_a_successor() {
|
||||
{
|
||||
Child server{server_args(r)};
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
||||
std::string burst;
|
||||
for (std::uint64_t tick = 1; tick <= 60; ++tick)
|
||||
burst += input_line(tick, false);
|
||||
server.send(burst);
|
||||
CHECK(server.wait_for([](const std::string& l) { return starts(l, "ack "); }, 15s));
|
||||
server.send(input_lines(1, 60, false));
|
||||
CHECK(server.wait_line("ack ", 15s));
|
||||
std::this_thread::sleep_for(300ms); // Ticks run past the last commit.
|
||||
server.signal(SIGKILL);
|
||||
CHECK(server.wait_exit(10s) == 128 + SIGKILL);
|
||||
for (const auto& [tick, sequence] : acks(server)) {
|
||||
static_cast<void>(sequence);
|
||||
acked_max = std::max(acked_max, tick);
|
||||
}
|
||||
acked_max = max_acked(server);
|
||||
}
|
||||
const auto committed = head_tick(r.key);
|
||||
CHECK(acked_max > 0 && committed >= acked_max); // No acknowledged tick was lost.
|
||||
CHECK(!head_of(r.key).ends_with(":released")); // The killed holder never released.
|
||||
const auto committed = head_of(r.key);
|
||||
CHECK(acked_max > 0 && committed.tick >= acked_max); // No acknowledged tick was lost.
|
||||
CHECK(!committed.released); // The killed holder never released.
|
||||
const auto started = Clock::now();
|
||||
Child successor{server_args(r)};
|
||||
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
||||
@@ -317,14 +77,15 @@ void sigkill_recovers_through_a_successor() {
|
||||
const auto running = std::ranges::find_if(
|
||||
successor.lines(), [](const std::string& l) { return starts(l, "phase running"); });
|
||||
CHECK(running != successor.lines().end());
|
||||
CHECK(running->find("committed=" + std::to_string(committed) + " ") != std::string::npos);
|
||||
CHECK(running->find("committed=" + std::to_string(committed.tick) + " ") != std::string::npos);
|
||||
CHECK(running->find("epoch=2 ") != std::string::npos);
|
||||
successor.signal(SIGTERM);
|
||||
CHECK(successor.wait_exit(20s) == 0);
|
||||
CHECK(head_tick(r.key) == committed && head_of(r.key).ends_with(":released"));
|
||||
const auto after = head_of(r.key);
|
||||
CHECK(after.tick == committed.tick && after.released);
|
||||
std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n",
|
||||
static_cast<unsigned long long>(acked_max),
|
||||
static_cast<unsigned long long>(committed), static_cast<long long>(to_ready));
|
||||
static_cast<unsigned long long>(committed.tick), static_cast<long long>(to_ready));
|
||||
}
|
||||
} // namespace
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha"
|
||||
-f "$root/migrations/0001_region_bootstrap.sql" > /dev/null
|
||||
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1
|
||||
PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh"
|
||||
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5
|
||||
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6
|
||||
# v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2.
|
||||
v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")"
|
||||
v2_sha="${v2_sha%% *}"
|
||||
@@ -67,7 +67,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \
|
||||
-c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \
|
||||
-c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null
|
||||
PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh"
|
||||
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 5:1,2,3,4,5
|
||||
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 6:1,2,3,4,5,6
|
||||
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1
|
||||
# Review P1: a v4 database holding a published (possibly terminal) checkpoint has
|
||||
# no durable decisions for it. The v5 upgrade must refuse before changing anything.
|
||||
@@ -136,9 +136,11 @@ expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.provision_region_check
|
||||
expect_audit_failure 'GRANT SELECT ON evolution.region_commits TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commits FROM evo_runtime_test'
|
||||
bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again.
|
||||
# Unknown future versions and checksum drift must not be repaired silently.
|
||||
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=6 WHERE version=5'
|
||||
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=7 WHERE version=6'
|
||||
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=6 WHERE version=7"
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=6"
|
||||
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=5 WHERE version=6"
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=5"
|
||||
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=4"
|
||||
|
||||
@@ -22,8 +22,11 @@ digest_4="$(sha256sum "$root/migrations/0004_lineage_profile.sql")"
|
||||
digest_4="${digest_4%% *}"
|
||||
digest_5="$(sha256sum "$root/migrations/0005_batch_decisions.sql")"
|
||||
digest_5="${digest_5%% *}"
|
||||
digest_6="$(sha256sum "$root/migrations/0006_commit_lookup.sql")"
|
||||
digest_6="${digest_6%% *}"
|
||||
psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \
|
||||
--set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \
|
||||
--set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \
|
||||
--set="migration_sha_6=$digest_6" \
|
||||
--set="runtime_role=$runtime_role" \
|
||||
--file="$root/migrations/apply.sql"
|
||||
|
||||
Reference in New Issue
Block a user