Add M10.4b durable action records (schema v11)
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m47s
Foundation / checks (network) (push) Successful in 9m30s
Foundation / checks (client) (push) Successful in 12m19s
Foundation / checks (tsan) (push) Successful in 7m43s
Foundation / checks (checks) (push) Successful in 31m29s
Foundation / checks (postgres) (push) Successful in 36m39s
Foundation / postgres-tools (push) Successful in 19s
Foundation / image (push) Successful in 2m15s
Foundation / server-image (push) Successful in 20m37s

Each batch's actions are recorded in the same statement as its checkpoint: one
region_actions row per (world, account, action id) and the block in an append-only
table, compared on exact retries and resolution. The region host attaches actions to
the ticks they ride and reports acknowledgements and drops, keeping unknown-outcome
batches unresolved. Outcomes come from the durable record via ActionLedger, so
crashes before or after a commit, unknown publications and cross-region moves all
resolve to one answer; a repeated id keeps its first record instead of stalling a
region. ADR-0092.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-03 22:53:38 -04:00
co-authored by Claude Opus 5.5
parent e2f6208433
commit 69f3ad7cf8
32 changed files with 1289 additions and 75 deletions
+17 -8
View File
@@ -49,7 +49,7 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra
const CommittedCheckpoint& predecessor,
const RegionBatch<typename Codec::State>& batch, Uuid commit,
const Sha256Digest& build, std::size_t budget, milliseconds timeout,
const PublishHooks& hooks) {
const PublishHooks& hooks, std::span<const ActionRecord> actions) {
const auto previous = decode_manifest(predecessor.manifest);
if (!previous || previous->profile != Codec::profile || !batch.sealed() ||
batch.steps().empty() || batch.begin_tick() != previous->end_tick)
@@ -81,7 +81,8 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra
batch.begin_tick(),
batch.end_tick()};
auto trace = encode_trace(header, batch.steps());
if (!decision_bytes || !trace)
auto action_bytes = encode_actions(actions, batch.begin_tick(), batch.end_tick());
if (!decision_bytes || !trace || !action_bytes)
return std::unexpected{fail(PublicationError::candidate)};
const auto payload_digest = sha256(*payload);
const auto trace_digest = sha256(*trace);
@@ -104,10 +105,10 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra
if (auto stored = put_verified(objects, *trace, {*trace_digest, trace->size()}); !stored)
return std::unexpected{stored.error()};
PreparedCommit prepared{previous->key, commit, *bytes, *decision_bytes};
PreparedCommit prepared{previous->key, commit, *bytes, *decision_bytes, *action_bytes};
if (hooks.before_transaction)
hooks.before_transaction();
auto published = publish_region_checkpoint(db, *bytes, *decision_bytes, timeout);
auto published = retry_prepared(db, prepared, timeout);
auto committed = confirm(published, std::move(*bytes), *manifest, std::move(prepared));
if (!committed)
return std::unexpected{committed.error()};
@@ -128,9 +129,10 @@ publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects,
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
const RegionBatch<processes::EnvironmentalCommunity>& batch,
Uuid commit, const Sha256Digest& build, std::size_t restore_budget,
milliseconds timeout, const PublishHooks& hooks) {
milliseconds timeout, const PublishHooks& hooks,
std::span<const ActionRecord> actions) {
return publish_batch<EnvironmentalCodec>(db, objects, lease, predecessor, batch, commit, build,
restore_budget, timeout, hooks);
restore_budget, timeout, hooks, actions);
}
std::expected<RecoveredCheckpoint, RecoveryFailure>
recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
@@ -152,9 +154,9 @@ publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const
const CommittedCheckpoint& predecessor,
const RegionBatch<processes::LineageRegion>& batch, Uuid commit,
const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout,
const PublishHooks& hooks) {
const PublishHooks& hooks, std::span<const ActionRecord> actions) {
return publish_batch<LineageCodec>(db, objects, lease, predecessor, batch, commit, build,
restore_budget, timeout, hooks);
restore_budget, timeout, hooks, actions);
}
std::expected<RecoveredLineageCheckpoint, RecoveryFailure>
recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
@@ -165,11 +167,18 @@ recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, R
std::expected<ResolvedCommit, LeaseStoreFailure>
resolve_prepared(Postgres& db, const PreparedCommit& prepared, milliseconds timeout) {
if (!prepared.actions.empty())
return resolve_region_commit(db, prepared.key, prepared.commit, prepared.manifest,
prepared.decisions, prepared.actions, timeout);
return resolve_region_commit(db, prepared.key, prepared.commit, prepared.manifest,
prepared.decisions, timeout);
}
std::expected<RegionHead, LeaseStoreFailure>
retry_prepared(Postgres& db, const PreparedCommit& prepared, milliseconds timeout) {
// Without actions the schema v5 publication is used, exactly as before v11.
if (!prepared.actions.empty())
return publish_region_checkpoint(db, prepared.manifest, prepared.decisions,
prepared.actions, timeout);
return publish_region_checkpoint(db, prepared.manifest, prepared.decisions, timeout);
}
} // namespace evo::persistence
+37
View File
@@ -1,5 +1,7 @@
#include "evo/persistence/region_batch.hpp"
#include <algorithm>
#include <tuple>
#include <utility>
namespace evo::persistence {
namespace {
@@ -223,4 +225,39 @@ std::vector<BatchDecision> derive_decisions(std::span<const TraceStep> steps,
}
return out;
}
std::expected<std::vector<std::uint8_t>, BatchError>
encode_actions(std::span<const ActionRecord> actions, std::uint64_t begin_tick,
std::uint64_t end_tick) {
if (actions.size() > max_batch_actions)
return std::unexpected{BatchError::format};
std::vector<ActionRecord> sorted(actions.begin(), actions.end());
const auto key = [](const ActionRecord& a) {
return std::tuple{a.tick, a.account, a.action_id};
};
std::ranges::sort(sorted, {}, key);
// An action id appears once, whatever its tick.
std::vector<std::pair<Uuid, std::uint64_t>> ids;
ids.reserve(sorted.size());
for (const auto& a : sorted)
ids.emplace_back(a.account, a.action_id);
std::ranges::sort(ids);
if (std::ranges::adjacent_find(ids) != ids.end())
return std::unexpected{BatchError::format};
std::vector<std::uint8_t> out;
out.reserve(sorted.size() * action_record_bytes);
for (std::size_t i = 0; i < sorted.size(); ++i) {
const auto& a = sorted[i];
if (a.account == Uuid{} || a.action_id == 0 || a.kind != 1 || a.tick <= begin_tick ||
a.tick > end_tick || (i > 0 && key(sorted[i - 1]) == key(a)))
return std::unexpected{BatchError::format};
out.insert(out.end(), a.account.begin(), a.account.end());
for (unsigned b = 0; b < 8; ++b)
out.push_back(static_cast<std::uint8_t>(a.action_id >> (8U * b)));
for (unsigned b = 0; b < 8; ++b)
out.push_back(static_cast<std::uint8_t>(a.tick >> (8U * b)));
out.push_back(static_cast<std::uint8_t>(a.kind));
out.push_back(static_cast<std::uint8_t>(a.kind >> 8U));
}
return out;
}
} // namespace evo::persistence
+71
View File
@@ -281,6 +281,77 @@ resolve_region_commit(Postgres& db, RegionKey key, const Uuid& commit,
return ResolvedCommit{Resolution::committed, *head};
}
std::expected<RegionHead, LeaseStoreFailure>
publish_region_checkpoint(Postgres& db, std::span<const std::uint8_t> manifest,
std::span<const std::uint8_t> decisions,
std::span<const std::uint8_t> actions, milliseconds timeout) {
const std::array<std::string, 3> p{bytea(manifest), bytea(decisions), bytea(actions)};
auto rows = db.query(
"SELECT * FROM evolution.publish_region_checkpoint($1::bytea, $2::bytea, $3::bytea)", p,
timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
auto head = parse_head(*rows);
if (!head)
return std::unexpected{malformed(true)};
return *head;
}
std::expected<ResolvedCommit, LeaseStoreFailure>
resolve_region_commit(Postgres& db, RegionKey key, const Uuid& commit,
std::span<const std::uint8_t> manifest,
std::span<const std::uint8_t> decisions,
std::span<const std::uint8_t> actions, milliseconds timeout) {
const std::array<std::string, 6> p{std::to_string(key.world), std::to_string(key.region),
uuid_text(commit), bytea(manifest),
bytea(decisions), bytea(actions)};
auto rows = db.query("SELECT * FROM evolution.resolve_region_commit($1::evolution.u64, "
"$2::evolution.u64, $3::uuid, $4::bytea, $5::bytea, $6::bytea)",
p, timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
if (rows->size() != 1 || (*rows)[0].size() != 6)
return std::unexpected{malformed(false)};
const auto& status_text = (*rows)[0][0];
if (!status_text)
return std::unexpected{malformed(false)};
const auto status = *status_text;
if (status == "absent")
return ResolvedCommit{Resolution::absent, {}};
if (status == "conflict")
return ResolvedCommit{Resolution::conflict, {}};
const PgRows receipt{
std::vector<std::optional<std::string>>((*rows)[0].begin() + 1, (*rows)[0].end())};
auto head = status == "committed" ? parse_head(receipt) : std::nullopt;
if (!head)
return std::unexpected{malformed(false)};
return ResolvedCommit{Resolution::committed, *head};
}
std::expected<std::optional<CommittedAction>, LeaseStoreFailure>
find_region_action(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account,
std::uint64_t action_id, milliseconds timeout) {
const std::array<std::string, 6> p{std::to_string(key.world), std::to_string(key.region),
uuid_text(lease.holder), std::to_string(lease.epoch),
uuid_text(account), std::to_string(action_id)};
auto rows = db.query("SELECT * FROM evolution.find_region_action($1::evolution.u64, "
"$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::uuid, "
"$6::evolution.u64)",
p, timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
if (rows->empty())
return std::optional<CommittedAction>{};
if (rows->size() != 1 || (*rows)[0].size() != 3)
return std::unexpected{malformed(false)};
const auto region = parse_integer<std::uint64_t>((*rows)[0][0]);
const auto tick = parse_integer<std::uint64_t>((*rows)[0][1]);
const auto sequence = parse_integer<std::uint64_t>((*rows)[0][2]);
if (!region || !tick || !sequence)
return std::unexpected{malformed(false)};
return std::optional<CommittedAction>{CommittedAction{*region, *tick, *sequence}};
}
namespace {
std::expected<std::vector<CommittedDecision>, LeaseStoreFailure>
decisions_from(const PgRows& rows) {