diff --git a/CMakeLists.txt b/CMakeLists.txt index b7edf5c..4ba432c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -307,6 +307,7 @@ if(EVO_BUILD_POSTGRES) add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp tests/cellular_publication_tests.cpp tests/population_publication_tests.cpp + tests/fixtures/postgres_relay.cpp tests/account_control_store_tests.cpp tests/probe_handoff_store_tests.cpp tests/life_reservation_store_tests.cpp tests/primitive_publication_tests.cpp diff --git a/README.md b/README.md index e4b1b25..5d5b70e 100644 --- a/README.md +++ b/README.md @@ -140,8 +140,11 @@ worker/session path. Opt-in exact population profile 5 is described in replacement owners; biological migration, environmental coupling and production qualification remain in progress. The portable [biological transfer candidates](docs/engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md) -preserve exact identity/history and regional boundary budgets; their durable -escrow is still required. The microscope client is the current M11 playtest path. +preserve exact identity/history and regional boundary budgets. Their scoped +durable preparation/staging, atomic paired publication and verified recovery are +implemented; [TLS transport fault checks](docs/engineering/M12_MIGRATION_TRANSPORT_FAULTS.md) +cover independent local path loss and owner replacement. Linked ecology and +deployment qualification remain open. The microscope client is the M11 playtest path. `evo::simulation` remains graphics-, network- and database-independent. diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index e9ac2fe..1e62065 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -156,7 +156,11 @@ RUN cmake --preset asan -B build/postgres-asan -DCMAKE_CXX_COMPILER=clang++-19 \ && cmake --build build/postgres-asan --target evo-postgres-tests --parallel "${PG_BUILD_JOBS}" FROM postgres-test-build AS ci-postgres -RUN useradd --uid 10001 --no-create-home evo +# Test-only ephemeral TLS credentials and a standard-library loopback port picker. +RUN apt-get update && apt-get install -y --no-install-recommends openssl python3 \ + && rm -rf /var/lib/apt/lists/* \ + && openssl version && python3 --version \ + && useradd --uid 10001 --no-create-home evo USER 10001:10001 # Full qualification once on the release build. ASan/UBSan reruns everything that # executes its instrumented binaries; the uninstrumented clang-tidy build reruns only diff --git a/docs/STATUS.md b/docs/STATUS.md index 4b11698..3ed90a5 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,23 @@ # Project status and next steps +## M12.4k independent migration TLS path faults - 2026-10-18 + +Test-only relays now exercise real loopback TCP/end-to-end TLS request/receipt +loss for raw and cohort migration. Independent observation proves unchanged or +atomically committed heads. Owning hosts retain lost prepare/stage/abort work, +fence after actual partitioned lease expiry, recover under replacement authority +and refuse healed stale owners. Their healthy peer keeps its epoch/freeze. +Wrong-hostname TLS connections fail; production adapter settings are unchanged. + +Native Release passes 78/78; GCC, ASan/UBSan, TSan, Clang analysis, format and +shell checks pass. Complete private persistence, process-fault, backup/DR and +migration/permission qualification, including the TLS matrix, passes in 540 s. +[Evidence and limits](engineering/VALIDATION.md) distinguish +these local transport faults from multi-host/kernel-network, database/object-store +and orchestrator drills. Linked ecology and later M12 integration/soak gates +remain open. No SQL, production code, biological rule/format, deployment or live +region changed. Full/sanitizer CI paths include the new disposable TLS matrix. + ## M12.4j atomic paired biological publication - 2026-10-18 ADR-0149/schema 49 commits both exact migration results and one immutable common diff --git a/docs/architecture/PERSISTENCE_ARCHITECTURE.md b/docs/architecture/PERSISTENCE_ARCHITECTURE.md index e4d47cc..17dd8a5 100644 --- a/docs/architecture/PERSISTENCE_ARCHITECTURE.md +++ b/docs/architecture/PERSISTENCE_ARCHITECTURE.md @@ -30,7 +30,11 @@ second-insert failure rolls back both heads and pointer settlement. Recovery replays both cuts and binds both results to the stage. Frozen hosts reload and verify a changed head before readiness, rather than inferring activation from pointer absence. See [ADR-0149](../adr/0149-atomic-cellular-migration-publication.md). -Independent network partitions and linked ecological boundaries remain separate. +M12.4k qualifies independent loopback TCP/TLS request/receipt loss and either +owner's actual lease expiry, replacement and healed stale fencing in both +representations. Independent database observation checks the outcomes. See +[transport scope](../engineering/M12_MIGRATION_TRANSPORT_FAULTS.md). Multi-host +network/kernel, object-store/orchestrator failures and linked ecology remain separate. M7.1 now scopes the first implementation in [ADR-0058](../adr/0058-region-durability-contract.md) and [REGION_DURABILITY.md](REGION_DURABILITY.md): full checkpoint batches, not diff --git a/docs/engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md b/docs/engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md index 9f342e1..8a6fd0a 100644 --- a/docs/engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md +++ b/docs/engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md @@ -190,3 +190,21 @@ See [the paired commit contract](../../schemas/ecology/cellular-migration-commit Independent network partitions, linked geometry/environment, maturation, world census/cuts, actual elapsed-day ecology and target-host capacity remain open. No automatic route, biological rule/format, golden or deployment changes. + +## M12.4k independent TLS transport fault qualification + +A test-owned opaque relay and separate disposable loopback TLS cluster exercise +the unchanged production adapter. Both representations lose real requests and +receipts; independent observation distinguishes unchanged heads from an exact +committed pair. Lost preparation/staging/abort replies retain durable work and +host pauses. Either owner's path can remain isolated through actual lease expiry +while its peer retains the same epoch/freeze. Replacement recovers the same work; +stale publication fails, explicit abort releases both pauses and healing does not +revive the old fence. Wrong-hostname TLS is refused. ASan/UBSan and TSan exercise +the matrix; the full/sanitizer CI paths include it in a separate cluster. + +See [fixture and qualification limits](M12_MIGRATION_TRANSPORT_FAULTS.md). This is +application-boundary byte loss over real local TCP/TLS, not multi-host/kernel +networking, database/object-store loss or orchestration. Linked ecology and later +M12 gates remain open. No SQL, production code, biological rule/format or deployment +changes; the added relays and credential tools belong to tests/CI only. diff --git a/docs/engineering/M12_MIGRATION_TRANSPORT_FAULTS.md b/docs/engineering/M12_MIGRATION_TRANSPORT_FAULTS.md new file mode 100644 index 0000000..ce1a1c8 --- /dev/null +++ b/docs/engineering/M12_MIGRATION_TRANSPORT_FAULTS.md @@ -0,0 +1,79 @@ +# M12.4k migration transport fault qualification + +This qualifies ADR-0147–0149 over real loopback TCP with end-to-end PostgreSQL TLS. +It changes tests and CI only: no SQL, biological format/rule, production adapter, +world configuration, dependency in the headless library or deployment changes. + +## Fixture and command + +```sh +bash tools/ci/postgres-check.sh --migration-faults "$PWD/build/m97-postgres/evo-postgres-tests" +``` + +The runner creates a fresh 0700 `/tmp/evo-pg-test.*` cluster and temporary TLS +certificate/key. Only this mode listens on `127.0.0.1` at an ephemeral port. The +owner stays on its protected Unix socket; only the least-privileged test login is +admitted over TLS. Ordinary fixtures retain disabled TCP. The full and sanitizer +qualification paths run this mode in a separate cluster, never a supplied/live +database. Setup failure is a failed test, without a silent skip. + +The test-owned relay forwards opaque encrypted bytes. It binds only loopback, +owns and joins its worker, bounds each direction's queue to 64 KiB, and records +discarded-byte counts. It can discard client requests or server responses +independently. The unchanged adapter still uses `sslmode=verify-full`; a wrong +hostname must fail. The fixture neither terminates TLS nor changes SQL results, +leases, clocks or biological stock. Timeouts retain the adapter's real unknown +outcome behavior. + +## Executed matrix + +Both raw owner-7/profile-4 and exact cohort/profile-5 cases use actual funded +player-origin wildlife from the existing publication fixture. + +- **Request path lost:** a paired request is discarded after TLS setup. The + caller gets unknown with the exact two manifests; independent Unix-socket + observation proves both heads stayed at their predecessors. +- **Receipt path lost:** the database receives and commits the pair while the + relay discards its reply. The caller gets unknown; independent observation + proves both exact result manifests committed. Existing exact retry/recovery + checks then prove no second transfer. +- **Preparation/staging replies lost:** the owning source/destination hosts + pause, retain exact work and reconnect to their durable freeze/ghost. No + candidate is activated from an unconfirmed reply. +- **Either owner isolated:** that owner's request path stays unavailable through + actual three-second lease expiry. The independent healthy peer keeps its same + epoch and retained pause. A replacement acquires a newer isolated-region epoch + and recovers the same preparation/stage. Old-fence publication is refused and + both heads remain unchanged. +- **Explicit abort reply lost:** current source authority decides abort despite + a lost response. Exact reconciliation releases both pointers. The healthy + owner resumes identical biology; the isolated old owner remains fenced when + its path heals while the replacement owns the region. + +The matrix runs alongside existing process-kill, second-insert rollback, +commit/abort race, full-payload replay and durable continuation tests. Numerical +stock/head equality is checked independently of relay counters and client errors. + +## Tooling evidence and limits + +The relay uses standard C++/POSIX facilities and the already approved PostgreSQL +TLS adapter. Certificate generation uses the existing OpenSSL tool family; +Python uses only its standard-library `socket` module to select a loopback port. +The CI-only image installs Ubuntu's `openssl`/`python3` packages and prints their +versions. Production image package lists do not change. + +Local evidence: OpenSSL 3.0.13 (`3.0.13-0ubuntu3.16`), Python 3.12.3 +(`python3 3.12.3-0ubuntu2.1`, `python3.12 3.12.3-1ubuntu0.17`). Packaged copyright +evidence is `/usr/share/doc/openssl/copyright` (Apache-2.0 for upstream OpenSSL) +and `/usr/share/doc/python3.12/copyright` (Python licensing and bundled notices). +These are distro-managed test utilities, without vendored code, a new package +registry, configure-time fetch or selected game dependency. CI records its actual +installed versions; local package versions do not promise future CI versions. + +The relay discards bytes at an application transport boundary on one machine. +This is not kernel packet loss, a multi-host routing/asymmetric firewall drill, +remote database/object-store failure, or a production orchestrator failover test. +The local object stores remain available. Linked geometry/resources, maturation, +global census/cuts, target-host cost and actual elapsed-day ecology remain open. +Executed compiler/sanitizer results and environmental limits belong in +[VALIDATION](VALIDATION.md). No World0 action or cellular clone/restore claim. diff --git a/docs/engineering/M12_REMAINING_GATES.md b/docs/engineering/M12_REMAINING_GATES.md index 2b87a77..c61076f 100644 --- a/docs/engineering/M12_REMAINING_GATES.md +++ b/docs/engineering/M12_REMAINING_GATES.md @@ -25,7 +25,7 @@ or authorize production enablement. |---|---|---| | 12.2 | Exact cellular model and paired matrix; local samples through 256 lives | Deployment-host tail costs including publication; larger wildlife cohorts, predation/twins and contention coverage; useful compression remains unimplemented. | | 12.3 | Dedicated population profile/trace/store/owning host; exact recovery and private PostgreSQL fault tests | Population-specific executable/network fault drills and target-host publication/capacity qualification. | -| 12.4 | Owned bounded intervals; exact transfer/accounting; durable preparation/staging, atomic paired publication and replayed host recovery | Independent network-partition drills, linked-boundary contracts and integrated ownership/stock audits at scale. | +| 12.4 | Owned intervals; exact transfer/accounting; durable atomic pair/recovery; independent loopback TLS path loss and owner expiry/replacement/heal | Multi-host/kernel-network and object-store/orchestrator drills, linked-boundary contracts and integrated ownership/stock audits at scale. | | 12.5 | Finite cellular contact feeding/decay and separate M6 producer/consumer resource owner | One jointly owned cellular/environment budget, explicit effective depth and source entitlements, versioned water/atmosphere species mapping and producer coefficients, plus closed/open and nutrient-exhaustion comparisons. | | 12.6 | Retained genome/organism archive with ancestry and common-ancestor queries | Append-only population census/history events in the same durable commit, bounded world history queries/indexing and client presentation; no account power or invented precise statistical ancestry. | | 12.7 | ADR-0133's viability/unknown semantics | Durable region-set/watermark reconciliation covering every owned representation, suspended/viable stock and migration escrow. Missing/stale regions must prevent terminal extinction. | @@ -73,8 +73,13 @@ second inserts roll back both heads; commit/abort races have one exclusive outcome. Exact retries preserve both manifests under fresh authority. Hosts replay and verify changed heads before readiness. Preparation remains an optimistic destination observation; staging/publication recheck captured heads. -Independent network-partition drills and linked ecological boundaries remain -open; shared-database fault tests do not supply those proofs. +M12.4k now exercises actual loopback TCP/end-to-end TLS with independent request +and response loss, lost prepare/stage/abort receipts, either owner's actual lease +expiry, replacement and healed stale-owner fencing in both representations. +Independent database observation proves unchanged or atomically committed heads; +the healthy peer keeps its epoch and retained pause. See +[the transport scope](M12_MIGRATION_TRANSPORT_FAULTS.md). Multi-host/kernel-network +partitions, object-store/orchestrator failures and linked ecology remain open. The transfer tests also expose the current rules-8 maturation gap: buds retain their subadult realization while reproduction requires a fully realized parent. Finite extra food alone does not close repeated wildlife generations. A versioned diff --git a/docs/engineering/TEST_STRATEGY.md b/docs/engineering/TEST_STRATEGY.md index 4466310..69dd2a5 100644 --- a/docs/engineering/TEST_STRATEGY.md +++ b/docs/engineering/TEST_STRATEGY.md @@ -58,6 +58,16 @@ retain complete recovery after replacement and continue biology afterwards. Independent network partitions, linked ecology and actual elapsed-day stability remain separate evidence; full schema/permission/backup/DR regression is required. +M12.4k adds an explicit disposable loopback TLS mode and test-owned opaque relays. +Both representations exercise request loss, committed receipt loss, wrong-host +rejection, lost preparation/staging/abort replies, either owner's actual lease +expiry, an unchanged healthy-peer epoch, replacement and healed stale-owner +fencing. Independent Unix-socket observations prove the database outcome; relay +byte counters alone are insufficient. Full and sanitizer CI paths run this mode +in a separate cluster. ASan/UBSan and TSan exercise the actual relay/host matrix. +Multi-host/kernel-network failures, object-store outages and orchestrator drills +remain separate; see [the scope](M12_MIGRATION_TRANSPORT_FAULTS.md). + Tests defend contracts, not directory coverage. All checks remain active in Release builds. Current proof has no actual network, biology or distributed recovery, so those categories below are milestone gates rather than passing diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 7e526b2..75043ab 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -1,5 +1,60 @@ # Validation report +## 2026-10-18 - M12.4k TLS migration request/receipt and owner partitions + +Test-only qualification extends ADR-0147–0149 with opaque loopback TCP relays and +the unchanged adapter's end-to-end `verify-full` PostgreSQL TLS. A separate +disposable cluster mode admits only the test runtime login over loopback TLS; +ordinary fixtures keep TCP disabled. Full/sanitizer CI paths run this matrix. +See [fixture, tooling and limits](M12_MIGRATION_TRANSPORT_FAULTS.md). + +Executed locally: + +- Native MSVC Release **78/78 passed**, 12.63 s. +- Actual GCC 14 optional targets rebuilt; final TLS matrix passed **35 s** + (34 s population phase), with both raw/cohort markers for request/reply loss + and independent owner partition/expiry/replacement/heal. +- Linux Clang 19 **ASan/UBSan** TLS matrix passed **62 s** (58 s population phase), + with no sanitizer reports. +- Separate Linux Clang 19 **ThreadSanitizer** optional PostgreSQL build and TLS + matrix passed **63 s** (60 s population phase), with no race reports. +- Explicit clang-tidy 19 on the relay and publication/fault tests passed with no + owned warnings. Repository clang-format 19 and shell syntax checks passed. +- Complete frozen-binary private PostgreSQL qualification passed with **exit 0**, + **540 s**: C++ integration/process/load/history (432 s), the separate TLS fault + matrix (36 s), life reservations, backup/restore, host-loss DR, upgrade, + migration history/rollback, runtime permission audit and checksum/future-version + rejection. The detached runner recorded its completion code. + +Wrong-hostname TLS connections fail. Discarded requests return unknown with the +exact prepared manifests while independent observation proves both heads stayed +unchanged. Discarded commit replies return unknown while independent observation +proves both exact results committed. Source preparation, destination staging and +explicit abort also lose real replies without losing retained work or activating +ghosts. Either owner remains isolated through actual lease expiry, while its +healthy peer keeps the same epoch and freeze. Replacement gets a newer epoch and +the same retained work; stale publication fails with both heads unchanged. +Explicit current-source abort reconciles lost receipts. The healthy host resumes +identical biology; a healed old owner stays fenced while replacement authority +is held. All of these cases run for raw and cohort representations with actual +player-origin wildlife and the existing full replay/continuation matrix. + +The initial analysis found avoidable stop-token copies in the test relay; both +callback/run arguments now use const references. No warning suppression or +production retry/TLS escape hatch was added. The new relay is test-owned and its +worker is joined; no production thread/service is introduced. + +Limits: byte discard at an application TCP boundary on one machine is not kernel +packet loss, a multi-host routing/firewall failure, a database host outage, object +store unavailability or an orchestrator drill. Existing local objects stay +available. The CI-only image adds distro OpenSSL/Python utilities for ephemeral +credentials/port selection; local versions/licensing evidence are recorded in +the fixture document. No Docker image build or remote CI result is claimed here. +Linked ecology, maturation, global census/cuts, target-host costs and actual +elapsed-day soak remain open. No SQL, production adapter, biological format/rule, +golden, live database, World0 or deployment changed. `Build_Errors.txt` and unrelated +local work remain untracked; prior captured CI runner failures remain separate. + ## 2026-10-18 - M12.4j atomic paired biological publication and recovery ADR-0149/schema 49 commits both exact bundle-10 no-tick results and one immutable diff --git a/docs/infrastructure/POSTGRESQL.md b/docs/infrastructure/POSTGRESQL.md index 6ea2d4a..b5cd739 100644 --- a/docs/infrastructure/POSTGRESQL.md +++ b/docs/infrastructure/POSTGRESQL.md @@ -9,8 +9,9 @@ distributed tooling. No custom hashing/cryptography is introduced. ## Local isolated verification Linux prerequisites: GCC14, CMake>=3.28, Ninja, make, bison, flex, curl, bzip2, -OpenSSL development headers and zlib development headers. Optional clang19 tools -support analysis/ASan. Run as a normal user, not root: +OpenSSL development headers, the OpenSSL CLI, Python 3 (standard library only) +and zlib development headers. Optional clang19 tools support analysis/ASan/TSan. +Run as a normal user, not root: ```sh bash tools/prepare-postgres.sh @@ -18,13 +19,18 @@ cmake --preset release -B build/postgres -DCMAKE_CXX_COMPILER=g++-14 \ -DEVO_BUILD_POSTGRES=ON -DPostgreSQL_ROOT="$PWD/.cache/postgres/install" cmake --build build/postgres --target evo-postgres-tests --parallel 2 bash tools/ci/postgres-check.sh "$PWD/build/postgres/evo-postgres-tests" +# Focused raw/cohort migration faults over real loopback TLS: +bash tools/ci/postgres-check.sh --migration-faults "$PWD/build/postgres/evo-postgres-tests" ``` The test script overrides all connection coordinates and creates a new private 0700 /tmp directory, private Unix socket and disposable local cluster. It creates the `evo_runtime_test` group and the `evo_runtime_login` member, then runs fencing -tests as that real runtime login. TCP is off; -local trust applies only inside that private directory. Cluster stops on exit; +tests as that real runtime login. Ordinary fixtures disable TCP; local trust +applies only inside that private directory. The migration-fault mode creates a +separate private cluster with ephemeral loopback TLS for the runtime login only; +the full and sanitizer paths include it. See [scope and prerequisites](../engineering/M12_MIGRATION_TRANSPORT_FAULTS.md). +Clusters stop on exit; logs/data are retained there for diagnosis. It never uses production PG credentials or deletes a pre-existing database. Do not point this script at a shared server. diff --git a/schemas/ecology/cellular-migration-commit-v1.md b/schemas/ecology/cellular-migration-commit-v1.md index dce9700..d405222 100644 --- a/schemas/ecology/cellular-migration-commit-v1.md +++ b/schemas/ecology/cellular-migration-commit-v1.md @@ -50,7 +50,9 @@ requires full recovery before readiness or advancement, including after restart. Pointer absence alone cannot resume the old source or activate an uploaded ghost. Unknown abort that loses to commit reconciles the actual committed head. -Independent network partitions, linked geometry/resources, world-wide cuts/census, +M12.4k's [loopback TCP/TLS matrix](../../docs/engineering/M12_MIGRATION_TRANSPORT_FAULTS.md) +qualifies request/receipt loss and independent owner expiry/replacement/heal. +Multi-host/kernel-network failures, linked geometry/resources, world-wide cuts/census, target-host costs and actual elapsed-day ecology remain separate gates. Backup/DR regression retains its supported fixtures and does not qualify cellular history cloning or a new cellular restore procedure. No automatic migration or deployment. diff --git a/tests/fixtures/postgres_relay.cpp b/tests/fixtures/postgres_relay.cpp new file mode 100644 index 0000000..6958773 --- /dev/null +++ b/tests/fixtures/postgres_relay.cpp @@ -0,0 +1,175 @@ +#include "postgres_relay.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace evo::test { +namespace { +struct Socket { + int fd{-1}; + Socket() = default; + explicit Socket(int value) : fd(value) {} + ~Socket() { + if (fd >= 0) + ::close(fd); + } + Socket(const Socket&) = delete; + Socket& operator=(const Socket&) = delete; + Socket(Socket&& other) noexcept : fd(std::exchange(other.fd, -1)) {} + Socket& operator=(Socket&& other) noexcept { + if (fd >= 0) + ::close(fd); + fd = std::exchange(other.fd, -1); + return *this; + } +}; +sockaddr_in loopback(std::uint16_t port) { + sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + address.sin_port = htons(port); + return address; +} +bool nonblocking(int fd) { + const int flags = ::fcntl(fd, F_GETFL); + return flags >= 0 && ::fcntl(fd, F_SETFL, flags | O_NONBLOCK) == 0; +} +bool receive(int fd, std::vector& queued, bool drop, std::atomic& discarded) { + if (queued.size() >= 65536) + return true; + std::array bytes{}; + const auto count = ::recv(fd, bytes.data(), std::min(bytes.size(), 65536 - queued.size()), 0); + if (count < 0) + return errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR; + if (count == 0) + return false; + if (drop) + discarded.fetch_add(static_cast(count)); + else + queued.insert(queued.end(), bytes.begin(), bytes.begin() + count); + return true; +} +bool transmit(int fd, std::vector& queued) { + if (queued.empty()) + return true; + const auto count = ::send(fd, queued.data(), queued.size(), MSG_NOSIGNAL); + if (count < 0) + return errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR; + if (count == 0) + return false; + queued.erase(queued.begin(), queued.begin() + count); + return true; +} +} // namespace +struct PostgresRelay::Impl { + Socket listener; + std::uint16_t port{}, backend{}; + std::atomic requests{}, responses{}, failed{}; + std::atomic request_bytes{}, response_bytes{}; + std::jthread worker; + explicit Impl(std::uint16_t target) + : listener(::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0)), backend(target) { + auto address = loopback(0); + socklen_t size = sizeof(address); + if (target == 0 || listener.fd < 0 || + ::bind(listener.fd, reinterpret_cast(&address), sizeof(address)) != + 0 || + ::listen(listener.fd, 8) != 0 || !nonblocking(listener.fd) || + ::getsockname(listener.fd, reinterpret_cast(&address), &size) != 0) + throw std::runtime_error("private PostgreSQL relay listener"); + port = ntohs(address.sin_port); + worker = std::jthread([this](const std::stop_token& stop) { + try { + run(stop); + } catch (...) { + failed = true; + } + }); + } + void run(const std::stop_token& stop) { + Socket client, server; + std::vector upstream, downstream; + while (!stop.stop_requested()) { + if (requests.load()) { + request_bytes.fetch_add(upstream.size()); + upstream.clear(); + } + if (responses.load()) { + response_bytes.fetch_add(downstream.size()); + downstream.clear(); + } + std::array fds{ + {{listener.fd, static_cast(client.fd < 0 ? POLLIN : 0), 0}, + {client.fd, static_cast(POLLIN | (downstream.empty() ? 0 : POLLOUT)), 0}, + {server.fd, static_cast(POLLIN | (upstream.empty() ? 0 : POLLOUT)), 0}}}; + const auto count = ::poll(fds.data(), fds.size(), 10); + if (count < 0) { + if (errno == EINTR) + continue; + failed = true; + return; + } + if (fds[0].revents & POLLIN) { + client = Socket(::accept4(listener.fd, nullptr, nullptr, SOCK_CLOEXEC)); + server = Socket(::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0)); + const auto target = loopback(backend); + if (client.fd < 0 || server.fd < 0 || + ::connect(server.fd, reinterpret_cast(&target), + sizeof(target)) != 0 || + !nonblocking(client.fd) || !nonblocking(server.fd)) { + failed = true; + return; + } + } + bool alive = true; + if (fds[1].revents & POLLIN) + alive = receive(client.fd, upstream, requests.load(), request_bytes); + if (alive && (fds[2].revents & POLLIN)) + alive = receive(server.fd, downstream, responses.load(), response_bytes); + if (alive && (fds[1].revents & POLLOUT)) + alive = transmit(client.fd, downstream); + if (alive && (fds[2].revents & POLLOUT)) + alive = transmit(server.fd, upstream); + if (!alive || (fds[1].revents & (POLLERR | POLLHUP | POLLNVAL)) || + (fds[2].revents & (POLLERR | POLLHUP | POLLNVAL))) { + client = Socket{}; + server = Socket{}; + upstream.clear(); + downstream.clear(); + } + } + } +}; +PostgresRelay::PostgresRelay(std::uint16_t backend_port) + : impl_(std::make_unique(backend_port)) {} +PostgresRelay::~PostgresRelay() = default; +std::string PostgresRelay::port() const { + return std::to_string(impl_->port); +} +void PostgresRelay::drop_requests(bool value) { + impl_->requests = value; +} +void PostgresRelay::drop_responses(bool value) { + impl_->responses = value; +} +std::uint64_t PostgresRelay::dropped_requests() const { + return impl_->request_bytes.load(); +} +std::uint64_t PostgresRelay::dropped_responses() const { + return impl_->response_bytes.load(); +} +bool PostgresRelay::healthy() const { + return !impl_->failed.load(); +} +} // namespace evo::test diff --git a/tests/fixtures/postgres_relay.hpp b/tests/fixtures/postgres_relay.hpp new file mode 100644 index 0000000..9313c0a --- /dev/null +++ b/tests/fixtures/postgres_relay.hpp @@ -0,0 +1,26 @@ +#pragma once + +// Test-only opaque TCP relay. TLS remains end-to-end between libpq and PostgreSQL. +#include +#include +#include + +namespace evo::test { +class PostgresRelay { + public: + explicit PostgresRelay(std::uint16_t backend_port); + ~PostgresRelay(); + PostgresRelay(const PostgresRelay&) = delete; + PostgresRelay& operator=(const PostgresRelay&) = delete; + [[nodiscard]] std::string port() const; + void drop_requests(bool); + void drop_responses(bool); + [[nodiscard]] std::uint64_t dropped_requests() const; + [[nodiscard]] std::uint64_t dropped_responses() const; + [[nodiscard]] bool healthy() const; + + private: + struct Impl; + std::unique_ptr impl_; +}; +} // namespace evo::test diff --git a/tests/population_publication_tests.cpp b/tests/population_publication_tests.cpp index 156500b..e8a634f 100644 --- a/tests/population_publication_tests.cpp +++ b/tests/population_publication_tests.cpp @@ -4,8 +4,10 @@ #include "evo/persistence/sha256.hpp" #include "evo/server/cellular_region_host.hpp" #include "fixtures/cellular_ecology.hpp" +#include "fixtures/postgres_relay.hpp" #include +#include #include #include #include @@ -96,6 +98,73 @@ struct Fixture { head.head.end_tick + count}; } }; +std::uint16_t migration_backend_port() { + const auto port = environment("EVO_MIGRATION_TCP_PORT"); + unsigned value{}; + const auto [end, error] = std::from_chars(port.data(), port.data() + port.size(), value); + require(error == std::errc{} && end == port.data() + port.size() && value > 0 && value <= 65535, + "invalid private migration TLS port"); + return static_cast(value); +} +PgOptions migration_tls_options(const test::PostgresRelay& relay) { + return {"localhost", + "127.0.0.1", + relay.port(), + environment("PGDATABASE"), + environment("EVO_PG_RUNTIME_USER"), + {}, + environment("EVO_MIGRATION_CA")}; +} +std::unique_ptr migration_tls_connection(const test::PostgresRelay& relay) { + auto db = take(Postgres::connect(migration_tls_options(relay), 2s), + "migration verify-full TLS connection"); + take(db->query("SELECT 1", {}, 2s), "migration TLS warmup"); + return db; +} +void migration_transport_commit(Fixture& f, const CellularMigrationCommitPrepared& prepared, + RegionKey sk, RegionKey dk, const LeaseGrant& sl, + const LeaseGrant& dl, const RegionHead& source_before, + const RegionHead& destination_before) { + { + test::PostgresRelay relay(migration_backend_port()); + auto wrong = migration_tls_options(relay); + wrong.host = "untrusted.evolution.test"; + require(!Postgres::connect(wrong, 1s), "relay bypassed TLS hostname verification"); + } + { + test::PostgresRelay relay(migration_backend_port()); + auto db = migration_tls_connection(relay); + relay.drop_requests(true); + const auto lost = retry_cellular_migration_commit(*db, sl, dl, prepared, 300ms); + require( + !lost && lost.error().kind == PublicationError::unknown && + lost.error().prepared.source_manifest == prepared.source_manifest && + lost.error().prepared.destination_manifest == prepared.destination_manifest && + relay.dropped_requests() > 0 && relay.healthy() && + take(load_region_checkpoint(*f.db, sk, 5s), "lost request source head").head == + source_before && + take(load_region_checkpoint(*f.db, dk, 5s), "lost request destination head").head == + destination_before, + "a dropped TLS request changed ownership or lost the exact pair"); + } + { + test::PostgresRelay relay(migration_backend_port()); + auto db = migration_tls_connection(relay); + relay.drop_responses(true); + const auto lost = retry_cellular_migration_commit(*db, sl, dl, prepared, 500ms); + require( + !lost && lost.error().kind == PublicationError::unknown && + lost.error().prepared.source_manifest == prepared.source_manifest && + lost.error().prepared.destination_manifest == prepared.destination_manifest && + relay.dropped_responses() > 0 && relay.healthy() && + take(load_region_checkpoint(*f.db, sk, 5s), "lost reply source head").manifest == + prepared.source_manifest && + take(load_region_checkpoint(*f.db, dk, 5s), "lost reply destination head") + .manifest == prepared.destination_manifest, + "lost TLS receipt was treated as rollback or committed only one head"); + } + std::puts("migration TLS request/reply loss and hostname verification passed"); +} void funded_source(Fixture& f) { f.build.fill(0xDA); const std::array build{hexadecimal(f.build)}; @@ -746,6 +815,196 @@ void boundary_upgrades(Fixture& f) { require(f.raw->format_version() == cellular_lives_migration_format, "reverse transition downgraded accounting owner"); } +void migration_owner_partitions(Fixture& f, RegionKey destination_key, + FilesystemObjectStore& destination_objects, + LeaseGrant& destination_lease, + CommittedCheckpoint& destination_head, std::uint64_t genome) { + if (!std::getenv("EVO_MIGRATION_TCP_PORT")) + return; + const std::array builds{f.build}; + for (unsigned lost_source = 0; lost_source < 2; ++lost_source) { + test::PostgresRelay source_relay(migration_backend_port()), + destination_relay(migration_backend_port()); + const auto make_host = [&](RegionKey key, const test::PostgresRelay& relay) { + auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), key, + max_cellular_population_bytes), + "partition owner objects"); + CellularHostConfig config{key, 1, {}, f.build, {f.build}}; + config.lease = {3000ms, 400ms, 300ms, 150ms, 250ms}; + const auto options = migration_tls_options(relay); + return take(CellularRegionHost::create( + config, [options] { return Postgres::connect(options, 500ms); }, + std::move(objects)), + "partition owner host"); + }; + take(release_region_lease(*f.db, f.key, f.lease, 5s), "release source to TLS owner"); + take(release_region_lease(*f.db, destination_key, destination_lease, 5s), + "release destination to TLS owner"); + auto source_host = make_host(f.key, source_relay), + destination_host = make_host(destination_key, destination_relay); + require(source_host->step().ready && destination_host->step().ready, + "TLS owners before freeze"); + const auto sl = take(source_host->authority(), "TLS source fence"), + dl = take(destination_host->authority(), "TLS destination fence"); + const auto source_bytes = source_host->state()->snapshot(), + destination_bytes = destination_host->state()->snapshot(); + const auto source_before = f.head, destination_before = destination_head; + const CellularMigrationRequest request{ + f.id(), + {f.key.region, f.head.head.commit, f.id(), f.head.head.digest}, + {destination_key.region, destination_head.head.commit, f.id(), + destination_head.head.digest}, + genome}; + source_relay.drop_responses(true); + const auto unknown_prepare = + source_host->prepare_migration_freeze(destination_head, destination_objects, request); + require(!unknown_prepare && + unknown_prepare.error().publication == PublicationError::unknown && + !source_host->status().ready && source_relay.dropped_responses() > 0, + "lost preparation receipt did not pause source ownership"); + const auto frozen = + take(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, sl, 5s), + "independent committed preparation observation"), + "missing preparation after TLS reply loss"); + source_relay.drop_responses(false); + const auto reconcile_source_until = LeaseClock::now() + 2s; + while (!source_host->status().migration_frozen && + LeaseClock::now() < reconcile_source_until) { + source_host->step(); + destination_host->step(); + std::this_thread::sleep_for(25ms); + } + require(source_host->status().migration_frozen && !source_host->status().ready, + "source lost the exact prepared request after TLS reconnect"); + destination_relay.drop_responses(true); + const auto unknown_stage = + destination_host->stage_migration(f.head, *f.objects, frozen.trace); + require(!unknown_stage && unknown_stage.error().publication == PublicationError::unknown && + !destination_host->status().ready && + destination_relay.dropped_responses() > 0 && + take(load_cellular_migration_stage(*f.db, destination_objects, destination_key, + dl, 5s), + "independent committed stage observation") + .has_value(), + "lost stage receipt activated the ghost or lost retained work"); + destination_relay.drop_responses(false); + const auto reconcile_destination_until = LeaseClock::now() + 2s; + while (!destination_host->status().migration_frozen && + LeaseClock::now() < reconcile_destination_until) { + source_host->step(); + destination_host->step(); + std::this_thread::sleep_for(25ms); + } + require(source_host->status().migration_frozen && + destination_host->status().migration_frozen, + "TLS owners failed exact stage reconciliation"); + auto& isolated = lost_source ? source_host : destination_host; + auto& healthy = lost_source ? destination_host : source_host; + auto& relay = lost_source ? source_relay : destination_relay; + const auto isolated_key = lost_source ? f.key : destination_key; + auto& replacement = lost_source ? f.lease : destination_lease; + const auto healthy_key = lost_source ? destination_key : f.key; + auto& healthy_lease = lost_source ? destination_lease : f.lease; + relay.drop_requests(true); + const auto expire_until = LeaseClock::now() + 3500ms; + while (LeaseClock::now() < expire_until) { + isolated->step(); + healthy->step(); + std::this_thread::sleep_for(25ms); + } + require(!isolated->status().ready && !isolated->authority().has_value() && + !isolated->advance_unattended(64) && relay.dropped_requests() > 0 && + healthy->status().phase == HostPhase::running && + healthy->status().epoch == (lost_source ? dl.epoch : sl.epoch) && + healthy->status().migration_frozen, + "isolated owner retained authority or healthy peer released a frozen ghost"); + replacement = take(acquire_region_lease(*f.db, isolated_key, f.build, 60000ms, 5s), + "replacement after actual TLS partition expiry"); + require(replacement.epoch > (lost_source ? sl.epoch : dl.epoch), + "replacement reused partitioned epoch"); + if (lost_source) + require(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, replacement, 5s), + "replacement source retains freeze") + .has_value(), + "source lease expiry implicitly aborted"); + else + require(take(load_cellular_migration_stage(*f.db, destination_objects, destination_key, + replacement, 5s), + "replacement destination retains stage") + .has_value(), + "destination lease expiry activated ghost"); + const auto stale = + publish_cellular_migration(*f.db, *f.objects, destination_objects, sl, dl, f.head, + destination_head, frozen.trace, f.build, 5s); + require(!stale && stale.error().store.kind == LeaseStoreError::fenced && + take(load_region_checkpoint(*f.db, f.key, 5s), "partition source head").head == + source_before.head && + take(load_region_checkpoint(*f.db, destination_key, 5s), + "partition destination head") + .head == destination_before.head, + "stale partitioned fence published a migration half"); + healthy->step(); + const auto current_source = lost_source ? replacement : sl; + if (lost_source) { + test::PostgresRelay abort_relay(migration_backend_port()); + auto db = migration_tls_connection(abort_relay); + abort_relay.drop_responses(true); + const auto unknown = + abort_cellular_migration_freeze(*db, f.key, current_source, request.request, 300ms); + require(!unknown && unknown.error().kind == LeaseStoreError::unknown && + abort_relay.dropped_responses() > 0 && abort_relay.healthy(), + "lost replacement abort receipt was not unknown"); + take(abort_cellular_migration_freeze(*f.db, f.key, current_source, request.request, 5s), + "exact replacement abort reconciliation"); + } else { + source_relay.drop_responses(true); + const auto unknown = source_host->abort_migration_freeze(request.request); + require(!unknown && !source_host->status().ready && + source_relay.dropped_responses() > 0, + "lost owning-source abort receipt resumed the region"); + source_relay.drop_responses(false); + } + require(!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, current_source, 5s), + "independent abort source observation") + .has_value(), + "lost abort receipt reopened source stock"); + const auto resume_until = LeaseClock::now() + 2s; + while (!healthy->step().ready && LeaseClock::now() < resume_until) + std::this_thread::sleep_for(25ms); + require(healthy->status().ready && healthy->state()->snapshot() == + (lost_source ? destination_bytes : source_bytes), + "healthy owner resumed different stock after abort"); + relay.drop_requests(false); + isolated->step(); + require(!isolated->status().ready && !isolated->advance_unattended(64), + "healed stale owner resumed despite a replacement fence"); + const auto isolated_manifest = + take(decode_manifest(lost_source ? f.head.manifest : destination_head.manifest), + "isolated head profile"); + const auto& objects = lost_source ? *f.objects : destination_objects; + std::vector recovered; + if (isolated_manifest.profile == CheckpointProfile::cellular_population_v1) + recovered = take(recover_population_checkpoint(*f.db, objects, isolated_key, + replacement, builds, 5s), + "replacement population replay") + .state->state() + .snapshot(); + else + recovered = take(recover_cellular_checkpoint(*f.db, objects, isolated_key, replacement, + builds, 5s), + "replacement raw replay") + .state->snapshot(); + require(recovered == (lost_source ? source_bytes : destination_bytes) && + source_relay.healthy() && destination_relay.healthy(), + "replacement or relay lost exact stock"); + source_host->drain(); + destination_host->drain(); + healthy_lease = take(acquire_region_lease(*f.db, healthy_key, f.build, 60000ms, 5s), + "return healthy TLS owner authority"); + } + std::puts("migration independent TLS owner partitions, expiry, replacement and heal passed"); +} + void migration_publication(Fixture& f, const RegionKey destination_key, FilesystemObjectStore& destination_objects, LeaseGrant& destination_lease, CommittedCheckpoint& destination_head) { @@ -920,6 +1179,9 @@ void migration_publication(Fixture& f, const RegionKey destination_key, take(load_region_checkpoint(*f.db, dk, 5s), "killed pair destination").head == destination_before.head, "pre-transaction paired death changed a head"); + if (direction == 0 && std::getenv("EVO_MIGRATION_TCP_PORT")) + migration_transport_commit(f, prepared, sk, dk, sl, dl, source_before.head, + destination_before.head); if (direction == 0) { auto locker = connection(true), timed = connection(); take(locker->query("BEGIN", {}, 5s), "pair ambiguity transaction"); @@ -1025,6 +1287,8 @@ void migration_publication(Fixture& f, const RegionKey destination_key, dl = take(acquire_region_lease(*f.db, dk, f.build, 60000ms, 5s), "return paired destination authority"); } + migration_owner_partitions(f, destination_key, destination_objects, destination_lease, + destination_head, genome); } // Race the actual two-head transaction against abort on independent connections. const auto prepare_pair = [&](bool reverse) { diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index c43d050..5ed6f2b 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -3,6 +3,7 @@ set -euo pipefail # Usage: postgres-check.sh [--full|--sanitizer|--suite-only] # postgres-check.sh --life-reservations-only [] # postgres-check.sh --population-only +# postgres-check.sh --migration-faults (private loopback TLS) # --full (default) is the complete qualification: the C++ suite, the backup, DR and # upgrade process checks, then the SQL-only migration, permission and checksum checks. # --sanitizer keeps everything that runs the binaries built beside the test binary @@ -12,6 +13,7 @@ set -euo pipefail root="$(cd "$(dirname "$0")/../.." && pwd)" mode=full case "${1:-}" in + --migration-faults) mode=migration-faults; shift ;; --population-only) mode=population; shift ;; --full) shift ;; --sanitizer) mode=sanitizer; shift ;; @@ -41,9 +43,26 @@ test_dir="$(mktemp -d /tmp/evo-pg-test.XXXXXXXX)" chmod 700 "$test_dir" export PGHOST="$test_dir" PGPORT=55432 PGDATABASE=postgres PGUSER="$(id -un)" unset PGHOSTADDR PGPASSWORD PGSERVICE PGSERVICEFILE +unset EVO_MIGRATION_TCP_PORT EVO_MIGRATION_CA initdb -D "$test_dir/data" --no-locale --encoding=UTF8 --auth-local=trust --auth-host=reject > "$test_dir/init.log" trap 'rc=$?; timing_summary "exit $rc" >&2; pg_ctl -D "$test_dir/data" -m immediate stop > /dev/null || true' EXIT -pg_ctl -D "$test_dir/data" -l "$test_dir/server.log" -o "-k $test_dir -p 55432 -c listen_addresses=''" -w start +if [ "$mode" = migration-faults ]; then + # One disposable cluster only; the owner remains Unix-socket-only. No production + # credential or adapter TLS exception. The relay forwards encrypted bytes. + openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj /CN=localhost \ + -addext 'subjectAltName=DNS:localhost,IP:127.0.0.1' \ + -keyout "$test_dir/server.key" -out "$test_dir/server.crt" > "$test_dir/tls.log" 2>&1 + chmod 600 "$test_dir/server.key" + PGPORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])')" + export PGPORT EVO_MIGRATION_TCP_PORT="$PGPORT" EVO_MIGRATION_CA="$test_dir/server.crt" + printf '%s\n' 'local all all trust' \ + 'hostssl postgres evo_runtime_login 127.0.0.1/32 trust' \ + 'host all all 127.0.0.1/32 reject' 'host all all ::1/128 reject' > "$test_dir/data/pg_hba.conf" + pg_ctl -D "$test_dir/data" -l "$test_dir/server.log" \ + -o "-k $test_dir -p $PGPORT -c listen_addresses=127.0.0.1 -c ssl=on -c ssl_cert_file=$test_dir/server.crt -c ssl_key_file=$test_dir/server.key" -w start +else + pg_ctl -D "$test_dir/data" -l "$test_dir/server.log" -o "-k $test_dir -p 55432 -c listen_addresses=''" -w start +fi # Runtime group role receives grants; a separate login inherits only that role. psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN' \ -c 'CREATE ROLE evo_runtime_login LOGIN IN ROLE evo_runtime_test' @@ -58,7 +77,7 @@ if [ "$mode" = life-reservations ]; then bash "$root/tools/ci/life-reservations-concurrency.sh" exit 0 fi -if [ "$mode" = population ]; then +if [ "$mode" = population ] || [ "$mode" = migration-faults ]; then phase population "$1" --population-only exit 0 @@ -85,6 +104,10 @@ if [ "$mode" = suite-only ]; then echo "postgres-check[suite-only]: $1 passed; process and SQL checks run in --full" exit 0 fi +if [ "$mode" = full ] || [ "$mode" = sanitizer ]; then + phase migration-partitions + bash "$root/tools/ci/postgres-check.sh" --migration-faults "$1" +fi if [ "$mode" = full ]; then phase life-reservations psql -X -v ON_ERROR_STOP=1 -f "$root/tools/ci/life-reservations-check.sql" > /dev/null