From 7aec68db8e14806d6224ba12691c4e9b00e6f6bd Mon Sep 17 00:00:00 2001 From: dingo Date: Mon, 28 Sep 2026 23:09:28 -0400 Subject: [PATCH] Complete M7.8 operations endpoints and region server image ADR-0066: evo-server --ops-listen serves /livez (loop responding, never authority), /readyz (200 only while the host holds the lease with a verified recovered head) and /metrics (Prometheus text: 30 fixed series for readiness, phase, lease epoch/remaining, simulated/committed ticks, commit lag, checkpoint age, tick debt, queue use and bounds, and counters; the only labels are a 6-value phase and 4-value drop reason). The dependency-free OpsServer runs on the host loop with no thread: GET on exact paths, 405/404/400/431 responses, 8 connections with excess refused on accept, 4 KiB requests, 1 s deadlines, 16 KiB responses. HostStatus/phase types moved to the portable evo::ops library with a Windows-tested renderer. evo-server now refuses to start without its required identity and connection settings. Adds the server-runtime Docker target (evo-server + pinned libpq, UID 10001, no credentials), a CI server-image job with smoke script and optional publishing, and a Kubernetes example outside Argo. Validated: Windows CTest 59/59; Linux Debug/Release/clang-tidy/ASan CTest 59/59; PostgreSQL suites under Release, clang-tidy and ASan with real process probes; two ops mutations caught. Docker build/smoke and Gitea CI not executed locally. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/build.yaml | 26 +++ CMakeLists.txt | 18 +- apps/evo-server/main.cpp | 111 ++++++---- deploy/README.md | 13 ++ deploy/docker/Dockerfile | 13 ++ .../region-server/deployment.yaml.example | 143 +++++++++++++ docs/ROADMAP.md | 8 + docs/STATUS.md | 21 ++ docs/adr/0015-observability.md | 2 + docs/adr/0066-operations-endpoints.md | 101 +++++++++ docs/adr/README.md | 1 + docs/engineering/VALIDATION.md | 41 ++++ docs/infrastructure/POSTGRESQL.md | 12 ++ include/evo/server/ops.hpp | 57 ++++++ include/evo/server/ops_http.hpp | 67 ++++++ include/evo/server/region_host.hpp | 26 +-- src/server/ops.cpp | 104 ++++++++++ src/server/ops_http.cpp | 175 ++++++++++++++++ src/server/region_host.cpp | 7 + tests/ops_tests.cpp | 119 +++++++++++ tests/postgres_tests.cpp | 3 + tests/region_ops_tests.cpp | 193 ++++++++++++++++++ tools/ci/server-image-smoke.sh | 26 +++ 23 files changed, 1222 insertions(+), 65 deletions(-) create mode 100644 deploy/kubernetes/region-server/deployment.yaml.example create mode 100644 docs/adr/0066-operations-endpoints.md create mode 100644 include/evo/server/ops.hpp create mode 100644 include/evo/server/ops_http.hpp create mode 100644 src/server/ops.cpp create mode 100644 src/server/ops_http.cpp create mode 100644 tests/ops_tests.cpp create mode 100644 tests/region_ops_tests.cpp create mode 100644 tools/ci/server-image-smoke.sh diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index 2f0592c..5bb87f5 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -76,6 +76,32 @@ jobs: export SOURCE_IMAGE="aevolution-postgres-tools:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" bash tools/ci/publish-image.sh + server-image: + needs: [checks] + runs-on: ${{ vars.CI_RUNNER_LABEL || 'ubuntu-22.04' }} + timeout-minutes: 60 + steps: + - name: Checkout + uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + - name: Build and smoke-test the region server image + run: | + set -eu + image="aevolution-server:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + docker build --file deploy/docker/Dockerfile --target server-runtime --build-arg "UBUNTU_IMAGE=$UBUNTU_IMAGE" --tag "$image" . + bash tools/ci/server-image-smoke.sh "$image" + - name: Publish configured region server image + if: ${{ gitea.ref == 'refs/heads/main' && gitea.event_name != 'pull_request' && vars.HARBOR_SERVER_REPOSITORY != '' }} + env: + HARBOR_PROJECT: ${{ vars.HARBOR_PROJECT }} + HARBOR_REPOSITORY: ${{ vars.HARBOR_SERVER_REPOSITORY }} + HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }} + HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }} + run: | + export SOURCE_IMAGE="aevolution-server:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + bash tools/ci/publish-image.sh + image: needs: [checks] runs-on: ${{ vars.CI_RUNNER_LABEL || 'ubuntu-22.04' }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 901fb99..40f09f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,6 +77,12 @@ add_library(evo::region_lease ALIAS evo_region_lease) target_include_directories(evo_region_lease PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") target_compile_features(evo_region_lease PUBLIC cxx_std_23) evo_owned_target(evo_region_lease) +# Portable host status and bounded operational exposition (M7.8). +add_library(evo_ops STATIC src/server/ops.cpp) +add_library(evo::ops ALIAS evo_ops) +target_include_directories(evo_ops PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") +target_compile_features(evo_ops PUBLIC cxx_std_23) +evo_owned_target(evo_ops) # Portable checkpoint manifest codec and environmental-v1 payload validation. add_library(evo_region_checkpoint STATIC src/persistence/region_checkpoint.cpp src/persistence/region_batch.cpp) add_library(evo::region_checkpoint ALIAS evo_region_checkpoint) @@ -104,9 +110,9 @@ if(EVO_BUILD_POSTGRES) PRIVATE OpenSSL::Crypto) evo_owned_target(evo_region_store) # M7.6 authoritative region host and its process (ADR-0064). - add_library(evo_region_host STATIC src/server/region_host.cpp) + add_library(evo_region_host STATIC src/server/region_host.cpp src/server/ops_http.cpp) add_library(evo::region_host ALIAS evo_region_host) - target_link_libraries(evo_region_host PUBLIC evo::region_store) + target_link_libraries(evo_region_host PUBLIC evo::region_store evo::ops) evo_owned_target(evo_region_host) add_executable(evo-server apps/evo-server/main.cpp) target_link_libraries(evo-server PRIVATE evo::region_host) @@ -115,7 +121,8 @@ if(EVO_BUILD_POSTGRES) # Contention tests use one thread and one connection per contender. add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp - tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp) + tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp + tests/region_ops_tests.cpp) target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads) add_dependencies(evo-postgres-tests evo-server) # Spawned by the process tests. evo_owned_target(evo-postgres-tests) @@ -133,6 +140,11 @@ if(BUILD_TESTING) evo_owned_target(evo-lineage-region-tests) add_test(NAME lineage-region-contracts COMMAND evo-lineage-region-tests) set_tests_properties(lineage-region-contracts PROPERTIES TIMEOUT 60) + add_executable(evo-ops-tests tests/ops_tests.cpp) + target_link_libraries(evo-ops-tests PRIVATE evo::ops) + evo_owned_target(evo-ops-tests) + add_test(NAME ops-contracts COMMAND evo-ops-tests) + set_tests_properties(ops-contracts PROPERTIES TIMEOUT 60) add_executable(evo-region-batch-tests tests/region_batch_tests.cpp) target_link_libraries(evo-region-batch-tests PRIVATE evo::region_checkpoint) evo_owned_target(evo-region-batch-tests) diff --git a/apps/evo-server/main.cpp b/apps/evo-server/main.cpp index 7caf485..a62993d 100644 --- a/apps/evo-server/main.cpp +++ b/apps/evo-server/main.cpp @@ -11,9 +11,14 @@ // stats max_queued= max_tick_debt= commits= store_failures= // Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined. // +// --ops-listen (M7.8) serves GET /livez (the loop is alive; says +// nothing about authority), /readyz (200 only with current authority) and /metrics +// (Prometheus text). It prints "ops listening " once bound. +// // --crash-at : (tests only, M7.7) SIGKILLs this process at the n-th // occurrence of before-upload, before-transaction, after-commit or after-ack (right // after a step's acknowledgements are written and flushed). +#include "evo/server/ops_http.hpp" #include "evo/server/region_host.hpp" #include @@ -48,6 +53,7 @@ struct Options { std::chrono::milliseconds drain{10000}; std::string crash_point; std::uint64_t crash_ordinal{}; + std::string ops_listen; }; // Kills this process without any cleanup: the real SIGKILL, at a chosen boundary. @@ -140,6 +146,8 @@ std::optional parse(int argc, char** argv) { o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms}; else if (flag == "--lease-profile" && value == "default") o.host.lease = {}; + else if (flag == "--ops-listen") + o.ops_listen = value; else if (flag == "--crash-at") { const auto colon = value.find(':'); const auto ordinal = @@ -154,6 +162,12 @@ std::optional parse(int argc, char** argv) { } else return std::nullopt; } + // Every identity and connection setting is required: there is no default + // region, database, object root or build to fall back on. + if (o.host.key.world == 0 || o.host.key.region == 0 || o.host.frame == 0 || o.pg.host.empty() || + o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() || + o.host.build == persistence::Sha256Digest{}) + return std::nullopt; if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end()) o.host.retained_builds.push_back(o.host.build); return o; @@ -194,23 +208,6 @@ std::optional parse_input(std::string_view line) { return in; } -const char* phase_name(server::HostPhase p) { - switch (p) { - case server::HostPhase::acquiring: - return "acquiring"; - case server::HostPhase::running: - return "running"; - case server::HostPhase::resolving: - return "resolving"; - case server::HostPhase::draining: - return "draining"; - case server::HostPhase::stopped: - return "stopped"; - case server::HostPhase::quarantined: - return "quarantined"; - } - return "unknown"; -} const char* admission_name(server::AdmissionError e) { switch (e) { case server::AdmissionError::not_ready: @@ -226,20 +223,6 @@ const char* admission_name(server::AdmissionError e) { } return "unknown"; } -const char* dropped_name(server::Unacknowledged u) { - switch (u) { - case server::Unacknowledged::refused: - return "refused"; - case server::Unacknowledged::drained: - return "drained"; - case server::Unacknowledged::fenced: - return "fenced"; - case server::Unacknowledged::quarantined: - return "quarantined"; - } - return "unknown"; -} - // Reads complete, bounded lines from stdin without blocking. class LineReader { public: @@ -294,6 +277,41 @@ template int serve(Options& o) { return 2; } auto& host = **made; + std::unique_ptr ops; + if (!o.ops_listen.empty()) { + auto listening = server::OpsServer::listen(o.ops_listen); + if (!listening) { + std::puts("error ops-listen"); + return 2; + } + ops = std::move(*listening); + std::printf("ops listening %u\n", static_cast(ops->port())); + std::fflush(stdout); + } + server::OpsCounters counters; + counters.queue_capacity_inputs = o.host.max_queued_inputs; + counters.queue_capacity_bytes = o.host.max_queued_bytes; + std::uint64_t last_sequence = 0; + auto last_checkpoint = std::chrono::steady_clock::now(); + bool recovered = false; + const auto handler = [&](std::string_view path) -> server::OpsResponse { + const auto& s = host.status(); + if (path == "/livez") + return {200, "live\n"}; + if (path == "/readyz") + return s.ready ? server::OpsResponse{200, "ready\n"} + : server::OpsResponse{503, std::string("not ready: ") + + server::phase_name(s.phase) + "\n"}; + if (path == "/metrics") { + counters.checkpoint_age_ms = + static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now() - last_checkpoint) + .count()); + return {200, server::render_metrics(s, counters), + "text/plain; version=0.0.4; charset=utf-8"}; + } + return {404, "not found\n"}; + }; LineReader reader; bool input_open = true, ready = false; std::optional pending; @@ -346,28 +364,46 @@ template int serve(Options& o) { } const auto& s = host.step(std::chrono::steady_clock::now()); max_queued = std::max(max_queued, s.queued); + // Checkpoint age restarts at recovery and at every confirmed commit. + if ((!recovered && s.phase == server::HostPhase::running) || + s.committed_sequence != last_sequence) { + recovered = recovered || s.phase == server::HostPhase::running; + last_sequence = s.committed_sequence; + last_checkpoint = std::chrono::steady_clock::now(); + } + if (s.phase == server::HostPhase::acquiring) + recovered = false; if (!phase || *phase != s.phase) { phase = s.phase; - std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n", phase_name(s.phase), - static_cast(s.epoch), + std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n", + server::phase_name(s.phase), static_cast(s.epoch), static_cast(s.committed_tick), static_cast(s.simulated_tick)); } set_ready(s.ready); const auto acknowledged = host.take_acks(); + counters.acks += acknowledged.size(); for (const auto& a : acknowledged) std::printf("ack %s %llu %llu\n", text(a.request).c_str(), static_cast(a.tick), static_cast(a.commit_sequence)); - for (const auto& x : host.take_outcomes()) + const auto outcomes = host.take_outcomes(); + counters.outcomes += outcomes.size(); + for (const auto& x : outcomes) std::printf("outcome %llu %u %llu %llu\n", static_cast(x.decision.tick), static_cast(x.decision.actor), static_cast(x.decision.organism), static_cast(x.commit_sequence)); - for (const auto& d : host.take_dropped()) - std::printf("dropped %s %s\n", text(d.input.request).c_str(), dropped_name(d.reason)); + const auto dropped = host.take_dropped(); + for (const auto& d : dropped) + ++counters.dropped[static_cast(d.reason)]; + for (const auto& d : dropped) + std::printf("dropped %s %s\n", text(d.input.request).c_str(), + server::unacknowledged_name(d.reason)); std::fflush(stdout); + if (ops) + ops->serve(handler); if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal) crash_now(); const auto finish = [&](int code) { @@ -398,7 +434,8 @@ int main(int argc, char** argv) { "--frame N --pg-host H --pg-port P --pg-database D --pg-user U " "--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A " "--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] " - "[--lease-profile default|fast] [--ready-file PATH]\n", + "[--lease-profile default|fast] [--ready-file PATH] " + "[--ops-listen ADDRESS:PORT]\n", stderr); return 2; } diff --git a/deploy/README.md b/deploy/README.md index e5978ad..e71aacc 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -134,3 +134,16 @@ migrator/runtime credentials. The example is outside active Argo resources until an operator supplies the database endpoint, role, Secret and egress rules. No region server or world-state migration is deployed by M7.2. Schema v2 (M7.3) additionally requires `EVO_PG_RUNTIME_ROLE`, naming an operator-created NOLOGIN runtime group role. + +## Region server image (M7.8) + +The `server-image` job builds the Docker target `server-runtime` (`evo-server` +plus the pinned libpq, UID 10001, no credentials). It smoke-tests the image with +`tools/ci/server-image-smoke.sh` and publishes only when Gitea variable +`HARBOR_SERVER_REPOSITORY` is set. + +`kubernetes/region-server/deployment.yaml.example` is outside Argo. It shows: +- probes: `/readyz` means authority, `/livez` means only liveness; +- the runtime Secret, object PVC and a NetworkPolicy. +Pin the digest and provision the region first. Without a gateway, the region +receives no inputs. diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index cc7854f..f5ca1dc 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -82,6 +82,19 @@ ENV PATH="/opt/postgres/bin:${PATH}" LD_LIBRARY_PATH=/opt/postgres/lib USER 10001:10001 ENTRYPOINT ["bash", "/opt/evo/tools/migrate-postgres.sh"] +# M7.8 development region server image (ADR-0066): evo-server with the pinned +# libpq, non-root, no credentials. Connection secrets are mounted at runtime only. +FROM ${UBUNTU_IMAGE} AS server-runtime +RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 libgcc-s1 libssl3t64 zlib1g ca-certificates && rm -rf /var/lib/apt/lists/* && groupadd --gid 10001 evo && useradd --uid 10001 --gid 10001 --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin evo +COPY --from=postgres-build /source/build/postgres/evo-server /usr/local/bin/evo-server +COPY --from=postgres-build /source/.cache/postgres/install/lib/libpq.so* /opt/postgres/lib/ +COPY --from=postgres-build /source/.cache/postgres/postgresql-18.6/COPYRIGHT /usr/share/doc/evo-postgres/COPYRIGHT +ENV LD_LIBRARY_PATH=/opt/postgres/lib +USER 10001:10001 +WORKDIR /tmp +EXPOSE 9100 +ENTRYPOINT ["/usr/local/bin/evo-server"] + FROM toolchain AS build RUN cmake --preset release -B build/container \ -DCMAKE_CXX_COMPILER=g++-14 -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=/opt/evo/bin \ diff --git a/deploy/kubernetes/region-server/deployment.yaml.example b/deploy/kubernetes/region-server/deployment.yaml.example new file mode 100644 index 0000000..4db8be2 --- /dev/null +++ b/deploy/kubernetes/region-server/deployment.yaml.example @@ -0,0 +1,143 @@ +# M7.8 development region server (ADR-0066). NOT in the active Argo kustomization. +# Before applying to evolution-dev: pin the server image digest, provision the region +# (owner-only), create the runtime Secret and object PVC, and add DB/DNS egress for +# the real endpoint. Never target World0 from this example. +# +# Authority, not liveness, gates traffic: /readyz is 200 only while this pod holds +# the region lease and has recovered and verified the committed head. /livez only +# says the loop is responding. A probe can wait up to one bounded I/O step (commit +# deadline 5 s), so probe timeouts below exceed it. +# +# There is no gateway yet: stdin is not attached, so the region hosts authority but +# receives no inputs (tick debt grows by design; nothing is invented). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: evolution-region-7-1 + namespace: evolution-dev + labels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"} +spec: + replicas: 1 + strategy: {type: Recreate} # Never two pods racing; the lease still arbitrates. + selector: + matchLabels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"} + template: + metadata: + labels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"} + spec: + automountServiceAccountToken: false + terminationGracePeriodSeconds: 30 # SIGTERM drains within --drain-ms 20000. + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: {type: RuntimeDefault} + initContainers: + # libpq rejects group-readable password files: copy to a private 0600 file. + - name: pgpass + image: registry.aeternalisgames.com/aeternalis/aevolution-server:replace-with-immutable-digest + command: ["/bin/sh", "-c", "install -m 0600 /run/postgres/pgpass /run/evo/pgpass"] + securityContext: {readOnlyRootFilesystem: true, allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}} + volumeMounts: + - {name: connection, mountPath: /run/postgres, readOnly: true} + - {name: private, mountPath: /run/evo} + containers: + - name: region + image: registry.aeternalisgames.com/aeternalis/aevolution-server:replace-with-immutable-digest + # Flags are "--name value" pairs; see docs/infrastructure/POSTGRESQL.md. + command: + - /usr/local/bin/evo-server + - --profile + - lineage + - --world + - "7" + - --region + - "1" + - --frame + - "9" + - --pg-host + - replace-with-tls-hostname + - --pg-hostaddr + - replace-with-numeric-address + - --pg-port + - "5432" + - --pg-database + - evolution + - --pg-user + - replace-with-runtime-login + - --pg-passfile + - /run/evo/pgpass + - --pg-ca + - /run/postgres/ca.crt + - --objects + - /var/lib/evolution/objects + - --build + - replace-with-retained-build-identity-hex + - --ops-listen + - 0.0.0.0:9100 + - --drain-ms + - "20000" + ports: + - {name: ops, containerPort: 9100} + startupProbe: + httpGet: {path: /livez, port: ops} + periodSeconds: 5 + timeoutSeconds: 8 + failureThreshold: 12 + readinessProbe: + httpGet: {path: /readyz, port: ops} + periodSeconds: 5 + timeoutSeconds: 8 + failureThreshold: 1 # Any doubt about authority removes the pod at once. + livenessProbe: + httpGet: {path: /livez, port: ops} + periodSeconds: 10 + timeoutSeconds: 8 + failureThreshold: 6 + resources: + requests: {cpu: 250m, memory: 128Mi} + limits: {cpu: "1", memory: 512Mi} + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + volumeMounts: + - {name: connection, mountPath: /run/postgres, readOnly: true} + - {name: private, mountPath: /run/evo, readOnly: true} + - {name: objects, mountPath: /var/lib/evolution/objects} + - {name: scratch, mountPath: /tmp} + volumes: + - name: connection + secret: + secretName: evolution-postgres-runtime # Runtime login only; never migrator. + defaultMode: 0440 + items: + - {key: pgpass, path: pgpass} + - {key: ca.crt, path: ca.crt} + - name: private + emptyDir: {medium: Memory, sizeLimit: 1Mi} + - name: objects + persistentVolumeClaim: {claimName: evolution-region-objects} + - name: scratch + emptyDir: {medium: Memory, sizeLimit: 8Mi} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: evolution-region-7-1 + namespace: evolution-dev +spec: + podSelector: + matchLabels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"} + policyTypes: [Ingress, Egress] + ingress: + # Metrics/probes only from the monitoring namespace (kubelet probes use the node). + - from: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: monitoring} + ports: [{port: 9100, protocol: TCP}] + egress: + # Replace with the actual database endpoint; no DNS is needed with a numeric hostaddr. + - to: [{ipBlock: {cidr: 192.0.2.10/32}}] + ports: [{port: 5432, protocol: TCP}] diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index bf8fad4..36ae547 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -576,3 +576,11 @@ M7.7 implemented and locally qualified (2026-09-28), under ADR-0065. - Real SIGKILL at five boundaries, a resumed SIGSTOPped owner, and damaged heads all behave as the crash matrix requires. Next: M7.8 operations endpoints and dev image. + +M7.8 implemented and locally qualified (2026-09-28), under ADR-0066. +- `evo-server --ops-listen` serves `/livez` (loop alive), `/readyz` (current + authority only) and `/metrics` (30 fixed series with bounded labels). +- The responder is dependency-free, single-threaded and bounded. +- The `server-runtime` image is non-root with no secrets; a CI smoke job and a + Kubernetes example are included. +Next: M7.9 clone, migrate and qualify durability. diff --git a/docs/STATUS.md b/docs/STATUS.md index 7b6306f..5cdd026 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -880,3 +880,24 @@ Tail-skip and replay-equality mutations were each caught. The PostgreSQL suites passed under Release, analysis and ASan/UBSan. Next: M7.8 operations endpoints and a dev image. + +## M7.8 operations endpoints and server image (2026-09-28) + +ADR-0066: `evo-server --ops-listen` serves three endpoints: +- `/livez`: the loop is responding; it is never a statement of authority. +- `/readyz`: 200 only with the current lease and a verified recovered head. +- `/metrics`: 30 fixed series whose only labels are a 6-value phase and a 4-value + drop reason. + +The responder is dependency-free, runs on the host's own loop with no thread, and +is bounded: 8 connections, 4 KiB requests, 1 s deadlines, GET on exact paths +only. The new `server-runtime` Docker target is non-root with no credentials, +covered by a CI smoke script, and a Kubernetes example ships outside Argo. + +Real process probes covered serving authority, an unavailable database (live, +503) and a resumed stale process (live, 503, fence counted). Readiness-as-liveness +and an unbounded connection pool were each caught by a test. Windows ran 59/59; +Linux ran 59/59 in four configurations; the PostgreSQL suites passed in three. + +Docker itself was not run locally; an emulated image layout was checked instead. +Next: M7.9. diff --git a/docs/adr/0015-observability.md b/docs/adr/0015-observability.md index 3c713c8..6ada508 100644 --- a/docs/adr/0015-observability.md +++ b/docs/adr/0015-observability.md @@ -3,6 +3,8 @@ ## Status Small proof boundary adopted; Prometheus/Grafana/Loki/Otel adapters proposed for M7–M8. +M7.8 (ADR-0066) implements the region server exposition with a tiny direct +responder and fixed-label metrics; client libraries remain unadopted. ## Context diff --git a/docs/adr/0066-operations-endpoints.md b/docs/adr/0066-operations-endpoints.md new file mode 100644 index 0000000..6c00ca4 --- /dev/null +++ b/docs/adr/0066-operations-endpoints.md @@ -0,0 +1,101 @@ +# ADR-0066: Operations endpoints and development server image + +## Status +Accepted for M7.8, 2026-09-28. It implements the M7 part of ADR-0015 +(observability). It builds on ADR-0064 (host) and ADR-0065 (recovery). There is +still no gateway, alerting rules or production promotion. + +## Context +M7.8 needs: +- separate liveness and readiness; +- metrics for tick debt, commit lag, lease safety, checkpoint age and queue bounds; +- a non-root image with no embedded secrets. + +The named risk is a healthy process mistaken for valid authority. ADR-0015 forbids +unbounded or entity labels, and allows a tiny direct exposition with format tests. +The constitution forbids hidden threads and unreviewed dependencies. + +## Decision +**Endpoints** (`evo-server --ops-listen `): +- `/livez` returns 200 whenever the host loop answers. It says nothing about + authority, and is used only for restarts. +- `/readyz` returns 200 only when `HostStatus::ready` holds: running, recovered and + tail-verified, not draining, and `LeaseAuthority::may_originate`. Otherwise it + returns 503 with the phase name. It is used for traffic, and later gateway + routing must additionally honor the lease epoch. +- `/metrics` returns Prometheus text 0.0.4. + +**Responder** (`OpsServer`, Linux, no dependency): +- Numeric bind only. GET only (405 otherwise), exact paths (404), malformed + requests 400, requests over 4 KiB 431. +- One request per connection, `Connection: close`, `Cache-Control: no-store`. +- At most 8 connections; any excess is closed on accept, never queued. +- Each connection has a 1 s deadline, and responses are capped at 16 KiB. +- `serve()` never blocks and runs on the host's own loop, so answers describe + current state and no thread is added. A complete request that waited behind one + bounded host step is still answered. + +**Metrics** (`evo::ops`, portable and unit-tested): + +| Group | Series | +|---|---| +| Authority | ready; one-hot phase (6 values); lease epoch; lease remaining (ms) | +| Progress | simulated and committed tick and sequence; commit lag; checkpoint age (ms) | +| Queue | tick debt, current and max; queued inputs and bytes; their capacities | +| Counters | commits; store failures; unknown outcomes; fences; acknowledgements; outcomes; dropped (4 reasons) | + +That is 30 series with fixed names. The only labels are `phase` and `reason`, +each from a fixed enum. There are no world, region, entity or request labels: +the scrape target carries region identity. The exposition is about 3.5 KB and at +most 8 KiB. Commit lag saturates at zero rather than wrapping. + +**Image** (`server-runtime` Docker target) contains `evo-server` and the pinned +libpq, and installs only the runtime libraries it needs (libstdc++, libssl3, +zlib, CA certificates): +- It runs as UID/GID 10001 with a read-only root filesystem and all capabilities + dropped. It exposes port 9100 and has no default configuration. +- With missing settings it exits 2 with usage; there is no default region, + database or build. +- CI builds it and smoke-tests it: the usage exit code under restrictions, + non-root identity, resolved libraries, and no credential files. Publishing + requires `HARBOR_SERVER_REPOSITORY`. + +**Deployment example** (`deploy/kubernetes/region-server`, not in Argo): +- One replica with the Recreate strategy. +- The runtime Secret is copied to a private 0600 file by an init container. +- An object PVC; startup, readiness and liveness probes with 8 s timeouts, + longer than one bounded I/O step. +- A NetworkPolicy allowing only monitoring ingress on 9100 and the database egress. + +## Alternatives Considered +- **prometheus-cpp or OpenTelemetry**: add dependencies and threads for about 30 + fixed series. +- **An HTTP library**: a dependency for three GET paths. +- **A separate probe thread**: would answer while the host is stuck in I/O, which + hides a blocked loop and adds shared state. +- **Readiness from process health or phase alone**: exactly the named risk. + +## Benefits +- Probes and metrics cannot claim authority the lease does not grant. +- Label cardinality is fixed by construction. +- The image carries nothing environment-specific. + +## Costs +- Probes can wait for one bounded host step (up to the 5 s commit deadline), so + probe timeouts must exceed it. +- The responder is intentionally minimal (no keep-alive, TLS or auth). +- The metrics endpoint must be network-restricted. + +## Risks +- Metrics and probes are unauthenticated plain HTTP; the NetworkPolicy is the + control. +- The build-tree RUNPATH stays in the binary; it is harmless in the image, where + the path does not exist, and `LD_LIBRARY_PATH` names libpq. +- Local Docker was not run in this gate; the CI job is configured but its first + execution is pending. + +## Consequences +- M7.9 measures recovery and batching latency through these metrics. +- A later gateway must honor readiness and the lease epoch before routing. +- Promoting to World0 needs a scanned, signed image and a reviewed alerting policy + (M8). diff --git a/docs/adr/README.md b/docs/adr/README.md index a30fbe0..f338b9d 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -76,3 +76,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0063: Batch outcomes](0063-batch-outcomes.md) - accepted for M7.5 manifest v2, validation trace replay, durable decisions/outbox and commit resolution. - [ADR-0064: Region host](0064-region-host.md) - accepted for M7.6 single-owner host state machine, bounded admission, final-only-after-commit and evo-server process. - [ADR-0065: Recovery before readiness](0065-recovery-before-readiness.md) - accepted for M7.7 head tail replay verification, schema v6 commit lookup and real kill-boundary tests. +- [ADR-0066: Operations endpoints](0066-operations-endpoints.md) - accepted for M7.8 liveness vs authority readiness, bounded metrics, dependency-free responder and the server image. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 343a32c..f43566b 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2381,3 +2381,44 @@ Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate - Not executed: Docker/Gitea CI, the production 20 s lease, host loss or VM freeze, multi-host network partitions, or verification of links older than the head. + +## M7.8 operations endpoints - 2026-09-28 + +- Windows MSVC Release CTest passed 59/59, including the new portable + `ops-contracts`: exposition format, HELP/TYPE per family, the fixed 30-series + set, bounded labels, one-hot phase and non-wrapping lag. The exposition is + 3485 bytes typically and 3836 at worst-case values (limit 8192). +- WSL Linux: the `checks` build set (GCC14 Debug and Release, clang19 analysis, + clang19 ASan/UBSan) passed CTest 59/59 each. +- The PostgreSQL suites (GCC Release, clang-tidy analysis, ASan/UBSan) passed + after fixing the ops test's optional-access and reserve findings. The earlier + PostgreSQL analysis runs used a stale binary and are not counted. The + repository format check passed after formatting the server. +- Process probe tests (real `evo-server`, loopback, ephemeral port): + - Serving: `/livez` 200 `live`, `/readyz` 200 `ready`. Metrics showed ready 1, + epoch 1, lease remaining above 0, capacities 64/65536 and the running phase. + After 30 inputs: committed tick 30, lag 0, 30 acknowledgements, checkpoint + age under 60 s. + - HTTP bounds: POST 405, unknown path and query string 404, malformed 400, + over 4 KiB 431. Eight idle connections do not block probes; a ninth is + closed within 500 ms; idle ones close at their deadline. + - Database unreachable: `/livez` 200, `/readyz` 503 `not ready: acquiring`, + ready 0, store failures above 0, never `ready 1`, exit 0 on SIGTERM. + - Stale process (SIGSTOP 4.5 s, then SIGCONT): `/livez` 200, `/readyz` 503, + ready 0, fences_total 1, lease remaining 0. The successor answers 200 at + epoch 2. +- Mutations (restored and rerun green): readiness that returns 200 whenever live + was caught. Raising the connection bound to 1000 was not caught at first, + because the check did not distinguish refusal from deadline expiry. The test + now requires the refusal within 500 ms and catches it. +- Image: the Docker engine was not running locally (Docker Desktop stopped), so + the `server-runtime` build and `tools/ci/server-image-smoke.sh` have not + executed. A filesystem emulation (binary plus the pinned libpq, `env -i`) + showed usage exit 2 with no settings and all libraries resolved. It also found + and fixed a bug: without arguments the server had reached object-store opening + instead of refusing with usage. The binary keeps a build-tree RUNPATH, which is + harmless in the image. +- The manifest example parses (Deployment and NetworkPolicy, three probes) and + the workflow YAML parses; `bash -n` passed for the smoke script. +- Not executed: the container build and smoke, the Gitea job, cluster apply, + Prometheus scraping, and TSan. diff --git a/docs/infrastructure/POSTGRESQL.md b/docs/infrastructure/POSTGRESQL.md index 7759105..778cc7e 100644 --- a/docs/infrastructure/POSTGRESQL.md +++ b/docs/infrastructure/POSTGRESQL.md @@ -191,3 +191,15 @@ boundary; never set it in shared environments. Use the default 20 s lease in any shared environment. `--lease-profile fast` exists only for local tests. The runtime credentials come from `--pg-*` flags and a passfile; never place passwords on the command line. + +### Operations endpoints (M7.8, ADR-0066) + +`--ops-listen ` serves three endpoints: +- `GET /livez`: the process loop is responding. It says nothing about authority. +- `GET /readyz`: 200 only while this process holds the lease and has a verified + recovered head; 503 with the phase otherwise. +- `GET /metrics`: Prometheus text. + +A probe may wait up to one bounded I/O step (the 5 s commit deadline), so probe +timeouts must exceed that. Expose port 9100 only to the monitoring namespace; +the endpoints are unauthenticated plain HTTP. diff --git a/include/evo/server/ops.hpp b/include/evo/server/ops.hpp new file mode 100644 index 0000000..866b7dd --- /dev/null +++ b/include/evo/server/ops.hpp @@ -0,0 +1,57 @@ +#pragma once +#include +#include +#include +#include + +// Region host status and its bounded operational exposition (M7.8, ADR-0066). +// Portable: no sockets, clocks or persistence types. Liveness is not authority: +// only `ready` means the host may originate output. +namespace evo::server { +enum class HostPhase : std::uint8_t { + acquiring, // Waiting for the lease (busy/unavailable) or recovering. + running, // Authoritative: simulating and committing. + resolving, // A commit outcome is unknown: paused until resolved. + draining, // Shutdown: committing pending work, then releasing. + stopped, // Drained (lease released when possible). + quarantined, // Fail closed: committed state or history cannot be trusted. +}; +inline constexpr std::size_t host_phase_count = 6; +// Why an accepted input will never be acknowledged by this host. +enum class Unacknowledged : std::uint8_t { + refused, // The simulation rejected it; no tick consumed it. + drained, // Still queued at shutdown. + fenced, // Authority was lost before its tick committed; the next owner never saw it. + quarantined, +}; +inline constexpr std::size_t unacknowledged_count = 4; + +struct HostStatus { + HostPhase phase{HostPhase::acquiring}; + bool ready{}; // Authoritative now: recovered, running, lease may originate. + std::uint64_t epoch{}; + std::uint64_t simulated_tick{}, committed_tick{}, committed_sequence{}; + std::size_t queued{}, queued_bytes{}; + std::uint64_t tick_debt{}, max_tick_debt{}; // Due minus simulated since running. + std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{}; + // Local lease safety: time until this host must stop originating (0 = none). + std::uint64_t lease_remaining_ms{}; +}; + +// Process-level counters and bounds the exposition adds to the host status. +struct OpsCounters { + std::size_t queue_capacity_inputs{}, queue_capacity_bytes{}; + std::uint64_t checkpoint_age_ms{}; // Since the last confirmed commit or recovery. + std::uint64_t acks{}, outcomes{}; + std::array dropped{}; // Indexed by Unacknowledged. +}; + +[[nodiscard]] const char* phase_name(HostPhase phase) noexcept; +[[nodiscard]] const char* unacknowledged_name(Unacknowledged reason) noexcept; + +// Prometheus text format 0.0.4. Fixed metric names; the only labels are `phase` +// (6 values) and `reason` (4 values), so cardinality is bounded by construction. +// No world/region/entity/request labels: scrape targets carry region identity. +[[nodiscard]] std::string render_metrics(const HostStatus& status, const OpsCounters& counters); +inline constexpr std::size_t max_metrics_bytes = 8192; +} // namespace evo::server diff --git a/include/evo/server/ops_http.hpp b/include/evo/server/ops_http.hpp new file mode 100644 index 0000000..8a68744 --- /dev/null +++ b/include/evo/server/ops_http.hpp @@ -0,0 +1,67 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Minimal operational HTTP/1.1 responder for probes and metrics (M7.8, ADR-0066). +// Linux. Not a general web server: GET only, one request per connection +// (Connection: close), bounded connections, request bytes, response bytes and +// per-connection time. serve() never blocks; it runs on the host's own loop so +// answers always describe the current state and no thread is added. +namespace evo::server { +struct OpsResponse { + int status{200}; + std::string body; + std::string content_type{"text/plain; charset=utf-8"}; +}; +// Receives the request path (no query handling: probes use exact paths). +using OpsHandler = std::function; +enum class OpsError : std::uint8_t { address, socket }; + +class OpsServer { + public: + static constexpr std::size_t max_connections = 8; + static constexpr std::size_t max_request_bytes = 4096; + static constexpr std::size_t max_response_bytes = 16384; + static constexpr std::chrono::milliseconds connection_deadline{1000}; + + // Numeric address only ("127.0.0.1:9100" or "[::1]:9100"); port 0 picks one. + [[nodiscard]] static std::expected, OpsError> + listen(std::string_view address); + OpsServer(const OpsServer&) = delete; + OpsServer& operator=(const OpsServer&) = delete; + OpsServer(OpsServer&&) = delete; + OpsServer& operator=(OpsServer&&) = delete; + ~OpsServer(); + [[nodiscard]] std::uint16_t port() const noexcept { + return port_; + } + // Accepts, reads, answers and expires connections without waiting. Connections + // beyond the bound are closed at once; slow or oversized ones are answered or + // dropped without delaying others. + void serve(const OpsHandler& handler); + [[nodiscard]] std::uint64_t refused() const noexcept { + return refused_; + } + + private: + struct Connection { + int fd{-1}; + std::string in, out; + std::size_t sent{}; + std::chrono::steady_clock::time_point deadline{}; + }; + OpsServer(int fd, std::uint16_t port) : fd_(fd), port_(port) {} + void respond(Connection& c, const OpsHandler& handler); + int fd_; + std::uint16_t port_; + std::vector connections_; + std::uint64_t refused_{}; +}; +} // namespace evo::server diff --git a/include/evo/server/region_host.hpp b/include/evo/server/region_host.hpp index 4818c92..3f6d6da 100644 --- a/include/evo/server/region_host.hpp +++ b/include/evo/server/region_host.hpp @@ -2,6 +2,7 @@ #include "evo/persistence/checkpoint_store.hpp" #include "evo/persistence/region_batch.hpp" #include "evo/persistence/region_lease.hpp" +#include "evo/server/ops.hpp" #include #include #include @@ -45,36 +46,11 @@ struct FinalOutcome { std::uint64_t commit_sequence{}; friend bool operator==(const FinalOutcome&, const FinalOutcome&) = default; }; -// Why an accepted input will never be acknowledged by this host. -enum class Unacknowledged : std::uint8_t { - refused, // The simulation rejected it; no tick consumed it. - drained, // Still queued at shutdown. - fenced, // Authority was lost before its tick committed; the next owner never saw it. - quarantined, -}; struct Dropped { TickInput input{}; Unacknowledged reason{}; }; -enum class HostPhase : std::uint8_t { - acquiring, // Waiting for the lease (busy/unavailable) or recovering. - running, // Authoritative: simulating and committing. - resolving, // A commit outcome is unknown: paused until resolved. - draining, // Shutdown: committing pending work, then releasing. - stopped, // Drained (lease released when possible). - quarantined, // Fail closed: committed state or history cannot be trusted. -}; -struct HostStatus { - HostPhase phase{HostPhase::acquiring}; - bool ready{}; // Authoritative now: recovered, running, lease may originate. - std::uint64_t epoch{}; - std::uint64_t simulated_tick{}, committed_tick{}, committed_sequence{}; - std::size_t queued{}, queued_bytes{}; - std::uint64_t tick_debt{}, max_tick_debt{}; // Due minus simulated since running. - std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{}; -}; - // Commit boundaries for fault injection (M7.7 kill tests). enum class CrashPoint : std::uint8_t { before_upload, // Batch sealed; nothing uploaded or sent. diff --git a/src/server/ops.cpp b/src/server/ops.cpp new file mode 100644 index 0000000..3d9507c --- /dev/null +++ b/src/server/ops.cpp @@ -0,0 +1,104 @@ +#include "evo/server/ops.hpp" +#include + +namespace evo::server { +const char* phase_name(HostPhase phase) noexcept { + switch (phase) { + case HostPhase::acquiring: + return "acquiring"; + case HostPhase::running: + return "running"; + case HostPhase::resolving: + return "resolving"; + case HostPhase::draining: + return "draining"; + case HostPhase::stopped: + return "stopped"; + case HostPhase::quarantined: + return "quarantined"; + } + return "unknown"; +} +const char* unacknowledged_name(Unacknowledged reason) noexcept { + switch (reason) { + case Unacknowledged::refused: + return "refused"; + case Unacknowledged::drained: + return "drained"; + case Unacknowledged::fenced: + return "fenced"; + case Unacknowledged::quarantined: + return "quarantined"; + } + return "unknown"; +} + +namespace { +void metric(std::string& out, std::string_view name, std::string_view type, std::string_view help, + std::uint64_t value) { + out.append("# HELP ").append(name).append(" ").append(help).append("\n"); + out.append("# TYPE ").append(name).append(" ").append(type).append("\n"); + out.append(name).append(" ").append(std::to_string(value)).append("\n"); +} +} // namespace + +std::string render_metrics(const HostStatus& s, const OpsCounters& c) { + std::string out; + out.reserve(4096); + metric(out, "evo_region_ready", "gauge", + "1 only while this host holds current authority and may originate output", s.ready); + out.append("# HELP evo_region_phase Current host phase (one-hot)\n"); + out.append("# TYPE evo_region_phase gauge\n"); + for (std::size_t i = 0; i < host_phase_count; ++i) { + const auto phase = static_cast(i); + out.append("evo_region_phase{phase=\"") + .append(phase_name(phase)) + .append("\"} ") + .append(phase == s.phase ? "1" : "0") + .append("\n"); + } + metric(out, "evo_region_lease_epoch", "gauge", "Lease epoch held or last held", s.epoch); + metric(out, "evo_region_lease_remaining_milliseconds", "gauge", + "Local time until this host must stop originating (0 without a lease)", + s.lease_remaining_ms); + metric(out, "evo_region_simulated_tick", "gauge", "Last simulated tick", s.simulated_tick); + metric(out, "evo_region_committed_tick", "gauge", "End tick of the committed head", + s.committed_tick); + metric(out, "evo_region_committed_sequence", "gauge", "Commit sequence of the head", + s.committed_sequence); + metric(out, "evo_region_commit_lag_ticks", "gauge", + "Simulated but not yet durably committed ticks", + s.simulated_tick >= s.committed_tick ? s.simulated_tick - s.committed_tick : 0); + metric(out, "evo_region_checkpoint_age_milliseconds", "gauge", + "Time since the last confirmed commit or recovery", c.checkpoint_age_ms); + metric(out, "evo_region_tick_debt", "gauge", "Due ticks not yet simulated", s.tick_debt); + metric(out, "evo_region_tick_debt_max", "gauge", "Largest tick debt since recovery", + s.max_tick_debt); + metric(out, "evo_region_queued_inputs", "gauge", "Admitted inputs awaiting a tick", s.queued); + metric(out, "evo_region_queued_bytes", "gauge", "Accounted bytes of queued inputs", + s.queued_bytes); + metric(out, "evo_region_queue_capacity_inputs", "gauge", "Input backlog bound", + c.queue_capacity_inputs); + metric(out, "evo_region_queue_capacity_bytes", "gauge", "Input backlog byte bound", + c.queue_capacity_bytes); + metric(out, "evo_region_commits_total", "counter", "Confirmed commits", s.commits); + metric(out, "evo_region_store_failures_total", "counter", + "Storage or database failures that paused progress", s.store_failures); + metric(out, "evo_region_unknown_outcomes_total", "counter", + "Commits whose outcome had to be resolved", s.unknown_outcomes); + metric(out, "evo_region_fences_total", "counter", "Times this host lost authority", s.fences); + metric(out, "evo_region_acknowledgements_total", "counter", "Final acknowledgements emitted", + c.acks); + metric(out, "evo_region_outcomes_total", "counter", "Final permanent outcomes emitted", + c.outcomes); + out.append("# HELP evo_region_dropped_total Accepted inputs that will never be acknowledged\n"); + out.append("# TYPE evo_region_dropped_total counter\n"); + for (std::size_t i = 0; i < unacknowledged_count; ++i) + out.append("evo_region_dropped_total{reason=\"") + .append(unacknowledged_name(static_cast(i))) + .append("\"} ") + .append(std::to_string(c.dropped[i])) + .append("\n"); + return out; +} +} // namespace evo::server diff --git a/src/server/ops_http.cpp b/src/server/ops_http.cpp new file mode 100644 index 0000000..dd393c2 --- /dev/null +++ b/src/server/ops_http.cpp @@ -0,0 +1,175 @@ +#include "evo/server/ops_http.hpp" +#include +#include +#include +#include +#include +#include + +namespace evo::server { +namespace { +using Clock = std::chrono::steady_clock; +const char* reason(int status) { + switch (status) { + case 200: + return "OK"; + case 400: + return "Bad Request"; + case 404: + return "Not Found"; + case 405: + return "Method Not Allowed"; + case 431: + return "Request Header Fields Too Large"; + case 503: + return "Service Unavailable"; + default: + return "Internal Server Error"; + } +} +std::string encode(const OpsResponse& r) { + std::string out = "HTTP/1.1 " + std::to_string(r.status) + " " + reason(r.status) + + "\r\nContent-Type: " + r.content_type + + "\r\nContent-Length: " + std::to_string(r.body.size()) + + "\r\nCache-Control: no-store\r\nConnection: close\r\n"; + if (r.status == 405) + out += "Allow: GET\r\n"; + return out + "\r\n" + r.body; +} +bool printable(std::string_view s) { + for (const char ch : s) + if (ch <= ' ' || ch > '~') + return false; + return true; +} +} // namespace + +std::expected, OpsError> OpsServer::listen(std::string_view address) { + const auto colon = address.rfind(':'); + if (colon == std::string_view::npos || colon + 1 >= address.size()) + return std::unexpected{OpsError::address}; + unsigned port{}; + const auto digits = address.substr(colon + 1); + const auto [end, error] = std::from_chars(digits.data(), digits.data() + digits.size(), port); + if (error != std::errc{} || end != digits.data() + digits.size() || port > 65535) + return std::unexpected{OpsError::address}; + auto host = std::string(address.substr(0, colon)); + sockaddr_storage storage{}; + socklen_t length{}; + int family{}; + if (host.size() >= 2 && host.front() == '[' && host.back() == ']') { + host = host.substr(1, host.size() - 2); + auto* v6 = reinterpret_cast(&storage); + if (inet_pton(AF_INET6, host.c_str(), &v6->sin6_addr) != 1) + return std::unexpected{OpsError::address}; + v6->sin6_family = AF_INET6; + v6->sin6_port = htons(static_cast(port)); + family = AF_INET6; + length = sizeof(sockaddr_in6); + } else { + auto* v4 = reinterpret_cast(&storage); + if (inet_pton(AF_INET, host.c_str(), &v4->sin_addr) != 1) + return std::unexpected{OpsError::address}; + v4->sin_family = AF_INET; + v4->sin_port = htons(static_cast(port)); + family = AF_INET; + length = sizeof(sockaddr_in); + } + const int fd = ::socket(family, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0); + if (fd < 0) + return std::unexpected{OpsError::socket}; + const int on = 1; + ::setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof on); + if (::bind(fd, reinterpret_cast(&storage), length) != 0 || ::listen(fd, 16) != 0) { + ::close(fd); + return std::unexpected{OpsError::socket}; + } + sockaddr_storage bound{}; + socklen_t bound_length = sizeof bound; + ::getsockname(fd, reinterpret_cast(&bound), &bound_length); + const auto actual = family == AF_INET6 + ? ntohs(reinterpret_cast(&bound)->sin6_port) + : ntohs(reinterpret_cast(&bound)->sin_port); + return std::unique_ptr(new OpsServer(fd, actual)); +} + +OpsServer::~OpsServer() { + for (auto& c : connections_) + ::close(c.fd); + ::close(fd_); +} + +void OpsServer::respond(Connection& c, const OpsHandler& handler) { + OpsResponse response; + const auto line_end = c.in.find("\r\n"); + const std::string_view line = std::string_view(c.in).substr(0, line_end); + const auto first = line.find(' '); + const auto second = first == std::string_view::npos ? first : line.find(' ', first + 1); + if (second == std::string_view::npos || line.substr(second + 1).rfind("HTTP/1.", 0) != 0) { + response = {400, "bad request\n"}; + } else { + const auto method = line.substr(0, first); + const auto path = line.substr(first + 1, second - first - 1); + if (method != "GET") + response = {405, "method not allowed\n"}; + else if (path.empty() || path.front() != '/' || path.size() > 256 || !printable(path)) + response = {400, "bad request\n"}; + else + response = handler(path); + } + if (response.body.size() > max_response_bytes) + response = {500, "response too large\n"}; + c.out = encode(response); +} + +void OpsServer::serve(const OpsHandler& handler) { + const auto now = Clock::now(); + for (;;) { + const int fd = ::accept4(fd_, nullptr, nullptr, SOCK_NONBLOCK | SOCK_CLOEXEC); + if (fd < 0) + break; // EAGAIN: nothing pending (other errors: retried next call). + if (connections_.size() >= max_connections) { + ::close(fd); // Bounded: excess connections are refused, never queued. + ++refused_; + continue; + } + connections_.push_back({fd, {}, {}, 0, now + connection_deadline}); + } + for (auto it = connections_.begin(); it != connections_.end();) { + auto& c = *it; + bool done = false; + // Read before expiring: a complete request that waited behind a bounded host + // step is still answered; only incomplete or unsent work expires. + if (c.out.empty()) { + char chunk[1024]; + const auto n = ::recv(c.fd, chunk, sizeof chunk, 0); + if (n == 0 || (n < 0 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR)) { + done = true; + } else if (n > 0) { + c.in.append(chunk, static_cast(n)); + if (c.in.size() > max_request_bytes) + c.out = encode({431, "request too large\n"}); + else if (c.in.find("\r\n\r\n") != std::string::npos) + respond(c, handler); + } + } + if (!done && !c.out.empty()) { + const auto n = ::send(c.fd, c.out.data() + c.sent, c.out.size() - c.sent, MSG_NOSIGNAL); + if (n > 0) + c.sent += static_cast(n); + done = c.sent == c.out.size() || + (n < 0 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR); + } + // Slow client: an incomplete request expires at the deadline; a response the + // client will not take gets one more deadline, then is dropped. + if (!done && now > c.deadline && (c.out.empty() || now > c.deadline + connection_deadline)) + done = true; + if (done) { + ::close(c.fd); + it = connections_.erase(it); + } else { + ++it; + } + } +} +} // namespace evo::server diff --git a/src/server/region_host.cpp b/src/server/region_host.cpp index 8398680..f8f626e 100644 --- a/src/server/region_host.cpp +++ b/src/server/region_host.cpp @@ -386,6 +386,13 @@ template void RegionHost::refresh(SchedulerTime now) { status_.queued_bytes = queue_.size() * tick_input_bytes; const auto& grant = authority_.grant(); status_.epoch = grant ? grant->epoch : 0; + const auto lease_now = LeaseClock::now(); + status_.lease_remaining_ms = + grant && authority_.may_originate(lease_now) + ? static_cast(std::chrono::duration_cast( + authority_.deadline() - lease_now) + .count()) + : 0; status_.committed_tick = committed_ ? committed_->head.end_tick : 0; status_.committed_sequence = committed_ ? committed_->head.sequence : 0; status_.simulated_tick = state_ ? Ops::tick(*state_) : status_.committed_tick; diff --git a/tests/ops_tests.cpp b/tests/ops_tests.cpp new file mode 100644 index 0000000..5a26077 --- /dev/null +++ b/tests/ops_tests.cpp @@ -0,0 +1,119 @@ +#include "evo/server/ops.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using namespace evo::server; +void check(bool ok, std::source_location at = std::source_location::current()) { + if (!ok) + throw std::runtime_error("ops line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) + +struct Sample { + std::string name, labels, value; +}; +// Parses exposition lines; every family must be introduced by HELP and TYPE. +std::vector parse(const std::string& text) { + std::vector out; + std::set typed, helped; + std::istringstream in(text); + for (std::string line; std::getline(in, line);) { + CHECK(!line.empty()); + if (line.rfind("# HELP ", 0) == 0) { + helped.insert(line.substr(7, line.find(' ', 7) - 7)); + continue; + } + if (line.rfind("# TYPE ", 0) == 0) { + const auto name = line.substr(7, line.find(' ', 7) - 7); + const auto type = line.substr(line.rfind(' ') + 1); + CHECK(type == "gauge" || type == "counter"); + CHECK(helped.contains(name) && !typed.contains(name)); // One family, documented. + typed.insert(name); + continue; + } + const auto space = line.rfind(' '); + const auto brace = line.find('{'); + Sample s{line.substr(0, std::min(space, brace)), "", line.substr(space + 1)}; + if (brace < space) + s.labels = line.substr(brace, space - brace); + CHECK(typed.contains(s.name)); + CHECK(!s.value.empty() && s.value.find_first_not_of("0123456789") == std::string::npos); + out.push_back(s); + } + return out; +} + +void exposition_is_bounded_and_well_formed() { + HostStatus s; + s.phase = HostPhase::running; + s.ready = true; + s.epoch = 3; + s.simulated_tick = 50; + s.committed_tick = 41; + s.lease_remaining_ms = 1234; + OpsCounters c; + c.queue_capacity_inputs = 64; + c.queue_capacity_bytes = 65536; + c.dropped[2] = 5; + const auto text = render_metrics(s, c); + const auto samples = parse(text); + const std::set label_names{"", "phase", "reason"}; + std::size_t phases_on = 0; + for (const auto& x : samples) { + CHECK(x.name.rfind("evo_region_", 0) == 0); + const auto key = + x.labels.empty() ? std::string{} : x.labels.substr(1, x.labels.find('=') - 1); + CHECK(label_names.contains(key)); // No world/region/entity/request labels. + if (x.name == "evo_region_phase") + phases_on += x.value == "1"; + } + CHECK(phases_on == 1); + CHECK(text.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos); + CHECK(text.find("evo_region_ready 1\n") != std::string::npos); + CHECK(text.find("evo_region_commit_lag_ticks 9\n") != std::string::npos); + CHECK(text.find("evo_region_lease_remaining_milliseconds 1234\n") != std::string::npos); + CHECK(text.find("evo_region_dropped_total{reason=\"fenced\"} 5\n") != std::string::npos); + // Series count is fixed: 6 phases + 4 reasons + 20 unlabelled families. + CHECK(samples.size() == 6 + 4 + 20); + + // Worst-case values keep the size bounded and never underflow the lag. + HostStatus big; + big.phase = HostPhase::quarantined; + constexpr auto most = std::numeric_limits::max(); + big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most; + big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most; + big.fences = big.lease_remaining_ms = most; + big.simulated_tick = 1; // Lag saturates at zero rather than wrapping. + OpsCounters bigc{most, most, most, most, most, {most, most, most, most}}; + const auto large = render_metrics(big, bigc); + CHECK(large.size() <= max_metrics_bytes); + CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos); + CHECK(large.find("evo_region_ready 0\n") != std::string::npos); + CHECK(parse(large).size() == samples.size()); + for (std::size_t i = 0; i < host_phase_count; ++i) + CHECK(std::string(phase_name(static_cast(i))) != "unknown"); + for (std::size_t i = 0; i < unacknowledged_count; ++i) + CHECK(std::string(unacknowledged_name(static_cast(i))) != "unknown"); + std::printf("ops_metrics_bytes typical=%zu worst=%zu\n", text.size(), large.size()); +} +} // namespace + +int main() { + try { + exposition_is_bounded_and_well_formed(); + std::puts("ops contracts passed"); + return 0; + } catch (const std::exception& e) { + std::fprintf(stderr, "%s\n", e.what()); + return 1; + } +} diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index fa41d77..25e0fc3 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -8,6 +8,7 @@ void run_region_checkpoint_store_tests(); void run_region_host_tests(); void run_region_server_process_tests(); void run_region_recovery_tests(); +void run_region_ops_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; @@ -84,6 +85,8 @@ int main() { std::puts("region server process integration passed"); run_region_recovery_tests(); std::puts("region recovery integration passed"); + run_region_ops_tests(); + std::puts("region ops integration passed"); return 0; } catch (const std::exception& e) { std::fprintf(stderr, "%s\n", e.what()); diff --git a/tests/region_ops_tests.cpp b/tests/region_ops_tests.cpp new file mode 100644 index 0000000..caee0b1 --- /dev/null +++ b/tests/region_ops_tests.cpp @@ -0,0 +1,193 @@ +// M7.8 operational endpoints on real evo-server processes (ADR-0066): liveness is +// never authority, readiness follows the lease, metrics are bounded, and the +// responder is bounded against idle, excess and malformed clients. +#include "fixtures/server_process.hpp" +#include +#include +#include +#include +#include +#include + +void run_region_ops_tests(); + +namespace { +using namespace evo::test::process; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("region ops test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) + +int connect_local(std::uint16_t port) { + const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); + CHECK(fd >= 0); + sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + CHECK(::connect(fd, reinterpret_cast(&address), sizeof address) == 0); + timeval timeout{3, 0}; + ::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout); + return fd; +} +struct Reply { + int status{}; + std::string body; +}; +// Sends raw bytes and reads until the server closes (Connection: close). +Reply http_exchange(std::uint16_t port, const std::string& request) { + const int fd = connect_local(port); + std::size_t sent = 0; + while (sent < request.size()) { + const auto n = ::send(fd, request.data() + sent, request.size() - sent, MSG_NOSIGNAL); + if (n <= 0) + break; + sent += static_cast(n); + } + std::string raw; + char chunk[4096]; + for (;;) { + const auto n = ::recv(fd, chunk, sizeof chunk, 0); + if (n <= 0) + break; + raw.append(chunk, static_cast(n)); + } + ::close(fd); + int status = 0; + CHECK(std::sscanf(raw.c_str(), "HTTP/1.1 %d", &status) == 1); + const auto body = raw.find("\r\n\r\n"); + return Reply{status, body == std::string::npos ? "" : raw.substr(body + 4)}; +} +Reply get(std::uint16_t port, const std::string& path) { + return http_exchange(port, "GET " + path + " HTTP/1.1\r\nHost: probe\r\n\r\n"); +} +std::uint64_t metric(std::uint16_t port, const std::string& series) { + const auto reply = get(port, "/metrics"); + CHECK(reply.status == 200); + const auto at = reply.body.find("\n" + series + " "); + CHECK(at != std::string::npos); + return std::stoull(reply.body.substr(at + series.size() + 2)); +} +std::uint16_t ops_port(Child& server) { + CHECK(server.wait_line("ops listening ", 10s)); + for (const auto& l : server.lines()) + if (starts(l, "ops listening ")) + return static_cast(std::stoul(l.substr(14))); + CHECK(false); + return 0; +} +std::vector with_ops(const ServerRegion& r, std::vector extra = {}) { + extra.insert(extra.begin(), {"--ops-listen", "127.0.0.1:0"}); + return server_args(r, std::move(extra)); +} + +void probes_follow_authority_and_stay_bounded() { + const auto r = provision(51, 0x0B); + Child server{with_ops(r)}; + const auto port = ops_port(server); + CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); + auto live = get(port, "/livez"); + auto ready = get(port, "/readyz"); + CHECK(live.status == 200 && live.body == "live\n"); + CHECK(ready.status == 200 && ready.body == "ready\n"); + CHECK(metric(port, "evo_region_ready") == 1 && metric(port, "evo_region_lease_epoch") == 1); + CHECK(metric(port, "evo_region_lease_remaining_milliseconds") > 0); + CHECK(metric(port, "evo_region_queue_capacity_inputs") == 64 && + metric(port, "evo_region_queue_capacity_bytes") == 65536); + auto page = get(port, "/metrics"); + CHECK(page.body.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos); + server.send(input_lines(1, 30, false)); + CHECK(server.wait_line("ack " + request_hex(30) + " 30 ", 15s)); + CHECK(metric(port, "evo_region_committed_tick") == 30 && + metric(port, "evo_region_commit_lag_ticks") == 0 && + metric(port, "evo_region_acknowledgements_total") == 30); + CHECK(metric(port, "evo_region_checkpoint_age_milliseconds") < 60000); + + // Bounded, strict responder. + CHECK(http_exchange(port, "POST /readyz HTTP/1.1\r\n\r\n").status == 405); + CHECK(get(port, "/nope").status == 404); + CHECK(get(port, "/readyz?x=1").status == 404); // Exact paths only. + CHECK(http_exchange(port, "HELLO\r\n\r\n").status == 400); + CHECK(http_exchange(port, "GET /livez HTTP/1.1\r\nX: " + std::string(5000, 'a') + "\r\n\r\n") + .status == 431); + // Idle connections neither block probes nor exceed the connection bound. + std::vector idle; + idle.reserve(8); + for (int i = 0; i < 8; ++i) + idle.push_back(connect_local(port)); + const int excess = connect_local(port); + char byte; + // Refused at once (well before the 1 s idle deadline), never queued behind others. + const auto refused_at = Clock::now(); + CHECK(::recv(excess, &byte, 1, 0) == 0); + CHECK(Clock::now() - refused_at < 500ms); + ::close(excess); + std::this_thread::sleep_for(1300ms); // Idle connections expire at their deadline. + const auto started = Clock::now(); + CHECK(get(port, "/livez").status == 200); + CHECK(Clock::now() - started < 1s); + for (const int fd : idle) { + CHECK(::recv(fd, &byte, 1, 0) == 0); // Closed by the server. + ::close(fd); + } + server.signal(SIGTERM); + CHECK(server.wait_exit(20s) == 0); +} + +// The database is unreachable: the process is alive but never ready. +void unavailable_database_is_live_not_ready() { + const ServerRegion r{{7, 52}, fs::path(env("PGHOST")) / "server-52"}; + fs::remove_all(r.root); + fs::create_directory(r.root); + auto args = with_ops(r); + const auto at = std::ranges::find(args, "--pg-port"); + CHECK(at != args.end()); + *(at + 1) = "1"; // No server socket there. + Child server{args}; + const auto port = ops_port(server); + std::this_thread::sleep_for(500ms); + CHECK(get(port, "/livez").status == 200); + auto ready = get(port, "/readyz"); + CHECK(ready.status == 503 && ready.body == "not ready: acquiring\n"); + CHECK(metric(port, "evo_region_ready") == 0 && + metric(port, "evo_region_store_failures_total") > 0); + CHECK(get(port, "/metrics").body.find("evo_region_phase{phase=\"acquiring\"} 1\n") != + std::string::npos); + CHECK(!server.printed("ready 1")); + server.signal(SIGTERM); + CHECK(server.wait_exit(20s) == 0); +} + +// A paused process that lost its lease is alive, but readiness and metrics say it +// no longer holds authority; the successor is the only ready process. +void stale_process_is_live_not_ready() { + const auto r = provision(53, 0x0C); + Child stale{with_ops(r)}; + const auto stale_port = ops_port(stale); + CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); + stale.signal(SIGSTOP); + std::this_thread::sleep_for(4500ms); + Child current{with_ops(r)}; + const auto current_port = ops_port(current); + CHECK(current.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); + stale.signal(SIGCONT); + CHECK(stale.wait_line("ready 0", 10s)); + CHECK(get(stale_port, "/livez").status == 200); + CHECK(get(stale_port, "/readyz").status == 503); + CHECK(metric(stale_port, "evo_region_ready") == 0 && + metric(stale_port, "evo_region_fences_total") == 1 && + metric(stale_port, "evo_region_lease_remaining_milliseconds") == 0); + CHECK(get(current_port, "/readyz").status == 200 && + metric(current_port, "evo_region_lease_epoch") == 2); + stale.signal(SIGTERM); + current.signal(SIGTERM); + CHECK(stale.wait_exit(20s) == 0 && current.wait_exit(20s) == 0); +} +} // namespace + +void run_region_ops_tests() { + probes_follow_authority_and_stay_bounded(); + unavailable_database_is_live_not_ready(); + stale_process_is_live_not_ready(); +} diff --git a/tools/ci/server-image-smoke.sh b/tools/ci/server-image-smoke.sh new file mode 100644 index 0000000..f1e88b2 --- /dev/null +++ b/tools/ci/server-image-smoke.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Smoke test for the evo-server image (M7.8, ADR-0066). Runs the image with the +# same restrictions a cluster uses; no network, no credentials, read-only root. +set -euo pipefail +image="${1:?image reference}" +run() { + docker run --rm --network none --read-only --cap-drop ALL \ + --security-opt no-new-privileges "$@" +} +# Without configuration the server refuses to start (usage, exit 2), and never +# falls back to any default region or database. +set +e +usage="$(run "$image" 2>&1)" +code=$? +set -e +test "$code" -eq 2 +grep -q '^usage: evo-server ' <<<"$usage" +# Non-root, every shared library resolved, no credential material in the image. +run --entrypoint /bin/sh "$image" -c ' + set -eu + test "$(id -u)" = 10001 && test "$(id -g)" = 10001 + ! ldd /usr/local/bin/evo-server | grep -q "not found" + test -z "$(find / -xdev \( -name "*pgpass*" -o -name "pg_service.conf" -o -name "*.pem" \ + -o -name "*.key" \) -not -path "/etc/ssl/*" -not -path "/usr/share/*" -print 2>/dev/null)" +' +echo "server image smoke passed: $image"