Complete M7.8 operations endpoints and region server image

ADR-0066: evo-server --ops-listen serves /livez (loop responding, never
authority), /readyz (200 only while the host holds the lease with a
verified recovered head) and /metrics (Prometheus text: 30 fixed
series for readiness, phase, lease epoch/remaining, simulated/committed
ticks, commit lag, checkpoint age, tick debt, queue use and bounds, and
counters; the only labels are a 6-value phase and 4-value drop reason).

The dependency-free OpsServer runs on the host loop with no thread:
GET on exact paths, 405/404/400/431 responses, 8 connections with excess
refused on accept, 4 KiB requests, 1 s deadlines, 16 KiB responses.
HostStatus/phase types moved to the portable evo::ops library with a
Windows-tested renderer. evo-server now refuses to start without its
required identity and connection settings.

Adds the server-runtime Docker target (evo-server + pinned libpq,
UID 10001, no credentials), a CI server-image job with smoke script and
optional publishing, and a Kubernetes example outside Argo.

Validated: Windows CTest 59/59; Linux Debug/Release/clang-tidy/ASan
CTest 59/59; PostgreSQL suites under Release, clang-tidy and ASan with
real process probes; two ops mutations caught. Docker build/smoke and
Gitea CI not executed locally.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-28 23:09:28 -04:00
co-authored by Claude Opus 5.5
parent 30411a83f9
commit 7aec68db8e
23 changed files with 1222 additions and 65 deletions
+26
View File
@@ -76,6 +76,32 @@ jobs:
export SOURCE_IMAGE="aevolution-postgres-tools:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
bash tools/ci/publish-image.sh
server-image:
needs: [checks]
runs-on: ${{ vars.CI_RUNNER_LABEL || 'ubuntu-22.04' }}
timeout-minutes: 60
steps:
- name: Checkout
uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
persist-credentials: false
- name: Build and smoke-test the region server image
run: |
set -eu
image="aevolution-server:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
docker build --file deploy/docker/Dockerfile --target server-runtime --build-arg "UBUNTU_IMAGE=$UBUNTU_IMAGE" --tag "$image" .
bash tools/ci/server-image-smoke.sh "$image"
- name: Publish configured region server image
if: ${{ gitea.ref == 'refs/heads/main' && gitea.event_name != 'pull_request' && vars.HARBOR_SERVER_REPOSITORY != '' }}
env:
HARBOR_PROJECT: ${{ vars.HARBOR_PROJECT }}
HARBOR_REPOSITORY: ${{ vars.HARBOR_SERVER_REPOSITORY }}
HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }}
run: |
export SOURCE_IMAGE="aevolution-server:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
bash tools/ci/publish-image.sh
image:
needs: [checks]
runs-on: ${{ vars.CI_RUNNER_LABEL || 'ubuntu-22.04' }}
+15 -3
View File
@@ -77,6 +77,12 @@ add_library(evo::region_lease ALIAS evo_region_lease)
target_include_directories(evo_region_lease PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
target_compile_features(evo_region_lease PUBLIC cxx_std_23)
evo_owned_target(evo_region_lease)
# Portable host status and bounded operational exposition (M7.8).
add_library(evo_ops STATIC src/server/ops.cpp)
add_library(evo::ops ALIAS evo_ops)
target_include_directories(evo_ops PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
target_compile_features(evo_ops PUBLIC cxx_std_23)
evo_owned_target(evo_ops)
# Portable checkpoint manifest codec and environmental-v1 payload validation.
add_library(evo_region_checkpoint STATIC src/persistence/region_checkpoint.cpp src/persistence/region_batch.cpp)
add_library(evo::region_checkpoint ALIAS evo_region_checkpoint)
@@ -104,9 +110,9 @@ if(EVO_BUILD_POSTGRES)
PRIVATE OpenSSL::Crypto)
evo_owned_target(evo_region_store)
# M7.6 authoritative region host and its process (ADR-0064).
add_library(evo_region_host STATIC src/server/region_host.cpp)
add_library(evo_region_host STATIC src/server/region_host.cpp src/server/ops_http.cpp)
add_library(evo::region_host ALIAS evo_region_host)
target_link_libraries(evo_region_host PUBLIC evo::region_store)
target_link_libraries(evo_region_host PUBLIC evo::region_store evo::ops)
evo_owned_target(evo_region_host)
add_executable(evo-server apps/evo-server/main.cpp)
target_link_libraries(evo-server PRIVATE evo::region_host)
@@ -115,7 +121,8 @@ if(EVO_BUILD_POSTGRES)
# Contention tests use one thread and one connection per contender.
add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp
tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp
tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp)
tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp
tests/region_ops_tests.cpp)
target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads)
add_dependencies(evo-postgres-tests evo-server) # Spawned by the process tests.
evo_owned_target(evo-postgres-tests)
@@ -133,6 +140,11 @@ if(BUILD_TESTING)
evo_owned_target(evo-lineage-region-tests)
add_test(NAME lineage-region-contracts COMMAND evo-lineage-region-tests)
set_tests_properties(lineage-region-contracts PROPERTIES TIMEOUT 60)
add_executable(evo-ops-tests tests/ops_tests.cpp)
target_link_libraries(evo-ops-tests PRIVATE evo::ops)
evo_owned_target(evo-ops-tests)
add_test(NAME ops-contracts COMMAND evo-ops-tests)
set_tests_properties(ops-contracts PROPERTIES TIMEOUT 60)
add_executable(evo-region-batch-tests tests/region_batch_tests.cpp)
target_link_libraries(evo-region-batch-tests PRIVATE evo::region_checkpoint)
evo_owned_target(evo-region-batch-tests)
+74 -37
View File
@@ -11,9 +11,14 @@
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
//
// --ops-listen <numeric address:port> (M7.8) serves GET /livez (the loop is alive; says
// nothing about authority), /readyz (200 only with current authority) and /metrics
// (Prometheus text). It prints "ops listening <port>" once bound.
//
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
// after a step's acknowledgements are written and flushed).
#include "evo/server/ops_http.hpp"
#include "evo/server/region_host.hpp"
#include <algorithm>
@@ -48,6 +53,7 @@ struct Options {
std::chrono::milliseconds drain{10000};
std::string crash_point;
std::uint64_t crash_ordinal{};
std::string ops_listen;
};
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
@@ -140,6 +146,8 @@ std::optional<Options> parse(int argc, char** argv) {
o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms};
else if (flag == "--lease-profile" && value == "default")
o.host.lease = {};
else if (flag == "--ops-listen")
o.ops_listen = value;
else if (flag == "--crash-at") {
const auto colon = value.find(':');
const auto ordinal =
@@ -154,6 +162,12 @@ std::optional<Options> parse(int argc, char** argv) {
} else
return std::nullopt;
}
// Every identity and connection setting is required: there is no default
// region, database, object root or build to fall back on.
if (o.host.key.world == 0 || o.host.key.region == 0 || o.host.frame == 0 || o.pg.host.empty() ||
o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() ||
o.host.build == persistence::Sha256Digest{})
return std::nullopt;
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
o.host.retained_builds.push_back(o.host.build);
return o;
@@ -194,23 +208,6 @@ std::optional<server::TickInput> parse_input(std::string_view line) {
return in;
}
const char* phase_name(server::HostPhase p) {
switch (p) {
case server::HostPhase::acquiring:
return "acquiring";
case server::HostPhase::running:
return "running";
case server::HostPhase::resolving:
return "resolving";
case server::HostPhase::draining:
return "draining";
case server::HostPhase::stopped:
return "stopped";
case server::HostPhase::quarantined:
return "quarantined";
}
return "unknown";
}
const char* admission_name(server::AdmissionError e) {
switch (e) {
case server::AdmissionError::not_ready:
@@ -226,20 +223,6 @@ const char* admission_name(server::AdmissionError e) {
}
return "unknown";
}
const char* dropped_name(server::Unacknowledged u) {
switch (u) {
case server::Unacknowledged::refused:
return "refused";
case server::Unacknowledged::drained:
return "drained";
case server::Unacknowledged::fenced:
return "fenced";
case server::Unacknowledged::quarantined:
return "quarantined";
}
return "unknown";
}
// Reads complete, bounded lines from stdin without blocking.
class LineReader {
public:
@@ -294,6 +277,41 @@ template <typename State> int serve(Options& o) {
return 2;
}
auto& host = **made;
std::unique_ptr<server::OpsServer> ops;
if (!o.ops_listen.empty()) {
auto listening = server::OpsServer::listen(o.ops_listen);
if (!listening) {
std::puts("error ops-listen");
return 2;
}
ops = std::move(*listening);
std::printf("ops listening %u\n", static_cast<unsigned>(ops->port()));
std::fflush(stdout);
}
server::OpsCounters counters;
counters.queue_capacity_inputs = o.host.max_queued_inputs;
counters.queue_capacity_bytes = o.host.max_queued_bytes;
std::uint64_t last_sequence = 0;
auto last_checkpoint = std::chrono::steady_clock::now();
bool recovered = false;
const auto handler = [&](std::string_view path) -> server::OpsResponse {
const auto& s = host.status();
if (path == "/livez")
return {200, "live\n"};
if (path == "/readyz")
return s.ready ? server::OpsResponse{200, "ready\n"}
: server::OpsResponse{503, std::string("not ready: ") +
server::phase_name(s.phase) + "\n"};
if (path == "/metrics") {
counters.checkpoint_age_ms =
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - last_checkpoint)
.count());
return {200, server::render_metrics(s, counters),
"text/plain; version=0.0.4; charset=utf-8"};
}
return {404, "not found\n"};
};
LineReader reader;
bool input_open = true, ready = false;
std::optional<std::string> pending;
@@ -346,28 +364,46 @@ template <typename State> int serve(Options& o) {
}
const auto& s = host.step(std::chrono::steady_clock::now());
max_queued = std::max(max_queued, s.queued);
// Checkpoint age restarts at recovery and at every confirmed commit.
if ((!recovered && s.phase == server::HostPhase::running) ||
s.committed_sequence != last_sequence) {
recovered = recovered || s.phase == server::HostPhase::running;
last_sequence = s.committed_sequence;
last_checkpoint = std::chrono::steady_clock::now();
}
if (s.phase == server::HostPhase::acquiring)
recovered = false;
if (!phase || *phase != s.phase) {
phase = s.phase;
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n", phase_name(s.phase),
static_cast<unsigned long long>(s.epoch),
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n",
server::phase_name(s.phase), static_cast<unsigned long long>(s.epoch),
static_cast<unsigned long long>(s.committed_tick),
static_cast<unsigned long long>(s.simulated_tick));
}
set_ready(s.ready);
const auto acknowledged = host.take_acks();
counters.acks += acknowledged.size();
for (const auto& a : acknowledged)
std::printf("ack %s %llu %llu\n", text(a.request).c_str(),
static_cast<unsigned long long>(a.tick),
static_cast<unsigned long long>(a.commit_sequence));
for (const auto& x : host.take_outcomes())
const auto outcomes = host.take_outcomes();
counters.outcomes += outcomes.size();
for (const auto& x : outcomes)
std::printf("outcome %llu %u %llu %llu\n",
static_cast<unsigned long long>(x.decision.tick),
static_cast<unsigned>(x.decision.actor),
static_cast<unsigned long long>(x.decision.organism),
static_cast<unsigned long long>(x.commit_sequence));
for (const auto& d : host.take_dropped())
std::printf("dropped %s %s\n", text(d.input.request).c_str(), dropped_name(d.reason));
const auto dropped = host.take_dropped();
for (const auto& d : dropped)
++counters.dropped[static_cast<std::size_t>(d.reason)];
for (const auto& d : dropped)
std::printf("dropped %s %s\n", text(d.input.request).c_str(),
server::unacknowledged_name(d.reason));
std::fflush(stdout);
if (ops)
ops->serve(handler);
if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal)
crash_now();
const auto finish = [&](int code) {
@@ -398,7 +434,8 @@ int main(int argc, char** argv) {
"--frame N --pg-host H --pg-port P --pg-database D --pg-user U "
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
"[--lease-profile default|fast] [--ready-file PATH]\n",
"[--lease-profile default|fast] [--ready-file PATH] "
"[--ops-listen ADDRESS:PORT]\n",
stderr);
return 2;
}
+13
View File
@@ -134,3 +134,16 @@ migrator/runtime credentials. The example is outside active Argo resources until
an operator supplies the database endpoint, role, Secret and egress rules. No region
server or world-state migration is deployed by M7.2. Schema v2 (M7.3) additionally
requires `EVO_PG_RUNTIME_ROLE`, naming an operator-created NOLOGIN runtime group role.
## Region server image (M7.8)
The `server-image` job builds the Docker target `server-runtime` (`evo-server`
plus the pinned libpq, UID 10001, no credentials). It smoke-tests the image with
`tools/ci/server-image-smoke.sh` and publishes only when Gitea variable
`HARBOR_SERVER_REPOSITORY` is set.
`kubernetes/region-server/deployment.yaml.example` is outside Argo. It shows:
- probes: `/readyz` means authority, `/livez` means only liveness;
- the runtime Secret, object PVC and a NetworkPolicy.
Pin the digest and provision the region first. Without a gateway, the region
receives no inputs.
+13
View File
@@ -82,6 +82,19 @@ ENV PATH="/opt/postgres/bin:${PATH}" LD_LIBRARY_PATH=/opt/postgres/lib
USER 10001:10001
ENTRYPOINT ["bash", "/opt/evo/tools/migrate-postgres.sh"]
# M7.8 development region server image (ADR-0066): evo-server with the pinned
# libpq, non-root, no credentials. Connection secrets are mounted at runtime only.
FROM ${UBUNTU_IMAGE} AS server-runtime
RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 libgcc-s1 libssl3t64 zlib1g ca-certificates && rm -rf /var/lib/apt/lists/* && groupadd --gid 10001 evo && useradd --uid 10001 --gid 10001 --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin evo
COPY --from=postgres-build /source/build/postgres/evo-server /usr/local/bin/evo-server
COPY --from=postgres-build /source/.cache/postgres/install/lib/libpq.so* /opt/postgres/lib/
COPY --from=postgres-build /source/.cache/postgres/postgresql-18.6/COPYRIGHT /usr/share/doc/evo-postgres/COPYRIGHT
ENV LD_LIBRARY_PATH=/opt/postgres/lib
USER 10001:10001
WORKDIR /tmp
EXPOSE 9100
ENTRYPOINT ["/usr/local/bin/evo-server"]
FROM toolchain AS build
RUN cmake --preset release -B build/container \
-DCMAKE_CXX_COMPILER=g++-14 -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=/opt/evo/bin \
@@ -0,0 +1,143 @@
# M7.8 development region server (ADR-0066). NOT in the active Argo kustomization.
# Before applying to evolution-dev: pin the server image digest, provision the region
# (owner-only), create the runtime Secret and object PVC, and add DB/DNS egress for
# the real endpoint. Never target World0 from this example.
#
# Authority, not liveness, gates traffic: /readyz is 200 only while this pod holds
# the region lease and has recovered and verified the committed head. /livez only
# says the loop is responding. A probe can wait up to one bounded I/O step (commit
# deadline 5 s), so probe timeouts below exceed it.
#
# There is no gateway yet: stdin is not attached, so the region hosts authority but
# receives no inputs (tick debt grows by design; nothing is invented).
apiVersion: apps/v1
kind: Deployment
metadata:
name: evolution-region-7-1
namespace: evolution-dev
labels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
spec:
replicas: 1
strategy: {type: Recreate} # Never two pods racing; the lease still arbitrates.
selector:
matchLabels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
template:
metadata:
labels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
spec:
automountServiceAccountToken: false
terminationGracePeriodSeconds: 30 # SIGTERM drains within --drain-ms 20000.
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
initContainers:
# libpq rejects group-readable password files: copy to a private 0600 file.
- name: pgpass
image: registry.aeternalisgames.com/aeternalis/aevolution-server:replace-with-immutable-digest
command: ["/bin/sh", "-c", "install -m 0600 /run/postgres/pgpass /run/evo/pgpass"]
securityContext: {readOnlyRootFilesystem: true, allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: private, mountPath: /run/evo}
containers:
- name: region
image: registry.aeternalisgames.com/aeternalis/aevolution-server:replace-with-immutable-digest
# Flags are "--name value" pairs; see docs/infrastructure/POSTGRESQL.md.
command:
- /usr/local/bin/evo-server
- --profile
- lineage
- --world
- "7"
- --region
- "1"
- --frame
- "9"
- --pg-host
- replace-with-tls-hostname
- --pg-hostaddr
- replace-with-numeric-address
- --pg-port
- "5432"
- --pg-database
- evolution
- --pg-user
- replace-with-runtime-login
- --pg-passfile
- /run/evo/pgpass
- --pg-ca
- /run/postgres/ca.crt
- --objects
- /var/lib/evolution/objects
- --build
- replace-with-retained-build-identity-hex
- --ops-listen
- 0.0.0.0:9100
- --drain-ms
- "20000"
ports:
- {name: ops, containerPort: 9100}
startupProbe:
httpGet: {path: /livez, port: ops}
periodSeconds: 5
timeoutSeconds: 8
failureThreshold: 12
readinessProbe:
httpGet: {path: /readyz, port: ops}
periodSeconds: 5
timeoutSeconds: 8
failureThreshold: 1 # Any doubt about authority removes the pod at once.
livenessProbe:
httpGet: {path: /livez, port: ops}
periodSeconds: 10
timeoutSeconds: 8
failureThreshold: 6
resources:
requests: {cpu: 250m, memory: 128Mi}
limits: {cpu: "1", memory: 512Mi}
securityContext:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: private, mountPath: /run/evo, readOnly: true}
- {name: objects, mountPath: /var/lib/evolution/objects}
- {name: scratch, mountPath: /tmp}
volumes:
- name: connection
secret:
secretName: evolution-postgres-runtime # Runtime login only; never migrator.
defaultMode: 0440
items:
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: private
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: objects
persistentVolumeClaim: {claimName: evolution-region-objects}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 8Mi}
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: evolution-region-7-1
namespace: evolution-dev
spec:
podSelector:
matchLabels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
policyTypes: [Ingress, Egress]
ingress:
# Metrics/probes only from the monitoring namespace (kubelet probes use the node).
- from:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: monitoring}
ports: [{port: 9100, protocol: TCP}]
egress:
# Replace with the actual database endpoint; no DNS is needed with a numeric hostaddr.
- to: [{ipBlock: {cidr: 192.0.2.10/32}}]
ports: [{port: 5432, protocol: TCP}]
+8
View File
@@ -576,3 +576,11 @@ M7.7 implemented and locally qualified (2026-09-28), under ADR-0065.
- Real SIGKILL at five boundaries, a resumed SIGSTOPped owner, and damaged heads
all behave as the crash matrix requires.
Next: M7.8 operations endpoints and dev image.
M7.8 implemented and locally qualified (2026-09-28), under ADR-0066.
- `evo-server --ops-listen` serves `/livez` (loop alive), `/readyz` (current
authority only) and `/metrics` (30 fixed series with bounded labels).
- The responder is dependency-free, single-threaded and bounded.
- The `server-runtime` image is non-root with no secrets; a CI smoke job and a
Kubernetes example are included.
Next: M7.9 clone, migrate and qualify durability.
+21
View File
@@ -880,3 +880,24 @@ Tail-skip and replay-equality mutations were each caught. The PostgreSQL suites
passed under Release, analysis and ASan/UBSan.
Next: M7.8 operations endpoints and a dev image.
## M7.8 operations endpoints and server image (2026-09-28)
ADR-0066: `evo-server --ops-listen` serves three endpoints:
- `/livez`: the loop is responding; it is never a statement of authority.
- `/readyz`: 200 only with the current lease and a verified recovered head.
- `/metrics`: 30 fixed series whose only labels are a 6-value phase and a 4-value
drop reason.
The responder is dependency-free, runs on the host's own loop with no thread, and
is bounded: 8 connections, 4 KiB requests, 1 s deadlines, GET on exact paths
only. The new `server-runtime` Docker target is non-root with no credentials,
covered by a CI smoke script, and a Kubernetes example ships outside Argo.
Real process probes covered serving authority, an unavailable database (live,
503) and a resumed stale process (live, 503, fence counted). Readiness-as-liveness
and an unbounded connection pool were each caught by a test. Windows ran 59/59;
Linux ran 59/59 in four configurations; the PostgreSQL suites passed in three.
Docker itself was not run locally; an emulated image layout was checked instead.
Next: M7.9.
+2
View File
@@ -3,6 +3,8 @@
## Status
Small proof boundary adopted; Prometheus/Grafana/Loki/Otel adapters proposed for M7–M8.
M7.8 (ADR-0066) implements the region server exposition with a tiny direct
responder and fixed-label metrics; client libraries remain unadopted.
## Context
+101
View File
@@ -0,0 +1,101 @@
# ADR-0066: Operations endpoints and development server image
## Status
Accepted for M7.8, 2026-09-28. It implements the M7 part of ADR-0015
(observability). It builds on ADR-0064 (host) and ADR-0065 (recovery). There is
still no gateway, alerting rules or production promotion.
## Context
M7.8 needs:
- separate liveness and readiness;
- metrics for tick debt, commit lag, lease safety, checkpoint age and queue bounds;
- a non-root image with no embedded secrets.
The named risk is a healthy process mistaken for valid authority. ADR-0015 forbids
unbounded or entity labels, and allows a tiny direct exposition with format tests.
The constitution forbids hidden threads and unreviewed dependencies.
## Decision
**Endpoints** (`evo-server --ops-listen <numeric address:port>`):
- `/livez` returns 200 whenever the host loop answers. It says nothing about
authority, and is used only for restarts.
- `/readyz` returns 200 only when `HostStatus::ready` holds: running, recovered and
tail-verified, not draining, and `LeaseAuthority::may_originate`. Otherwise it
returns 503 with the phase name. It is used for traffic, and later gateway
routing must additionally honor the lease epoch.
- `/metrics` returns Prometheus text 0.0.4.
**Responder** (`OpsServer`, Linux, no dependency):
- Numeric bind only. GET only (405 otherwise), exact paths (404), malformed
requests 400, requests over 4 KiB 431.
- One request per connection, `Connection: close`, `Cache-Control: no-store`.
- At most 8 connections; any excess is closed on accept, never queued.
- Each connection has a 1 s deadline, and responses are capped at 16 KiB.
- `serve()` never blocks and runs on the host's own loop, so answers describe
current state and no thread is added. A complete request that waited behind one
bounded host step is still answered.
**Metrics** (`evo::ops`, portable and unit-tested):
| Group | Series |
|---|---|
| Authority | ready; one-hot phase (6 values); lease epoch; lease remaining (ms) |
| Progress | simulated and committed tick and sequence; commit lag; checkpoint age (ms) |
| Queue | tick debt, current and max; queued inputs and bytes; their capacities |
| Counters | commits; store failures; unknown outcomes; fences; acknowledgements; outcomes; dropped (4 reasons) |
That is 30 series with fixed names. The only labels are `phase` and `reason`,
each from a fixed enum. There are no world, region, entity or request labels:
the scrape target carries region identity. The exposition is about 3.5 KB and at
most 8 KiB. Commit lag saturates at zero rather than wrapping.
**Image** (`server-runtime` Docker target) contains `evo-server` and the pinned
libpq, and installs only the runtime libraries it needs (libstdc++, libssl3,
zlib, CA certificates):
- It runs as UID/GID 10001 with a read-only root filesystem and all capabilities
dropped. It exposes port 9100 and has no default configuration.
- With missing settings it exits 2 with usage; there is no default region,
database or build.
- CI builds it and smoke-tests it: the usage exit code under restrictions,
non-root identity, resolved libraries, and no credential files. Publishing
requires `HARBOR_SERVER_REPOSITORY`.
**Deployment example** (`deploy/kubernetes/region-server`, not in Argo):
- One replica with the Recreate strategy.
- The runtime Secret is copied to a private 0600 file by an init container.
- An object PVC; startup, readiness and liveness probes with 8 s timeouts,
longer than one bounded I/O step.
- A NetworkPolicy allowing only monitoring ingress on 9100 and the database egress.
## Alternatives Considered
- **prometheus-cpp or OpenTelemetry**: add dependencies and threads for about 30
fixed series.
- **An HTTP library**: a dependency for three GET paths.
- **A separate probe thread**: would answer while the host is stuck in I/O, which
hides a blocked loop and adds shared state.
- **Readiness from process health or phase alone**: exactly the named risk.
## Benefits
- Probes and metrics cannot claim authority the lease does not grant.
- Label cardinality is fixed by construction.
- The image carries nothing environment-specific.
## Costs
- Probes can wait for one bounded host step (up to the 5 s commit deadline), so
probe timeouts must exceed it.
- The responder is intentionally minimal (no keep-alive, TLS or auth).
- The metrics endpoint must be network-restricted.
## Risks
- Metrics and probes are unauthenticated plain HTTP; the NetworkPolicy is the
control.
- The build-tree RUNPATH stays in the binary; it is harmless in the image, where
the path does not exist, and `LD_LIBRARY_PATH` names libpq.
- Local Docker was not run in this gate; the CI job is configured but its first
execution is pending.
## Consequences
- M7.9 measures recovery and batching latency through these metrics.
- A later gateway must honor readiness and the lease epoch before routing.
- Promoting to World0 needs a scanned, signed image and a reviewed alerting policy
(M8).
+1
View File
@@ -76,3 +76,4 @@ design. No unadopted candidate is a link dependency of the proof.
- [ADR-0063: Batch outcomes](0063-batch-outcomes.md) - accepted for M7.5 manifest v2, validation trace replay, durable decisions/outbox and commit resolution.
- [ADR-0064: Region host](0064-region-host.md) - accepted for M7.6 single-owner host state machine, bounded admission, final-only-after-commit and evo-server process.
- [ADR-0065: Recovery before readiness](0065-recovery-before-readiness.md) - accepted for M7.7 head tail replay verification, schema v6 commit lookup and real kill-boundary tests.
- [ADR-0066: Operations endpoints](0066-operations-endpoints.md) - accepted for M7.8 liveness vs authority readiness, bounded metrics, dependency-free responder and the server image.
+41
View File
@@ -2381,3 +2381,44 @@ Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate
- Not executed: Docker/Gitea CI, the production 20 s lease, host loss or VM
freeze, multi-host network partitions, or verification of links older than
the head.
## M7.8 operations endpoints - 2026-09-28
- Windows MSVC Release CTest passed 59/59, including the new portable
`ops-contracts`: exposition format, HELP/TYPE per family, the fixed 30-series
set, bounded labels, one-hot phase and non-wrapping lag. The exposition is
3485 bytes typically and 3836 at worst-case values (limit 8192).
- WSL Linux: the `checks` build set (GCC14 Debug and Release, clang19 analysis,
clang19 ASan/UBSan) passed CTest 59/59 each.
- The PostgreSQL suites (GCC Release, clang-tidy analysis, ASan/UBSan) passed
after fixing the ops test's optional-access and reserve findings. The earlier
PostgreSQL analysis runs used a stale binary and are not counted. The
repository format check passed after formatting the server.
- Process probe tests (real `evo-server`, loopback, ephemeral port):
- Serving: `/livez` 200 `live`, `/readyz` 200 `ready`. Metrics showed ready 1,
epoch 1, lease remaining above 0, capacities 64/65536 and the running phase.
After 30 inputs: committed tick 30, lag 0, 30 acknowledgements, checkpoint
age under 60 s.
- HTTP bounds: POST 405, unknown path and query string 404, malformed 400,
over 4 KiB 431. Eight idle connections do not block probes; a ninth is
closed within 500 ms; idle ones close at their deadline.
- Database unreachable: `/livez` 200, `/readyz` 503 `not ready: acquiring`,
ready 0, store failures above 0, never `ready 1`, exit 0 on SIGTERM.
- Stale process (SIGSTOP 4.5 s, then SIGCONT): `/livez` 200, `/readyz` 503,
ready 0, fences_total 1, lease remaining 0. The successor answers 200 at
epoch 2.
- Mutations (restored and rerun green): readiness that returns 200 whenever live
was caught. Raising the connection bound to 1000 was not caught at first,
because the check did not distinguish refusal from deadline expiry. The test
now requires the refusal within 500 ms and catches it.
- Image: the Docker engine was not running locally (Docker Desktop stopped), so
the `server-runtime` build and `tools/ci/server-image-smoke.sh` have not
executed. A filesystem emulation (binary plus the pinned libpq, `env -i`)
showed usage exit 2 with no settings and all libraries resolved. It also found
and fixed a bug: without arguments the server had reached object-store opening
instead of refusing with usage. The binary keeps a build-tree RUNPATH, which is
harmless in the image.
- The manifest example parses (Deployment and NetworkPolicy, three probes) and
the workflow YAML parses; `bash -n` passed for the smoke script.
- Not executed: the container build and smoke, the Gitea job, cluster apply,
Prometheus scraping, and TSan.
+12
View File
@@ -191,3 +191,15 @@ boundary; never set it in shared environments. Use the default 20 s lease in any
shared environment. `--lease-profile fast` exists
only for local tests. The runtime credentials come from `--pg-*` flags and a
passfile; never place passwords on the command line.
### Operations endpoints (M7.8, ADR-0066)
`--ops-listen <numeric address:port>` serves three endpoints:
- `GET /livez`: the process loop is responding. It says nothing about authority.
- `GET /readyz`: 200 only while this process holds the lease and has a verified
recovered head; 503 with the phase otherwise.
- `GET /metrics`: Prometheus text.
A probe may wait up to one bounded I/O step (the 5 s commit deadline), so probe
timeouts must exceed that. Expose port 9100 only to the monitoring namespace;
the endpoints are unauthenticated plain HTTP.
+57
View File
@@ -0,0 +1,57 @@
#pragma once
#include <array>
#include <cstddef>
#include <cstdint>
#include <string>
// Region host status and its bounded operational exposition (M7.8, ADR-0066).
// Portable: no sockets, clocks or persistence types. Liveness is not authority:
// only `ready` means the host may originate output.
namespace evo::server {
enum class HostPhase : std::uint8_t {
acquiring, // Waiting for the lease (busy/unavailable) or recovering.
running, // Authoritative: simulating and committing.
resolving, // A commit outcome is unknown: paused until resolved.
draining, // Shutdown: committing pending work, then releasing.
stopped, // Drained (lease released when possible).
quarantined, // Fail closed: committed state or history cannot be trusted.
};
inline constexpr std::size_t host_phase_count = 6;
// Why an accepted input will never be acknowledged by this host.
enum class Unacknowledged : std::uint8_t {
refused, // The simulation rejected it; no tick consumed it.
drained, // Still queued at shutdown.
fenced, // Authority was lost before its tick committed; the next owner never saw it.
quarantined,
};
inline constexpr std::size_t unacknowledged_count = 4;
struct HostStatus {
HostPhase phase{HostPhase::acquiring};
bool ready{}; // Authoritative now: recovered, running, lease may originate.
std::uint64_t epoch{};
std::uint64_t simulated_tick{}, committed_tick{}, committed_sequence{};
std::size_t queued{}, queued_bytes{};
std::uint64_t tick_debt{}, max_tick_debt{}; // Due minus simulated since running.
std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{};
// Local lease safety: time until this host must stop originating (0 = none).
std::uint64_t lease_remaining_ms{};
};
// Process-level counters and bounds the exposition adds to the host status.
struct OpsCounters {
std::size_t queue_capacity_inputs{}, queue_capacity_bytes{};
std::uint64_t checkpoint_age_ms{}; // Since the last confirmed commit or recovery.
std::uint64_t acks{}, outcomes{};
std::array<std::uint64_t, unacknowledged_count> dropped{}; // Indexed by Unacknowledged.
};
[[nodiscard]] const char* phase_name(HostPhase phase) noexcept;
[[nodiscard]] const char* unacknowledged_name(Unacknowledged reason) noexcept;
// Prometheus text format 0.0.4. Fixed metric names; the only labels are `phase`
// (6 values) and `reason` (4 values), so cardinality is bounded by construction.
// No world/region/entity/request labels: scrape targets carry region identity.
[[nodiscard]] std::string render_metrics(const HostStatus& status, const OpsCounters& counters);
inline constexpr std::size_t max_metrics_bytes = 8192;
} // namespace evo::server
+67
View File
@@ -0,0 +1,67 @@
#pragma once
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <expected>
#include <functional>
#include <memory>
#include <string>
#include <string_view>
#include <vector>
// Minimal operational HTTP/1.1 responder for probes and metrics (M7.8, ADR-0066).
// Linux. Not a general web server: GET only, one request per connection
// (Connection: close), bounded connections, request bytes, response bytes and
// per-connection time. serve() never blocks; it runs on the host's own loop so
// answers always describe the current state and no thread is added.
namespace evo::server {
struct OpsResponse {
int status{200};
std::string body;
std::string content_type{"text/plain; charset=utf-8"};
};
// Receives the request path (no query handling: probes use exact paths).
using OpsHandler = std::function<OpsResponse(std::string_view path)>;
enum class OpsError : std::uint8_t { address, socket };
class OpsServer {
public:
static constexpr std::size_t max_connections = 8;
static constexpr std::size_t max_request_bytes = 4096;
static constexpr std::size_t max_response_bytes = 16384;
static constexpr std::chrono::milliseconds connection_deadline{1000};
// Numeric address only ("127.0.0.1:9100" or "[::1]:9100"); port 0 picks one.
[[nodiscard]] static std::expected<std::unique_ptr<OpsServer>, OpsError>
listen(std::string_view address);
OpsServer(const OpsServer&) = delete;
OpsServer& operator=(const OpsServer&) = delete;
OpsServer(OpsServer&&) = delete;
OpsServer& operator=(OpsServer&&) = delete;
~OpsServer();
[[nodiscard]] std::uint16_t port() const noexcept {
return port_;
}
// Accepts, reads, answers and expires connections without waiting. Connections
// beyond the bound are closed at once; slow or oversized ones are answered or
// dropped without delaying others.
void serve(const OpsHandler& handler);
[[nodiscard]] std::uint64_t refused() const noexcept {
return refused_;
}
private:
struct Connection {
int fd{-1};
std::string in, out;
std::size_t sent{};
std::chrono::steady_clock::time_point deadline{};
};
OpsServer(int fd, std::uint16_t port) : fd_(fd), port_(port) {}
void respond(Connection& c, const OpsHandler& handler);
int fd_;
std::uint16_t port_;
std::vector<Connection> connections_;
std::uint64_t refused_{};
};
} // namespace evo::server
+1 -25
View File
@@ -2,6 +2,7 @@
#include "evo/persistence/checkpoint_store.hpp"
#include "evo/persistence/region_batch.hpp"
#include "evo/persistence/region_lease.hpp"
#include "evo/server/ops.hpp"
#include <chrono>
#include <cstddef>
#include <cstdint>
@@ -45,36 +46,11 @@ struct FinalOutcome {
std::uint64_t commit_sequence{};
friend bool operator==(const FinalOutcome&, const FinalOutcome&) = default;
};
// Why an accepted input will never be acknowledged by this host.
enum class Unacknowledged : std::uint8_t {
refused, // The simulation rejected it; no tick consumed it.
drained, // Still queued at shutdown.
fenced, // Authority was lost before its tick committed; the next owner never saw it.
quarantined,
};
struct Dropped {
TickInput input{};
Unacknowledged reason{};
};
enum class HostPhase : std::uint8_t {
acquiring, // Waiting for the lease (busy/unavailable) or recovering.
running, // Authoritative: simulating and committing.
resolving, // A commit outcome is unknown: paused until resolved.
draining, // Shutdown: committing pending work, then releasing.
stopped, // Drained (lease released when possible).
quarantined, // Fail closed: committed state or history cannot be trusted.
};
struct HostStatus {
HostPhase phase{HostPhase::acquiring};
bool ready{}; // Authoritative now: recovered, running, lease may originate.
std::uint64_t epoch{};
std::uint64_t simulated_tick{}, committed_tick{}, committed_sequence{};
std::size_t queued{}, queued_bytes{};
std::uint64_t tick_debt{}, max_tick_debt{}; // Due minus simulated since running.
std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{};
};
// Commit boundaries for fault injection (M7.7 kill tests).
enum class CrashPoint : std::uint8_t {
before_upload, // Batch sealed; nothing uploaded or sent.
+104
View File
@@ -0,0 +1,104 @@
#include "evo/server/ops.hpp"
#include <string_view>
namespace evo::server {
const char* phase_name(HostPhase phase) noexcept {
switch (phase) {
case HostPhase::acquiring:
return "acquiring";
case HostPhase::running:
return "running";
case HostPhase::resolving:
return "resolving";
case HostPhase::draining:
return "draining";
case HostPhase::stopped:
return "stopped";
case HostPhase::quarantined:
return "quarantined";
}
return "unknown";
}
const char* unacknowledged_name(Unacknowledged reason) noexcept {
switch (reason) {
case Unacknowledged::refused:
return "refused";
case Unacknowledged::drained:
return "drained";
case Unacknowledged::fenced:
return "fenced";
case Unacknowledged::quarantined:
return "quarantined";
}
return "unknown";
}
namespace {
void metric(std::string& out, std::string_view name, std::string_view type, std::string_view help,
std::uint64_t value) {
out.append("# HELP ").append(name).append(" ").append(help).append("\n");
out.append("# TYPE ").append(name).append(" ").append(type).append("\n");
out.append(name).append(" ").append(std::to_string(value)).append("\n");
}
} // namespace
std::string render_metrics(const HostStatus& s, const OpsCounters& c) {
std::string out;
out.reserve(4096);
metric(out, "evo_region_ready", "gauge",
"1 only while this host holds current authority and may originate output", s.ready);
out.append("# HELP evo_region_phase Current host phase (one-hot)\n");
out.append("# TYPE evo_region_phase gauge\n");
for (std::size_t i = 0; i < host_phase_count; ++i) {
const auto phase = static_cast<HostPhase>(i);
out.append("evo_region_phase{phase=\"")
.append(phase_name(phase))
.append("\"} ")
.append(phase == s.phase ? "1" : "0")
.append("\n");
}
metric(out, "evo_region_lease_epoch", "gauge", "Lease epoch held or last held", s.epoch);
metric(out, "evo_region_lease_remaining_milliseconds", "gauge",
"Local time until this host must stop originating (0 without a lease)",
s.lease_remaining_ms);
metric(out, "evo_region_simulated_tick", "gauge", "Last simulated tick", s.simulated_tick);
metric(out, "evo_region_committed_tick", "gauge", "End tick of the committed head",
s.committed_tick);
metric(out, "evo_region_committed_sequence", "gauge", "Commit sequence of the head",
s.committed_sequence);
metric(out, "evo_region_commit_lag_ticks", "gauge",
"Simulated but not yet durably committed ticks",
s.simulated_tick >= s.committed_tick ? s.simulated_tick - s.committed_tick : 0);
metric(out, "evo_region_checkpoint_age_milliseconds", "gauge",
"Time since the last confirmed commit or recovery", c.checkpoint_age_ms);
metric(out, "evo_region_tick_debt", "gauge", "Due ticks not yet simulated", s.tick_debt);
metric(out, "evo_region_tick_debt_max", "gauge", "Largest tick debt since recovery",
s.max_tick_debt);
metric(out, "evo_region_queued_inputs", "gauge", "Admitted inputs awaiting a tick", s.queued);
metric(out, "evo_region_queued_bytes", "gauge", "Accounted bytes of queued inputs",
s.queued_bytes);
metric(out, "evo_region_queue_capacity_inputs", "gauge", "Input backlog bound",
c.queue_capacity_inputs);
metric(out, "evo_region_queue_capacity_bytes", "gauge", "Input backlog byte bound",
c.queue_capacity_bytes);
metric(out, "evo_region_commits_total", "counter", "Confirmed commits", s.commits);
metric(out, "evo_region_store_failures_total", "counter",
"Storage or database failures that paused progress", s.store_failures);
metric(out, "evo_region_unknown_outcomes_total", "counter",
"Commits whose outcome had to be resolved", s.unknown_outcomes);
metric(out, "evo_region_fences_total", "counter", "Times this host lost authority", s.fences);
metric(out, "evo_region_acknowledgements_total", "counter", "Final acknowledgements emitted",
c.acks);
metric(out, "evo_region_outcomes_total", "counter", "Final permanent outcomes emitted",
c.outcomes);
out.append("# HELP evo_region_dropped_total Accepted inputs that will never be acknowledged\n");
out.append("# TYPE evo_region_dropped_total counter\n");
for (std::size_t i = 0; i < unacknowledged_count; ++i)
out.append("evo_region_dropped_total{reason=\"")
.append(unacknowledged_name(static_cast<Unacknowledged>(i)))
.append("\"} ")
.append(std::to_string(c.dropped[i]))
.append("\n");
return out;
}
} // namespace evo::server
+175
View File
@@ -0,0 +1,175 @@
#include "evo/server/ops_http.hpp"
#include <arpa/inet.h>
#include <cerrno>
#include <charconv>
#include <netinet/in.h>
#include <sys/socket.h>
#include <unistd.h>
namespace evo::server {
namespace {
using Clock = std::chrono::steady_clock;
const char* reason(int status) {
switch (status) {
case 200:
return "OK";
case 400:
return "Bad Request";
case 404:
return "Not Found";
case 405:
return "Method Not Allowed";
case 431:
return "Request Header Fields Too Large";
case 503:
return "Service Unavailable";
default:
return "Internal Server Error";
}
}
std::string encode(const OpsResponse& r) {
std::string out = "HTTP/1.1 " + std::to_string(r.status) + " " + reason(r.status) +
"\r\nContent-Type: " + r.content_type +
"\r\nContent-Length: " + std::to_string(r.body.size()) +
"\r\nCache-Control: no-store\r\nConnection: close\r\n";
if (r.status == 405)
out += "Allow: GET\r\n";
return out + "\r\n" + r.body;
}
bool printable(std::string_view s) {
for (const char ch : s)
if (ch <= ' ' || ch > '~')
return false;
return true;
}
} // namespace
std::expected<std::unique_ptr<OpsServer>, OpsError> OpsServer::listen(std::string_view address) {
const auto colon = address.rfind(':');
if (colon == std::string_view::npos || colon + 1 >= address.size())
return std::unexpected{OpsError::address};
unsigned port{};
const auto digits = address.substr(colon + 1);
const auto [end, error] = std::from_chars(digits.data(), digits.data() + digits.size(), port);
if (error != std::errc{} || end != digits.data() + digits.size() || port > 65535)
return std::unexpected{OpsError::address};
auto host = std::string(address.substr(0, colon));
sockaddr_storage storage{};
socklen_t length{};
int family{};
if (host.size() >= 2 && host.front() == '[' && host.back() == ']') {
host = host.substr(1, host.size() - 2);
auto* v6 = reinterpret_cast<sockaddr_in6*>(&storage);
if (inet_pton(AF_INET6, host.c_str(), &v6->sin6_addr) != 1)
return std::unexpected{OpsError::address};
v6->sin6_family = AF_INET6;
v6->sin6_port = htons(static_cast<std::uint16_t>(port));
family = AF_INET6;
length = sizeof(sockaddr_in6);
} else {
auto* v4 = reinterpret_cast<sockaddr_in*>(&storage);
if (inet_pton(AF_INET, host.c_str(), &v4->sin_addr) != 1)
return std::unexpected{OpsError::address};
v4->sin_family = AF_INET;
v4->sin_port = htons(static_cast<std::uint16_t>(port));
family = AF_INET;
length = sizeof(sockaddr_in);
}
const int fd = ::socket(family, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0);
if (fd < 0)
return std::unexpected{OpsError::socket};
const int on = 1;
::setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof on);
if (::bind(fd, reinterpret_cast<sockaddr*>(&storage), length) != 0 || ::listen(fd, 16) != 0) {
::close(fd);
return std::unexpected{OpsError::socket};
}
sockaddr_storage bound{};
socklen_t bound_length = sizeof bound;
::getsockname(fd, reinterpret_cast<sockaddr*>(&bound), &bound_length);
const auto actual = family == AF_INET6
? ntohs(reinterpret_cast<sockaddr_in6*>(&bound)->sin6_port)
: ntohs(reinterpret_cast<sockaddr_in*>(&bound)->sin_port);
return std::unique_ptr<OpsServer>(new OpsServer(fd, actual));
}
OpsServer::~OpsServer() {
for (auto& c : connections_)
::close(c.fd);
::close(fd_);
}
void OpsServer::respond(Connection& c, const OpsHandler& handler) {
OpsResponse response;
const auto line_end = c.in.find("\r\n");
const std::string_view line = std::string_view(c.in).substr(0, line_end);
const auto first = line.find(' ');
const auto second = first == std::string_view::npos ? first : line.find(' ', first + 1);
if (second == std::string_view::npos || line.substr(second + 1).rfind("HTTP/1.", 0) != 0) {
response = {400, "bad request\n"};
} else {
const auto method = line.substr(0, first);
const auto path = line.substr(first + 1, second - first - 1);
if (method != "GET")
response = {405, "method not allowed\n"};
else if (path.empty() || path.front() != '/' || path.size() > 256 || !printable(path))
response = {400, "bad request\n"};
else
response = handler(path);
}
if (response.body.size() > max_response_bytes)
response = {500, "response too large\n"};
c.out = encode(response);
}
void OpsServer::serve(const OpsHandler& handler) {
const auto now = Clock::now();
for (;;) {
const int fd = ::accept4(fd_, nullptr, nullptr, SOCK_NONBLOCK | SOCK_CLOEXEC);
if (fd < 0)
break; // EAGAIN: nothing pending (other errors: retried next call).
if (connections_.size() >= max_connections) {
::close(fd); // Bounded: excess connections are refused, never queued.
++refused_;
continue;
}
connections_.push_back({fd, {}, {}, 0, now + connection_deadline});
}
for (auto it = connections_.begin(); it != connections_.end();) {
auto& c = *it;
bool done = false;
// Read before expiring: a complete request that waited behind a bounded host
// step is still answered; only incomplete or unsent work expires.
if (c.out.empty()) {
char chunk[1024];
const auto n = ::recv(c.fd, chunk, sizeof chunk, 0);
if (n == 0 || (n < 0 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR)) {
done = true;
} else if (n > 0) {
c.in.append(chunk, static_cast<std::size_t>(n));
if (c.in.size() > max_request_bytes)
c.out = encode({431, "request too large\n"});
else if (c.in.find("\r\n\r\n") != std::string::npos)
respond(c, handler);
}
}
if (!done && !c.out.empty()) {
const auto n = ::send(c.fd, c.out.data() + c.sent, c.out.size() - c.sent, MSG_NOSIGNAL);
if (n > 0)
c.sent += static_cast<std::size_t>(n);
done = c.sent == c.out.size() ||
(n < 0 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR);
}
// Slow client: an incomplete request expires at the deadline; a response the
// client will not take gets one more deadline, then is dropped.
if (!done && now > c.deadline && (c.out.empty() || now > c.deadline + connection_deadline))
done = true;
if (done) {
::close(c.fd);
it = connections_.erase(it);
} else {
++it;
}
}
}
} // namespace evo::server
+7
View File
@@ -386,6 +386,13 @@ template <typename State> void RegionHost<State>::refresh(SchedulerTime now) {
status_.queued_bytes = queue_.size() * tick_input_bytes;
const auto& grant = authority_.grant();
status_.epoch = grant ? grant->epoch : 0;
const auto lease_now = LeaseClock::now();
status_.lease_remaining_ms =
grant && authority_.may_originate(lease_now)
? static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
authority_.deadline() - lease_now)
.count())
: 0;
status_.committed_tick = committed_ ? committed_->head.end_tick : 0;
status_.committed_sequence = committed_ ? committed_->head.sequence : 0;
status_.simulated_tick = state_ ? Ops<State>::tick(*state_) : status_.committed_tick;
+119
View File
@@ -0,0 +1,119 @@
#include "evo/server/ops.hpp"
#include <algorithm>
#include <cstdio>
#include <limits>
#include <set>
#include <source_location>
#include <sstream>
#include <stdexcept>
#include <string>
#include <vector>
namespace {
using namespace evo::server;
void check(bool ok, std::source_location at = std::source_location::current()) {
if (!ok)
throw std::runtime_error("ops line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
struct Sample {
std::string name, labels, value;
};
// Parses exposition lines; every family must be introduced by HELP and TYPE.
std::vector<Sample> parse(const std::string& text) {
std::vector<Sample> out;
std::set<std::string> typed, helped;
std::istringstream in(text);
for (std::string line; std::getline(in, line);) {
CHECK(!line.empty());
if (line.rfind("# HELP ", 0) == 0) {
helped.insert(line.substr(7, line.find(' ', 7) - 7));
continue;
}
if (line.rfind("# TYPE ", 0) == 0) {
const auto name = line.substr(7, line.find(' ', 7) - 7);
const auto type = line.substr(line.rfind(' ') + 1);
CHECK(type == "gauge" || type == "counter");
CHECK(helped.contains(name) && !typed.contains(name)); // One family, documented.
typed.insert(name);
continue;
}
const auto space = line.rfind(' ');
const auto brace = line.find('{');
Sample s{line.substr(0, std::min(space, brace)), "", line.substr(space + 1)};
if (brace < space)
s.labels = line.substr(brace, space - brace);
CHECK(typed.contains(s.name));
CHECK(!s.value.empty() && s.value.find_first_not_of("0123456789") == std::string::npos);
out.push_back(s);
}
return out;
}
void exposition_is_bounded_and_well_formed() {
HostStatus s;
s.phase = HostPhase::running;
s.ready = true;
s.epoch = 3;
s.simulated_tick = 50;
s.committed_tick = 41;
s.lease_remaining_ms = 1234;
OpsCounters c;
c.queue_capacity_inputs = 64;
c.queue_capacity_bytes = 65536;
c.dropped[2] = 5;
const auto text = render_metrics(s, c);
const auto samples = parse(text);
const std::set<std::string> label_names{"", "phase", "reason"};
std::size_t phases_on = 0;
for (const auto& x : samples) {
CHECK(x.name.rfind("evo_region_", 0) == 0);
const auto key =
x.labels.empty() ? std::string{} : x.labels.substr(1, x.labels.find('=') - 1);
CHECK(label_names.contains(key)); // No world/region/entity/request labels.
if (x.name == "evo_region_phase")
phases_on += x.value == "1";
}
CHECK(phases_on == 1);
CHECK(text.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos);
CHECK(text.find("evo_region_ready 1\n") != std::string::npos);
CHECK(text.find("evo_region_commit_lag_ticks 9\n") != std::string::npos);
CHECK(text.find("evo_region_lease_remaining_milliseconds 1234\n") != std::string::npos);
CHECK(text.find("evo_region_dropped_total{reason=\"fenced\"} 5\n") != std::string::npos);
// Series count is fixed: 6 phases + 4 reasons + 20 unlabelled families.
CHECK(samples.size() == 6 + 4 + 20);
// Worst-case values keep the size bounded and never underflow the lag.
HostStatus big;
big.phase = HostPhase::quarantined;
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most;
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
big.fences = big.lease_remaining_ms = most;
big.simulated_tick = 1; // Lag saturates at zero rather than wrapping.
OpsCounters bigc{most, most, most, most, most, {most, most, most, most}};
const auto large = render_metrics(big, bigc);
CHECK(large.size() <= max_metrics_bytes);
CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos);
CHECK(large.find("evo_region_ready 0\n") != std::string::npos);
CHECK(parse(large).size() == samples.size());
for (std::size_t i = 0; i < host_phase_count; ++i)
CHECK(std::string(phase_name(static_cast<HostPhase>(i))) != "unknown");
for (std::size_t i = 0; i < unacknowledged_count; ++i)
CHECK(std::string(unacknowledged_name(static_cast<Unacknowledged>(i))) != "unknown");
std::printf("ops_metrics_bytes typical=%zu worst=%zu\n", text.size(), large.size());
}
} // namespace
int main() {
try {
exposition_is_bounded_and_well_formed();
std::puts("ops contracts passed");
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "%s\n", e.what());
return 1;
}
}
+3
View File
@@ -8,6 +8,7 @@ void run_region_checkpoint_store_tests();
void run_region_host_tests();
void run_region_server_process_tests();
void run_region_recovery_tests();
void run_region_ops_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
@@ -84,6 +85,8 @@ int main() {
std::puts("region server process integration passed");
run_region_recovery_tests();
std::puts("region recovery integration passed");
run_region_ops_tests();
std::puts("region ops integration passed");
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "%s\n", e.what());
+193
View File
@@ -0,0 +1,193 @@
// M7.8 operational endpoints on real evo-server processes (ADR-0066): liveness is
// never authority, readiness follows the lease, metrics are bounded, and the
// responder is bounded against idle, excess and malformed clients.
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <source_location>
#include <sys/socket.h>
#include <thread>
void run_region_ops_tests();
namespace {
using namespace evo::test::process;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region ops test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
int connect_local(std::uint16_t port) {
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
CHECK(fd >= 0);
sockaddr_in address{};
address.sin_family = AF_INET;
address.sin_port = htons(port);
address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
CHECK(::connect(fd, reinterpret_cast<sockaddr*>(&address), sizeof address) == 0);
timeval timeout{3, 0};
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
return fd;
}
struct Reply {
int status{};
std::string body;
};
// Sends raw bytes and reads until the server closes (Connection: close).
Reply http_exchange(std::uint16_t port, const std::string& request) {
const int fd = connect_local(port);
std::size_t sent = 0;
while (sent < request.size()) {
const auto n = ::send(fd, request.data() + sent, request.size() - sent, MSG_NOSIGNAL);
if (n <= 0)
break;
sent += static_cast<std::size_t>(n);
}
std::string raw;
char chunk[4096];
for (;;) {
const auto n = ::recv(fd, chunk, sizeof chunk, 0);
if (n <= 0)
break;
raw.append(chunk, static_cast<std::size_t>(n));
}
::close(fd);
int status = 0;
CHECK(std::sscanf(raw.c_str(), "HTTP/1.1 %d", &status) == 1);
const auto body = raw.find("\r\n\r\n");
return Reply{status, body == std::string::npos ? "" : raw.substr(body + 4)};
}
Reply get(std::uint16_t port, const std::string& path) {
return http_exchange(port, "GET " + path + " HTTP/1.1\r\nHost: probe\r\n\r\n");
}
std::uint64_t metric(std::uint16_t port, const std::string& series) {
const auto reply = get(port, "/metrics");
CHECK(reply.status == 200);
const auto at = reply.body.find("\n" + series + " ");
CHECK(at != std::string::npos);
return std::stoull(reply.body.substr(at + series.size() + 2));
}
std::uint16_t ops_port(Child& server) {
CHECK(server.wait_line("ops listening ", 10s));
for (const auto& l : server.lines())
if (starts(l, "ops listening "))
return static_cast<std::uint16_t>(std::stoul(l.substr(14)));
CHECK(false);
return 0;
}
std::vector<std::string> with_ops(const ServerRegion& r, std::vector<std::string> extra = {}) {
extra.insert(extra.begin(), {"--ops-listen", "127.0.0.1:0"});
return server_args(r, std::move(extra));
}
void probes_follow_authority_and_stay_bounded() {
const auto r = provision(51, 0x0B);
Child server{with_ops(r)};
const auto port = ops_port(server);
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
auto live = get(port, "/livez");
auto ready = get(port, "/readyz");
CHECK(live.status == 200 && live.body == "live\n");
CHECK(ready.status == 200 && ready.body == "ready\n");
CHECK(metric(port, "evo_region_ready") == 1 && metric(port, "evo_region_lease_epoch") == 1);
CHECK(metric(port, "evo_region_lease_remaining_milliseconds") > 0);
CHECK(metric(port, "evo_region_queue_capacity_inputs") == 64 &&
metric(port, "evo_region_queue_capacity_bytes") == 65536);
auto page = get(port, "/metrics");
CHECK(page.body.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos);
server.send(input_lines(1, 30, false));
CHECK(server.wait_line("ack " + request_hex(30) + " 30 ", 15s));
CHECK(metric(port, "evo_region_committed_tick") == 30 &&
metric(port, "evo_region_commit_lag_ticks") == 0 &&
metric(port, "evo_region_acknowledgements_total") == 30);
CHECK(metric(port, "evo_region_checkpoint_age_milliseconds") < 60000);
// Bounded, strict responder.
CHECK(http_exchange(port, "POST /readyz HTTP/1.1\r\n\r\n").status == 405);
CHECK(get(port, "/nope").status == 404);
CHECK(get(port, "/readyz?x=1").status == 404); // Exact paths only.
CHECK(http_exchange(port, "HELLO\r\n\r\n").status == 400);
CHECK(http_exchange(port, "GET /livez HTTP/1.1\r\nX: " + std::string(5000, 'a') + "\r\n\r\n")
.status == 431);
// Idle connections neither block probes nor exceed the connection bound.
std::vector<int> idle;
idle.reserve(8);
for (int i = 0; i < 8; ++i)
idle.push_back(connect_local(port));
const int excess = connect_local(port);
char byte;
// Refused at once (well before the 1 s idle deadline), never queued behind others.
const auto refused_at = Clock::now();
CHECK(::recv(excess, &byte, 1, 0) == 0);
CHECK(Clock::now() - refused_at < 500ms);
::close(excess);
std::this_thread::sleep_for(1300ms); // Idle connections expire at their deadline.
const auto started = Clock::now();
CHECK(get(port, "/livez").status == 200);
CHECK(Clock::now() - started < 1s);
for (const int fd : idle) {
CHECK(::recv(fd, &byte, 1, 0) == 0); // Closed by the server.
::close(fd);
}
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
}
// The database is unreachable: the process is alive but never ready.
void unavailable_database_is_live_not_ready() {
const ServerRegion r{{7, 52}, fs::path(env("PGHOST")) / "server-52"};
fs::remove_all(r.root);
fs::create_directory(r.root);
auto args = with_ops(r);
const auto at = std::ranges::find(args, "--pg-port");
CHECK(at != args.end());
*(at + 1) = "1"; // No server socket there.
Child server{args};
const auto port = ops_port(server);
std::this_thread::sleep_for(500ms);
CHECK(get(port, "/livez").status == 200);
auto ready = get(port, "/readyz");
CHECK(ready.status == 503 && ready.body == "not ready: acquiring\n");
CHECK(metric(port, "evo_region_ready") == 0 &&
metric(port, "evo_region_store_failures_total") > 0);
CHECK(get(port, "/metrics").body.find("evo_region_phase{phase=\"acquiring\"} 1\n") !=
std::string::npos);
CHECK(!server.printed("ready 1"));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
}
// A paused process that lost its lease is alive, but readiness and metrics say it
// no longer holds authority; the successor is the only ready process.
void stale_process_is_live_not_ready() {
const auto r = provision(53, 0x0C);
Child stale{with_ops(r)};
const auto stale_port = ops_port(stale);
CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
stale.signal(SIGSTOP);
std::this_thread::sleep_for(4500ms);
Child current{with_ops(r)};
const auto current_port = ops_port(current);
CHECK(current.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
stale.signal(SIGCONT);
CHECK(stale.wait_line("ready 0", 10s));
CHECK(get(stale_port, "/livez").status == 200);
CHECK(get(stale_port, "/readyz").status == 503);
CHECK(metric(stale_port, "evo_region_ready") == 0 &&
metric(stale_port, "evo_region_fences_total") == 1 &&
metric(stale_port, "evo_region_lease_remaining_milliseconds") == 0);
CHECK(get(current_port, "/readyz").status == 200 &&
metric(current_port, "evo_region_lease_epoch") == 2);
stale.signal(SIGTERM);
current.signal(SIGTERM);
CHECK(stale.wait_exit(20s) == 0 && current.wait_exit(20s) == 0);
}
} // namespace
void run_region_ops_tests() {
probes_follow_authority_and_stay_bounded();
unavailable_database_is_live_not_ready();
stale_process_is_live_not_ready();
}
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
# Smoke test for the evo-server image (M7.8, ADR-0066). Runs the image with the
# same restrictions a cluster uses; no network, no credentials, read-only root.
set -euo pipefail
image="${1:?image reference}"
run() {
docker run --rm --network none --read-only --cap-drop ALL \
--security-opt no-new-privileges "$@"
}
# Without configuration the server refuses to start (usage, exit 2), and never
# falls back to any default region or database.
set +e
usage="$(run "$image" 2>&1)"
code=$?
set -e
test "$code" -eq 2
grep -q '^usage: evo-server ' <<<"$usage"
# Non-root, every shared library resolved, no credential material in the image.
run --entrypoint /bin/sh "$image" -c '
set -eu
test "$(id -u)" = 10001 && test "$(id -g)" = 10001
! ldd /usr/local/bin/evo-server | grep -q "not found"
test -z "$(find / -xdev \( -name "*pgpass*" -o -name "pg_service.conf" -o -name "*.pem" \
-o -name "*.key" \) -not -path "/etc/ssl/*" -not -path "/usr/share/*" -print 2>/dev/null)"
'
echo "server image smoke passed: $image"