Complete M7.8 operations endpoints and region server image
ADR-0066: evo-server --ops-listen serves /livez (loop responding, never authority), /readyz (200 only while the host holds the lease with a verified recovered head) and /metrics (Prometheus text: 30 fixed series for readiness, phase, lease epoch/remaining, simulated/committed ticks, commit lag, checkpoint age, tick debt, queue use and bounds, and counters; the only labels are a 6-value phase and 4-value drop reason). The dependency-free OpsServer runs on the host loop with no thread: GET on exact paths, 405/404/400/431 responses, 8 connections with excess refused on accept, 4 KiB requests, 1 s deadlines, 16 KiB responses. HostStatus/phase types moved to the portable evo::ops library with a Windows-tested renderer. evo-server now refuses to start without its required identity and connection settings. Adds the server-runtime Docker target (evo-server + pinned libpq, UID 10001, no credentials), a CI server-image job with smoke script and optional publishing, and a Kubernetes example outside Argo. Validated: Windows CTest 59/59; Linux Debug/Release/clang-tidy/ASan CTest 59/59; PostgreSQL suites under Release, clang-tidy and ASan with real process probes; two ops mutations caught. Docker build/smoke and Gitea CI not executed locally. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -76,6 +76,32 @@ jobs:
|
||||
export SOURCE_IMAGE="aevolution-postgres-tools:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
bash tools/ci/publish-image.sh
|
||||
|
||||
server-image:
|
||||
needs: [checks]
|
||||
runs-on: ${{ vars.CI_RUNNER_LABEL || 'ubuntu-22.04' }}
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Build and smoke-test the region server image
|
||||
run: |
|
||||
set -eu
|
||||
image="aevolution-server:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
docker build --file deploy/docker/Dockerfile --target server-runtime --build-arg "UBUNTU_IMAGE=$UBUNTU_IMAGE" --tag "$image" .
|
||||
bash tools/ci/server-image-smoke.sh "$image"
|
||||
- name: Publish configured region server image
|
||||
if: ${{ gitea.ref == 'refs/heads/main' && gitea.event_name != 'pull_request' && vars.HARBOR_SERVER_REPOSITORY != '' }}
|
||||
env:
|
||||
HARBOR_PROJECT: ${{ vars.HARBOR_PROJECT }}
|
||||
HARBOR_REPOSITORY: ${{ vars.HARBOR_SERVER_REPOSITORY }}
|
||||
HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
|
||||
HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }}
|
||||
run: |
|
||||
export SOURCE_IMAGE="aevolution-server:${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
bash tools/ci/publish-image.sh
|
||||
|
||||
image:
|
||||
needs: [checks]
|
||||
runs-on: ${{ vars.CI_RUNNER_LABEL || 'ubuntu-22.04' }}
|
||||
|
||||
+15
-3
@@ -77,6 +77,12 @@ add_library(evo::region_lease ALIAS evo_region_lease)
|
||||
target_include_directories(evo_region_lease PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
|
||||
target_compile_features(evo_region_lease PUBLIC cxx_std_23)
|
||||
evo_owned_target(evo_region_lease)
|
||||
# Portable host status and bounded operational exposition (M7.8).
|
||||
add_library(evo_ops STATIC src/server/ops.cpp)
|
||||
add_library(evo::ops ALIAS evo_ops)
|
||||
target_include_directories(evo_ops PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
|
||||
target_compile_features(evo_ops PUBLIC cxx_std_23)
|
||||
evo_owned_target(evo_ops)
|
||||
# Portable checkpoint manifest codec and environmental-v1 payload validation.
|
||||
add_library(evo_region_checkpoint STATIC src/persistence/region_checkpoint.cpp src/persistence/region_batch.cpp)
|
||||
add_library(evo::region_checkpoint ALIAS evo_region_checkpoint)
|
||||
@@ -104,9 +110,9 @@ if(EVO_BUILD_POSTGRES)
|
||||
PRIVATE OpenSSL::Crypto)
|
||||
evo_owned_target(evo_region_store)
|
||||
# M7.6 authoritative region host and its process (ADR-0064).
|
||||
add_library(evo_region_host STATIC src/server/region_host.cpp)
|
||||
add_library(evo_region_host STATIC src/server/region_host.cpp src/server/ops_http.cpp)
|
||||
add_library(evo::region_host ALIAS evo_region_host)
|
||||
target_link_libraries(evo_region_host PUBLIC evo::region_store)
|
||||
target_link_libraries(evo_region_host PUBLIC evo::region_store evo::ops)
|
||||
evo_owned_target(evo_region_host)
|
||||
add_executable(evo-server apps/evo-server/main.cpp)
|
||||
target_link_libraries(evo-server PRIVATE evo::region_host)
|
||||
@@ -115,7 +121,8 @@ if(EVO_BUILD_POSTGRES)
|
||||
# Contention tests use one thread and one connection per contender.
|
||||
add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp
|
||||
tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp
|
||||
tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp)
|
||||
tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp
|
||||
tests/region_ops_tests.cpp)
|
||||
target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads)
|
||||
add_dependencies(evo-postgres-tests evo-server) # Spawned by the process tests.
|
||||
evo_owned_target(evo-postgres-tests)
|
||||
@@ -133,6 +140,11 @@ if(BUILD_TESTING)
|
||||
evo_owned_target(evo-lineage-region-tests)
|
||||
add_test(NAME lineage-region-contracts COMMAND evo-lineage-region-tests)
|
||||
set_tests_properties(lineage-region-contracts PROPERTIES TIMEOUT 60)
|
||||
add_executable(evo-ops-tests tests/ops_tests.cpp)
|
||||
target_link_libraries(evo-ops-tests PRIVATE evo::ops)
|
||||
evo_owned_target(evo-ops-tests)
|
||||
add_test(NAME ops-contracts COMMAND evo-ops-tests)
|
||||
set_tests_properties(ops-contracts PROPERTIES TIMEOUT 60)
|
||||
add_executable(evo-region-batch-tests tests/region_batch_tests.cpp)
|
||||
target_link_libraries(evo-region-batch-tests PRIVATE evo::region_checkpoint)
|
||||
evo_owned_target(evo-region-batch-tests)
|
||||
|
||||
+74
-37
@@ -11,9 +11,14 @@
|
||||
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
|
||||
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
|
||||
//
|
||||
// --ops-listen <numeric address:port> (M7.8) serves GET /livez (the loop is alive; says
|
||||
// nothing about authority), /readyz (200 only with current authority) and /metrics
|
||||
// (Prometheus text). It prints "ops listening <port>" once bound.
|
||||
//
|
||||
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
|
||||
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
|
||||
// after a step's acknowledgements are written and flushed).
|
||||
#include "evo/server/ops_http.hpp"
|
||||
#include "evo/server/region_host.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -48,6 +53,7 @@ struct Options {
|
||||
std::chrono::milliseconds drain{10000};
|
||||
std::string crash_point;
|
||||
std::uint64_t crash_ordinal{};
|
||||
std::string ops_listen;
|
||||
};
|
||||
|
||||
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
|
||||
@@ -140,6 +146,8 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms};
|
||||
else if (flag == "--lease-profile" && value == "default")
|
||||
o.host.lease = {};
|
||||
else if (flag == "--ops-listen")
|
||||
o.ops_listen = value;
|
||||
else if (flag == "--crash-at") {
|
||||
const auto colon = value.find(':');
|
||||
const auto ordinal =
|
||||
@@ -154,6 +162,12 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
} else
|
||||
return std::nullopt;
|
||||
}
|
||||
// Every identity and connection setting is required: there is no default
|
||||
// region, database, object root or build to fall back on.
|
||||
if (o.host.key.world == 0 || o.host.key.region == 0 || o.host.frame == 0 || o.pg.host.empty() ||
|
||||
o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() ||
|
||||
o.host.build == persistence::Sha256Digest{})
|
||||
return std::nullopt;
|
||||
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
|
||||
o.host.retained_builds.push_back(o.host.build);
|
||||
return o;
|
||||
@@ -194,23 +208,6 @@ std::optional<server::TickInput> parse_input(std::string_view line) {
|
||||
return in;
|
||||
}
|
||||
|
||||
const char* phase_name(server::HostPhase p) {
|
||||
switch (p) {
|
||||
case server::HostPhase::acquiring:
|
||||
return "acquiring";
|
||||
case server::HostPhase::running:
|
||||
return "running";
|
||||
case server::HostPhase::resolving:
|
||||
return "resolving";
|
||||
case server::HostPhase::draining:
|
||||
return "draining";
|
||||
case server::HostPhase::stopped:
|
||||
return "stopped";
|
||||
case server::HostPhase::quarantined:
|
||||
return "quarantined";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
const char* admission_name(server::AdmissionError e) {
|
||||
switch (e) {
|
||||
case server::AdmissionError::not_ready:
|
||||
@@ -226,20 +223,6 @@ const char* admission_name(server::AdmissionError e) {
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
const char* dropped_name(server::Unacknowledged u) {
|
||||
switch (u) {
|
||||
case server::Unacknowledged::refused:
|
||||
return "refused";
|
||||
case server::Unacknowledged::drained:
|
||||
return "drained";
|
||||
case server::Unacknowledged::fenced:
|
||||
return "fenced";
|
||||
case server::Unacknowledged::quarantined:
|
||||
return "quarantined";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
// Reads complete, bounded lines from stdin without blocking.
|
||||
class LineReader {
|
||||
public:
|
||||
@@ -294,6 +277,41 @@ template <typename State> int serve(Options& o) {
|
||||
return 2;
|
||||
}
|
||||
auto& host = **made;
|
||||
std::unique_ptr<server::OpsServer> ops;
|
||||
if (!o.ops_listen.empty()) {
|
||||
auto listening = server::OpsServer::listen(o.ops_listen);
|
||||
if (!listening) {
|
||||
std::puts("error ops-listen");
|
||||
return 2;
|
||||
}
|
||||
ops = std::move(*listening);
|
||||
std::printf("ops listening %u\n", static_cast<unsigned>(ops->port()));
|
||||
std::fflush(stdout);
|
||||
}
|
||||
server::OpsCounters counters;
|
||||
counters.queue_capacity_inputs = o.host.max_queued_inputs;
|
||||
counters.queue_capacity_bytes = o.host.max_queued_bytes;
|
||||
std::uint64_t last_sequence = 0;
|
||||
auto last_checkpoint = std::chrono::steady_clock::now();
|
||||
bool recovered = false;
|
||||
const auto handler = [&](std::string_view path) -> server::OpsResponse {
|
||||
const auto& s = host.status();
|
||||
if (path == "/livez")
|
||||
return {200, "live\n"};
|
||||
if (path == "/readyz")
|
||||
return s.ready ? server::OpsResponse{200, "ready\n"}
|
||||
: server::OpsResponse{503, std::string("not ready: ") +
|
||||
server::phase_name(s.phase) + "\n"};
|
||||
if (path == "/metrics") {
|
||||
counters.checkpoint_age_ms =
|
||||
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - last_checkpoint)
|
||||
.count());
|
||||
return {200, server::render_metrics(s, counters),
|
||||
"text/plain; version=0.0.4; charset=utf-8"};
|
||||
}
|
||||
return {404, "not found\n"};
|
||||
};
|
||||
LineReader reader;
|
||||
bool input_open = true, ready = false;
|
||||
std::optional<std::string> pending;
|
||||
@@ -346,28 +364,46 @@ template <typename State> int serve(Options& o) {
|
||||
}
|
||||
const auto& s = host.step(std::chrono::steady_clock::now());
|
||||
max_queued = std::max(max_queued, s.queued);
|
||||
// Checkpoint age restarts at recovery and at every confirmed commit.
|
||||
if ((!recovered && s.phase == server::HostPhase::running) ||
|
||||
s.committed_sequence != last_sequence) {
|
||||
recovered = recovered || s.phase == server::HostPhase::running;
|
||||
last_sequence = s.committed_sequence;
|
||||
last_checkpoint = std::chrono::steady_clock::now();
|
||||
}
|
||||
if (s.phase == server::HostPhase::acquiring)
|
||||
recovered = false;
|
||||
if (!phase || *phase != s.phase) {
|
||||
phase = s.phase;
|
||||
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n", phase_name(s.phase),
|
||||
static_cast<unsigned long long>(s.epoch),
|
||||
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n",
|
||||
server::phase_name(s.phase), static_cast<unsigned long long>(s.epoch),
|
||||
static_cast<unsigned long long>(s.committed_tick),
|
||||
static_cast<unsigned long long>(s.simulated_tick));
|
||||
}
|
||||
set_ready(s.ready);
|
||||
const auto acknowledged = host.take_acks();
|
||||
counters.acks += acknowledged.size();
|
||||
for (const auto& a : acknowledged)
|
||||
std::printf("ack %s %llu %llu\n", text(a.request).c_str(),
|
||||
static_cast<unsigned long long>(a.tick),
|
||||
static_cast<unsigned long long>(a.commit_sequence));
|
||||
for (const auto& x : host.take_outcomes())
|
||||
const auto outcomes = host.take_outcomes();
|
||||
counters.outcomes += outcomes.size();
|
||||
for (const auto& x : outcomes)
|
||||
std::printf("outcome %llu %u %llu %llu\n",
|
||||
static_cast<unsigned long long>(x.decision.tick),
|
||||
static_cast<unsigned>(x.decision.actor),
|
||||
static_cast<unsigned long long>(x.decision.organism),
|
||||
static_cast<unsigned long long>(x.commit_sequence));
|
||||
for (const auto& d : host.take_dropped())
|
||||
std::printf("dropped %s %s\n", text(d.input.request).c_str(), dropped_name(d.reason));
|
||||
const auto dropped = host.take_dropped();
|
||||
for (const auto& d : dropped)
|
||||
++counters.dropped[static_cast<std::size_t>(d.reason)];
|
||||
for (const auto& d : dropped)
|
||||
std::printf("dropped %s %s\n", text(d.input.request).c_str(),
|
||||
server::unacknowledged_name(d.reason));
|
||||
std::fflush(stdout);
|
||||
if (ops)
|
||||
ops->serve(handler);
|
||||
if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal)
|
||||
crash_now();
|
||||
const auto finish = [&](int code) {
|
||||
@@ -398,7 +434,8 @@ int main(int argc, char** argv) {
|
||||
"--frame N --pg-host H --pg-port P --pg-database D --pg-user U "
|
||||
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
|
||||
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
|
||||
"[--lease-profile default|fast] [--ready-file PATH]\n",
|
||||
"[--lease-profile default|fast] [--ready-file PATH] "
|
||||
"[--ops-listen ADDRESS:PORT]\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
|
||||
@@ -134,3 +134,16 @@ migrator/runtime credentials. The example is outside active Argo resources until
|
||||
an operator supplies the database endpoint, role, Secret and egress rules. No region
|
||||
server or world-state migration is deployed by M7.2. Schema v2 (M7.3) additionally
|
||||
requires `EVO_PG_RUNTIME_ROLE`, naming an operator-created NOLOGIN runtime group role.
|
||||
|
||||
## Region server image (M7.8)
|
||||
|
||||
The `server-image` job builds the Docker target `server-runtime` (`evo-server`
|
||||
plus the pinned libpq, UID 10001, no credentials). It smoke-tests the image with
|
||||
`tools/ci/server-image-smoke.sh` and publishes only when Gitea variable
|
||||
`HARBOR_SERVER_REPOSITORY` is set.
|
||||
|
||||
`kubernetes/region-server/deployment.yaml.example` is outside Argo. It shows:
|
||||
- probes: `/readyz` means authority, `/livez` means only liveness;
|
||||
- the runtime Secret, object PVC and a NetworkPolicy.
|
||||
Pin the digest and provision the region first. Without a gateway, the region
|
||||
receives no inputs.
|
||||
|
||||
@@ -82,6 +82,19 @@ ENV PATH="/opt/postgres/bin:${PATH}" LD_LIBRARY_PATH=/opt/postgres/lib
|
||||
USER 10001:10001
|
||||
ENTRYPOINT ["bash", "/opt/evo/tools/migrate-postgres.sh"]
|
||||
|
||||
# M7.8 development region server image (ADR-0066): evo-server with the pinned
|
||||
# libpq, non-root, no credentials. Connection secrets are mounted at runtime only.
|
||||
FROM ${UBUNTU_IMAGE} AS server-runtime
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 libgcc-s1 libssl3t64 zlib1g ca-certificates && rm -rf /var/lib/apt/lists/* && groupadd --gid 10001 evo && useradd --uid 10001 --gid 10001 --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin evo
|
||||
COPY --from=postgres-build /source/build/postgres/evo-server /usr/local/bin/evo-server
|
||||
COPY --from=postgres-build /source/.cache/postgres/install/lib/libpq.so* /opt/postgres/lib/
|
||||
COPY --from=postgres-build /source/.cache/postgres/postgresql-18.6/COPYRIGHT /usr/share/doc/evo-postgres/COPYRIGHT
|
||||
ENV LD_LIBRARY_PATH=/opt/postgres/lib
|
||||
USER 10001:10001
|
||||
WORKDIR /tmp
|
||||
EXPOSE 9100
|
||||
ENTRYPOINT ["/usr/local/bin/evo-server"]
|
||||
|
||||
FROM toolchain AS build
|
||||
RUN cmake --preset release -B build/container \
|
||||
-DCMAKE_CXX_COMPILER=g++-14 -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=/opt/evo/bin \
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
# M7.8 development region server (ADR-0066). NOT in the active Argo kustomization.
|
||||
# Before applying to evolution-dev: pin the server image digest, provision the region
|
||||
# (owner-only), create the runtime Secret and object PVC, and add DB/DNS egress for
|
||||
# the real endpoint. Never target World0 from this example.
|
||||
#
|
||||
# Authority, not liveness, gates traffic: /readyz is 200 only while this pod holds
|
||||
# the region lease and has recovered and verified the committed head. /livez only
|
||||
# says the loop is responding. A probe can wait up to one bounded I/O step (commit
|
||||
# deadline 5 s), so probe timeouts below exceed it.
|
||||
#
|
||||
# There is no gateway yet: stdin is not attached, so the region hosts authority but
|
||||
# receives no inputs (tick debt grows by design; nothing is invented).
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: evolution-region-7-1
|
||||
namespace: evolution-dev
|
||||
labels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {type: Recreate} # Never two pods racing; the lease still arbitrates.
|
||||
selector:
|
||||
matchLabels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
terminationGracePeriodSeconds: 30 # SIGTERM drains within --drain-ms 20000.
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
initContainers:
|
||||
# libpq rejects group-readable password files: copy to a private 0600 file.
|
||||
- name: pgpass
|
||||
image: registry.aeternalisgames.com/aeternalis/aevolution-server:replace-with-immutable-digest
|
||||
command: ["/bin/sh", "-c", "install -m 0600 /run/postgres/pgpass /run/evo/pgpass"]
|
||||
securityContext: {readOnlyRootFilesystem: true, allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}}
|
||||
volumeMounts:
|
||||
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
||||
- {name: private, mountPath: /run/evo}
|
||||
containers:
|
||||
- name: region
|
||||
image: registry.aeternalisgames.com/aeternalis/aevolution-server:replace-with-immutable-digest
|
||||
# Flags are "--name value" pairs; see docs/infrastructure/POSTGRESQL.md.
|
||||
command:
|
||||
- /usr/local/bin/evo-server
|
||||
- --profile
|
||||
- lineage
|
||||
- --world
|
||||
- "7"
|
||||
- --region
|
||||
- "1"
|
||||
- --frame
|
||||
- "9"
|
||||
- --pg-host
|
||||
- replace-with-tls-hostname
|
||||
- --pg-hostaddr
|
||||
- replace-with-numeric-address
|
||||
- --pg-port
|
||||
- "5432"
|
||||
- --pg-database
|
||||
- evolution
|
||||
- --pg-user
|
||||
- replace-with-runtime-login
|
||||
- --pg-passfile
|
||||
- /run/evo/pgpass
|
||||
- --pg-ca
|
||||
- /run/postgres/ca.crt
|
||||
- --objects
|
||||
- /var/lib/evolution/objects
|
||||
- --build
|
||||
- replace-with-retained-build-identity-hex
|
||||
- --ops-listen
|
||||
- 0.0.0.0:9100
|
||||
- --drain-ms
|
||||
- "20000"
|
||||
ports:
|
||||
- {name: ops, containerPort: 9100}
|
||||
startupProbe:
|
||||
httpGet: {path: /livez, port: ops}
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 8
|
||||
failureThreshold: 12
|
||||
readinessProbe:
|
||||
httpGet: {path: /readyz, port: ops}
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 8
|
||||
failureThreshold: 1 # Any doubt about authority removes the pod at once.
|
||||
livenessProbe:
|
||||
httpGet: {path: /livez, port: ops}
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 8
|
||||
failureThreshold: 6
|
||||
resources:
|
||||
requests: {cpu: 250m, memory: 128Mi}
|
||||
limits: {cpu: "1", memory: 512Mi}
|
||||
securityContext:
|
||||
readOnlyRootFilesystem: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
||||
- {name: private, mountPath: /run/evo, readOnly: true}
|
||||
- {name: objects, mountPath: /var/lib/evolution/objects}
|
||||
- {name: scratch, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: connection
|
||||
secret:
|
||||
secretName: evolution-postgres-runtime # Runtime login only; never migrator.
|
||||
defaultMode: 0440
|
||||
items:
|
||||
- {key: pgpass, path: pgpass}
|
||||
- {key: ca.crt, path: ca.crt}
|
||||
- name: private
|
||||
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
||||
- name: objects
|
||||
persistentVolumeClaim: {claimName: evolution-region-objects}
|
||||
- name: scratch
|
||||
emptyDir: {medium: Memory, sizeLimit: 8Mi}
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: evolution-region-7-1
|
||||
namespace: evolution-dev
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress:
|
||||
# Metrics/probes only from the monitoring namespace (kubelet probes use the node).
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels: {kubernetes.io/metadata.name: monitoring}
|
||||
ports: [{port: 9100, protocol: TCP}]
|
||||
egress:
|
||||
# Replace with the actual database endpoint; no DNS is needed with a numeric hostaddr.
|
||||
- to: [{ipBlock: {cidr: 192.0.2.10/32}}]
|
||||
ports: [{port: 5432, protocol: TCP}]
|
||||
@@ -576,3 +576,11 @@ M7.7 implemented and locally qualified (2026-09-28), under ADR-0065.
|
||||
- Real SIGKILL at five boundaries, a resumed SIGSTOPped owner, and damaged heads
|
||||
all behave as the crash matrix requires.
|
||||
Next: M7.8 operations endpoints and dev image.
|
||||
|
||||
M7.8 implemented and locally qualified (2026-09-28), under ADR-0066.
|
||||
- `evo-server --ops-listen` serves `/livez` (loop alive), `/readyz` (current
|
||||
authority only) and `/metrics` (30 fixed series with bounded labels).
|
||||
- The responder is dependency-free, single-threaded and bounded.
|
||||
- The `server-runtime` image is non-root with no secrets; a CI smoke job and a
|
||||
Kubernetes example are included.
|
||||
Next: M7.9 clone, migrate and qualify durability.
|
||||
|
||||
@@ -880,3 +880,24 @@ Tail-skip and replay-equality mutations were each caught. The PostgreSQL suites
|
||||
passed under Release, analysis and ASan/UBSan.
|
||||
|
||||
Next: M7.8 operations endpoints and a dev image.
|
||||
|
||||
## M7.8 operations endpoints and server image (2026-09-28)
|
||||
|
||||
ADR-0066: `evo-server --ops-listen` serves three endpoints:
|
||||
- `/livez`: the loop is responding; it is never a statement of authority.
|
||||
- `/readyz`: 200 only with the current lease and a verified recovered head.
|
||||
- `/metrics`: 30 fixed series whose only labels are a 6-value phase and a 4-value
|
||||
drop reason.
|
||||
|
||||
The responder is dependency-free, runs on the host's own loop with no thread, and
|
||||
is bounded: 8 connections, 4 KiB requests, 1 s deadlines, GET on exact paths
|
||||
only. The new `server-runtime` Docker target is non-root with no credentials,
|
||||
covered by a CI smoke script, and a Kubernetes example ships outside Argo.
|
||||
|
||||
Real process probes covered serving authority, an unavailable database (live,
|
||||
503) and a resumed stale process (live, 503, fence counted). Readiness-as-liveness
|
||||
and an unbounded connection pool were each caught by a test. Windows ran 59/59;
|
||||
Linux ran 59/59 in four configurations; the PostgreSQL suites passed in three.
|
||||
|
||||
Docker itself was not run locally; an emulated image layout was checked instead.
|
||||
Next: M7.9.
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
## Status
|
||||
|
||||
Small proof boundary adopted; Prometheus/Grafana/Loki/Otel adapters proposed for M7–M8.
|
||||
M7.8 (ADR-0066) implements the region server exposition with a tiny direct
|
||||
responder and fixed-label metrics; client libraries remain unadopted.
|
||||
|
||||
## Context
|
||||
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
# ADR-0066: Operations endpoints and development server image
|
||||
|
||||
## Status
|
||||
Accepted for M7.8, 2026-09-28. It implements the M7 part of ADR-0015
|
||||
(observability). It builds on ADR-0064 (host) and ADR-0065 (recovery). There is
|
||||
still no gateway, alerting rules or production promotion.
|
||||
|
||||
## Context
|
||||
M7.8 needs:
|
||||
- separate liveness and readiness;
|
||||
- metrics for tick debt, commit lag, lease safety, checkpoint age and queue bounds;
|
||||
- a non-root image with no embedded secrets.
|
||||
|
||||
The named risk is a healthy process mistaken for valid authority. ADR-0015 forbids
|
||||
unbounded or entity labels, and allows a tiny direct exposition with format tests.
|
||||
The constitution forbids hidden threads and unreviewed dependencies.
|
||||
|
||||
## Decision
|
||||
**Endpoints** (`evo-server --ops-listen <numeric address:port>`):
|
||||
- `/livez` returns 200 whenever the host loop answers. It says nothing about
|
||||
authority, and is used only for restarts.
|
||||
- `/readyz` returns 200 only when `HostStatus::ready` holds: running, recovered and
|
||||
tail-verified, not draining, and `LeaseAuthority::may_originate`. Otherwise it
|
||||
returns 503 with the phase name. It is used for traffic, and later gateway
|
||||
routing must additionally honor the lease epoch.
|
||||
- `/metrics` returns Prometheus text 0.0.4.
|
||||
|
||||
**Responder** (`OpsServer`, Linux, no dependency):
|
||||
- Numeric bind only. GET only (405 otherwise), exact paths (404), malformed
|
||||
requests 400, requests over 4 KiB 431.
|
||||
- One request per connection, `Connection: close`, `Cache-Control: no-store`.
|
||||
- At most 8 connections; any excess is closed on accept, never queued.
|
||||
- Each connection has a 1 s deadline, and responses are capped at 16 KiB.
|
||||
- `serve()` never blocks and runs on the host's own loop, so answers describe
|
||||
current state and no thread is added. A complete request that waited behind one
|
||||
bounded host step is still answered.
|
||||
|
||||
**Metrics** (`evo::ops`, portable and unit-tested):
|
||||
|
||||
| Group | Series |
|
||||
|---|---|
|
||||
| Authority | ready; one-hot phase (6 values); lease epoch; lease remaining (ms) |
|
||||
| Progress | simulated and committed tick and sequence; commit lag; checkpoint age (ms) |
|
||||
| Queue | tick debt, current and max; queued inputs and bytes; their capacities |
|
||||
| Counters | commits; store failures; unknown outcomes; fences; acknowledgements; outcomes; dropped (4 reasons) |
|
||||
|
||||
That is 30 series with fixed names. The only labels are `phase` and `reason`,
|
||||
each from a fixed enum. There are no world, region, entity or request labels:
|
||||
the scrape target carries region identity. The exposition is about 3.5 KB and at
|
||||
most 8 KiB. Commit lag saturates at zero rather than wrapping.
|
||||
|
||||
**Image** (`server-runtime` Docker target) contains `evo-server` and the pinned
|
||||
libpq, and installs only the runtime libraries it needs (libstdc++, libssl3,
|
||||
zlib, CA certificates):
|
||||
- It runs as UID/GID 10001 with a read-only root filesystem and all capabilities
|
||||
dropped. It exposes port 9100 and has no default configuration.
|
||||
- With missing settings it exits 2 with usage; there is no default region,
|
||||
database or build.
|
||||
- CI builds it and smoke-tests it: the usage exit code under restrictions,
|
||||
non-root identity, resolved libraries, and no credential files. Publishing
|
||||
requires `HARBOR_SERVER_REPOSITORY`.
|
||||
|
||||
**Deployment example** (`deploy/kubernetes/region-server`, not in Argo):
|
||||
- One replica with the Recreate strategy.
|
||||
- The runtime Secret is copied to a private 0600 file by an init container.
|
||||
- An object PVC; startup, readiness and liveness probes with 8 s timeouts,
|
||||
longer than one bounded I/O step.
|
||||
- A NetworkPolicy allowing only monitoring ingress on 9100 and the database egress.
|
||||
|
||||
## Alternatives Considered
|
||||
- **prometheus-cpp or OpenTelemetry**: add dependencies and threads for about 30
|
||||
fixed series.
|
||||
- **An HTTP library**: a dependency for three GET paths.
|
||||
- **A separate probe thread**: would answer while the host is stuck in I/O, which
|
||||
hides a blocked loop and adds shared state.
|
||||
- **Readiness from process health or phase alone**: exactly the named risk.
|
||||
|
||||
## Benefits
|
||||
- Probes and metrics cannot claim authority the lease does not grant.
|
||||
- Label cardinality is fixed by construction.
|
||||
- The image carries nothing environment-specific.
|
||||
|
||||
## Costs
|
||||
- Probes can wait for one bounded host step (up to the 5 s commit deadline), so
|
||||
probe timeouts must exceed it.
|
||||
- The responder is intentionally minimal (no keep-alive, TLS or auth).
|
||||
- The metrics endpoint must be network-restricted.
|
||||
|
||||
## Risks
|
||||
- Metrics and probes are unauthenticated plain HTTP; the NetworkPolicy is the
|
||||
control.
|
||||
- The build-tree RUNPATH stays in the binary; it is harmless in the image, where
|
||||
the path does not exist, and `LD_LIBRARY_PATH` names libpq.
|
||||
- Local Docker was not run in this gate; the CI job is configured but its first
|
||||
execution is pending.
|
||||
|
||||
## Consequences
|
||||
- M7.9 measures recovery and batching latency through these metrics.
|
||||
- A later gateway must honor readiness and the lease epoch before routing.
|
||||
- Promoting to World0 needs a scanned, signed image and a reviewed alerting policy
|
||||
(M8).
|
||||
@@ -76,3 +76,4 @@ design. No unadopted candidate is a link dependency of the proof.
|
||||
- [ADR-0063: Batch outcomes](0063-batch-outcomes.md) - accepted for M7.5 manifest v2, validation trace replay, durable decisions/outbox and commit resolution.
|
||||
- [ADR-0064: Region host](0064-region-host.md) - accepted for M7.6 single-owner host state machine, bounded admission, final-only-after-commit and evo-server process.
|
||||
- [ADR-0065: Recovery before readiness](0065-recovery-before-readiness.md) - accepted for M7.7 head tail replay verification, schema v6 commit lookup and real kill-boundary tests.
|
||||
- [ADR-0066: Operations endpoints](0066-operations-endpoints.md) - accepted for M7.8 liveness vs authority readiness, bounded metrics, dependency-free responder and the server image.
|
||||
|
||||
@@ -2381,3 +2381,44 @@ Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate
|
||||
- Not executed: Docker/Gitea CI, the production 20 s lease, host loss or VM
|
||||
freeze, multi-host network partitions, or verification of links older than
|
||||
the head.
|
||||
|
||||
## M7.8 operations endpoints - 2026-09-28
|
||||
|
||||
- Windows MSVC Release CTest passed 59/59, including the new portable
|
||||
`ops-contracts`: exposition format, HELP/TYPE per family, the fixed 30-series
|
||||
set, bounded labels, one-hot phase and non-wrapping lag. The exposition is
|
||||
3485 bytes typically and 3836 at worst-case values (limit 8192).
|
||||
- WSL Linux: the `checks` build set (GCC14 Debug and Release, clang19 analysis,
|
||||
clang19 ASan/UBSan) passed CTest 59/59 each.
|
||||
- The PostgreSQL suites (GCC Release, clang-tidy analysis, ASan/UBSan) passed
|
||||
after fixing the ops test's optional-access and reserve findings. The earlier
|
||||
PostgreSQL analysis runs used a stale binary and are not counted. The
|
||||
repository format check passed after formatting the server.
|
||||
- Process probe tests (real `evo-server`, loopback, ephemeral port):
|
||||
- Serving: `/livez` 200 `live`, `/readyz` 200 `ready`. Metrics showed ready 1,
|
||||
epoch 1, lease remaining above 0, capacities 64/65536 and the running phase.
|
||||
After 30 inputs: committed tick 30, lag 0, 30 acknowledgements, checkpoint
|
||||
age under 60 s.
|
||||
- HTTP bounds: POST 405, unknown path and query string 404, malformed 400,
|
||||
over 4 KiB 431. Eight idle connections do not block probes; a ninth is
|
||||
closed within 500 ms; idle ones close at their deadline.
|
||||
- Database unreachable: `/livez` 200, `/readyz` 503 `not ready: acquiring`,
|
||||
ready 0, store failures above 0, never `ready 1`, exit 0 on SIGTERM.
|
||||
- Stale process (SIGSTOP 4.5 s, then SIGCONT): `/livez` 200, `/readyz` 503,
|
||||
ready 0, fences_total 1, lease remaining 0. The successor answers 200 at
|
||||
epoch 2.
|
||||
- Mutations (restored and rerun green): readiness that returns 200 whenever live
|
||||
was caught. Raising the connection bound to 1000 was not caught at first,
|
||||
because the check did not distinguish refusal from deadline expiry. The test
|
||||
now requires the refusal within 500 ms and catches it.
|
||||
- Image: the Docker engine was not running locally (Docker Desktop stopped), so
|
||||
the `server-runtime` build and `tools/ci/server-image-smoke.sh` have not
|
||||
executed. A filesystem emulation (binary plus the pinned libpq, `env -i`)
|
||||
showed usage exit 2 with no settings and all libraries resolved. It also found
|
||||
and fixed a bug: without arguments the server had reached object-store opening
|
||||
instead of refusing with usage. The binary keeps a build-tree RUNPATH, which is
|
||||
harmless in the image.
|
||||
- The manifest example parses (Deployment and NetworkPolicy, three probes) and
|
||||
the workflow YAML parses; `bash -n` passed for the smoke script.
|
||||
- Not executed: the container build and smoke, the Gitea job, cluster apply,
|
||||
Prometheus scraping, and TSan.
|
||||
|
||||
@@ -191,3 +191,15 @@ boundary; never set it in shared environments. Use the default 20 s lease in any
|
||||
shared environment. `--lease-profile fast` exists
|
||||
only for local tests. The runtime credentials come from `--pg-*` flags and a
|
||||
passfile; never place passwords on the command line.
|
||||
|
||||
### Operations endpoints (M7.8, ADR-0066)
|
||||
|
||||
`--ops-listen <numeric address:port>` serves three endpoints:
|
||||
- `GET /livez`: the process loop is responding. It says nothing about authority.
|
||||
- `GET /readyz`: 200 only while this process holds the lease and has a verified
|
||||
recovered head; 503 with the phase otherwise.
|
||||
- `GET /metrics`: Prometheus text.
|
||||
|
||||
A probe may wait up to one bounded I/O step (the 5 s commit deadline), so probe
|
||||
timeouts must exceed that. Expose port 9100 only to the monitoring namespace;
|
||||
the endpoints are unauthenticated plain HTTP.
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
#pragma once
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
// Region host status and its bounded operational exposition (M7.8, ADR-0066).
|
||||
// Portable: no sockets, clocks or persistence types. Liveness is not authority:
|
||||
// only `ready` means the host may originate output.
|
||||
namespace evo::server {
|
||||
enum class HostPhase : std::uint8_t {
|
||||
acquiring, // Waiting for the lease (busy/unavailable) or recovering.
|
||||
running, // Authoritative: simulating and committing.
|
||||
resolving, // A commit outcome is unknown: paused until resolved.
|
||||
draining, // Shutdown: committing pending work, then releasing.
|
||||
stopped, // Drained (lease released when possible).
|
||||
quarantined, // Fail closed: committed state or history cannot be trusted.
|
||||
};
|
||||
inline constexpr std::size_t host_phase_count = 6;
|
||||
// Why an accepted input will never be acknowledged by this host.
|
||||
enum class Unacknowledged : std::uint8_t {
|
||||
refused, // The simulation rejected it; no tick consumed it.
|
||||
drained, // Still queued at shutdown.
|
||||
fenced, // Authority was lost before its tick committed; the next owner never saw it.
|
||||
quarantined,
|
||||
};
|
||||
inline constexpr std::size_t unacknowledged_count = 4;
|
||||
|
||||
struct HostStatus {
|
||||
HostPhase phase{HostPhase::acquiring};
|
||||
bool ready{}; // Authoritative now: recovered, running, lease may originate.
|
||||
std::uint64_t epoch{};
|
||||
std::uint64_t simulated_tick{}, committed_tick{}, committed_sequence{};
|
||||
std::size_t queued{}, queued_bytes{};
|
||||
std::uint64_t tick_debt{}, max_tick_debt{}; // Due minus simulated since running.
|
||||
std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{};
|
||||
// Local lease safety: time until this host must stop originating (0 = none).
|
||||
std::uint64_t lease_remaining_ms{};
|
||||
};
|
||||
|
||||
// Process-level counters and bounds the exposition adds to the host status.
|
||||
struct OpsCounters {
|
||||
std::size_t queue_capacity_inputs{}, queue_capacity_bytes{};
|
||||
std::uint64_t checkpoint_age_ms{}; // Since the last confirmed commit or recovery.
|
||||
std::uint64_t acks{}, outcomes{};
|
||||
std::array<std::uint64_t, unacknowledged_count> dropped{}; // Indexed by Unacknowledged.
|
||||
};
|
||||
|
||||
[[nodiscard]] const char* phase_name(HostPhase phase) noexcept;
|
||||
[[nodiscard]] const char* unacknowledged_name(Unacknowledged reason) noexcept;
|
||||
|
||||
// Prometheus text format 0.0.4. Fixed metric names; the only labels are `phase`
|
||||
// (6 values) and `reason` (4 values), so cardinality is bounded by construction.
|
||||
// No world/region/entity/request labels: scrape targets carry region identity.
|
||||
[[nodiscard]] std::string render_metrics(const HostStatus& status, const OpsCounters& counters);
|
||||
inline constexpr std::size_t max_metrics_bytes = 8192;
|
||||
} // namespace evo::server
|
||||
@@ -0,0 +1,67 @@
|
||||
#pragma once
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <expected>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
// Minimal operational HTTP/1.1 responder for probes and metrics (M7.8, ADR-0066).
|
||||
// Linux. Not a general web server: GET only, one request per connection
|
||||
// (Connection: close), bounded connections, request bytes, response bytes and
|
||||
// per-connection time. serve() never blocks; it runs on the host's own loop so
|
||||
// answers always describe the current state and no thread is added.
|
||||
namespace evo::server {
|
||||
struct OpsResponse {
|
||||
int status{200};
|
||||
std::string body;
|
||||
std::string content_type{"text/plain; charset=utf-8"};
|
||||
};
|
||||
// Receives the request path (no query handling: probes use exact paths).
|
||||
using OpsHandler = std::function<OpsResponse(std::string_view path)>;
|
||||
enum class OpsError : std::uint8_t { address, socket };
|
||||
|
||||
class OpsServer {
|
||||
public:
|
||||
static constexpr std::size_t max_connections = 8;
|
||||
static constexpr std::size_t max_request_bytes = 4096;
|
||||
static constexpr std::size_t max_response_bytes = 16384;
|
||||
static constexpr std::chrono::milliseconds connection_deadline{1000};
|
||||
|
||||
// Numeric address only ("127.0.0.1:9100" or "[::1]:9100"); port 0 picks one.
|
||||
[[nodiscard]] static std::expected<std::unique_ptr<OpsServer>, OpsError>
|
||||
listen(std::string_view address);
|
||||
OpsServer(const OpsServer&) = delete;
|
||||
OpsServer& operator=(const OpsServer&) = delete;
|
||||
OpsServer(OpsServer&&) = delete;
|
||||
OpsServer& operator=(OpsServer&&) = delete;
|
||||
~OpsServer();
|
||||
[[nodiscard]] std::uint16_t port() const noexcept {
|
||||
return port_;
|
||||
}
|
||||
// Accepts, reads, answers and expires connections without waiting. Connections
|
||||
// beyond the bound are closed at once; slow or oversized ones are answered or
|
||||
// dropped without delaying others.
|
||||
void serve(const OpsHandler& handler);
|
||||
[[nodiscard]] std::uint64_t refused() const noexcept {
|
||||
return refused_;
|
||||
}
|
||||
|
||||
private:
|
||||
struct Connection {
|
||||
int fd{-1};
|
||||
std::string in, out;
|
||||
std::size_t sent{};
|
||||
std::chrono::steady_clock::time_point deadline{};
|
||||
};
|
||||
OpsServer(int fd, std::uint16_t port) : fd_(fd), port_(port) {}
|
||||
void respond(Connection& c, const OpsHandler& handler);
|
||||
int fd_;
|
||||
std::uint16_t port_;
|
||||
std::vector<Connection> connections_;
|
||||
std::uint64_t refused_{};
|
||||
};
|
||||
} // namespace evo::server
|
||||
@@ -2,6 +2,7 @@
|
||||
#include "evo/persistence/checkpoint_store.hpp"
|
||||
#include "evo/persistence/region_batch.hpp"
|
||||
#include "evo/persistence/region_lease.hpp"
|
||||
#include "evo/server/ops.hpp"
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
@@ -45,36 +46,11 @@ struct FinalOutcome {
|
||||
std::uint64_t commit_sequence{};
|
||||
friend bool operator==(const FinalOutcome&, const FinalOutcome&) = default;
|
||||
};
|
||||
// Why an accepted input will never be acknowledged by this host.
|
||||
enum class Unacknowledged : std::uint8_t {
|
||||
refused, // The simulation rejected it; no tick consumed it.
|
||||
drained, // Still queued at shutdown.
|
||||
fenced, // Authority was lost before its tick committed; the next owner never saw it.
|
||||
quarantined,
|
||||
};
|
||||
struct Dropped {
|
||||
TickInput input{};
|
||||
Unacknowledged reason{};
|
||||
};
|
||||
|
||||
enum class HostPhase : std::uint8_t {
|
||||
acquiring, // Waiting for the lease (busy/unavailable) or recovering.
|
||||
running, // Authoritative: simulating and committing.
|
||||
resolving, // A commit outcome is unknown: paused until resolved.
|
||||
draining, // Shutdown: committing pending work, then releasing.
|
||||
stopped, // Drained (lease released when possible).
|
||||
quarantined, // Fail closed: committed state or history cannot be trusted.
|
||||
};
|
||||
struct HostStatus {
|
||||
HostPhase phase{HostPhase::acquiring};
|
||||
bool ready{}; // Authoritative now: recovered, running, lease may originate.
|
||||
std::uint64_t epoch{};
|
||||
std::uint64_t simulated_tick{}, committed_tick{}, committed_sequence{};
|
||||
std::size_t queued{}, queued_bytes{};
|
||||
std::uint64_t tick_debt{}, max_tick_debt{}; // Due minus simulated since running.
|
||||
std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{};
|
||||
};
|
||||
|
||||
// Commit boundaries for fault injection (M7.7 kill tests).
|
||||
enum class CrashPoint : std::uint8_t {
|
||||
before_upload, // Batch sealed; nothing uploaded or sent.
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
#include "evo/server/ops.hpp"
|
||||
#include <string_view>
|
||||
|
||||
namespace evo::server {
|
||||
const char* phase_name(HostPhase phase) noexcept {
|
||||
switch (phase) {
|
||||
case HostPhase::acquiring:
|
||||
return "acquiring";
|
||||
case HostPhase::running:
|
||||
return "running";
|
||||
case HostPhase::resolving:
|
||||
return "resolving";
|
||||
case HostPhase::draining:
|
||||
return "draining";
|
||||
case HostPhase::stopped:
|
||||
return "stopped";
|
||||
case HostPhase::quarantined:
|
||||
return "quarantined";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
const char* unacknowledged_name(Unacknowledged reason) noexcept {
|
||||
switch (reason) {
|
||||
case Unacknowledged::refused:
|
||||
return "refused";
|
||||
case Unacknowledged::drained:
|
||||
return "drained";
|
||||
case Unacknowledged::fenced:
|
||||
return "fenced";
|
||||
case Unacknowledged::quarantined:
|
||||
return "quarantined";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
namespace {
|
||||
void metric(std::string& out, std::string_view name, std::string_view type, std::string_view help,
|
||||
std::uint64_t value) {
|
||||
out.append("# HELP ").append(name).append(" ").append(help).append("\n");
|
||||
out.append("# TYPE ").append(name).append(" ").append(type).append("\n");
|
||||
out.append(name).append(" ").append(std::to_string(value)).append("\n");
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::string render_metrics(const HostStatus& s, const OpsCounters& c) {
|
||||
std::string out;
|
||||
out.reserve(4096);
|
||||
metric(out, "evo_region_ready", "gauge",
|
||||
"1 only while this host holds current authority and may originate output", s.ready);
|
||||
out.append("# HELP evo_region_phase Current host phase (one-hot)\n");
|
||||
out.append("# TYPE evo_region_phase gauge\n");
|
||||
for (std::size_t i = 0; i < host_phase_count; ++i) {
|
||||
const auto phase = static_cast<HostPhase>(i);
|
||||
out.append("evo_region_phase{phase=\"")
|
||||
.append(phase_name(phase))
|
||||
.append("\"} ")
|
||||
.append(phase == s.phase ? "1" : "0")
|
||||
.append("\n");
|
||||
}
|
||||
metric(out, "evo_region_lease_epoch", "gauge", "Lease epoch held or last held", s.epoch);
|
||||
metric(out, "evo_region_lease_remaining_milliseconds", "gauge",
|
||||
"Local time until this host must stop originating (0 without a lease)",
|
||||
s.lease_remaining_ms);
|
||||
metric(out, "evo_region_simulated_tick", "gauge", "Last simulated tick", s.simulated_tick);
|
||||
metric(out, "evo_region_committed_tick", "gauge", "End tick of the committed head",
|
||||
s.committed_tick);
|
||||
metric(out, "evo_region_committed_sequence", "gauge", "Commit sequence of the head",
|
||||
s.committed_sequence);
|
||||
metric(out, "evo_region_commit_lag_ticks", "gauge",
|
||||
"Simulated but not yet durably committed ticks",
|
||||
s.simulated_tick >= s.committed_tick ? s.simulated_tick - s.committed_tick : 0);
|
||||
metric(out, "evo_region_checkpoint_age_milliseconds", "gauge",
|
||||
"Time since the last confirmed commit or recovery", c.checkpoint_age_ms);
|
||||
metric(out, "evo_region_tick_debt", "gauge", "Due ticks not yet simulated", s.tick_debt);
|
||||
metric(out, "evo_region_tick_debt_max", "gauge", "Largest tick debt since recovery",
|
||||
s.max_tick_debt);
|
||||
metric(out, "evo_region_queued_inputs", "gauge", "Admitted inputs awaiting a tick", s.queued);
|
||||
metric(out, "evo_region_queued_bytes", "gauge", "Accounted bytes of queued inputs",
|
||||
s.queued_bytes);
|
||||
metric(out, "evo_region_queue_capacity_inputs", "gauge", "Input backlog bound",
|
||||
c.queue_capacity_inputs);
|
||||
metric(out, "evo_region_queue_capacity_bytes", "gauge", "Input backlog byte bound",
|
||||
c.queue_capacity_bytes);
|
||||
metric(out, "evo_region_commits_total", "counter", "Confirmed commits", s.commits);
|
||||
metric(out, "evo_region_store_failures_total", "counter",
|
||||
"Storage or database failures that paused progress", s.store_failures);
|
||||
metric(out, "evo_region_unknown_outcomes_total", "counter",
|
||||
"Commits whose outcome had to be resolved", s.unknown_outcomes);
|
||||
metric(out, "evo_region_fences_total", "counter", "Times this host lost authority", s.fences);
|
||||
metric(out, "evo_region_acknowledgements_total", "counter", "Final acknowledgements emitted",
|
||||
c.acks);
|
||||
metric(out, "evo_region_outcomes_total", "counter", "Final permanent outcomes emitted",
|
||||
c.outcomes);
|
||||
out.append("# HELP evo_region_dropped_total Accepted inputs that will never be acknowledged\n");
|
||||
out.append("# TYPE evo_region_dropped_total counter\n");
|
||||
for (std::size_t i = 0; i < unacknowledged_count; ++i)
|
||||
out.append("evo_region_dropped_total{reason=\"")
|
||||
.append(unacknowledged_name(static_cast<Unacknowledged>(i)))
|
||||
.append("\"} ")
|
||||
.append(std::to_string(c.dropped[i]))
|
||||
.append("\n");
|
||||
return out;
|
||||
}
|
||||
} // namespace evo::server
|
||||
@@ -0,0 +1,175 @@
|
||||
#include "evo/server/ops_http.hpp"
|
||||
#include <arpa/inet.h>
|
||||
#include <cerrno>
|
||||
#include <charconv>
|
||||
#include <netinet/in.h>
|
||||
#include <sys/socket.h>
|
||||
#include <unistd.h>
|
||||
|
||||
namespace evo::server {
|
||||
namespace {
|
||||
using Clock = std::chrono::steady_clock;
|
||||
const char* reason(int status) {
|
||||
switch (status) {
|
||||
case 200:
|
||||
return "OK";
|
||||
case 400:
|
||||
return "Bad Request";
|
||||
case 404:
|
||||
return "Not Found";
|
||||
case 405:
|
||||
return "Method Not Allowed";
|
||||
case 431:
|
||||
return "Request Header Fields Too Large";
|
||||
case 503:
|
||||
return "Service Unavailable";
|
||||
default:
|
||||
return "Internal Server Error";
|
||||
}
|
||||
}
|
||||
std::string encode(const OpsResponse& r) {
|
||||
std::string out = "HTTP/1.1 " + std::to_string(r.status) + " " + reason(r.status) +
|
||||
"\r\nContent-Type: " + r.content_type +
|
||||
"\r\nContent-Length: " + std::to_string(r.body.size()) +
|
||||
"\r\nCache-Control: no-store\r\nConnection: close\r\n";
|
||||
if (r.status == 405)
|
||||
out += "Allow: GET\r\n";
|
||||
return out + "\r\n" + r.body;
|
||||
}
|
||||
bool printable(std::string_view s) {
|
||||
for (const char ch : s)
|
||||
if (ch <= ' ' || ch > '~')
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<std::unique_ptr<OpsServer>, OpsError> OpsServer::listen(std::string_view address) {
|
||||
const auto colon = address.rfind(':');
|
||||
if (colon == std::string_view::npos || colon + 1 >= address.size())
|
||||
return std::unexpected{OpsError::address};
|
||||
unsigned port{};
|
||||
const auto digits = address.substr(colon + 1);
|
||||
const auto [end, error] = std::from_chars(digits.data(), digits.data() + digits.size(), port);
|
||||
if (error != std::errc{} || end != digits.data() + digits.size() || port > 65535)
|
||||
return std::unexpected{OpsError::address};
|
||||
auto host = std::string(address.substr(0, colon));
|
||||
sockaddr_storage storage{};
|
||||
socklen_t length{};
|
||||
int family{};
|
||||
if (host.size() >= 2 && host.front() == '[' && host.back() == ']') {
|
||||
host = host.substr(1, host.size() - 2);
|
||||
auto* v6 = reinterpret_cast<sockaddr_in6*>(&storage);
|
||||
if (inet_pton(AF_INET6, host.c_str(), &v6->sin6_addr) != 1)
|
||||
return std::unexpected{OpsError::address};
|
||||
v6->sin6_family = AF_INET6;
|
||||
v6->sin6_port = htons(static_cast<std::uint16_t>(port));
|
||||
family = AF_INET6;
|
||||
length = sizeof(sockaddr_in6);
|
||||
} else {
|
||||
auto* v4 = reinterpret_cast<sockaddr_in*>(&storage);
|
||||
if (inet_pton(AF_INET, host.c_str(), &v4->sin_addr) != 1)
|
||||
return std::unexpected{OpsError::address};
|
||||
v4->sin_family = AF_INET;
|
||||
v4->sin_port = htons(static_cast<std::uint16_t>(port));
|
||||
family = AF_INET;
|
||||
length = sizeof(sockaddr_in);
|
||||
}
|
||||
const int fd = ::socket(family, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0);
|
||||
if (fd < 0)
|
||||
return std::unexpected{OpsError::socket};
|
||||
const int on = 1;
|
||||
::setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof on);
|
||||
if (::bind(fd, reinterpret_cast<sockaddr*>(&storage), length) != 0 || ::listen(fd, 16) != 0) {
|
||||
::close(fd);
|
||||
return std::unexpected{OpsError::socket};
|
||||
}
|
||||
sockaddr_storage bound{};
|
||||
socklen_t bound_length = sizeof bound;
|
||||
::getsockname(fd, reinterpret_cast<sockaddr*>(&bound), &bound_length);
|
||||
const auto actual = family == AF_INET6
|
||||
? ntohs(reinterpret_cast<sockaddr_in6*>(&bound)->sin6_port)
|
||||
: ntohs(reinterpret_cast<sockaddr_in*>(&bound)->sin_port);
|
||||
return std::unique_ptr<OpsServer>(new OpsServer(fd, actual));
|
||||
}
|
||||
|
||||
OpsServer::~OpsServer() {
|
||||
for (auto& c : connections_)
|
||||
::close(c.fd);
|
||||
::close(fd_);
|
||||
}
|
||||
|
||||
void OpsServer::respond(Connection& c, const OpsHandler& handler) {
|
||||
OpsResponse response;
|
||||
const auto line_end = c.in.find("\r\n");
|
||||
const std::string_view line = std::string_view(c.in).substr(0, line_end);
|
||||
const auto first = line.find(' ');
|
||||
const auto second = first == std::string_view::npos ? first : line.find(' ', first + 1);
|
||||
if (second == std::string_view::npos || line.substr(second + 1).rfind("HTTP/1.", 0) != 0) {
|
||||
response = {400, "bad request\n"};
|
||||
} else {
|
||||
const auto method = line.substr(0, first);
|
||||
const auto path = line.substr(first + 1, second - first - 1);
|
||||
if (method != "GET")
|
||||
response = {405, "method not allowed\n"};
|
||||
else if (path.empty() || path.front() != '/' || path.size() > 256 || !printable(path))
|
||||
response = {400, "bad request\n"};
|
||||
else
|
||||
response = handler(path);
|
||||
}
|
||||
if (response.body.size() > max_response_bytes)
|
||||
response = {500, "response too large\n"};
|
||||
c.out = encode(response);
|
||||
}
|
||||
|
||||
void OpsServer::serve(const OpsHandler& handler) {
|
||||
const auto now = Clock::now();
|
||||
for (;;) {
|
||||
const int fd = ::accept4(fd_, nullptr, nullptr, SOCK_NONBLOCK | SOCK_CLOEXEC);
|
||||
if (fd < 0)
|
||||
break; // EAGAIN: nothing pending (other errors: retried next call).
|
||||
if (connections_.size() >= max_connections) {
|
||||
::close(fd); // Bounded: excess connections are refused, never queued.
|
||||
++refused_;
|
||||
continue;
|
||||
}
|
||||
connections_.push_back({fd, {}, {}, 0, now + connection_deadline});
|
||||
}
|
||||
for (auto it = connections_.begin(); it != connections_.end();) {
|
||||
auto& c = *it;
|
||||
bool done = false;
|
||||
// Read before expiring: a complete request that waited behind a bounded host
|
||||
// step is still answered; only incomplete or unsent work expires.
|
||||
if (c.out.empty()) {
|
||||
char chunk[1024];
|
||||
const auto n = ::recv(c.fd, chunk, sizeof chunk, 0);
|
||||
if (n == 0 || (n < 0 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR)) {
|
||||
done = true;
|
||||
} else if (n > 0) {
|
||||
c.in.append(chunk, static_cast<std::size_t>(n));
|
||||
if (c.in.size() > max_request_bytes)
|
||||
c.out = encode({431, "request too large\n"});
|
||||
else if (c.in.find("\r\n\r\n") != std::string::npos)
|
||||
respond(c, handler);
|
||||
}
|
||||
}
|
||||
if (!done && !c.out.empty()) {
|
||||
const auto n = ::send(c.fd, c.out.data() + c.sent, c.out.size() - c.sent, MSG_NOSIGNAL);
|
||||
if (n > 0)
|
||||
c.sent += static_cast<std::size_t>(n);
|
||||
done = c.sent == c.out.size() ||
|
||||
(n < 0 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR);
|
||||
}
|
||||
// Slow client: an incomplete request expires at the deadline; a response the
|
||||
// client will not take gets one more deadline, then is dropped.
|
||||
if (!done && now > c.deadline && (c.out.empty() || now > c.deadline + connection_deadline))
|
||||
done = true;
|
||||
if (done) {
|
||||
::close(c.fd);
|
||||
it = connections_.erase(it);
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace evo::server
|
||||
@@ -386,6 +386,13 @@ template <typename State> void RegionHost<State>::refresh(SchedulerTime now) {
|
||||
status_.queued_bytes = queue_.size() * tick_input_bytes;
|
||||
const auto& grant = authority_.grant();
|
||||
status_.epoch = grant ? grant->epoch : 0;
|
||||
const auto lease_now = LeaseClock::now();
|
||||
status_.lease_remaining_ms =
|
||||
grant && authority_.may_originate(lease_now)
|
||||
? static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
authority_.deadline() - lease_now)
|
||||
.count())
|
||||
: 0;
|
||||
status_.committed_tick = committed_ ? committed_->head.end_tick : 0;
|
||||
status_.committed_sequence = committed_ ? committed_->head.sequence : 0;
|
||||
status_.simulated_tick = state_ ? Ops<State>::tick(*state_) : status_.committed_tick;
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
#include "evo/server/ops.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <limits>
|
||||
#include <set>
|
||||
#include <source_location>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
using namespace evo::server;
|
||||
void check(bool ok, std::source_location at = std::source_location::current()) {
|
||||
if (!ok)
|
||||
throw std::runtime_error("ops line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
|
||||
struct Sample {
|
||||
std::string name, labels, value;
|
||||
};
|
||||
// Parses exposition lines; every family must be introduced by HELP and TYPE.
|
||||
std::vector<Sample> parse(const std::string& text) {
|
||||
std::vector<Sample> out;
|
||||
std::set<std::string> typed, helped;
|
||||
std::istringstream in(text);
|
||||
for (std::string line; std::getline(in, line);) {
|
||||
CHECK(!line.empty());
|
||||
if (line.rfind("# HELP ", 0) == 0) {
|
||||
helped.insert(line.substr(7, line.find(' ', 7) - 7));
|
||||
continue;
|
||||
}
|
||||
if (line.rfind("# TYPE ", 0) == 0) {
|
||||
const auto name = line.substr(7, line.find(' ', 7) - 7);
|
||||
const auto type = line.substr(line.rfind(' ') + 1);
|
||||
CHECK(type == "gauge" || type == "counter");
|
||||
CHECK(helped.contains(name) && !typed.contains(name)); // One family, documented.
|
||||
typed.insert(name);
|
||||
continue;
|
||||
}
|
||||
const auto space = line.rfind(' ');
|
||||
const auto brace = line.find('{');
|
||||
Sample s{line.substr(0, std::min(space, brace)), "", line.substr(space + 1)};
|
||||
if (brace < space)
|
||||
s.labels = line.substr(brace, space - brace);
|
||||
CHECK(typed.contains(s.name));
|
||||
CHECK(!s.value.empty() && s.value.find_first_not_of("0123456789") == std::string::npos);
|
||||
out.push_back(s);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
void exposition_is_bounded_and_well_formed() {
|
||||
HostStatus s;
|
||||
s.phase = HostPhase::running;
|
||||
s.ready = true;
|
||||
s.epoch = 3;
|
||||
s.simulated_tick = 50;
|
||||
s.committed_tick = 41;
|
||||
s.lease_remaining_ms = 1234;
|
||||
OpsCounters c;
|
||||
c.queue_capacity_inputs = 64;
|
||||
c.queue_capacity_bytes = 65536;
|
||||
c.dropped[2] = 5;
|
||||
const auto text = render_metrics(s, c);
|
||||
const auto samples = parse(text);
|
||||
const std::set<std::string> label_names{"", "phase", "reason"};
|
||||
std::size_t phases_on = 0;
|
||||
for (const auto& x : samples) {
|
||||
CHECK(x.name.rfind("evo_region_", 0) == 0);
|
||||
const auto key =
|
||||
x.labels.empty() ? std::string{} : x.labels.substr(1, x.labels.find('=') - 1);
|
||||
CHECK(label_names.contains(key)); // No world/region/entity/request labels.
|
||||
if (x.name == "evo_region_phase")
|
||||
phases_on += x.value == "1";
|
||||
}
|
||||
CHECK(phases_on == 1);
|
||||
CHECK(text.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos);
|
||||
CHECK(text.find("evo_region_ready 1\n") != std::string::npos);
|
||||
CHECK(text.find("evo_region_commit_lag_ticks 9\n") != std::string::npos);
|
||||
CHECK(text.find("evo_region_lease_remaining_milliseconds 1234\n") != std::string::npos);
|
||||
CHECK(text.find("evo_region_dropped_total{reason=\"fenced\"} 5\n") != std::string::npos);
|
||||
// Series count is fixed: 6 phases + 4 reasons + 20 unlabelled families.
|
||||
CHECK(samples.size() == 6 + 4 + 20);
|
||||
|
||||
// Worst-case values keep the size bounded and never underflow the lag.
|
||||
HostStatus big;
|
||||
big.phase = HostPhase::quarantined;
|
||||
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
|
||||
big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most;
|
||||
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
|
||||
big.fences = big.lease_remaining_ms = most;
|
||||
big.simulated_tick = 1; // Lag saturates at zero rather than wrapping.
|
||||
OpsCounters bigc{most, most, most, most, most, {most, most, most, most}};
|
||||
const auto large = render_metrics(big, bigc);
|
||||
CHECK(large.size() <= max_metrics_bytes);
|
||||
CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos);
|
||||
CHECK(large.find("evo_region_ready 0\n") != std::string::npos);
|
||||
CHECK(parse(large).size() == samples.size());
|
||||
for (std::size_t i = 0; i < host_phase_count; ++i)
|
||||
CHECK(std::string(phase_name(static_cast<HostPhase>(i))) != "unknown");
|
||||
for (std::size_t i = 0; i < unacknowledged_count; ++i)
|
||||
CHECK(std::string(unacknowledged_name(static_cast<Unacknowledged>(i))) != "unknown");
|
||||
std::printf("ops_metrics_bytes typical=%zu worst=%zu\n", text.size(), large.size());
|
||||
}
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
try {
|
||||
exposition_is_bounded_and_well_formed();
|
||||
std::puts("ops contracts passed");
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
std::fprintf(stderr, "%s\n", e.what());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ void run_region_checkpoint_store_tests();
|
||||
void run_region_host_tests();
|
||||
void run_region_server_process_tests();
|
||||
void run_region_recovery_tests();
|
||||
void run_region_ops_tests();
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
@@ -84,6 +85,8 @@ int main() {
|
||||
std::puts("region server process integration passed");
|
||||
run_region_recovery_tests();
|
||||
std::puts("region recovery integration passed");
|
||||
run_region_ops_tests();
|
||||
std::puts("region ops integration passed");
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
std::fprintf(stderr, "%s\n", e.what());
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
// M7.8 operational endpoints on real evo-server processes (ADR-0066): liveness is
|
||||
// never authority, readiness follows the lease, metrics are bounded, and the
|
||||
// responder is bounded against idle, excess and malformed clients.
|
||||
#include "fixtures/server_process.hpp"
|
||||
#include <algorithm>
|
||||
#include <arpa/inet.h>
|
||||
#include <netinet/in.h>
|
||||
#include <source_location>
|
||||
#include <sys/socket.h>
|
||||
#include <thread>
|
||||
|
||||
void run_region_ops_tests();
|
||||
|
||||
namespace {
|
||||
using namespace evo::test::process;
|
||||
void check(bool value, std::source_location at = std::source_location::current()) {
|
||||
if (!value)
|
||||
throw std::runtime_error("region ops test line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
|
||||
int connect_local(std::uint16_t port) {
|
||||
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
|
||||
CHECK(fd >= 0);
|
||||
sockaddr_in address{};
|
||||
address.sin_family = AF_INET;
|
||||
address.sin_port = htons(port);
|
||||
address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
||||
CHECK(::connect(fd, reinterpret_cast<sockaddr*>(&address), sizeof address) == 0);
|
||||
timeval timeout{3, 0};
|
||||
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
|
||||
return fd;
|
||||
}
|
||||
struct Reply {
|
||||
int status{};
|
||||
std::string body;
|
||||
};
|
||||
// Sends raw bytes and reads until the server closes (Connection: close).
|
||||
Reply http_exchange(std::uint16_t port, const std::string& request) {
|
||||
const int fd = connect_local(port);
|
||||
std::size_t sent = 0;
|
||||
while (sent < request.size()) {
|
||||
const auto n = ::send(fd, request.data() + sent, request.size() - sent, MSG_NOSIGNAL);
|
||||
if (n <= 0)
|
||||
break;
|
||||
sent += static_cast<std::size_t>(n);
|
||||
}
|
||||
std::string raw;
|
||||
char chunk[4096];
|
||||
for (;;) {
|
||||
const auto n = ::recv(fd, chunk, sizeof chunk, 0);
|
||||
if (n <= 0)
|
||||
break;
|
||||
raw.append(chunk, static_cast<std::size_t>(n));
|
||||
}
|
||||
::close(fd);
|
||||
int status = 0;
|
||||
CHECK(std::sscanf(raw.c_str(), "HTTP/1.1 %d", &status) == 1);
|
||||
const auto body = raw.find("\r\n\r\n");
|
||||
return Reply{status, body == std::string::npos ? "" : raw.substr(body + 4)};
|
||||
}
|
||||
Reply get(std::uint16_t port, const std::string& path) {
|
||||
return http_exchange(port, "GET " + path + " HTTP/1.1\r\nHost: probe\r\n\r\n");
|
||||
}
|
||||
std::uint64_t metric(std::uint16_t port, const std::string& series) {
|
||||
const auto reply = get(port, "/metrics");
|
||||
CHECK(reply.status == 200);
|
||||
const auto at = reply.body.find("\n" + series + " ");
|
||||
CHECK(at != std::string::npos);
|
||||
return std::stoull(reply.body.substr(at + series.size() + 2));
|
||||
}
|
||||
std::uint16_t ops_port(Child& server) {
|
||||
CHECK(server.wait_line("ops listening ", 10s));
|
||||
for (const auto& l : server.lines())
|
||||
if (starts(l, "ops listening "))
|
||||
return static_cast<std::uint16_t>(std::stoul(l.substr(14)));
|
||||
CHECK(false);
|
||||
return 0;
|
||||
}
|
||||
std::vector<std::string> with_ops(const ServerRegion& r, std::vector<std::string> extra = {}) {
|
||||
extra.insert(extra.begin(), {"--ops-listen", "127.0.0.1:0"});
|
||||
return server_args(r, std::move(extra));
|
||||
}
|
||||
|
||||
void probes_follow_authority_and_stay_bounded() {
|
||||
const auto r = provision(51, 0x0B);
|
||||
Child server{with_ops(r)};
|
||||
const auto port = ops_port(server);
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
||||
auto live = get(port, "/livez");
|
||||
auto ready = get(port, "/readyz");
|
||||
CHECK(live.status == 200 && live.body == "live\n");
|
||||
CHECK(ready.status == 200 && ready.body == "ready\n");
|
||||
CHECK(metric(port, "evo_region_ready") == 1 && metric(port, "evo_region_lease_epoch") == 1);
|
||||
CHECK(metric(port, "evo_region_lease_remaining_milliseconds") > 0);
|
||||
CHECK(metric(port, "evo_region_queue_capacity_inputs") == 64 &&
|
||||
metric(port, "evo_region_queue_capacity_bytes") == 65536);
|
||||
auto page = get(port, "/metrics");
|
||||
CHECK(page.body.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos);
|
||||
server.send(input_lines(1, 30, false));
|
||||
CHECK(server.wait_line("ack " + request_hex(30) + " 30 ", 15s));
|
||||
CHECK(metric(port, "evo_region_committed_tick") == 30 &&
|
||||
metric(port, "evo_region_commit_lag_ticks") == 0 &&
|
||||
metric(port, "evo_region_acknowledgements_total") == 30);
|
||||
CHECK(metric(port, "evo_region_checkpoint_age_milliseconds") < 60000);
|
||||
|
||||
// Bounded, strict responder.
|
||||
CHECK(http_exchange(port, "POST /readyz HTTP/1.1\r\n\r\n").status == 405);
|
||||
CHECK(get(port, "/nope").status == 404);
|
||||
CHECK(get(port, "/readyz?x=1").status == 404); // Exact paths only.
|
||||
CHECK(http_exchange(port, "HELLO\r\n\r\n").status == 400);
|
||||
CHECK(http_exchange(port, "GET /livez HTTP/1.1\r\nX: " + std::string(5000, 'a') + "\r\n\r\n")
|
||||
.status == 431);
|
||||
// Idle connections neither block probes nor exceed the connection bound.
|
||||
std::vector<int> idle;
|
||||
idle.reserve(8);
|
||||
for (int i = 0; i < 8; ++i)
|
||||
idle.push_back(connect_local(port));
|
||||
const int excess = connect_local(port);
|
||||
char byte;
|
||||
// Refused at once (well before the 1 s idle deadline), never queued behind others.
|
||||
const auto refused_at = Clock::now();
|
||||
CHECK(::recv(excess, &byte, 1, 0) == 0);
|
||||
CHECK(Clock::now() - refused_at < 500ms);
|
||||
::close(excess);
|
||||
std::this_thread::sleep_for(1300ms); // Idle connections expire at their deadline.
|
||||
const auto started = Clock::now();
|
||||
CHECK(get(port, "/livez").status == 200);
|
||||
CHECK(Clock::now() - started < 1s);
|
||||
for (const int fd : idle) {
|
||||
CHECK(::recv(fd, &byte, 1, 0) == 0); // Closed by the server.
|
||||
::close(fd);
|
||||
}
|
||||
server.signal(SIGTERM);
|
||||
CHECK(server.wait_exit(20s) == 0);
|
||||
}
|
||||
|
||||
// The database is unreachable: the process is alive but never ready.
|
||||
void unavailable_database_is_live_not_ready() {
|
||||
const ServerRegion r{{7, 52}, fs::path(env("PGHOST")) / "server-52"};
|
||||
fs::remove_all(r.root);
|
||||
fs::create_directory(r.root);
|
||||
auto args = with_ops(r);
|
||||
const auto at = std::ranges::find(args, "--pg-port");
|
||||
CHECK(at != args.end());
|
||||
*(at + 1) = "1"; // No server socket there.
|
||||
Child server{args};
|
||||
const auto port = ops_port(server);
|
||||
std::this_thread::sleep_for(500ms);
|
||||
CHECK(get(port, "/livez").status == 200);
|
||||
auto ready = get(port, "/readyz");
|
||||
CHECK(ready.status == 503 && ready.body == "not ready: acquiring\n");
|
||||
CHECK(metric(port, "evo_region_ready") == 0 &&
|
||||
metric(port, "evo_region_store_failures_total") > 0);
|
||||
CHECK(get(port, "/metrics").body.find("evo_region_phase{phase=\"acquiring\"} 1\n") !=
|
||||
std::string::npos);
|
||||
CHECK(!server.printed("ready 1"));
|
||||
server.signal(SIGTERM);
|
||||
CHECK(server.wait_exit(20s) == 0);
|
||||
}
|
||||
|
||||
// A paused process that lost its lease is alive, but readiness and metrics say it
|
||||
// no longer holds authority; the successor is the only ready process.
|
||||
void stale_process_is_live_not_ready() {
|
||||
const auto r = provision(53, 0x0C);
|
||||
Child stale{with_ops(r)};
|
||||
const auto stale_port = ops_port(stale);
|
||||
CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
||||
stale.signal(SIGSTOP);
|
||||
std::this_thread::sleep_for(4500ms);
|
||||
Child current{with_ops(r)};
|
||||
const auto current_port = ops_port(current);
|
||||
CHECK(current.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
||||
stale.signal(SIGCONT);
|
||||
CHECK(stale.wait_line("ready 0", 10s));
|
||||
CHECK(get(stale_port, "/livez").status == 200);
|
||||
CHECK(get(stale_port, "/readyz").status == 503);
|
||||
CHECK(metric(stale_port, "evo_region_ready") == 0 &&
|
||||
metric(stale_port, "evo_region_fences_total") == 1 &&
|
||||
metric(stale_port, "evo_region_lease_remaining_milliseconds") == 0);
|
||||
CHECK(get(current_port, "/readyz").status == 200 &&
|
||||
metric(current_port, "evo_region_lease_epoch") == 2);
|
||||
stale.signal(SIGTERM);
|
||||
current.signal(SIGTERM);
|
||||
CHECK(stale.wait_exit(20s) == 0 && current.wait_exit(20s) == 0);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void run_region_ops_tests() {
|
||||
probes_follow_authority_and_stay_bounded();
|
||||
unavailable_database_is_live_not_ready();
|
||||
stale_process_is_live_not_ready();
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
# Smoke test for the evo-server image (M7.8, ADR-0066). Runs the image with the
|
||||
# same restrictions a cluster uses; no network, no credentials, read-only root.
|
||||
set -euo pipefail
|
||||
image="${1:?image reference}"
|
||||
run() {
|
||||
docker run --rm --network none --read-only --cap-drop ALL \
|
||||
--security-opt no-new-privileges "$@"
|
||||
}
|
||||
# Without configuration the server refuses to start (usage, exit 2), and never
|
||||
# falls back to any default region or database.
|
||||
set +e
|
||||
usage="$(run "$image" 2>&1)"
|
||||
code=$?
|
||||
set -e
|
||||
test "$code" -eq 2
|
||||
grep -q '^usage: evo-server ' <<<"$usage"
|
||||
# Non-root, every shared library resolved, no credential material in the image.
|
||||
run --entrypoint /bin/sh "$image" -c '
|
||||
set -eu
|
||||
test "$(id -u)" = 10001 && test "$(id -g)" = 10001
|
||||
! ldd /usr/local/bin/evo-server | grep -q "not found"
|
||||
test -z "$(find / -xdev \( -name "*pgpass*" -o -name "pg_service.conf" -o -name "*.pem" \
|
||||
-o -name "*.key" \) -not -path "/etc/ssl/*" -not -path "/usr/share/*" -print 2>/dev/null)"
|
||||
'
|
||||
echo "server image smoke passed: $image"
|
||||
Reference in New Issue
Block a user