Add M8.4 region host deployment, provisioning hook and drill

Deploy evo-region per environment (ADR-0071): one replica with the
Recreate strategy (a rolling update would deadlock on the lease),
/readyz readiness, /livez liveness, no CPU limit and a drain inside the
grace period, with the image digest doubling as the build identity. A
new idempotent evo-world-clone provision subcommand creates genesis from
development_opening (byte-identical to the qualification fixture) as a
Sync hook after the migration. Dev and stage paths include their region;
a dev AppProject allows Deployment and Service; region-drill.sh times
rollout and forced-deletion recovery; the policy check enforces the
host's invariants.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-29 21:32:56 -04:00
co-authored by Claude Opus 5.5
parent 98240ef96e
commit 8ef69fab1d
27 changed files with 879 additions and 159 deletions
+1 -1
View File
@@ -62,7 +62,7 @@ target_link_libraries(evo_ecology PUBLIC evo::accounting evo::biology)
target_compile_features(evo_ecology PUBLIC cxx_std_23)
evo_owned_target(evo_ecology)
add_library(evo_processes STATIC src/processes/producer.cpp src/processes/feeding.cpp src/processes/ecosystem.cpp src/processes/genome_strategy.cpp src/processes/ensemble.cpp src/processes/environmental_community.cpp src/processes/environmental_codec.cpp src/processes/environmental_schedule.cpp src/processes/lineage_region.cpp)
add_library(evo_processes STATIC src/processes/producer.cpp src/processes/feeding.cpp src/processes/ecosystem.cpp src/processes/genome_strategy.cpp src/processes/ensemble.cpp src/processes/environmental_community.cpp src/processes/environmental_codec.cpp src/processes/environmental_schedule.cpp src/processes/lineage_region.cpp src/processes/development_opening.cpp)
add_library(evo::processes ALIAS evo_processes)
target_link_libraries(evo_processes PUBLIC evo::accounting evo::biology evo::world)
evo_owned_target(evo_processes)
+80 -2
View File
@@ -9,6 +9,8 @@
// [--max-commits N]
// evo-world-clone check-store --world W --region R --pg-* ... --objects DIR
// [--reserve-mib N]
// evo-world-clone provision --world W --region R --opening development-v1 --pg-* ...
// --objects DIR --build HEX [--retain-build]
//
// --*pg-* is host, hostaddr, port, database, user, passfile, ca, or service (a libpq
// service file whose [evolution] section supplies everything but the password file). The source
@@ -31,10 +33,18 @@
// Output: objects available_mib=A reserve_mib=R / probe created|verified digest=HEX
// database version=V user=U ssl=S / database tables=denied functions=reachable
//
// provision (M8.4) creates a region's genesis with the owner (migrator) login, once:
// an already provisioned region is reported and never touched. --retain-build first
// records the build identity (the server image digest) in retained_builds. The only
// opening is development-v1 (evo/processes/development_opening.hpp), for dev and
// stage; World0's opening is a separate, reviewed decision.
// Output: provisioned|already-provisioned world=W region=R sequence=S tick=T
//
// Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused,
// 5 target unknown.
#include "evo/persistence/region_clone.hpp"
#include "evo/persistence/sha256.hpp"
#include "evo/processes/development_opening.hpp"
#include <array>
#include <charconv>
@@ -65,6 +75,9 @@ struct Options {
std::vector<persistence::Sha256Digest> retained;
std::optional<persistence::Uuid> commit;
std::uint64_t audit_max{}, max_commits{1u << 20U}, reserve_mib{1024};
std::string opening;
std::optional<persistence::Sha256Digest> build;
bool retain_build{};
bool crash{};
};
@@ -136,7 +149,8 @@ std::optional<Options> parse(int argc, char** argv) {
return std::nullopt;
Options o;
o.command = argv[1];
if (o.command != "clone" && o.command != "audit" && o.command != "check-store")
if (o.command != "clone" && o.command != "audit" && o.command != "check-store" &&
o.command != "provision")
return std::nullopt;
const bool clone = o.command == "clone";
for (int i = 2; i < argc; ++i) {
@@ -178,12 +192,20 @@ std::optional<Options> parse(int argc, char** argv) {
o.max_commits = *n;
else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576)
o.reserve_mib = *n;
else if (o.command == "provision" && flag == "--opening" && value == "development-v1")
o.opening = value;
else if (o.command == "provision" && flag == "--build" && hex<32>(value))
o.build = hex<32>(value);
else if (o.command == "provision" && flag == "--retain-build" && value == "true")
o.retain_build = true;
else
return std::nullopt;
}
if (clone && o.target.key.region == 0)
o.target.key.region = o.source.key.region;
const bool needs_builds = o.command != "check-store";
const bool needs_builds = o.command != "check-store" && o.command != "provision";
if (o.command == "provision" && (o.opening.empty() || !o.build))
return std::nullopt;
if (!complete(o.source) || (needs_builds && o.retained.empty()) ||
(clone && !complete(o.target)))
return std::nullopt;
@@ -274,6 +296,60 @@ std::vector<std::uint8_t> probe_bytes() {
return out;
}
int provision(const Options& o) {
auto store = open_objects(o.source);
if (!store) {
std::puts("objects refused");
return 4;
}
auto db = connect(o.source);
if (!db) {
std::puts("database refused");
return 3;
}
if (!o.build)
return 2; // Required by parse(); checked again for the reader.
const auto& build = *o.build;
if (o.retain_build) {
const std::array<std::string, 1> p{"\\x" + text(build)};
if (!db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES ($1::bytea) "
"ON CONFLICT DO NOTHING",
p, timeout)) {
std::puts("retain-build refused");
return 3;
}
}
const auto report = [&](const char* what, const persistence::RegionHead& head) {
std::printf("%s world=%llu region=%llu sequence=%llu tick=%llu\n", what,
static_cast<unsigned long long>(o.source.key.world),
static_cast<unsigned long long>(o.source.key.region),
static_cast<unsigned long long>(head.sequence),
static_cast<unsigned long long>(head.end_tick));
return 0;
};
// Never re-provision: an existing region, whatever its state, is left alone.
auto existing = persistence::load_region_head(*db, o.source.key, timeout);
if (existing)
return report("already-provisioned", *existing);
if (existing.error().kind != persistence::LeaseStoreError::not_provisioned) {
std::puts("database refused");
return 3;
}
auto opening = processes::development_opening(o.source.key.world);
persistence::Uuid commit{};
if (!opening ||
::getrandom(commit.data(), commit.size(), 0) != static_cast<ssize_t>(commit.size()))
return 4;
auto made = persistence::provision_lineage_region(
*db, *store, o.source.key, processes::development_opening_frame, **opening, commit, build,
restore_budget, timeout);
if (!made) {
std::printf("provision refused %u\n", static_cast<unsigned>(made.error().kind));
return 4;
}
return report("provisioned", made->head);
}
int check_store(const Options& o) {
auto store = persistence::FilesystemObjectStore::open(o.source.objects, o.source.key,
persistence::max_lineage_payload_bytes,
@@ -441,5 +517,7 @@ int main(int argc, char** argv) {
}
if (options->command == "check-store")
return check_store(*options);
if (options->command == "provision")
return provision(*options);
return options->command == "clone" ? clone(*options) : audit(*options);
}
+8 -6
View File
@@ -147,17 +147,19 @@ plus the pinned libpq, UID 10001, no credentials). It smoke-tests the image with
`tools/ci/server-image-smoke.sh` and publishes only when Gitea variable
`HARBOR_SERVER_REPOSITORY` is set.
`kubernetes/region-server/deployment.yaml.example` is outside Argo. It shows:
- probes: `/readyz` means authority, `/livez` means only liveness;
- the runtime Secret, object PVC and a NetworkPolicy.
Pin the digest and provision the region first. Without a gateway, the region
receives no inputs.
The region host is deployed from `kubernetes/region/<env>` (M8.4, ADR-0071),
included in each environment's Argo path. It replaces the former
`region-server` example. Probes: `/readyz` means authority, and `/livez` means
only that the process is alive. Without a gateway, the region receives no inputs.
## Environments and World0 protection (M8.2)
`kubernetes/platform/<env>` holds each environment's namespace, quota, limit range,
default-deny policies and service accounts. `kubernetes/platform/admission` holds
the World0 admission policy. A cluster administrator applies these by hand; they
the World0 admission policy. Since the M8.3 amendment they are delivered by the
`aevolution-platform` Argo Application (`kubernetes/platform/cluster`), which
self-heals and never prunes. Originally a cluster administrator applied them by
hand; they
are never part of an Argo Application. The stage and World0 Argo projects and
applications are in `argocd/stage` and `argocd/world0` as examples.
+25
View File
@@ -0,0 +1,25 @@
# M8.4 dev workload project (replaces deploy/argocd/project.yaml.example). Apply by hand
# (kubectl apply -f deploy/argocd/dev/project.yaml): Argo CD does not manage its own
# projects here. One namespace, no cluster kinds; ServiceAccounts only for the
# foundation proof Job (the platform owns the environment's service accounts).
apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
name: aevolution-dev
namespace: argocd
spec:
description: AEvolution dev (evolution-dev only)
sourceRepos:
- https://git.aeternalisgames.com/dingo/AEvolution.git
destinations:
- server: https://kubernetes.default.svc
namespace: evolution-dev
clusterResourceWhitelist: []
namespaceResourceWhitelist:
- {group: "", kind: ServiceAccount}
- {group: "", kind: Service}
- {group: apps, kind: Deployment}
- {group: batch, kind: Job}
- {group: networking.k8s.io, kind: NetworkPolicy}
orphanedResources:
warn: true
+1
View File
@@ -6,6 +6,7 @@ resources:
- networkpolicy.yaml
- job.yaml
- ../stores/dev
- ../region/dev
images:
- name: evo-sim
newName: registry.aeternalisgames.com/aeternalis/aevolution
@@ -1,143 +0,0 @@
# M7.8 development region server (ADR-0066). NOT in the active Argo kustomization.
# Before applying to evolution-dev: pin the server image digest, provision the region
# (owner-only), create the runtime Secret and object PVC, and add DB/DNS egress for
# the real endpoint. Never target World0 from this example.
#
# Authority, not liveness, gates traffic: /readyz is 200 only while this pod holds
# the region lease and has recovered and verified the committed head. /livez only
# says the loop is responding. A probe can wait up to one bounded I/O step (commit
# deadline 5 s), so probe timeouts below exceed it.
#
# There is no gateway yet: stdin is not attached, so the region hosts authority but
# receives no inputs (tick debt grows by design; nothing is invented).
apiVersion: apps/v1
kind: Deployment
metadata:
name: evolution-region-7-1
namespace: evolution-dev
labels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
spec:
replicas: 1
strategy: {type: Recreate} # Never two pods racing; the lease still arbitrates.
selector:
matchLabels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
template:
metadata:
labels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
spec:
automountServiceAccountToken: false
terminationGracePeriodSeconds: 30 # SIGTERM drains within --drain-ms 20000.
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
initContainers:
# libpq rejects group-readable password files: copy to a private 0600 file.
- name: pgpass
image: registry.aeternalisgames.com/aeternalis/aevolution-server:replace-with-immutable-digest
command: ["/bin/sh", "-c", "install -m 0600 /run/postgres/pgpass /run/evo/pgpass"]
securityContext: {readOnlyRootFilesystem: true, allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: private, mountPath: /run/evo}
containers:
- name: region
image: registry.aeternalisgames.com/aeternalis/aevolution-server:replace-with-immutable-digest
# Flags are "--name value" pairs; see docs/infrastructure/POSTGRESQL.md.
command:
- /usr/local/bin/evo-server
- --profile
- lineage
- --world
- "7"
- --region
- "1"
- --frame
- "9"
- --pg-host
- replace-with-tls-hostname
- --pg-hostaddr
- replace-with-numeric-address
- --pg-port
- "5432"
- --pg-database
- evolution
- --pg-user
- replace-with-runtime-login
- --pg-passfile
- /run/evo/pgpass
- --pg-ca
- /run/postgres/ca.crt
- --objects
- /var/lib/evolution/objects
- --build
- replace-with-retained-build-identity-hex
- --ops-listen
- 0.0.0.0:9100
- --drain-ms
- "20000"
ports:
- {name: ops, containerPort: 9100}
startupProbe:
httpGet: {path: /livez, port: ops}
periodSeconds: 5
timeoutSeconds: 8
failureThreshold: 12
readinessProbe:
httpGet: {path: /readyz, port: ops}
periodSeconds: 5
timeoutSeconds: 8
failureThreshold: 1 # Any doubt about authority removes the pod at once.
livenessProbe:
httpGet: {path: /livez, port: ops}
periodSeconds: 10
timeoutSeconds: 8
failureThreshold: 6
resources:
requests: {cpu: 250m, memory: 128Mi}
limits: {cpu: "1", memory: 512Mi}
securityContext:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
capabilities: {drop: [ALL]}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: private, mountPath: /run/evo, readOnly: true}
- {name: objects, mountPath: /var/lib/evolution/objects}
- {name: scratch, mountPath: /tmp}
volumes:
- name: connection
secret:
secretName: evolution-postgres-runtime # Runtime login only; never migrator.
defaultMode: 0440
items:
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: private
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: objects
persistentVolumeClaim: {claimName: evolution-region-objects}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 8Mi}
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: evolution-region-7-1
namespace: evolution-dev
spec:
podSelector:
matchLabels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
policyTypes: [Ingress, Egress]
ingress:
# Metrics/probes only from the monitoring namespace (kubelet probes use the node).
- from:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: monitoring}
ports: [{port: 9100, protocol: TCP}]
egress:
# Replace with the actual database endpoint; no DNS is needed with a numeric hostaddr.
- to: [{ipBlock: {cidr: 192.0.2.10/32}}]
ports: [{port: 5432, protocol: TCP}]
@@ -0,0 +1,103 @@
# M8.4 region host (ADR-0071). One replica, Recreate: a rolling update would start
# the new pod while the old one still holds the lease; the new pod could never become
# ready, so the old one would never be removed. Recreate drains the old pod (commits,
# releases the lease) before the new one starts and recovers. Readiness is /readyz
# (current lease + verified recovered head), liveness is /livez only. No CPU limit:
# throttling would delay lease renewals.
apiVersion: apps/v1
kind: Deployment
metadata:
name: evo-region
annotations:
argocd.argoproj.io/sync-wave: "1" # After migration and provisioning.
labels:
app.kubernetes.io/name: evo-region
app.kubernetes.io/component: region
spec:
replicas: 1
strategy:
type: Recreate
revisionHistoryLimit: 3
progressDeadlineSeconds: 180
selector:
matchLabels:
app.kubernetes.io/name: evo-region
template:
metadata:
labels:
app.kubernetes.io/name: evo-region
app.kubernetes.io/component: region
app.kubernetes.io/part-of: aevolution
spec:
serviceAccountName: aevolution-region
automountServiceAccountToken: false
nodeSelector:
aevolution.io/stores: "true"
# Longer than --drain-ms, so a graceful stop commits and releases the lease.
terminationGracePeriodSeconds: 45
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: evo-server
image: aevolution-server
# libpq ignores group-readable password files: copy to private memory, then
# exec so evo-server is PID 1 and receives SIGTERM directly.
command: ["/bin/bash", "-c"]
args:
- |
set -eu
umask 077
cp /run/postgres/pgpass /tmp/pgpass
exec /usr/local/bin/evo-server --profile lineage \
--world "$REGION_WORLD" --region "$REGION_ID" --frame "$REGION_FRAME" \
--pg-service /run/postgres/pg_service.conf --pg-passfile /tmp/pgpass \
--objects /objects --objects-reserve-mib 1024 \
--build "$REGION_BUILD" --lease-profile default \
--flush-ms 1000 --drain-ms 20000 --ops-listen 0.0.0.0:9100
env:
- {name: REGION_WORLD, value: set-by-overlay}
- {name: REGION_ID, value: set-by-overlay}
- {name: REGION_FRAME, value: "9"}
- {name: REGION_BUILD, value: set-by-overlay}
ports:
- {name: ops, containerPort: 9100, protocol: TCP}
readinessProbe:
httpGet: {path: /readyz, port: ops}
periodSeconds: 2
timeoutSeconds: 1
failureThreshold: 1
livenessProbe:
httpGet: {path: /livez, port: ops}
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 2
failureThreshold: 3
resources:
requests: {cpu: 50m, memory: 64Mi}
limits: {memory: 256Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: objects, mountPath: /objects}
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: objects
persistentVolumeClaim:
claimName: aevolution-objects
- name: connection
secret:
secretName: evolution-postgres-runtime
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 1Mi}
@@ -0,0 +1,13 @@
# M8.4 region host component (ADR-0071), included by an environment's workload path
# through region/<env>, which sets the region identity and the server image digest
# (also its build identity). Needs stores/<env> (database egress, migration) and the
# platform layer (namespace, service accounts, object claim).
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- provision-job.yaml
- deployment.yaml
- service.yaml
labels:
- pairs:
app.kubernetes.io/part-of: aevolution
@@ -0,0 +1,77 @@
# Sync hook at wave 0: after the database egress policy (wave -2) and the migration
# (wave -1), before the region host (wave 1). Creates the region's genesis from
# the development opening once, with the migrator (owner) login, and records the
# server image digest as a retained build. Every later sync reports
# "already-provisioned" and changes nothing.
apiVersion: batch/v1
kind: Job
metadata:
name: aevolution-region-provision
labels:
app.kubernetes.io/component: migrate
annotations:
argocd.argoproj.io/hook: Sync
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
argocd.argoproj.io/sync-wave: "0"
spec:
backoffLimit: 0
activeDeadlineSeconds: 120
template:
metadata:
labels:
app.kubernetes.io/component: migrate
app.kubernetes.io/part-of: aevolution
spec:
serviceAccountName: aevolution-migrator
automountServiceAccountToken: false
restartPolicy: Never
nodeSelector:
aevolution.io/stores: "true"
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: provision
image: aevolution-server
command: ["/bin/bash", "-c"]
args:
- |
set -eu
umask 077
cp /run/postgres/pgpass /tmp/pgpass
exec /usr/local/bin/evo-world-clone provision \
--world "$REGION_WORLD" --region "$REGION_ID" --opening development-v1 \
--pg-service /run/postgres/pg_service.conf --pg-passfile /tmp/pgpass \
--objects /objects --build "$REGION_BUILD" --retain-build true
env:
- {name: REGION_WORLD, value: set-by-overlay}
- {name: REGION_ID, value: set-by-overlay}
- {name: REGION_BUILD, value: set-by-overlay}
resources:
requests: {cpu: 50m, memory: 64Mi}
limits: {memory: 256Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: objects, mountPath: /objects}
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: objects
persistentVolumeClaim:
claimName: aevolution-objects
- name: connection
secret:
secretName: evolution-postgres-migrator
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 1Mi}
@@ -0,0 +1,16 @@
# Operations endpoints (/livez, /readyz, /metrics) inside the cluster only; no
# NodePort or LoadBalancer (the quota forbids both). Scraping is enabled in M8.5.
apiVersion: v1
kind: Service
metadata:
name: evo-region-ops
annotations:
argocd.argoproj.io/sync-wave: "1" # After migration and provisioning.
labels:
app.kubernetes.io/name: evo-region
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: evo-region
ports:
- {name: ops, port: 9100, targetPort: ops, protocol: TCP}
@@ -0,0 +1,14 @@
# Region host for evolution-dev: world 100, region 1, frame 9 (development opening).
# The server image digest appears twice on purpose: as the image and as the build
# identity the region records in every commit. The policy check requires they match.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: evolution-dev
resources:
- ../base
images:
- name: aevolution-server
newName: registry.aeternalisgames.com/aeternalis/aevolution-server
digest: sha256:661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075
patches:
- path: region-values.yaml
@@ -0,0 +1,28 @@
# Region identity and build (the server image digest without "sha256:").
apiVersion: apps/v1
kind: Deployment
metadata:
name: evo-region
spec:
template:
spec:
containers:
- name: evo-server
env:
- {name: REGION_WORLD, value: "100"}
- {name: REGION_ID, value: "1"}
- {name: REGION_BUILD, value: 661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075}
---
apiVersion: batch/v1
kind: Job
metadata:
name: aevolution-region-provision
spec:
template:
spec:
containers:
- name: provision
env:
- {name: REGION_WORLD, value: "100"}
- {name: REGION_ID, value: "1"}
- {name: REGION_BUILD, value: 661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075}
@@ -0,0 +1,14 @@
# Region host for evolution-stage: world 200, region 1, frame 9 (development opening).
# The server image digest appears twice on purpose: as the image and as the build
# identity the region records in every commit. The policy check requires they match.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: evolution-stage
resources:
- ../base
images:
- name: aevolution-server
newName: registry.aeternalisgames.com/aeternalis/aevolution-server
digest: sha256:661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075
patches:
- path: region-values.yaml
@@ -0,0 +1,28 @@
# Region identity and build (the server image digest without "sha256:").
apiVersion: apps/v1
kind: Deployment
metadata:
name: evo-region
spec:
template:
spec:
containers:
- name: evo-server
env:
- {name: REGION_WORLD, value: "200"}
- {name: REGION_ID, value: "1"}
- {name: REGION_BUILD, value: 661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075}
---
apiVersion: batch/v1
kind: Job
metadata:
name: aevolution-region-provision
spec:
template:
spec:
containers:
- name: provision
env:
- {name: REGION_WORLD, value: "200"}
- {name: REGION_ID, value: "1"}
- {name: REGION_BUILD, value: 661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075}
+2 -1
View File
@@ -2,4 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: evolution-stage
resources:
- ../stores/stage
- ../stores/stage
- ../region/stage
+11
View File
@@ -624,3 +624,14 @@ M8.3 implemented and locally qualified (2026-09-29), under ADR-0070:
- `evo-world-clone check-store` for in-cluster qualification.
The in-cluster evidence and the backup destination wait on the maintainer
(docs/BACKLOG.md).
M8.3 delivered through Argo CD by the maintainer (2026-09-29): the dev, stage and
platform applications are synced and connected to PostgreSQL.
M8.4 implemented and locally qualified (2026-09-29), under ADR-0071:
- one `evo-region` host per environment, with the Recreate strategy and the image
digest as its build identity;
- an idempotent provisioning hook from the development opening;
- `region-drill.sh` for rollout and forced-deletion evidence.
The in-cluster sync and drill wait on the maintainer. No director yet: assignment
is static and lives in Git.
+20
View File
@@ -1044,3 +1044,23 @@ M8.3 delivery moved to Argo CD, at the maintainer's request:
- **Store Jobs:** they become sync hooks in each environment's path once their
digests and runtime roles are supplied.
- **Outside Argo:** Secrets and the store node label.
## M8.4 region host (2026-09-29)
The maintainer synced the dev, stage and platform applications through Argo CD,
and they are connected to PostgreSQL, with digests `661e433e…` (server) and
`9abb06ef…` (tools).
ADR-0071 deploys `evo-region` as one replica per environment:
- the Recreate strategy, since a rolling update would deadlock on the lease;
- readiness `/readyz`, liveness `/livez`;
- no CPU limit;
- a 20 s drain inside a 45 s grace period.
Each sync runs, in order: egress, migration, provisioning (idempotent, from the
new `development_opening`, which is byte-identical to the qualification fixture),
the host, then the store check. The image digest is also the build identity, and
the policy check enforces this. Dev is world 100 and stage world 200, both region 1.
Next: apply `deploy/argocd/dev/project.yaml`, sync, and run
`tools/ops/region-drill.sh dev`.
+92
View File
@@ -0,0 +1,92 @@
# ADR-0071: Region host deployment in dev and stage
## Status
Accepted for M8.4, 2026-09-29. It builds on:
- ADR-0064 to ADR-0066 (host, recovery and operations endpoints);
- ADR-0068 (build-bound authority);
- ADR-0069 and ADR-0070 (environments, stores and Argo delivery).
It amends the M8.4 roadmap row: there is **no director** yet.
## Context
M8.4 asks for:
- the host deployed through the existing GitOps path, with an immutable digest;
- measured resources, probes, a security context and a drain budget;
- evidence that a region is ready only after lease and recovery;
- a graceful rollout, and forced deletion of a worker.
The named risk is a successful rollout hiding a failed region recovery. Regions
could so far only be provisioned from test code.
M7.3 deferred the director because it would hold no authority, and would be a
placeholder. With one region per environment, the assignment is static and lives
in Git.
## Decision
**Deployment `evo-region`** (`deploy/kubernetes/region`), one per environment.
- **Rollout.** `replicas: 1` with the **Recreate** strategy. A rolling update would
start a pod that cannot become ready while the old one holds the lease, and the
old one is never removed. Recreate drains the old pod (`--drain-ms 20000`,
inside a 45 s grace period) before the replacement starts. The replacement then
acquires, recovers, verifies the tail, and only then becomes ready.
- **Probes.** Readiness is `/readyz` (current lease and verified head). Liveness is
`/livez` only, so a process waiting out a lease is never restarted for being
unready.
- **Resources.** These come from the measured profile: a 64 Mi / 50m request and a
256 Mi memory limit. There is **no CPU limit**, because throttling would delay
renewals. The production lease policy and the 1024 MiB object reserve apply.
- **Security.** Pod security `restricted`, a read-only root filesystem, no token,
and no capabilities.
- **Placement and access.** Pinned to the store node (`aevolution.io/stores=true`).
Its object claim is node-local, and the replacement lands where the data is.
- **Credentials.** The runtime login comes from its service file. The password file
is copied to private memory, and `exec` makes `evo-server` PID 1, so SIGTERM
reaches it.
- **Service.** A ClusterIP Service `evo-region-ops` on port 9100. Scraping is M8.5.
- **Identity.** Dev is world 100, region 1; stage is world 200, region 1. Both use
frame 9 and the development opening.
**Build identity is the image digest.** Each overlay sets the server image digest
twice: as the image, and as `--build`, the identity recorded in every commit. The
policy check requires them to be equal. An image upgrade is therefore also a
visible build change, handled by the ADR-0068 barrier and `--retained`.
**Provisioning.** `evo-world-clone provision` runs as a Sync hook (wave 0) with the
migrator login. The order is:
1. the database egress policy (wave −2);
2. the migration (wave −1);
3. provisioning (wave 0);
4. the Deployment and Service (wave 1);
5. the PostSync store check.
Provisioning records the digest in `retained_builds`, then creates genesis from
`development-v1` only when the region does not exist. A region that exists, in any
state, is reported and never touched, so every sync can run it. The opening is
`evo::processes::development_opening`, byte-identical to the qualification fixture
(proved by `lineage-region-contracts`). World0's opening remains a separate,
reviewed decision.
**Evidence tooling.** `tools/ops/region-drill.sh <dev|stage>` (World0 refused)
reads epoch and committed tick from the pod's own `/metrics`, then:
- performs `rollout restart`, and times the replacement to Ready;
- force-deletes the pod, and times the replacement to Ready.
After each step, the replacement must report a newer epoch at a committed tick
that is no lower than before.
## Consequences
- No inputs reach the region until a gateway exists. The host runs idle, holding
and renewing its lease, and tick debt accrues visibly. Commit-level fault
evidence under load is M8.6.
- A forced deletion's replacement waits out up to 20 s of lease. That is correct,
not a fault.
- Losing the store node leaves the region unavailable until the node returns
(ADR-0070); it is never re-created empty.
- Each environment's AppProject must allow Deployment and Service.
`deploy/argocd/dev/project.yaml` does, and supersedes the old example.
## Alternatives rejected
- **RollingUpdate with maxSurge.** It deadlocks on the lease, as described above.
- **A StatefulSet.** It adds no guarantee the lease does not already provide.
- **A director.** With static single-region assignment, it would be a placeholder
with no authority.
+1
View File
@@ -81,3 +81,4 @@ design. No unadopted candidate is a link dependency of the proof.
- [ADR-0068: Upgrade barrier and durability qualification](0068-upgrade-barrier.md) - accepted for M7.9b build-bound authority, retirement barrier fencing old writers, explicit refusal of rule/format changes and the measured recovery fixture.
- [ADR-0069: Environment isolation and World0 protection](0069-environment-isolation.md) - accepted for M8.2 per-environment guardrails, native World0 admission protection, scoped Argo projects and verification.
- [ADR-0070: Durable stores for dev and stage](0070-dev-stage-stores.md) - accepted for M8.3 external PostgreSQL per environment, Retain-class node-local object volumes, free-space reserve and in-cluster store qualification.
- [ADR-0071: Region host deployment in dev and stage](0071-region-host-deployment.md) - accepted for M8.4 single-replica Recreate host, digest-as-build identity, idempotent provisioning hook and the region drill; no director yet.
+30
View File
@@ -2781,3 +2781,33 @@ admission policy and the stage and World0 AppProjects, then ran
- The digests the maintainer pasted are local image IDs from the build step, not
registry digests. The registry digest is printed by the Publish step.
- Not executed: bootstrapping the platform Application, or any Argo sync.
## M8.4 region host - 2026-09-29
- Windows MSVC Release CTest 59/59, including the new opening code.
`lineage-region-contracts` proves `development_opening(7)` is byte-identical
to the fixture, and that world 100 differs only by identity.
- WSL Linux `checks` set: 59/59 in each configuration. The format check passed.
- PostgreSQL suites: Release, analysis and ASan/UBSan passed. Analysis passed after
one optional-access fix in `provision`.
- Provisioning test:
- A runtime login cannot provision.
- The owner provisions `world=7 region=86 sequence=0 tick=0`, and a rerun reports
`already-provisioned`.
- A server then became ready and acknowledged ticks 1–5, after which the tool
reported `sequence=1 tick=5` and left the region untouched.
- Mutation: removing the already-provisioned early return was caught.
- `deploy-policy-check.py`: 50 objects, 0 failures. Four new mutations were each
caught:
- a build identity differing from the image digest;
- a RollingUpdate strategy;
- a CPU limit;
- a grace period shorter than the drain.
The checker first misread overlay patches as whole Deployments; it now checks
only the base.
- Kustomize renders dev and stage in order: egress policy (−2), migration hook
(−1), provisioning hook (0), Deployment and Service (1), then the store check
(PostSync).
- `region-drill.sh` passed `bash -n`, and refuses World0 before kubectl (also
enforced by the policy check).
- Not executed: the in-cluster region sync and the drill.
+23
View File
@@ -116,6 +116,29 @@ this evidence in docs/engineering/VALIDATION.md:
the Secret. The check must exit 3 with `database refused connect`.
- Update the Secret's `pgpass` and sync again. It must exit 0.
## Region host (M8.4, ADR-0071)
`deploy/kubernetes/<env>` includes `stores/<env>` and `region/<env>`. A sync runs:
1. the database egress policy (wave −2);
2. the migration hook (wave −1);
3. the provisioning hook (wave 0, idempotent);
4. the `evo-region` Deployment and `evo-region-ops` Service (wave 1);
5. the store check (PostSync).
Dev is world 100, region 1; stage is world 200, region 1. The server image digest
in `region/<env>` is also the build identity.
Before the first region sync, apply `deploy/argocd/dev/project.yaml` (it allows
Deployment and Service), and label the store node.
Evidence: `bash tools/ops/region-drill.sh dev` (dev or stage only; World0 is
refused). It records the time to Ready after a rollout restart and after a forced
pod deletion, and checks that the new epoch resumed at the committed tick. Also
record:
- the Argo sync result;
- the first pod's `phase running epoch=... committed=...` line;
- that `/readyz` stays 503 while a replacement waits out the lease.
## Break-glass
How membership is granted is not decided yet; see docs/BACKLOG.md. Until it is,
@@ -0,0 +1,18 @@
#pragma once
#include "evo/processes/lineage_region.hpp"
#include <expected>
#include <memory>
// M8.4 development opening (ADR-0071): the deterministic lineage region used to
// provision dev and stage regions, identical in content to the qualification fixture
// every M7 test runs (tests/fixtures/environmental_region.hpp). It is an engineering
// opening, not a world design: one habitat site, two bound organisms (producer 10,
// consumer 11), founder genome 1 with descendant 2 and one retained dead organism,
// rules 2, a 50 ms tick. Never provision World0 with it.
namespace evo::processes {
inline constexpr std::uint64_t development_opening_frame = 9;
enum class DevelopmentOpeningError : std::uint8_t { identity, build };
// Built for `world` in frame development_opening_frame; the world must be nonzero.
[[nodiscard]] std::expected<std::unique_ptr<LineageRegion>, DevelopmentOpeningError>
development_opening(std::uint64_t world);
} // namespace evo::processes
+96
View File
@@ -0,0 +1,96 @@
#include "evo/processes/development_opening.hpp"
#include <array>
#include <limits>
#include <optional>
namespace evo::processes {
namespace {
// Mirrors tests/fixtures/environmental_region.hpp exactly; the lineage-region
// contracts test proves byte equality with the fixture for world 7.
EnvironmentalConfig config(world::WorldIdentity id) {
EnvironmentalConfig c;
c.rules = 2;
c.world = id.world;
c.frame = id.frame;
c.producer = 1;
c.consumer = 2;
c.source = 3;
c.tick_microseconds = 50000;
c.area_um2 = 1;
c.irradiance_numerator = 11;
c.irradiance_denominator = 3;
c.efficiency = 600;
c.growth_numerator = 3;
c.growth_denominator = 2;
c.growth_cost = 1;
c.recipe = {1, 2, 1, 1, 1};
c.intake.fill(100);
c.assimilation = 1000;
c.producer_min_mK = c.consumer_min_mK = 270000;
c.producer_max_mK = c.consumer_max_mK = 310000;
c.decay.fill(1);
return c;
}
std::unique_ptr<EnvironmentalCommunity> community(world::WorldIdentity id) {
using namespace world;
constexpr auto maximum = std::numeric_limits<std::uint64_t>::max();
std::array<std::uint64_t, 10> environment{200, 6, 6, 400, 0, 0, 200, 0, 2000, 0};
const FieldSite site{{0, 0, -16000}, {{0}, {0}, {0}, {10}, {20}, {3}}, {1}};
std::array<FieldOpening, 10> opening{};
for (std::size_t k = 0; k < 10; ++k)
opening[k] = {static_cast<Field>(k), {&environment[k], 1}};
const FieldDefaults defaults{
{accounting::Femtograms{0}, accounting::Femtograms{maximum}, accounting::Femtograms{0}}};
auto f = FieldStore::create(id, 1, defaults, {&site, 1}, opening, 2000000);
auto t = ChunkStore::create({}, 1, 2000000, {{VolumeUnits{100}, VolumeUnits{100}}});
if (!f || !t || !(*t)->load({0, 0, -1000}))
return nullptr;
std::array<OrganismResources, 2> accounts{};
accounts[0].id = {1};
accounts[1].id = {2};
accounts[0].capacities.fill(100000000);
accounts[1].capacities.fill(100000000);
auto r = EnvironmentalTransfers::create(id, std::move(*f), std::move(*t), accounts, 4000000);
if (!r)
return nullptr;
auto o = EnvironmentalCommunity::create(config(id), std::move(*r), 8000000);
return o ? std::move(*o) : nullptr;
}
std::optional<biology::Genome> genome(std::uint64_t id, std::uint64_t parent) {
biology::GenomeDraft draft;
draft.id = id;
draft.origin_event = id;
draft.development_version = 1;
if (parent != 0)
draft.parents = {parent};
draft.capabilities = {{biology::Capability::energy_conversion, 400, 0},
{biology::Capability::membrane, 120, 0},
{biology::Capability::target_size, 8'000'000, 0}};
auto made = biology::Genome::create(draft);
if (!made)
return std::nullopt;
return *made;
}
} // namespace
std::expected<std::unique_ptr<LineageRegion>, DevelopmentOpeningError>
development_opening(std::uint64_t world) {
if (world == 0)
return std::unexpected{DevelopmentOpeningError::identity};
const world::WorldIdentity id{world, development_opening_frame};
auto founder = genome(1, 0);
auto descendant = genome(2, 1);
auto opening = community(id);
if (!founder || !descendant || !opening)
return std::unexpected{DevelopmentOpeningError::build};
biology::LineageArchive archive;
if (!archive.admit(*founder) || !archive.admit(*descendant) ||
!archive.record_birth(5, 1, Tick{0}) || !archive.record_death(5, Tick{0}) ||
!archive.record_birth(10, 1, Tick{0}) || !archive.record_birth(11, 2, Tick{0}))
return std::unexpected{DevelopmentOpeningError::build};
auto region = LineageRegion::create(std::move(opening), archive, {10, 11, 12, 3});
if (!region)
return std::unexpected{DevelopmentOpeningError::build};
return std::move(*region);
}
} // namespace evo::processes
+15
View File
@@ -1,3 +1,4 @@
#include "evo/processes/development_opening.hpp"
#include "evo/processes/lineage_region.hpp"
#include "fixtures/environmental_region.hpp"
@@ -164,6 +165,19 @@ void rebinding_changes_only_the_world() {
CHECK(community->rebind_world(9) && community->rebind_world(region_identity.world));
CHECK(community->snapshot() == before);
}
// M8.4: the provisioning opening is byte-identical to the qualification fixture, and
// differs for another world only by the recorded identity.
void development_opening_matches_the_fixture() {
auto opening = development_opening(region_identity.world);
CHECK(opening && (*opening)->snapshot() == lineage_region()->snapshot());
auto elsewhere = development_opening(100);
CHECK(elsewhere && (*elsewhere)->community().world() == 100);
CHECK((*elsewhere)->rebind_world(region_identity.world));
CHECK((*elsewhere)->snapshot() == lineage_region()->snapshot());
CHECK(!development_opening(0));
CHECK(development_opening_frame == region_identity.frame);
}
} // namespace
int main() {
@@ -172,6 +186,7 @@ int main() {
deaths_are_recorded_in_the_same_tick();
snapshots_restore_and_continue();
rebinding_changes_only_the_world();
development_opening_matches_the_fixture();
std::puts("lineage region contracts passed");
return 0;
} catch (const std::exception& e) {
+47
View File
@@ -199,9 +199,56 @@ void full_volume_server_acknowledges_nothing() {
const auto after = head_of(r.key);
CHECK(after.tick == before.tick && after.sequence == before.sequence);
}
// M8.4: the provisioning tool creates a region once from the development opening;
// a rerun (every Argo sync) leaves it untouched, and a server then serves it.
void provisioning_is_idempotent() {
const ServerRegion r{{7, 86}, fresh("provision-86")};
const auto run = [&](const std::string& user, int& code) {
Child tool{{env("EVO_WORLD_CLONE_BIN"),
"provision",
"--world",
"7",
"--region",
"86",
"--opening",
"development-v1",
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
user,
"--objects",
r.root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--retain-build",
"true"}};
code = tool.wait_exit(30s);
return tool.lines();
};
int code = -1;
auto denied = run(env("EVO_PG_RUNTIME_USER"), code);
CHECK(code != 0 && !has(denied, "provisioned "));
auto first = run(env("PGUSER"), code);
CHECK(code == 0 && has(first, "provisioned world=7 region=86 sequence=0 tick=0"));
auto second = run(env("PGUSER"), code);
CHECK(code == 0 && has(second, "already-provisioned world=7 region=86 sequence=0 tick=0"));
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
server.send(input_lines(1, 5, false));
CHECK(server.wait_line("ack " + request_hex(5) + " 5 ", 15s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
auto after = run(env("PGUSER"), code);
CHECK(code == 0 && has(after, "already-provisioned world=7 region=86 sequence=1 tick=5"));
}
} // namespace
void run_region_store_tests() {
provisioning_is_idempotent();
full_volumes_publish_nothing();
failing_writes_leave_nothing();
check_store_qualifies_the_stores();
+37 -6
View File
@@ -192,6 +192,36 @@ for path, doc in objects:
check(doc['metadata'].get('namespace') != 'evolution-world0',
f'{path}: store Jobs never target World0')
# M8.4 region host (ADR-0071): one owner at a time, readiness means authority.
for path, doc in objects:
# Overlay patches (region-values.yaml) are partial objects; the base is complete.
if (doc['kind'] == 'Deployment' and doc['metadata'].get('name') == 'evo-region'
and not path.endswith('region-values.yaml')):
spec = doc['spec']
pod = spec['template']['spec']
container = pod['containers'][0]
check(spec.get('replicas') == 1 and spec.get('strategy', {}).get('type') == 'Recreate',
f'{path}: the region host runs one replica with the Recreate strategy')
check(container.get('readinessProbe', {}).get('httpGet', {}).get('path') == '/readyz'
and container.get('livenessProbe', {}).get('httpGet', {}).get('path') == '/livez',
f'{path}: readiness must be /readyz and liveness /livez')
limits = container.get('resources', {}).get('limits', {})
check('cpu' not in limits and 'memory' in limits,
f'{path}: memory limit required, no CPU limit (throttling delays renewals)')
script = ' '.join(container.get('args', []))
drain = [int(x) for x in __import__('re').findall(r'--drain-ms (\d+)', script)]
check(len(drain) == 1 and pod.get('terminationGracePeriodSeconds', 30) * 1000 > drain[0],
f'{path}: the termination grace period must exceed --drain-ms')
for env in ('dev', 'stage'):
overlay = f'deploy/kubernetes/region/{env}/'
images = manifests.get(overlay + 'kustomization.yaml', [{}])[0].get('images', [])
server = [i.get('digest', '') for i in images if i.get('name') == 'aevolution-server']
builds = [v.get('value') for d in manifests.get(overlay + 'region-values.yaml', [])
for c in d['spec']['template']['spec']['containers'] for v in c.get('env', [])
if v.get('name') == 'REGION_BUILD']
check(len(server) == 1 and len(builds) == 2 and all('sha256:' + b == server[0] for b in builds),
f'{overlay}: the build identity must equal the server image digest')
# The platform bundle: every rendered object is protected from prune and deletion.
bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0]
check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false',
@@ -243,12 +273,13 @@ for path, doc in objects:
# The cleanup tool refuses World0 before kubectl runs (no kubectl on PATH here).
with tempfile.TemporaryDirectory() as empty:
env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable))
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, 'tools/ops/cleanup-environment.sh'), bad],
capture_output=True, text=True, env=env)
check(result.returncode == 2 and ('refused' in result.stderr or 'usage' in result.stderr)
and 'kubectl' not in result.stderr,
f'cleanup must refuse {bad!r} before kubectl (rc={result.returncode})')
for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh'):
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad],
capture_output=True, text=True, env=env)
check(result.returncode == 2 and ('refused' in result.stderr or 'usage' in result.stderr)
and 'kubectl' not in result.stderr,
f'{tool} must refuse {bad!r} before kubectl (rc={result.returncode})')
for message in failures:
print('FAIL', message)
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env bash
# M8.4 region host drill (ADR-0071) for a disposable environment only.
#
# tools/ops/region-drill.sh <dev|stage> [--context <name>]
#
# 1. Reads the ready region's authority from /metrics (epoch, committed tick).
# 2. Graceful rollout: `kubectl rollout restart`. With the Recreate strategy the old
# pod drains (commits, releases the lease) before the new one starts; measures the
# time until the new pod is Ready and checks it resumed at the committed tick with
# a newer epoch.
# 3. Forced worker deletion: deletes the pod with --grace-period=0 --force; the
# replacement must wait out the dead holder's lease (up to 20 s) and recover; the
# time to Ready is measured the same way.
# Refuses anything but dev or stage before touching the cluster; World0 is never
# drilled from here. Prints one line per measurement; exits nonzero on any failure.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> [--context <name>]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
k=(kubectl --request-timeout=30s)
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
ns="evolution-$env"
selector=app.kubernetes.io/name=evo-region
pod_name() { "${k[@]}" get pods -n "$ns" -l "$selector" \
-o jsonpath='{range .items[?(@.metadata.deletionTimestamp == "")]}{.metadata.name}{"\n"}{end}' |
head -1; }
ready_pod() { # Waits (up to 120 s) for a Ready pod other than $1; prints its name.
local old="$1"
for _ in $(seq 1 600); do
for p in $("${k[@]}" get pods -n "$ns" -l "$selector" -o name 2>/dev/null); do
p="${p#pod/}"
[ "$p" = "$old" ] && continue
ready="$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
[ "$ready" = True ] && { echo "$p"; return 0; }
done
sleep 0.2
done
return 1
}
metric() { # metric <pod> <series>: one value from the pod's own /metrics (bash /dev/tcp).
"${k[@]}" exec -n "$ns" "$1" -- /bin/bash -c \
'exec 3<>/dev/tcp/127.0.0.1/9100; printf "GET /metrics HTTP/1.1\r\nHost: drill\r\n\r\n" >&3; cat <&3' |
awk -v s="$2" '$1 == s {print $2}'
}
now_ms() { date +%s%3N; }
first="$(pod_name)"
[ -n "$first" ] || { echo "FAIL no evo-region pod in $ns" >&2; exit 1; }
first="$(ready_pod "")" || { echo "FAIL no Ready evo-region pod in $ns" >&2; exit 1; }
epoch0="$(metric "$first" evo_region_lease_epoch)"
tick0="$(metric "$first" evo_region_committed_tick)"
echo "start pod=$first epoch=$epoch0 committed_tick=$tick0"
start="$(now_ms)"
"${k[@]}" rollout restart -n "$ns" deployment/evo-region > /dev/null
second="$(ready_pod "$first")" || { echo "FAIL rollout: no Ready replacement" >&2; exit 1; }
rollout_ms=$(( $(now_ms) - start ))
epoch1="$(metric "$second" evo_region_lease_epoch)"
tick1="$(metric "$second" evo_region_committed_tick)"
echo "rollout ready_ms=$rollout_ms pod=$second epoch=$epoch1 committed_tick=$tick1"
[ "$epoch1" -gt "$epoch0" ] && [ "$tick1" -ge "$tick0" ] ||
{ echo "FAIL rollout did not resume with a newer epoch at the committed tick" >&2; exit 1; }
start="$(now_ms)"
"${k[@]}" delete pod -n "$ns" "$second" --grace-period=0 --force > /dev/null 2>&1
third="$(ready_pod "$second")" || { echo "FAIL forced deletion: no Ready replacement" >&2; exit 1; }
forced_ms=$(( $(now_ms) - start ))
epoch2="$(metric "$third" evo_region_lease_epoch)"
tick2="$(metric "$third" evo_region_committed_tick)"
echo "forced ready_ms=$forced_ms pod=$third epoch=$epoch2 committed_tick=$tick2"
[ "$epoch2" -gt "$epoch1" ] && [ "$tick2" -ge "$tick1" ] ||
{ echo "FAIL forced deletion did not resume with a newer epoch at the committed tick" >&2; exit 1; }
echo "PASS region drill in $ns"