Add M8.4 region host deployment, provisioning hook and drill
Deploy evo-region per environment (ADR-0071): one replica with the Recreate strategy (a rolling update would deadlock on the lease), /readyz readiness, /livez liveness, no CPU limit and a drain inside the grace period, with the image digest doubling as the build identity. A new idempotent evo-world-clone provision subcommand creates genesis from development_opening (byte-identical to the qualification fixture) as a Sync hook after the migration. Dev and stage paths include their region; a dev AppProject allows Deployment and Service; region-drill.sh times rollout and forced-deletion recovery; the policy check enforces the host's invariants. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+1
-1
@@ -62,7 +62,7 @@ target_link_libraries(evo_ecology PUBLIC evo::accounting evo::biology)
|
||||
target_compile_features(evo_ecology PUBLIC cxx_std_23)
|
||||
evo_owned_target(evo_ecology)
|
||||
|
||||
add_library(evo_processes STATIC src/processes/producer.cpp src/processes/feeding.cpp src/processes/ecosystem.cpp src/processes/genome_strategy.cpp src/processes/ensemble.cpp src/processes/environmental_community.cpp src/processes/environmental_codec.cpp src/processes/environmental_schedule.cpp src/processes/lineage_region.cpp)
|
||||
add_library(evo_processes STATIC src/processes/producer.cpp src/processes/feeding.cpp src/processes/ecosystem.cpp src/processes/genome_strategy.cpp src/processes/ensemble.cpp src/processes/environmental_community.cpp src/processes/environmental_codec.cpp src/processes/environmental_schedule.cpp src/processes/lineage_region.cpp src/processes/development_opening.cpp)
|
||||
add_library(evo::processes ALIAS evo_processes)
|
||||
target_link_libraries(evo_processes PUBLIC evo::accounting evo::biology evo::world)
|
||||
evo_owned_target(evo_processes)
|
||||
|
||||
@@ -9,6 +9,8 @@
|
||||
// [--max-commits N]
|
||||
// evo-world-clone check-store --world W --region R --pg-* ... --objects DIR
|
||||
// [--reserve-mib N]
|
||||
// evo-world-clone provision --world W --region R --opening development-v1 --pg-* ...
|
||||
// --objects DIR --build HEX [--retain-build]
|
||||
//
|
||||
// --*pg-* is host, hostaddr, port, database, user, passfile, ca, or service (a libpq
|
||||
// service file whose [evolution] section supplies everything but the password file). The source
|
||||
@@ -31,10 +33,18 @@
|
||||
// Output: objects available_mib=A reserve_mib=R / probe created|verified digest=HEX
|
||||
// database version=V user=U ssl=S / database tables=denied functions=reachable
|
||||
//
|
||||
// provision (M8.4) creates a region's genesis with the owner (migrator) login, once:
|
||||
// an already provisioned region is reported and never touched. --retain-build first
|
||||
// records the build identity (the server image digest) in retained_builds. The only
|
||||
// opening is development-v1 (evo/processes/development_opening.hpp), for dev and
|
||||
// stage; World0's opening is a separate, reviewed decision.
|
||||
// Output: provisioned|already-provisioned world=W region=R sequence=S tick=T
|
||||
//
|
||||
// Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused,
|
||||
// 5 target unknown.
|
||||
#include "evo/persistence/region_clone.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include "evo/processes/development_opening.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <charconv>
|
||||
@@ -65,6 +75,9 @@ struct Options {
|
||||
std::vector<persistence::Sha256Digest> retained;
|
||||
std::optional<persistence::Uuid> commit;
|
||||
std::uint64_t audit_max{}, max_commits{1u << 20U}, reserve_mib{1024};
|
||||
std::string opening;
|
||||
std::optional<persistence::Sha256Digest> build;
|
||||
bool retain_build{};
|
||||
bool crash{};
|
||||
};
|
||||
|
||||
@@ -136,7 +149,8 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
return std::nullopt;
|
||||
Options o;
|
||||
o.command = argv[1];
|
||||
if (o.command != "clone" && o.command != "audit" && o.command != "check-store")
|
||||
if (o.command != "clone" && o.command != "audit" && o.command != "check-store" &&
|
||||
o.command != "provision")
|
||||
return std::nullopt;
|
||||
const bool clone = o.command == "clone";
|
||||
for (int i = 2; i < argc; ++i) {
|
||||
@@ -178,12 +192,20 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.max_commits = *n;
|
||||
else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576)
|
||||
o.reserve_mib = *n;
|
||||
else if (o.command == "provision" && flag == "--opening" && value == "development-v1")
|
||||
o.opening = value;
|
||||
else if (o.command == "provision" && flag == "--build" && hex<32>(value))
|
||||
o.build = hex<32>(value);
|
||||
else if (o.command == "provision" && flag == "--retain-build" && value == "true")
|
||||
o.retain_build = true;
|
||||
else
|
||||
return std::nullopt;
|
||||
}
|
||||
if (clone && o.target.key.region == 0)
|
||||
o.target.key.region = o.source.key.region;
|
||||
const bool needs_builds = o.command != "check-store";
|
||||
const bool needs_builds = o.command != "check-store" && o.command != "provision";
|
||||
if (o.command == "provision" && (o.opening.empty() || !o.build))
|
||||
return std::nullopt;
|
||||
if (!complete(o.source) || (needs_builds && o.retained.empty()) ||
|
||||
(clone && !complete(o.target)))
|
||||
return std::nullopt;
|
||||
@@ -274,6 +296,60 @@ std::vector<std::uint8_t> probe_bytes() {
|
||||
return out;
|
||||
}
|
||||
|
||||
int provision(const Options& o) {
|
||||
auto store = open_objects(o.source);
|
||||
if (!store) {
|
||||
std::puts("objects refused");
|
||||
return 4;
|
||||
}
|
||||
auto db = connect(o.source);
|
||||
if (!db) {
|
||||
std::puts("database refused");
|
||||
return 3;
|
||||
}
|
||||
if (!o.build)
|
||||
return 2; // Required by parse(); checked again for the reader.
|
||||
const auto& build = *o.build;
|
||||
if (o.retain_build) {
|
||||
const std::array<std::string, 1> p{"\\x" + text(build)};
|
||||
if (!db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES ($1::bytea) "
|
||||
"ON CONFLICT DO NOTHING",
|
||||
p, timeout)) {
|
||||
std::puts("retain-build refused");
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
const auto report = [&](const char* what, const persistence::RegionHead& head) {
|
||||
std::printf("%s world=%llu region=%llu sequence=%llu tick=%llu\n", what,
|
||||
static_cast<unsigned long long>(o.source.key.world),
|
||||
static_cast<unsigned long long>(o.source.key.region),
|
||||
static_cast<unsigned long long>(head.sequence),
|
||||
static_cast<unsigned long long>(head.end_tick));
|
||||
return 0;
|
||||
};
|
||||
// Never re-provision: an existing region, whatever its state, is left alone.
|
||||
auto existing = persistence::load_region_head(*db, o.source.key, timeout);
|
||||
if (existing)
|
||||
return report("already-provisioned", *existing);
|
||||
if (existing.error().kind != persistence::LeaseStoreError::not_provisioned) {
|
||||
std::puts("database refused");
|
||||
return 3;
|
||||
}
|
||||
auto opening = processes::development_opening(o.source.key.world);
|
||||
persistence::Uuid commit{};
|
||||
if (!opening ||
|
||||
::getrandom(commit.data(), commit.size(), 0) != static_cast<ssize_t>(commit.size()))
|
||||
return 4;
|
||||
auto made = persistence::provision_lineage_region(
|
||||
*db, *store, o.source.key, processes::development_opening_frame, **opening, commit, build,
|
||||
restore_budget, timeout);
|
||||
if (!made) {
|
||||
std::printf("provision refused %u\n", static_cast<unsigned>(made.error().kind));
|
||||
return 4;
|
||||
}
|
||||
return report("provisioned", made->head);
|
||||
}
|
||||
|
||||
int check_store(const Options& o) {
|
||||
auto store = persistence::FilesystemObjectStore::open(o.source.objects, o.source.key,
|
||||
persistence::max_lineage_payload_bytes,
|
||||
@@ -441,5 +517,7 @@ int main(int argc, char** argv) {
|
||||
}
|
||||
if (options->command == "check-store")
|
||||
return check_store(*options);
|
||||
if (options->command == "provision")
|
||||
return provision(*options);
|
||||
return options->command == "clone" ? clone(*options) : audit(*options);
|
||||
}
|
||||
|
||||
+8
-6
@@ -147,17 +147,19 @@ plus the pinned libpq, UID 10001, no credentials). It smoke-tests the image with
|
||||
`tools/ci/server-image-smoke.sh` and publishes only when Gitea variable
|
||||
`HARBOR_SERVER_REPOSITORY` is set.
|
||||
|
||||
`kubernetes/region-server/deployment.yaml.example` is outside Argo. It shows:
|
||||
- probes: `/readyz` means authority, `/livez` means only liveness;
|
||||
- the runtime Secret, object PVC and a NetworkPolicy.
|
||||
Pin the digest and provision the region first. Without a gateway, the region
|
||||
receives no inputs.
|
||||
The region host is deployed from `kubernetes/region/<env>` (M8.4, ADR-0071),
|
||||
included in each environment's Argo path. It replaces the former
|
||||
`region-server` example. Probes: `/readyz` means authority, and `/livez` means
|
||||
only that the process is alive. Without a gateway, the region receives no inputs.
|
||||
|
||||
## Environments and World0 protection (M8.2)
|
||||
|
||||
`kubernetes/platform/<env>` holds each environment's namespace, quota, limit range,
|
||||
default-deny policies and service accounts. `kubernetes/platform/admission` holds
|
||||
the World0 admission policy. A cluster administrator applies these by hand; they
|
||||
the World0 admission policy. Since the M8.3 amendment they are delivered by the
|
||||
`aevolution-platform` Argo Application (`kubernetes/platform/cluster`), which
|
||||
self-heals and never prunes. Originally a cluster administrator applied them by
|
||||
hand; they
|
||||
are never part of an Argo Application. The stage and World0 Argo projects and
|
||||
applications are in `argocd/stage` and `argocd/world0` as examples.
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# M8.4 dev workload project (replaces deploy/argocd/project.yaml.example). Apply by hand
|
||||
# (kubectl apply -f deploy/argocd/dev/project.yaml): Argo CD does not manage its own
|
||||
# projects here. One namespace, no cluster kinds; ServiceAccounts only for the
|
||||
# foundation proof Job (the platform owns the environment's service accounts).
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: AppProject
|
||||
metadata:
|
||||
name: aevolution-dev
|
||||
namespace: argocd
|
||||
spec:
|
||||
description: AEvolution dev (evolution-dev only)
|
||||
sourceRepos:
|
||||
- https://git.aeternalisgames.com/dingo/AEvolution.git
|
||||
destinations:
|
||||
- server: https://kubernetes.default.svc
|
||||
namespace: evolution-dev
|
||||
clusterResourceWhitelist: []
|
||||
namespaceResourceWhitelist:
|
||||
- {group: "", kind: ServiceAccount}
|
||||
- {group: "", kind: Service}
|
||||
- {group: apps, kind: Deployment}
|
||||
- {group: batch, kind: Job}
|
||||
- {group: networking.k8s.io, kind: NetworkPolicy}
|
||||
orphanedResources:
|
||||
warn: true
|
||||
@@ -6,6 +6,7 @@ resources:
|
||||
- networkpolicy.yaml
|
||||
- job.yaml
|
||||
- ../stores/dev
|
||||
- ../region/dev
|
||||
images:
|
||||
- name: evo-sim
|
||||
newName: registry.aeternalisgames.com/aeternalis/aevolution
|
||||
|
||||
@@ -1,143 +0,0 @@
|
||||
# M7.8 development region server (ADR-0066). NOT in the active Argo kustomization.
|
||||
# Before applying to evolution-dev: pin the server image digest, provision the region
|
||||
# (owner-only), create the runtime Secret and object PVC, and add DB/DNS egress for
|
||||
# the real endpoint. Never target World0 from this example.
|
||||
#
|
||||
# Authority, not liveness, gates traffic: /readyz is 200 only while this pod holds
|
||||
# the region lease and has recovered and verified the committed head. /livez only
|
||||
# says the loop is responding. A probe can wait up to one bounded I/O step (commit
|
||||
# deadline 5 s), so probe timeouts below exceed it.
|
||||
#
|
||||
# There is no gateway yet: stdin is not attached, so the region hosts authority but
|
||||
# receives no inputs (tick debt grows by design; nothing is invented).
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: evolution-region-7-1
|
||||
namespace: evolution-dev
|
||||
labels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {type: Recreate} # Never two pods racing; the lease still arbitrates.
|
||||
selector:
|
||||
matchLabels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
terminationGracePeriodSeconds: 30 # SIGTERM drains within --drain-ms 20000.
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
initContainers:
|
||||
# libpq rejects group-readable password files: copy to a private 0600 file.
|
||||
- name: pgpass
|
||||
image: registry.aeternalisgames.com/aeternalis/aevolution-server:replace-with-immutable-digest
|
||||
command: ["/bin/sh", "-c", "install -m 0600 /run/postgres/pgpass /run/evo/pgpass"]
|
||||
securityContext: {readOnlyRootFilesystem: true, allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}}
|
||||
volumeMounts:
|
||||
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
||||
- {name: private, mountPath: /run/evo}
|
||||
containers:
|
||||
- name: region
|
||||
image: registry.aeternalisgames.com/aeternalis/aevolution-server:replace-with-immutable-digest
|
||||
# Flags are "--name value" pairs; see docs/infrastructure/POSTGRESQL.md.
|
||||
command:
|
||||
- /usr/local/bin/evo-server
|
||||
- --profile
|
||||
- lineage
|
||||
- --world
|
||||
- "7"
|
||||
- --region
|
||||
- "1"
|
||||
- --frame
|
||||
- "9"
|
||||
- --pg-host
|
||||
- replace-with-tls-hostname
|
||||
- --pg-hostaddr
|
||||
- replace-with-numeric-address
|
||||
- --pg-port
|
||||
- "5432"
|
||||
- --pg-database
|
||||
- evolution
|
||||
- --pg-user
|
||||
- replace-with-runtime-login
|
||||
- --pg-passfile
|
||||
- /run/evo/pgpass
|
||||
- --pg-ca
|
||||
- /run/postgres/ca.crt
|
||||
- --objects
|
||||
- /var/lib/evolution/objects
|
||||
- --build
|
||||
- replace-with-retained-build-identity-hex
|
||||
- --ops-listen
|
||||
- 0.0.0.0:9100
|
||||
- --drain-ms
|
||||
- "20000"
|
||||
ports:
|
||||
- {name: ops, containerPort: 9100}
|
||||
startupProbe:
|
||||
httpGet: {path: /livez, port: ops}
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 8
|
||||
failureThreshold: 12
|
||||
readinessProbe:
|
||||
httpGet: {path: /readyz, port: ops}
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 8
|
||||
failureThreshold: 1 # Any doubt about authority removes the pod at once.
|
||||
livenessProbe:
|
||||
httpGet: {path: /livez, port: ops}
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 8
|
||||
failureThreshold: 6
|
||||
resources:
|
||||
requests: {cpu: 250m, memory: 128Mi}
|
||||
limits: {cpu: "1", memory: 512Mi}
|
||||
securityContext:
|
||||
readOnlyRootFilesystem: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
||||
- {name: private, mountPath: /run/evo, readOnly: true}
|
||||
- {name: objects, mountPath: /var/lib/evolution/objects}
|
||||
- {name: scratch, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: connection
|
||||
secret:
|
||||
secretName: evolution-postgres-runtime # Runtime login only; never migrator.
|
||||
defaultMode: 0440
|
||||
items:
|
||||
- {key: pgpass, path: pgpass}
|
||||
- {key: ca.crt, path: ca.crt}
|
||||
- name: private
|
||||
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
||||
- name: objects
|
||||
persistentVolumeClaim: {claimName: evolution-region-objects}
|
||||
- name: scratch
|
||||
emptyDir: {medium: Memory, sizeLimit: 8Mi}
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: evolution-region-7-1
|
||||
namespace: evolution-dev
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"}
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress:
|
||||
# Metrics/probes only from the monitoring namespace (kubelet probes use the node).
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels: {kubernetes.io/metadata.name: monitoring}
|
||||
ports: [{port: 9100, protocol: TCP}]
|
||||
egress:
|
||||
# Replace with the actual database endpoint; no DNS is needed with a numeric hostaddr.
|
||||
- to: [{ipBlock: {cidr: 192.0.2.10/32}}]
|
||||
ports: [{port: 5432, protocol: TCP}]
|
||||
@@ -0,0 +1,103 @@
|
||||
# M8.4 region host (ADR-0071). One replica, Recreate: a rolling update would start
|
||||
# the new pod while the old one still holds the lease; the new pod could never become
|
||||
# ready, so the old one would never be removed. Recreate drains the old pod (commits,
|
||||
# releases the lease) before the new one starts and recovers. Readiness is /readyz
|
||||
# (current lease + verified recovered head), liveness is /livez only. No CPU limit:
|
||||
# throttling would delay lease renewals.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: evo-region
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "1" # After migration and provisioning.
|
||||
labels:
|
||||
app.kubernetes.io/name: evo-region
|
||||
app.kubernetes.io/component: region
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
revisionHistoryLimit: 3
|
||||
progressDeadlineSeconds: 180
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: evo-region
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: evo-region
|
||||
app.kubernetes.io/component: region
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
spec:
|
||||
serviceAccountName: aevolution-region
|
||||
automountServiceAccountToken: false
|
||||
nodeSelector:
|
||||
aevolution.io/stores: "true"
|
||||
# Longer than --drain-ms, so a graceful stop commits and releases the lease.
|
||||
terminationGracePeriodSeconds: 45
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: evo-server
|
||||
image: aevolution-server
|
||||
# libpq ignores group-readable password files: copy to private memory, then
|
||||
# exec so evo-server is PID 1 and receives SIGTERM directly.
|
||||
command: ["/bin/bash", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -eu
|
||||
umask 077
|
||||
cp /run/postgres/pgpass /tmp/pgpass
|
||||
exec /usr/local/bin/evo-server --profile lineage \
|
||||
--world "$REGION_WORLD" --region "$REGION_ID" --frame "$REGION_FRAME" \
|
||||
--pg-service /run/postgres/pg_service.conf --pg-passfile /tmp/pgpass \
|
||||
--objects /objects --objects-reserve-mib 1024 \
|
||||
--build "$REGION_BUILD" --lease-profile default \
|
||||
--flush-ms 1000 --drain-ms 20000 --ops-listen 0.0.0.0:9100
|
||||
env:
|
||||
- {name: REGION_WORLD, value: set-by-overlay}
|
||||
- {name: REGION_ID, value: set-by-overlay}
|
||||
- {name: REGION_FRAME, value: "9"}
|
||||
- {name: REGION_BUILD, value: set-by-overlay}
|
||||
ports:
|
||||
- {name: ops, containerPort: 9100, protocol: TCP}
|
||||
readinessProbe:
|
||||
httpGet: {path: /readyz, port: ops}
|
||||
periodSeconds: 2
|
||||
timeoutSeconds: 1
|
||||
failureThreshold: 1
|
||||
livenessProbe:
|
||||
httpGet: {path: /livez, port: ops}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
resources:
|
||||
requests: {cpu: 50m, memory: 64Mi}
|
||||
limits: {memory: 256Mi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: objects, mountPath: /objects}
|
||||
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
||||
- {name: scratch, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: objects
|
||||
persistentVolumeClaim:
|
||||
claimName: aevolution-objects
|
||||
- name: connection
|
||||
secret:
|
||||
secretName: evolution-postgres-runtime
|
||||
defaultMode: 0440
|
||||
items:
|
||||
- {key: pg_service.conf, path: pg_service.conf}
|
||||
- {key: pgpass, path: pgpass}
|
||||
- {key: ca.crt, path: ca.crt}
|
||||
- name: scratch
|
||||
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
||||
@@ -0,0 +1,13 @@
|
||||
# M8.4 region host component (ADR-0071), included by an environment's workload path
|
||||
# through region/<env>, which sets the region identity and the server image digest
|
||||
# (also its build identity). Needs stores/<env> (database egress, migration) and the
|
||||
# platform layer (namespace, service accounts, object claim).
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- provision-job.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
labels:
|
||||
- pairs:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
@@ -0,0 +1,77 @@
|
||||
# Sync hook at wave 0: after the database egress policy (wave -2) and the migration
|
||||
# (wave -1), before the region host (wave 1). Creates the region's genesis from
|
||||
# the development opening once, with the migrator (owner) login, and records the
|
||||
# server image digest as a retained build. Every later sync reports
|
||||
# "already-provisioned" and changes nothing.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: aevolution-region-provision
|
||||
labels:
|
||||
app.kubernetes.io/component: migrate
|
||||
annotations:
|
||||
argocd.argoproj.io/hook: Sync
|
||||
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
|
||||
argocd.argoproj.io/sync-wave: "0"
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
activeDeadlineSeconds: 120
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/component: migrate
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
spec:
|
||||
serviceAccountName: aevolution-migrator
|
||||
automountServiceAccountToken: false
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
aevolution.io/stores: "true"
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: provision
|
||||
image: aevolution-server
|
||||
command: ["/bin/bash", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -eu
|
||||
umask 077
|
||||
cp /run/postgres/pgpass /tmp/pgpass
|
||||
exec /usr/local/bin/evo-world-clone provision \
|
||||
--world "$REGION_WORLD" --region "$REGION_ID" --opening development-v1 \
|
||||
--pg-service /run/postgres/pg_service.conf --pg-passfile /tmp/pgpass \
|
||||
--objects /objects --build "$REGION_BUILD" --retain-build true
|
||||
env:
|
||||
- {name: REGION_WORLD, value: set-by-overlay}
|
||||
- {name: REGION_ID, value: set-by-overlay}
|
||||
- {name: REGION_BUILD, value: set-by-overlay}
|
||||
resources:
|
||||
requests: {cpu: 50m, memory: 64Mi}
|
||||
limits: {memory: 256Mi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: objects, mountPath: /objects}
|
||||
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
||||
- {name: scratch, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: objects
|
||||
persistentVolumeClaim:
|
||||
claimName: aevolution-objects
|
||||
- name: connection
|
||||
secret:
|
||||
secretName: evolution-postgres-migrator
|
||||
defaultMode: 0440
|
||||
items:
|
||||
- {key: pg_service.conf, path: pg_service.conf}
|
||||
- {key: pgpass, path: pgpass}
|
||||
- {key: ca.crt, path: ca.crt}
|
||||
- name: scratch
|
||||
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
||||
@@ -0,0 +1,16 @@
|
||||
# Operations endpoints (/livez, /readyz, /metrics) inside the cluster only; no
|
||||
# NodePort or LoadBalancer (the quota forbids both). Scraping is enabled in M8.5.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: evo-region-ops
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "1" # After migration and provisioning.
|
||||
labels:
|
||||
app.kubernetes.io/name: evo-region
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: evo-region
|
||||
ports:
|
||||
- {name: ops, port: 9100, targetPort: ops, protocol: TCP}
|
||||
@@ -0,0 +1,14 @@
|
||||
# Region host for evolution-dev: world 100, region 1, frame 9 (development opening).
|
||||
# The server image digest appears twice on purpose: as the image and as the build
|
||||
# identity the region records in every commit. The policy check requires they match.
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: evolution-dev
|
||||
resources:
|
||||
- ../base
|
||||
images:
|
||||
- name: aevolution-server
|
||||
newName: registry.aeternalisgames.com/aeternalis/aevolution-server
|
||||
digest: sha256:661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075
|
||||
patches:
|
||||
- path: region-values.yaml
|
||||
@@ -0,0 +1,28 @@
|
||||
# Region identity and build (the server image digest without "sha256:").
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: evo-region
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: evo-server
|
||||
env:
|
||||
- {name: REGION_WORLD, value: "100"}
|
||||
- {name: REGION_ID, value: "1"}
|
||||
- {name: REGION_BUILD, value: 661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: aevolution-region-provision
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: provision
|
||||
env:
|
||||
- {name: REGION_WORLD, value: "100"}
|
||||
- {name: REGION_ID, value: "1"}
|
||||
- {name: REGION_BUILD, value: 661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075}
|
||||
@@ -0,0 +1,14 @@
|
||||
# Region host for evolution-stage: world 200, region 1, frame 9 (development opening).
|
||||
# The server image digest appears twice on purpose: as the image and as the build
|
||||
# identity the region records in every commit. The policy check requires they match.
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: evolution-stage
|
||||
resources:
|
||||
- ../base
|
||||
images:
|
||||
- name: aevolution-server
|
||||
newName: registry.aeternalisgames.com/aeternalis/aevolution-server
|
||||
digest: sha256:661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075
|
||||
patches:
|
||||
- path: region-values.yaml
|
||||
@@ -0,0 +1,28 @@
|
||||
# Region identity and build (the server image digest without "sha256:").
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: evo-region
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: evo-server
|
||||
env:
|
||||
- {name: REGION_WORLD, value: "200"}
|
||||
- {name: REGION_ID, value: "1"}
|
||||
- {name: REGION_BUILD, value: 661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: aevolution-region-provision
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: provision
|
||||
env:
|
||||
- {name: REGION_WORLD, value: "200"}
|
||||
- {name: REGION_ID, value: "1"}
|
||||
- {name: REGION_BUILD, value: 661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075}
|
||||
@@ -2,4 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: evolution-stage
|
||||
resources:
|
||||
- ../stores/stage
|
||||
- ../stores/stage
|
||||
- ../region/stage
|
||||
|
||||
@@ -624,3 +624,14 @@ M8.3 implemented and locally qualified (2026-09-29), under ADR-0070:
|
||||
- `evo-world-clone check-store` for in-cluster qualification.
|
||||
The in-cluster evidence and the backup destination wait on the maintainer
|
||||
(docs/BACKLOG.md).
|
||||
|
||||
M8.3 delivered through Argo CD by the maintainer (2026-09-29): the dev, stage and
|
||||
platform applications are synced and connected to PostgreSQL.
|
||||
|
||||
M8.4 implemented and locally qualified (2026-09-29), under ADR-0071:
|
||||
- one `evo-region` host per environment, with the Recreate strategy and the image
|
||||
digest as its build identity;
|
||||
- an idempotent provisioning hook from the development opening;
|
||||
- `region-drill.sh` for rollout and forced-deletion evidence.
|
||||
The in-cluster sync and drill wait on the maintainer. No director yet: assignment
|
||||
is static and lives in Git.
|
||||
|
||||
@@ -1044,3 +1044,23 @@ M8.3 delivery moved to Argo CD, at the maintainer's request:
|
||||
- **Store Jobs:** they become sync hooks in each environment's path once their
|
||||
digests and runtime roles are supplied.
|
||||
- **Outside Argo:** Secrets and the store node label.
|
||||
|
||||
## M8.4 region host (2026-09-29)
|
||||
|
||||
The maintainer synced the dev, stage and platform applications through Argo CD,
|
||||
and they are connected to PostgreSQL, with digests `661e433e…` (server) and
|
||||
`9abb06ef…` (tools).
|
||||
|
||||
ADR-0071 deploys `evo-region` as one replica per environment:
|
||||
- the Recreate strategy, since a rolling update would deadlock on the lease;
|
||||
- readiness `/readyz`, liveness `/livez`;
|
||||
- no CPU limit;
|
||||
- a 20 s drain inside a 45 s grace period.
|
||||
|
||||
Each sync runs, in order: egress, migration, provisioning (idempotent, from the
|
||||
new `development_opening`, which is byte-identical to the qualification fixture),
|
||||
the host, then the store check. The image digest is also the build identity, and
|
||||
the policy check enforces this. Dev is world 100 and stage world 200, both region 1.
|
||||
|
||||
Next: apply `deploy/argocd/dev/project.yaml`, sync, and run
|
||||
`tools/ops/region-drill.sh dev`.
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# ADR-0071: Region host deployment in dev and stage
|
||||
|
||||
## Status
|
||||
Accepted for M8.4, 2026-09-29. It builds on:
|
||||
- ADR-0064 to ADR-0066 (host, recovery and operations endpoints);
|
||||
- ADR-0068 (build-bound authority);
|
||||
- ADR-0069 and ADR-0070 (environments, stores and Argo delivery).
|
||||
|
||||
It amends the M8.4 roadmap row: there is **no director** yet.
|
||||
|
||||
## Context
|
||||
M8.4 asks for:
|
||||
- the host deployed through the existing GitOps path, with an immutable digest;
|
||||
- measured resources, probes, a security context and a drain budget;
|
||||
- evidence that a region is ready only after lease and recovery;
|
||||
- a graceful rollout, and forced deletion of a worker.
|
||||
|
||||
The named risk is a successful rollout hiding a failed region recovery. Regions
|
||||
could so far only be provisioned from test code.
|
||||
|
||||
M7.3 deferred the director because it would hold no authority, and would be a
|
||||
placeholder. With one region per environment, the assignment is static and lives
|
||||
in Git.
|
||||
|
||||
## Decision
|
||||
**Deployment `evo-region`** (`deploy/kubernetes/region`), one per environment.
|
||||
- **Rollout.** `replicas: 1` with the **Recreate** strategy. A rolling update would
|
||||
start a pod that cannot become ready while the old one holds the lease, and the
|
||||
old one is never removed. Recreate drains the old pod (`--drain-ms 20000`,
|
||||
inside a 45 s grace period) before the replacement starts. The replacement then
|
||||
acquires, recovers, verifies the tail, and only then becomes ready.
|
||||
- **Probes.** Readiness is `/readyz` (current lease and verified head). Liveness is
|
||||
`/livez` only, so a process waiting out a lease is never restarted for being
|
||||
unready.
|
||||
- **Resources.** These come from the measured profile: a 64 Mi / 50m request and a
|
||||
256 Mi memory limit. There is **no CPU limit**, because throttling would delay
|
||||
renewals. The production lease policy and the 1024 MiB object reserve apply.
|
||||
- **Security.** Pod security `restricted`, a read-only root filesystem, no token,
|
||||
and no capabilities.
|
||||
- **Placement and access.** Pinned to the store node (`aevolution.io/stores=true`).
|
||||
Its object claim is node-local, and the replacement lands where the data is.
|
||||
- **Credentials.** The runtime login comes from its service file. The password file
|
||||
is copied to private memory, and `exec` makes `evo-server` PID 1, so SIGTERM
|
||||
reaches it.
|
||||
- **Service.** A ClusterIP Service `evo-region-ops` on port 9100. Scraping is M8.5.
|
||||
- **Identity.** Dev is world 100, region 1; stage is world 200, region 1. Both use
|
||||
frame 9 and the development opening.
|
||||
|
||||
**Build identity is the image digest.** Each overlay sets the server image digest
|
||||
twice: as the image, and as `--build`, the identity recorded in every commit. The
|
||||
policy check requires them to be equal. An image upgrade is therefore also a
|
||||
visible build change, handled by the ADR-0068 barrier and `--retained`.
|
||||
|
||||
**Provisioning.** `evo-world-clone provision` runs as a Sync hook (wave 0) with the
|
||||
migrator login. The order is:
|
||||
1. the database egress policy (wave −2);
|
||||
2. the migration (wave −1);
|
||||
3. provisioning (wave 0);
|
||||
4. the Deployment and Service (wave 1);
|
||||
5. the PostSync store check.
|
||||
|
||||
Provisioning records the digest in `retained_builds`, then creates genesis from
|
||||
`development-v1` only when the region does not exist. A region that exists, in any
|
||||
state, is reported and never touched, so every sync can run it. The opening is
|
||||
`evo::processes::development_opening`, byte-identical to the qualification fixture
|
||||
(proved by `lineage-region-contracts`). World0's opening remains a separate,
|
||||
reviewed decision.
|
||||
|
||||
**Evidence tooling.** `tools/ops/region-drill.sh <dev|stage>` (World0 refused)
|
||||
reads epoch and committed tick from the pod's own `/metrics`, then:
|
||||
- performs `rollout restart`, and times the replacement to Ready;
|
||||
- force-deletes the pod, and times the replacement to Ready.
|
||||
|
||||
After each step, the replacement must report a newer epoch at a committed tick
|
||||
that is no lower than before.
|
||||
|
||||
## Consequences
|
||||
- No inputs reach the region until a gateway exists. The host runs idle, holding
|
||||
and renewing its lease, and tick debt accrues visibly. Commit-level fault
|
||||
evidence under load is M8.6.
|
||||
- A forced deletion's replacement waits out up to 20 s of lease. That is correct,
|
||||
not a fault.
|
||||
- Losing the store node leaves the region unavailable until the node returns
|
||||
(ADR-0070); it is never re-created empty.
|
||||
- Each environment's AppProject must allow Deployment and Service.
|
||||
`deploy/argocd/dev/project.yaml` does, and supersedes the old example.
|
||||
|
||||
## Alternatives rejected
|
||||
- **RollingUpdate with maxSurge.** It deadlocks on the lease, as described above.
|
||||
- **A StatefulSet.** It adds no guarantee the lease does not already provide.
|
||||
- **A director.** With static single-region assignment, it would be a placeholder
|
||||
with no authority.
|
||||
@@ -81,3 +81,4 @@ design. No unadopted candidate is a link dependency of the proof.
|
||||
- [ADR-0068: Upgrade barrier and durability qualification](0068-upgrade-barrier.md) - accepted for M7.9b build-bound authority, retirement barrier fencing old writers, explicit refusal of rule/format changes and the measured recovery fixture.
|
||||
- [ADR-0069: Environment isolation and World0 protection](0069-environment-isolation.md) - accepted for M8.2 per-environment guardrails, native World0 admission protection, scoped Argo projects and verification.
|
||||
- [ADR-0070: Durable stores for dev and stage](0070-dev-stage-stores.md) - accepted for M8.3 external PostgreSQL per environment, Retain-class node-local object volumes, free-space reserve and in-cluster store qualification.
|
||||
- [ADR-0071: Region host deployment in dev and stage](0071-region-host-deployment.md) - accepted for M8.4 single-replica Recreate host, digest-as-build identity, idempotent provisioning hook and the region drill; no director yet.
|
||||
|
||||
@@ -2781,3 +2781,33 @@ admission policy and the stage and World0 AppProjects, then ran
|
||||
- The digests the maintainer pasted are local image IDs from the build step, not
|
||||
registry digests. The registry digest is printed by the Publish step.
|
||||
- Not executed: bootstrapping the platform Application, or any Argo sync.
|
||||
|
||||
## M8.4 region host - 2026-09-29
|
||||
|
||||
- Windows MSVC Release CTest 59/59, including the new opening code.
|
||||
`lineage-region-contracts` proves `development_opening(7)` is byte-identical
|
||||
to the fixture, and that world 100 differs only by identity.
|
||||
- WSL Linux `checks` set: 59/59 in each configuration. The format check passed.
|
||||
- PostgreSQL suites: Release, analysis and ASan/UBSan passed. Analysis passed after
|
||||
one optional-access fix in `provision`.
|
||||
- Provisioning test:
|
||||
- A runtime login cannot provision.
|
||||
- The owner provisions `world=7 region=86 sequence=0 tick=0`, and a rerun reports
|
||||
`already-provisioned`.
|
||||
- A server then became ready and acknowledged ticks 1–5, after which the tool
|
||||
reported `sequence=1 tick=5` and left the region untouched.
|
||||
- Mutation: removing the already-provisioned early return was caught.
|
||||
- `deploy-policy-check.py`: 50 objects, 0 failures. Four new mutations were each
|
||||
caught:
|
||||
- a build identity differing from the image digest;
|
||||
- a RollingUpdate strategy;
|
||||
- a CPU limit;
|
||||
- a grace period shorter than the drain.
|
||||
The checker first misread overlay patches as whole Deployments; it now checks
|
||||
only the base.
|
||||
- Kustomize renders dev and stage in order: egress policy (−2), migration hook
|
||||
(−1), provisioning hook (0), Deployment and Service (1), then the store check
|
||||
(PostSync).
|
||||
- `region-drill.sh` passed `bash -n`, and refuses World0 before kubectl (also
|
||||
enforced by the policy check).
|
||||
- Not executed: the in-cluster region sync and the drill.
|
||||
|
||||
@@ -116,6 +116,29 @@ this evidence in docs/engineering/VALIDATION.md:
|
||||
the Secret. The check must exit 3 with `database refused connect`.
|
||||
- Update the Secret's `pgpass` and sync again. It must exit 0.
|
||||
|
||||
## Region host (M8.4, ADR-0071)
|
||||
|
||||
`deploy/kubernetes/<env>` includes `stores/<env>` and `region/<env>`. A sync runs:
|
||||
1. the database egress policy (wave −2);
|
||||
2. the migration hook (wave −1);
|
||||
3. the provisioning hook (wave 0, idempotent);
|
||||
4. the `evo-region` Deployment and `evo-region-ops` Service (wave 1);
|
||||
5. the store check (PostSync).
|
||||
|
||||
Dev is world 100, region 1; stage is world 200, region 1. The server image digest
|
||||
in `region/<env>` is also the build identity.
|
||||
|
||||
Before the first region sync, apply `deploy/argocd/dev/project.yaml` (it allows
|
||||
Deployment and Service), and label the store node.
|
||||
|
||||
Evidence: `bash tools/ops/region-drill.sh dev` (dev or stage only; World0 is
|
||||
refused). It records the time to Ready after a rollout restart and after a forced
|
||||
pod deletion, and checks that the new epoch resumed at the committed tick. Also
|
||||
record:
|
||||
- the Argo sync result;
|
||||
- the first pod's `phase running epoch=... committed=...` line;
|
||||
- that `/readyz` stays 503 while a replacement waits out the lease.
|
||||
|
||||
## Break-glass
|
||||
|
||||
How membership is granted is not decided yet; see docs/BACKLOG.md. Until it is,
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
#include "evo/processes/lineage_region.hpp"
|
||||
#include <expected>
|
||||
#include <memory>
|
||||
|
||||
// M8.4 development opening (ADR-0071): the deterministic lineage region used to
|
||||
// provision dev and stage regions, identical in content to the qualification fixture
|
||||
// every M7 test runs (tests/fixtures/environmental_region.hpp). It is an engineering
|
||||
// opening, not a world design: one habitat site, two bound organisms (producer 10,
|
||||
// consumer 11), founder genome 1 with descendant 2 and one retained dead organism,
|
||||
// rules 2, a 50 ms tick. Never provision World0 with it.
|
||||
namespace evo::processes {
|
||||
inline constexpr std::uint64_t development_opening_frame = 9;
|
||||
enum class DevelopmentOpeningError : std::uint8_t { identity, build };
|
||||
// Built for `world` in frame development_opening_frame; the world must be nonzero.
|
||||
[[nodiscard]] std::expected<std::unique_ptr<LineageRegion>, DevelopmentOpeningError>
|
||||
development_opening(std::uint64_t world);
|
||||
} // namespace evo::processes
|
||||
@@ -0,0 +1,96 @@
|
||||
#include "evo/processes/development_opening.hpp"
|
||||
#include <array>
|
||||
#include <limits>
|
||||
#include <optional>
|
||||
|
||||
namespace evo::processes {
|
||||
namespace {
|
||||
// Mirrors tests/fixtures/environmental_region.hpp exactly; the lineage-region
|
||||
// contracts test proves byte equality with the fixture for world 7.
|
||||
EnvironmentalConfig config(world::WorldIdentity id) {
|
||||
EnvironmentalConfig c;
|
||||
c.rules = 2;
|
||||
c.world = id.world;
|
||||
c.frame = id.frame;
|
||||
c.producer = 1;
|
||||
c.consumer = 2;
|
||||
c.source = 3;
|
||||
c.tick_microseconds = 50000;
|
||||
c.area_um2 = 1;
|
||||
c.irradiance_numerator = 11;
|
||||
c.irradiance_denominator = 3;
|
||||
c.efficiency = 600;
|
||||
c.growth_numerator = 3;
|
||||
c.growth_denominator = 2;
|
||||
c.growth_cost = 1;
|
||||
c.recipe = {1, 2, 1, 1, 1};
|
||||
c.intake.fill(100);
|
||||
c.assimilation = 1000;
|
||||
c.producer_min_mK = c.consumer_min_mK = 270000;
|
||||
c.producer_max_mK = c.consumer_max_mK = 310000;
|
||||
c.decay.fill(1);
|
||||
return c;
|
||||
}
|
||||
std::unique_ptr<EnvironmentalCommunity> community(world::WorldIdentity id) {
|
||||
using namespace world;
|
||||
constexpr auto maximum = std::numeric_limits<std::uint64_t>::max();
|
||||
std::array<std::uint64_t, 10> environment{200, 6, 6, 400, 0, 0, 200, 0, 2000, 0};
|
||||
const FieldSite site{{0, 0, -16000}, {{0}, {0}, {0}, {10}, {20}, {3}}, {1}};
|
||||
std::array<FieldOpening, 10> opening{};
|
||||
for (std::size_t k = 0; k < 10; ++k)
|
||||
opening[k] = {static_cast<Field>(k), {&environment[k], 1}};
|
||||
const FieldDefaults defaults{
|
||||
{accounting::Femtograms{0}, accounting::Femtograms{maximum}, accounting::Femtograms{0}}};
|
||||
auto f = FieldStore::create(id, 1, defaults, {&site, 1}, opening, 2000000);
|
||||
auto t = ChunkStore::create({}, 1, 2000000, {{VolumeUnits{100}, VolumeUnits{100}}});
|
||||
if (!f || !t || !(*t)->load({0, 0, -1000}))
|
||||
return nullptr;
|
||||
std::array<OrganismResources, 2> accounts{};
|
||||
accounts[0].id = {1};
|
||||
accounts[1].id = {2};
|
||||
accounts[0].capacities.fill(100000000);
|
||||
accounts[1].capacities.fill(100000000);
|
||||
auto r = EnvironmentalTransfers::create(id, std::move(*f), std::move(*t), accounts, 4000000);
|
||||
if (!r)
|
||||
return nullptr;
|
||||
auto o = EnvironmentalCommunity::create(config(id), std::move(*r), 8000000);
|
||||
return o ? std::move(*o) : nullptr;
|
||||
}
|
||||
std::optional<biology::Genome> genome(std::uint64_t id, std::uint64_t parent) {
|
||||
biology::GenomeDraft draft;
|
||||
draft.id = id;
|
||||
draft.origin_event = id;
|
||||
draft.development_version = 1;
|
||||
if (parent != 0)
|
||||
draft.parents = {parent};
|
||||
draft.capabilities = {{biology::Capability::energy_conversion, 400, 0},
|
||||
{biology::Capability::membrane, 120, 0},
|
||||
{biology::Capability::target_size, 8'000'000, 0}};
|
||||
auto made = biology::Genome::create(draft);
|
||||
if (!made)
|
||||
return std::nullopt;
|
||||
return *made;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<std::unique_ptr<LineageRegion>, DevelopmentOpeningError>
|
||||
development_opening(std::uint64_t world) {
|
||||
if (world == 0)
|
||||
return std::unexpected{DevelopmentOpeningError::identity};
|
||||
const world::WorldIdentity id{world, development_opening_frame};
|
||||
auto founder = genome(1, 0);
|
||||
auto descendant = genome(2, 1);
|
||||
auto opening = community(id);
|
||||
if (!founder || !descendant || !opening)
|
||||
return std::unexpected{DevelopmentOpeningError::build};
|
||||
biology::LineageArchive archive;
|
||||
if (!archive.admit(*founder) || !archive.admit(*descendant) ||
|
||||
!archive.record_birth(5, 1, Tick{0}) || !archive.record_death(5, Tick{0}) ||
|
||||
!archive.record_birth(10, 1, Tick{0}) || !archive.record_birth(11, 2, Tick{0}))
|
||||
return std::unexpected{DevelopmentOpeningError::build};
|
||||
auto region = LineageRegion::create(std::move(opening), archive, {10, 11, 12, 3});
|
||||
if (!region)
|
||||
return std::unexpected{DevelopmentOpeningError::build};
|
||||
return std::move(*region);
|
||||
}
|
||||
} // namespace evo::processes
|
||||
@@ -1,3 +1,4 @@
|
||||
#include "evo/processes/development_opening.hpp"
|
||||
#include "evo/processes/lineage_region.hpp"
|
||||
#include "fixtures/environmental_region.hpp"
|
||||
|
||||
@@ -164,6 +165,19 @@ void rebinding_changes_only_the_world() {
|
||||
CHECK(community->rebind_world(9) && community->rebind_world(region_identity.world));
|
||||
CHECK(community->snapshot() == before);
|
||||
}
|
||||
|
||||
// M8.4: the provisioning opening is byte-identical to the qualification fixture, and
|
||||
// differs for another world only by the recorded identity.
|
||||
void development_opening_matches_the_fixture() {
|
||||
auto opening = development_opening(region_identity.world);
|
||||
CHECK(opening && (*opening)->snapshot() == lineage_region()->snapshot());
|
||||
auto elsewhere = development_opening(100);
|
||||
CHECK(elsewhere && (*elsewhere)->community().world() == 100);
|
||||
CHECK((*elsewhere)->rebind_world(region_identity.world));
|
||||
CHECK((*elsewhere)->snapshot() == lineage_region()->snapshot());
|
||||
CHECK(!development_opening(0));
|
||||
CHECK(development_opening_frame == region_identity.frame);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
@@ -172,6 +186,7 @@ int main() {
|
||||
deaths_are_recorded_in_the_same_tick();
|
||||
snapshots_restore_and_continue();
|
||||
rebinding_changes_only_the_world();
|
||||
development_opening_matches_the_fixture();
|
||||
std::puts("lineage region contracts passed");
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
|
||||
@@ -199,9 +199,56 @@ void full_volume_server_acknowledges_nothing() {
|
||||
const auto after = head_of(r.key);
|
||||
CHECK(after.tick == before.tick && after.sequence == before.sequence);
|
||||
}
|
||||
// M8.4: the provisioning tool creates a region once from the development opening;
|
||||
// a rerun (every Argo sync) leaves it untouched, and a server then serves it.
|
||||
void provisioning_is_idempotent() {
|
||||
const ServerRegion r{{7, 86}, fresh("provision-86")};
|
||||
const auto run = [&](const std::string& user, int& code) {
|
||||
Child tool{{env("EVO_WORLD_CLONE_BIN"),
|
||||
"provision",
|
||||
"--world",
|
||||
"7",
|
||||
"--region",
|
||||
"86",
|
||||
"--opening",
|
||||
"development-v1",
|
||||
"--pg-host",
|
||||
env("PGHOST"),
|
||||
"--pg-port",
|
||||
env("PGPORT"),
|
||||
"--pg-database",
|
||||
env("PGDATABASE"),
|
||||
"--pg-user",
|
||||
user,
|
||||
"--objects",
|
||||
r.root.string(),
|
||||
"--build",
|
||||
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
|
||||
"--retain-build",
|
||||
"true"}};
|
||||
code = tool.wait_exit(30s);
|
||||
return tool.lines();
|
||||
};
|
||||
int code = -1;
|
||||
auto denied = run(env("EVO_PG_RUNTIME_USER"), code);
|
||||
CHECK(code != 0 && !has(denied, "provisioned "));
|
||||
auto first = run(env("PGUSER"), code);
|
||||
CHECK(code == 0 && has(first, "provisioned world=7 region=86 sequence=0 tick=0"));
|
||||
auto second = run(env("PGUSER"), code);
|
||||
CHECK(code == 0 && has(second, "already-provisioned world=7 region=86 sequence=0 tick=0"));
|
||||
Child server{server_args(r)};
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
||||
server.send(input_lines(1, 5, false));
|
||||
CHECK(server.wait_line("ack " + request_hex(5) + " 5 ", 15s));
|
||||
server.signal(SIGTERM);
|
||||
CHECK(server.wait_exit(20s) == 0);
|
||||
auto after = run(env("PGUSER"), code);
|
||||
CHECK(code == 0 && has(after, "already-provisioned world=7 region=86 sequence=1 tick=5"));
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void run_region_store_tests() {
|
||||
provisioning_is_idempotent();
|
||||
full_volumes_publish_nothing();
|
||||
failing_writes_leave_nothing();
|
||||
check_store_qualifies_the_stores();
|
||||
|
||||
@@ -192,6 +192,36 @@ for path, doc in objects:
|
||||
check(doc['metadata'].get('namespace') != 'evolution-world0',
|
||||
f'{path}: store Jobs never target World0')
|
||||
|
||||
# M8.4 region host (ADR-0071): one owner at a time, readiness means authority.
|
||||
for path, doc in objects:
|
||||
# Overlay patches (region-values.yaml) are partial objects; the base is complete.
|
||||
if (doc['kind'] == 'Deployment' and doc['metadata'].get('name') == 'evo-region'
|
||||
and not path.endswith('region-values.yaml')):
|
||||
spec = doc['spec']
|
||||
pod = spec['template']['spec']
|
||||
container = pod['containers'][0]
|
||||
check(spec.get('replicas') == 1 and spec.get('strategy', {}).get('type') == 'Recreate',
|
||||
f'{path}: the region host runs one replica with the Recreate strategy')
|
||||
check(container.get('readinessProbe', {}).get('httpGet', {}).get('path') == '/readyz'
|
||||
and container.get('livenessProbe', {}).get('httpGet', {}).get('path') == '/livez',
|
||||
f'{path}: readiness must be /readyz and liveness /livez')
|
||||
limits = container.get('resources', {}).get('limits', {})
|
||||
check('cpu' not in limits and 'memory' in limits,
|
||||
f'{path}: memory limit required, no CPU limit (throttling delays renewals)')
|
||||
script = ' '.join(container.get('args', []))
|
||||
drain = [int(x) for x in __import__('re').findall(r'--drain-ms (\d+)', script)]
|
||||
check(len(drain) == 1 and pod.get('terminationGracePeriodSeconds', 30) * 1000 > drain[0],
|
||||
f'{path}: the termination grace period must exceed --drain-ms')
|
||||
for env in ('dev', 'stage'):
|
||||
overlay = f'deploy/kubernetes/region/{env}/'
|
||||
images = manifests.get(overlay + 'kustomization.yaml', [{}])[0].get('images', [])
|
||||
server = [i.get('digest', '') for i in images if i.get('name') == 'aevolution-server']
|
||||
builds = [v.get('value') for d in manifests.get(overlay + 'region-values.yaml', [])
|
||||
for c in d['spec']['template']['spec']['containers'] for v in c.get('env', [])
|
||||
if v.get('name') == 'REGION_BUILD']
|
||||
check(len(server) == 1 and len(builds) == 2 and all('sha256:' + b == server[0] for b in builds),
|
||||
f'{overlay}: the build identity must equal the server image digest')
|
||||
|
||||
# The platform bundle: every rendered object is protected from prune and deletion.
|
||||
bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0]
|
||||
check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false',
|
||||
@@ -243,12 +273,13 @@ for path, doc in objects:
|
||||
# The cleanup tool refuses World0 before kubectl runs (no kubectl on PATH here).
|
||||
with tempfile.TemporaryDirectory() as empty:
|
||||
env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable))
|
||||
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
|
||||
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, 'tools/ops/cleanup-environment.sh'), bad],
|
||||
capture_output=True, text=True, env=env)
|
||||
check(result.returncode == 2 and ('refused' in result.stderr or 'usage' in result.stderr)
|
||||
and 'kubectl' not in result.stderr,
|
||||
f'cleanup must refuse {bad!r} before kubectl (rc={result.returncode})')
|
||||
for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh'):
|
||||
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
|
||||
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad],
|
||||
capture_output=True, text=True, env=env)
|
||||
check(result.returncode == 2 and ('refused' in result.stderr or 'usage' in result.stderr)
|
||||
and 'kubectl' not in result.stderr,
|
||||
f'{tool} must refuse {bad!r} before kubectl (rc={result.returncode})')
|
||||
|
||||
for message in failures:
|
||||
print('FAIL', message)
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env bash
|
||||
# M8.4 region host drill (ADR-0071) for a disposable environment only.
|
||||
#
|
||||
# tools/ops/region-drill.sh <dev|stage> [--context <name>]
|
||||
#
|
||||
# 1. Reads the ready region's authority from /metrics (epoch, committed tick).
|
||||
# 2. Graceful rollout: `kubectl rollout restart`. With the Recreate strategy the old
|
||||
# pod drains (commits, releases the lease) before the new one starts; measures the
|
||||
# time until the new pod is Ready and checks it resumed at the committed tick with
|
||||
# a newer epoch.
|
||||
# 3. Forced worker deletion: deletes the pod with --grace-period=0 --force; the
|
||||
# replacement must wait out the dead holder's lease (up to 20 s) and recover; the
|
||||
# time to Ready is measured the same way.
|
||||
# Refuses anything but dev or stage before touching the cluster; World0 is never
|
||||
# drilled from here. Prints one line per measurement; exits nonzero on any failure.
|
||||
set -euo pipefail
|
||||
usage() { echo "usage: $0 <dev|stage> [--context <name>]" >&2; exit 2; }
|
||||
[ $# -ge 1 ] || usage
|
||||
env="$1"; shift
|
||||
case "$env" in
|
||||
dev|stage) ;;
|
||||
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
k=(kubectl --request-timeout=30s)
|
||||
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
|
||||
ns="evolution-$env"
|
||||
selector=app.kubernetes.io/name=evo-region
|
||||
|
||||
pod_name() { "${k[@]}" get pods -n "$ns" -l "$selector" \
|
||||
-o jsonpath='{range .items[?(@.metadata.deletionTimestamp == "")]}{.metadata.name}{"\n"}{end}' |
|
||||
head -1; }
|
||||
ready_pod() { # Waits (up to 120 s) for a Ready pod other than $1; prints its name.
|
||||
local old="$1"
|
||||
for _ in $(seq 1 600); do
|
||||
for p in $("${k[@]}" get pods -n "$ns" -l "$selector" -o name 2>/dev/null); do
|
||||
p="${p#pod/}"
|
||||
[ "$p" = "$old" ] && continue
|
||||
ready="$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
|
||||
[ "$ready" = True ] && { echo "$p"; return 0; }
|
||||
done
|
||||
sleep 0.2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
metric() { # metric <pod> <series>: one value from the pod's own /metrics (bash /dev/tcp).
|
||||
"${k[@]}" exec -n "$ns" "$1" -- /bin/bash -c \
|
||||
'exec 3<>/dev/tcp/127.0.0.1/9100; printf "GET /metrics HTTP/1.1\r\nHost: drill\r\n\r\n" >&3; cat <&3' |
|
||||
awk -v s="$2" '$1 == s {print $2}'
|
||||
}
|
||||
now_ms() { date +%s%3N; }
|
||||
|
||||
first="$(pod_name)"
|
||||
[ -n "$first" ] || { echo "FAIL no evo-region pod in $ns" >&2; exit 1; }
|
||||
first="$(ready_pod "")" || { echo "FAIL no Ready evo-region pod in $ns" >&2; exit 1; }
|
||||
epoch0="$(metric "$first" evo_region_lease_epoch)"
|
||||
tick0="$(metric "$first" evo_region_committed_tick)"
|
||||
echo "start pod=$first epoch=$epoch0 committed_tick=$tick0"
|
||||
|
||||
start="$(now_ms)"
|
||||
"${k[@]}" rollout restart -n "$ns" deployment/evo-region > /dev/null
|
||||
second="$(ready_pod "$first")" || { echo "FAIL rollout: no Ready replacement" >&2; exit 1; }
|
||||
rollout_ms=$(( $(now_ms) - start ))
|
||||
epoch1="$(metric "$second" evo_region_lease_epoch)"
|
||||
tick1="$(metric "$second" evo_region_committed_tick)"
|
||||
echo "rollout ready_ms=$rollout_ms pod=$second epoch=$epoch1 committed_tick=$tick1"
|
||||
[ "$epoch1" -gt "$epoch0" ] && [ "$tick1" -ge "$tick0" ] ||
|
||||
{ echo "FAIL rollout did not resume with a newer epoch at the committed tick" >&2; exit 1; }
|
||||
|
||||
start="$(now_ms)"
|
||||
"${k[@]}" delete pod -n "$ns" "$second" --grace-period=0 --force > /dev/null 2>&1
|
||||
third="$(ready_pod "$second")" || { echo "FAIL forced deletion: no Ready replacement" >&2; exit 1; }
|
||||
forced_ms=$(( $(now_ms) - start ))
|
||||
epoch2="$(metric "$third" evo_region_lease_epoch)"
|
||||
tick2="$(metric "$third" evo_region_committed_tick)"
|
||||
echo "forced ready_ms=$forced_ms pod=$third epoch=$epoch2 committed_tick=$tick2"
|
||||
[ "$epoch2" -gt "$epoch1" ] && [ "$tick2" -ge "$tick1" ] ||
|
||||
{ echo "FAIL forced deletion did not resume with a newer epoch at the committed tick" >&2; exit 1; }
|
||||
echo "PASS region drill in $ns"
|
||||
Reference in New Issue
Block a user