diff --git a/CMakeLists.txt b/CMakeLists.txt index c5d326d..2502fce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,7 +62,7 @@ target_link_libraries(evo_ecology PUBLIC evo::accounting evo::biology) target_compile_features(evo_ecology PUBLIC cxx_std_23) evo_owned_target(evo_ecology) -add_library(evo_processes STATIC src/processes/producer.cpp src/processes/feeding.cpp src/processes/ecosystem.cpp src/processes/genome_strategy.cpp src/processes/ensemble.cpp src/processes/environmental_community.cpp src/processes/environmental_codec.cpp src/processes/environmental_schedule.cpp src/processes/lineage_region.cpp) +add_library(evo_processes STATIC src/processes/producer.cpp src/processes/feeding.cpp src/processes/ecosystem.cpp src/processes/genome_strategy.cpp src/processes/ensemble.cpp src/processes/environmental_community.cpp src/processes/environmental_codec.cpp src/processes/environmental_schedule.cpp src/processes/lineage_region.cpp src/processes/development_opening.cpp) add_library(evo::processes ALIAS evo_processes) target_link_libraries(evo_processes PUBLIC evo::accounting evo::biology evo::world) evo_owned_target(evo_processes) diff --git a/apps/evo-world-clone/main.cpp b/apps/evo-world-clone/main.cpp index 4e0cf2c..28a0dad 100644 --- a/apps/evo-world-clone/main.cpp +++ b/apps/evo-world-clone/main.cpp @@ -9,6 +9,8 @@ // [--max-commits N] // evo-world-clone check-store --world W --region R --pg-* ... --objects DIR // [--reserve-mib N] +// evo-world-clone provision --world W --region R --opening development-v1 --pg-* ... +// --objects DIR --build HEX [--retain-build] // // --*pg-* is host, hostaddr, port, database, user, passfile, ca, or service (a libpq // service file whose [evolution] section supplies everything but the password file). The source @@ -31,10 +33,18 @@ // Output: objects available_mib=A reserve_mib=R / probe created|verified digest=HEX // database version=V user=U ssl=S / database tables=denied functions=reachable // +// provision (M8.4) creates a region's genesis with the owner (migrator) login, once: +// an already provisioned region is reported and never touched. --retain-build first +// records the build identity (the server image digest) in retained_builds. The only +// opening is development-v1 (evo/processes/development_opening.hpp), for dev and +// stage; World0's opening is a separate, reviewed decision. +// Output: provisioned|already-provisioned world=W region=R sequence=S tick=T +// // Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused, // 5 target unknown. #include "evo/persistence/region_clone.hpp" #include "evo/persistence/sha256.hpp" +#include "evo/processes/development_opening.hpp" #include #include @@ -65,6 +75,9 @@ struct Options { std::vector retained; std::optional commit; std::uint64_t audit_max{}, max_commits{1u << 20U}, reserve_mib{1024}; + std::string opening; + std::optional build; + bool retain_build{}; bool crash{}; }; @@ -136,7 +149,8 @@ std::optional parse(int argc, char** argv) { return std::nullopt; Options o; o.command = argv[1]; - if (o.command != "clone" && o.command != "audit" && o.command != "check-store") + if (o.command != "clone" && o.command != "audit" && o.command != "check-store" && + o.command != "provision") return std::nullopt; const bool clone = o.command == "clone"; for (int i = 2; i < argc; ++i) { @@ -178,12 +192,20 @@ std::optional parse(int argc, char** argv) { o.max_commits = *n; else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576) o.reserve_mib = *n; + else if (o.command == "provision" && flag == "--opening" && value == "development-v1") + o.opening = value; + else if (o.command == "provision" && flag == "--build" && hex<32>(value)) + o.build = hex<32>(value); + else if (o.command == "provision" && flag == "--retain-build" && value == "true") + o.retain_build = true; else return std::nullopt; } if (clone && o.target.key.region == 0) o.target.key.region = o.source.key.region; - const bool needs_builds = o.command != "check-store"; + const bool needs_builds = o.command != "check-store" && o.command != "provision"; + if (o.command == "provision" && (o.opening.empty() || !o.build)) + return std::nullopt; if (!complete(o.source) || (needs_builds && o.retained.empty()) || (clone && !complete(o.target))) return std::nullopt; @@ -274,6 +296,60 @@ std::vector probe_bytes() { return out; } +int provision(const Options& o) { + auto store = open_objects(o.source); + if (!store) { + std::puts("objects refused"); + return 4; + } + auto db = connect(o.source); + if (!db) { + std::puts("database refused"); + return 3; + } + if (!o.build) + return 2; // Required by parse(); checked again for the reader. + const auto& build = *o.build; + if (o.retain_build) { + const std::array p{"\\x" + text(build)}; + if (!db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES ($1::bytea) " + "ON CONFLICT DO NOTHING", + p, timeout)) { + std::puts("retain-build refused"); + return 3; + } + } + const auto report = [&](const char* what, const persistence::RegionHead& head) { + std::printf("%s world=%llu region=%llu sequence=%llu tick=%llu\n", what, + static_cast(o.source.key.world), + static_cast(o.source.key.region), + static_cast(head.sequence), + static_cast(head.end_tick)); + return 0; + }; + // Never re-provision: an existing region, whatever its state, is left alone. + auto existing = persistence::load_region_head(*db, o.source.key, timeout); + if (existing) + return report("already-provisioned", *existing); + if (existing.error().kind != persistence::LeaseStoreError::not_provisioned) { + std::puts("database refused"); + return 3; + } + auto opening = processes::development_opening(o.source.key.world); + persistence::Uuid commit{}; + if (!opening || + ::getrandom(commit.data(), commit.size(), 0) != static_cast(commit.size())) + return 4; + auto made = persistence::provision_lineage_region( + *db, *store, o.source.key, processes::development_opening_frame, **opening, commit, build, + restore_budget, timeout); + if (!made) { + std::printf("provision refused %u\n", static_cast(made.error().kind)); + return 4; + } + return report("provisioned", made->head); +} + int check_store(const Options& o) { auto store = persistence::FilesystemObjectStore::open(o.source.objects, o.source.key, persistence::max_lineage_payload_bytes, @@ -441,5 +517,7 @@ int main(int argc, char** argv) { } if (options->command == "check-store") return check_store(*options); + if (options->command == "provision") + return provision(*options); return options->command == "clone" ? clone(*options) : audit(*options); } diff --git a/deploy/README.md b/deploy/README.md index e27d4b5..1b37362 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -147,17 +147,19 @@ plus the pinned libpq, UID 10001, no credentials). It smoke-tests the image with `tools/ci/server-image-smoke.sh` and publishes only when Gitea variable `HARBOR_SERVER_REPOSITORY` is set. -`kubernetes/region-server/deployment.yaml.example` is outside Argo. It shows: -- probes: `/readyz` means authority, `/livez` means only liveness; -- the runtime Secret, object PVC and a NetworkPolicy. -Pin the digest and provision the region first. Without a gateway, the region -receives no inputs. +The region host is deployed from `kubernetes/region/` (M8.4, ADR-0071), +included in each environment's Argo path. It replaces the former +`region-server` example. Probes: `/readyz` means authority, and `/livez` means +only that the process is alive. Without a gateway, the region receives no inputs. ## Environments and World0 protection (M8.2) `kubernetes/platform/` holds each environment's namespace, quota, limit range, default-deny policies and service accounts. `kubernetes/platform/admission` holds -the World0 admission policy. A cluster administrator applies these by hand; they +the World0 admission policy. Since the M8.3 amendment they are delivered by the +`aevolution-platform` Argo Application (`kubernetes/platform/cluster`), which +self-heals and never prunes. Originally a cluster administrator applied them by +hand; they are never part of an Argo Application. The stage and World0 Argo projects and applications are in `argocd/stage` and `argocd/world0` as examples. diff --git a/deploy/argocd/dev/project.yaml b/deploy/argocd/dev/project.yaml new file mode 100644 index 0000000..d9602c4 --- /dev/null +++ b/deploy/argocd/dev/project.yaml @@ -0,0 +1,25 @@ +# M8.4 dev workload project (replaces deploy/argocd/project.yaml.example). Apply by hand +# (kubectl apply -f deploy/argocd/dev/project.yaml): Argo CD does not manage its own +# projects here. One namespace, no cluster kinds; ServiceAccounts only for the +# foundation proof Job (the platform owns the environment's service accounts). +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: aevolution-dev + namespace: argocd +spec: + description: AEvolution dev (evolution-dev only) + sourceRepos: + - https://git.aeternalisgames.com/dingo/AEvolution.git + destinations: + - server: https://kubernetes.default.svc + namespace: evolution-dev + clusterResourceWhitelist: [] + namespaceResourceWhitelist: + - {group: "", kind: ServiceAccount} + - {group: "", kind: Service} + - {group: apps, kind: Deployment} + - {group: batch, kind: Job} + - {group: networking.k8s.io, kind: NetworkPolicy} + orphanedResources: + warn: true diff --git a/deploy/kubernetes/dev/kustomization.yaml b/deploy/kubernetes/dev/kustomization.yaml index 7be194c..17a85a1 100644 --- a/deploy/kubernetes/dev/kustomization.yaml +++ b/deploy/kubernetes/dev/kustomization.yaml @@ -6,6 +6,7 @@ resources: - networkpolicy.yaml - job.yaml - ../stores/dev + - ../region/dev images: - name: evo-sim newName: registry.aeternalisgames.com/aeternalis/aevolution diff --git a/deploy/kubernetes/region-server/deployment.yaml.example b/deploy/kubernetes/region-server/deployment.yaml.example deleted file mode 100644 index 4db8be2..0000000 --- a/deploy/kubernetes/region-server/deployment.yaml.example +++ /dev/null @@ -1,143 +0,0 @@ -# M7.8 development region server (ADR-0066). NOT in the active Argo kustomization. -# Before applying to evolution-dev: pin the server image digest, provision the region -# (owner-only), create the runtime Secret and object PVC, and add DB/DNS egress for -# the real endpoint. Never target World0 from this example. -# -# Authority, not liveness, gates traffic: /readyz is 200 only while this pod holds -# the region lease and has recovered and verified the committed head. /livez only -# says the loop is responding. A probe can wait up to one bounded I/O step (commit -# deadline 5 s), so probe timeouts below exceed it. -# -# There is no gateway yet: stdin is not attached, so the region hosts authority but -# receives no inputs (tick debt grows by design; nothing is invented). -apiVersion: apps/v1 -kind: Deployment -metadata: - name: evolution-region-7-1 - namespace: evolution-dev - labels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"} -spec: - replicas: 1 - strategy: {type: Recreate} # Never two pods racing; the lease still arbitrates. - selector: - matchLabels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"} - template: - metadata: - labels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"} - spec: - automountServiceAccountToken: false - terminationGracePeriodSeconds: 30 # SIGTERM drains within --drain-ms 20000. - securityContext: - runAsNonRoot: true - runAsUser: 10001 - runAsGroup: 10001 - fsGroup: 10001 - seccompProfile: {type: RuntimeDefault} - initContainers: - # libpq rejects group-readable password files: copy to a private 0600 file. - - name: pgpass - image: registry.aeternalisgames.com/aeternalis/aevolution-server:replace-with-immutable-digest - command: ["/bin/sh", "-c", "install -m 0600 /run/postgres/pgpass /run/evo/pgpass"] - securityContext: {readOnlyRootFilesystem: true, allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}} - volumeMounts: - - {name: connection, mountPath: /run/postgres, readOnly: true} - - {name: private, mountPath: /run/evo} - containers: - - name: region - image: registry.aeternalisgames.com/aeternalis/aevolution-server:replace-with-immutable-digest - # Flags are "--name value" pairs; see docs/infrastructure/POSTGRESQL.md. - command: - - /usr/local/bin/evo-server - - --profile - - lineage - - --world - - "7" - - --region - - "1" - - --frame - - "9" - - --pg-host - - replace-with-tls-hostname - - --pg-hostaddr - - replace-with-numeric-address - - --pg-port - - "5432" - - --pg-database - - evolution - - --pg-user - - replace-with-runtime-login - - --pg-passfile - - /run/evo/pgpass - - --pg-ca - - /run/postgres/ca.crt - - --objects - - /var/lib/evolution/objects - - --build - - replace-with-retained-build-identity-hex - - --ops-listen - - 0.0.0.0:9100 - - --drain-ms - - "20000" - ports: - - {name: ops, containerPort: 9100} - startupProbe: - httpGet: {path: /livez, port: ops} - periodSeconds: 5 - timeoutSeconds: 8 - failureThreshold: 12 - readinessProbe: - httpGet: {path: /readyz, port: ops} - periodSeconds: 5 - timeoutSeconds: 8 - failureThreshold: 1 # Any doubt about authority removes the pod at once. - livenessProbe: - httpGet: {path: /livez, port: ops} - periodSeconds: 10 - timeoutSeconds: 8 - failureThreshold: 6 - resources: - requests: {cpu: 250m, memory: 128Mi} - limits: {cpu: "1", memory: 512Mi} - securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: {drop: [ALL]} - volumeMounts: - - {name: connection, mountPath: /run/postgres, readOnly: true} - - {name: private, mountPath: /run/evo, readOnly: true} - - {name: objects, mountPath: /var/lib/evolution/objects} - - {name: scratch, mountPath: /tmp} - volumes: - - name: connection - secret: - secretName: evolution-postgres-runtime # Runtime login only; never migrator. - defaultMode: 0440 - items: - - {key: pgpass, path: pgpass} - - {key: ca.crt, path: ca.crt} - - name: private - emptyDir: {medium: Memory, sizeLimit: 1Mi} - - name: objects - persistentVolumeClaim: {claimName: evolution-region-objects} - - name: scratch - emptyDir: {medium: Memory, sizeLimit: 8Mi} ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: evolution-region-7-1 - namespace: evolution-dev -spec: - podSelector: - matchLabels: {app.kubernetes.io/name: evolution-region, evolution.aeternalis/region: "7-1"} - policyTypes: [Ingress, Egress] - ingress: - # Metrics/probes only from the monitoring namespace (kubelet probes use the node). - - from: - - namespaceSelector: - matchLabels: {kubernetes.io/metadata.name: monitoring} - ports: [{port: 9100, protocol: TCP}] - egress: - # Replace with the actual database endpoint; no DNS is needed with a numeric hostaddr. - - to: [{ipBlock: {cidr: 192.0.2.10/32}}] - ports: [{port: 5432, protocol: TCP}] diff --git a/deploy/kubernetes/region/base/deployment.yaml b/deploy/kubernetes/region/base/deployment.yaml new file mode 100644 index 0000000..1809a33 --- /dev/null +++ b/deploy/kubernetes/region/base/deployment.yaml @@ -0,0 +1,103 @@ +# M8.4 region host (ADR-0071). One replica, Recreate: a rolling update would start +# the new pod while the old one still holds the lease; the new pod could never become +# ready, so the old one would never be removed. Recreate drains the old pod (commits, +# releases the lease) before the new one starts and recovers. Readiness is /readyz +# (current lease + verified recovered head), liveness is /livez only. No CPU limit: +# throttling would delay lease renewals. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: evo-region + annotations: + argocd.argoproj.io/sync-wave: "1" # After migration and provisioning. + labels: + app.kubernetes.io/name: evo-region + app.kubernetes.io/component: region +spec: + replicas: 1 + strategy: + type: Recreate + revisionHistoryLimit: 3 + progressDeadlineSeconds: 180 + selector: + matchLabels: + app.kubernetes.io/name: evo-region + template: + metadata: + labels: + app.kubernetes.io/name: evo-region + app.kubernetes.io/component: region + app.kubernetes.io/part-of: aevolution + spec: + serviceAccountName: aevolution-region + automountServiceAccountToken: false + nodeSelector: + aevolution.io/stores: "true" + # Longer than --drain-ms, so a graceful stop commits and releases the lease. + terminationGracePeriodSeconds: 45 + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: {type: RuntimeDefault} + containers: + - name: evo-server + image: aevolution-server + # libpq ignores group-readable password files: copy to private memory, then + # exec so evo-server is PID 1 and receives SIGTERM directly. + command: ["/bin/bash", "-c"] + args: + - | + set -eu + umask 077 + cp /run/postgres/pgpass /tmp/pgpass + exec /usr/local/bin/evo-server --profile lineage \ + --world "$REGION_WORLD" --region "$REGION_ID" --frame "$REGION_FRAME" \ + --pg-service /run/postgres/pg_service.conf --pg-passfile /tmp/pgpass \ + --objects /objects --objects-reserve-mib 1024 \ + --build "$REGION_BUILD" --lease-profile default \ + --flush-ms 1000 --drain-ms 20000 --ops-listen 0.0.0.0:9100 + env: + - {name: REGION_WORLD, value: set-by-overlay} + - {name: REGION_ID, value: set-by-overlay} + - {name: REGION_FRAME, value: "9"} + - {name: REGION_BUILD, value: set-by-overlay} + ports: + - {name: ops, containerPort: 9100, protocol: TCP} + readinessProbe: + httpGet: {path: /readyz, port: ops} + periodSeconds: 2 + timeoutSeconds: 1 + failureThreshold: 1 + livenessProbe: + httpGet: {path: /livez, port: ops} + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + resources: + requests: {cpu: 50m, memory: 64Mi} + limits: {memory: 256Mi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: objects, mountPath: /objects} + - {name: connection, mountPath: /run/postgres, readOnly: true} + - {name: scratch, mountPath: /tmp} + volumes: + - name: objects + persistentVolumeClaim: + claimName: aevolution-objects + - name: connection + secret: + secretName: evolution-postgres-runtime + defaultMode: 0440 + items: + - {key: pg_service.conf, path: pg_service.conf} + - {key: pgpass, path: pgpass} + - {key: ca.crt, path: ca.crt} + - name: scratch + emptyDir: {medium: Memory, sizeLimit: 1Mi} diff --git a/deploy/kubernetes/region/base/kustomization.yaml b/deploy/kubernetes/region/base/kustomization.yaml new file mode 100644 index 0000000..393abb5 --- /dev/null +++ b/deploy/kubernetes/region/base/kustomization.yaml @@ -0,0 +1,13 @@ +# M8.4 region host component (ADR-0071), included by an environment's workload path +# through region/, which sets the region identity and the server image digest +# (also its build identity). Needs stores/ (database egress, migration) and the +# platform layer (namespace, service accounts, object claim). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - provision-job.yaml + - deployment.yaml + - service.yaml +labels: + - pairs: + app.kubernetes.io/part-of: aevolution diff --git a/deploy/kubernetes/region/base/provision-job.yaml b/deploy/kubernetes/region/base/provision-job.yaml new file mode 100644 index 0000000..156c226 --- /dev/null +++ b/deploy/kubernetes/region/base/provision-job.yaml @@ -0,0 +1,77 @@ +# Sync hook at wave 0: after the database egress policy (wave -2) and the migration +# (wave -1), before the region host (wave 1). Creates the region's genesis from +# the development opening once, with the migrator (owner) login, and records the +# server image digest as a retained build. Every later sync reports +# "already-provisioned" and changes nothing. +apiVersion: batch/v1 +kind: Job +metadata: + name: aevolution-region-provision + labels: + app.kubernetes.io/component: migrate + annotations: + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation + argocd.argoproj.io/sync-wave: "0" +spec: + backoffLimit: 0 + activeDeadlineSeconds: 120 + template: + metadata: + labels: + app.kubernetes.io/component: migrate + app.kubernetes.io/part-of: aevolution + spec: + serviceAccountName: aevolution-migrator + automountServiceAccountToken: false + restartPolicy: Never + nodeSelector: + aevolution.io/stores: "true" + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: {type: RuntimeDefault} + containers: + - name: provision + image: aevolution-server + command: ["/bin/bash", "-c"] + args: + - | + set -eu + umask 077 + cp /run/postgres/pgpass /tmp/pgpass + exec /usr/local/bin/evo-world-clone provision \ + --world "$REGION_WORLD" --region "$REGION_ID" --opening development-v1 \ + --pg-service /run/postgres/pg_service.conf --pg-passfile /tmp/pgpass \ + --objects /objects --build "$REGION_BUILD" --retain-build true + env: + - {name: REGION_WORLD, value: set-by-overlay} + - {name: REGION_ID, value: set-by-overlay} + - {name: REGION_BUILD, value: set-by-overlay} + resources: + requests: {cpu: 50m, memory: 64Mi} + limits: {memory: 256Mi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: objects, mountPath: /objects} + - {name: connection, mountPath: /run/postgres, readOnly: true} + - {name: scratch, mountPath: /tmp} + volumes: + - name: objects + persistentVolumeClaim: + claimName: aevolution-objects + - name: connection + secret: + secretName: evolution-postgres-migrator + defaultMode: 0440 + items: + - {key: pg_service.conf, path: pg_service.conf} + - {key: pgpass, path: pgpass} + - {key: ca.crt, path: ca.crt} + - name: scratch + emptyDir: {medium: Memory, sizeLimit: 1Mi} diff --git a/deploy/kubernetes/region/base/service.yaml b/deploy/kubernetes/region/base/service.yaml new file mode 100644 index 0000000..46c955e --- /dev/null +++ b/deploy/kubernetes/region/base/service.yaml @@ -0,0 +1,16 @@ +# Operations endpoints (/livez, /readyz, /metrics) inside the cluster only; no +# NodePort or LoadBalancer (the quota forbids both). Scraping is enabled in M8.5. +apiVersion: v1 +kind: Service +metadata: + name: evo-region-ops + annotations: + argocd.argoproj.io/sync-wave: "1" # After migration and provisioning. + labels: + app.kubernetes.io/name: evo-region +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: evo-region + ports: + - {name: ops, port: 9100, targetPort: ops, protocol: TCP} diff --git a/deploy/kubernetes/region/dev/kustomization.yaml b/deploy/kubernetes/region/dev/kustomization.yaml new file mode 100644 index 0000000..fa85066 --- /dev/null +++ b/deploy/kubernetes/region/dev/kustomization.yaml @@ -0,0 +1,14 @@ +# Region host for evolution-dev: world 100, region 1, frame 9 (development opening). +# The server image digest appears twice on purpose: as the image and as the build +# identity the region records in every commit. The policy check requires they match. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: evolution-dev +resources: + - ../base +images: + - name: aevolution-server + newName: registry.aeternalisgames.com/aeternalis/aevolution-server + digest: sha256:661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075 +patches: + - path: region-values.yaml diff --git a/deploy/kubernetes/region/dev/region-values.yaml b/deploy/kubernetes/region/dev/region-values.yaml new file mode 100644 index 0000000..ff1cea4 --- /dev/null +++ b/deploy/kubernetes/region/dev/region-values.yaml @@ -0,0 +1,28 @@ +# Region identity and build (the server image digest without "sha256:"). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: evo-region +spec: + template: + spec: + containers: + - name: evo-server + env: + - {name: REGION_WORLD, value: "100"} + - {name: REGION_ID, value: "1"} + - {name: REGION_BUILD, value: 661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: aevolution-region-provision +spec: + template: + spec: + containers: + - name: provision + env: + - {name: REGION_WORLD, value: "100"} + - {name: REGION_ID, value: "1"} + - {name: REGION_BUILD, value: 661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075} diff --git a/deploy/kubernetes/region/stage/kustomization.yaml b/deploy/kubernetes/region/stage/kustomization.yaml new file mode 100644 index 0000000..3af1c89 --- /dev/null +++ b/deploy/kubernetes/region/stage/kustomization.yaml @@ -0,0 +1,14 @@ +# Region host for evolution-stage: world 200, region 1, frame 9 (development opening). +# The server image digest appears twice on purpose: as the image and as the build +# identity the region records in every commit. The policy check requires they match. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: evolution-stage +resources: + - ../base +images: + - name: aevolution-server + newName: registry.aeternalisgames.com/aeternalis/aevolution-server + digest: sha256:661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075 +patches: + - path: region-values.yaml diff --git a/deploy/kubernetes/region/stage/region-values.yaml b/deploy/kubernetes/region/stage/region-values.yaml new file mode 100644 index 0000000..41412fa --- /dev/null +++ b/deploy/kubernetes/region/stage/region-values.yaml @@ -0,0 +1,28 @@ +# Region identity and build (the server image digest without "sha256:"). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: evo-region +spec: + template: + spec: + containers: + - name: evo-server + env: + - {name: REGION_WORLD, value: "200"} + - {name: REGION_ID, value: "1"} + - {name: REGION_BUILD, value: 661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: aevolution-region-provision +spec: + template: + spec: + containers: + - name: provision + env: + - {name: REGION_WORLD, value: "200"} + - {name: REGION_ID, value: "1"} + - {name: REGION_BUILD, value: 661e433ef0cf4fcb619f1e2ed623297ca20ade896aa16d29e67f47e6ccfa2075} diff --git a/deploy/kubernetes/stage/kustomization.yaml b/deploy/kubernetes/stage/kustomization.yaml index 21187d9..367db81 100644 --- a/deploy/kubernetes/stage/kustomization.yaml +++ b/deploy/kubernetes/stage/kustomization.yaml @@ -2,4 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: evolution-stage resources: - - ../stores/stage \ No newline at end of file + - ../stores/stage + - ../region/stage diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 0ad47ad..f5d424a 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -624,3 +624,14 @@ M8.3 implemented and locally qualified (2026-09-29), under ADR-0070: - `evo-world-clone check-store` for in-cluster qualification. The in-cluster evidence and the backup destination wait on the maintainer (docs/BACKLOG.md). + +M8.3 delivered through Argo CD by the maintainer (2026-09-29): the dev, stage and +platform applications are synced and connected to PostgreSQL. + +M8.4 implemented and locally qualified (2026-09-29), under ADR-0071: +- one `evo-region` host per environment, with the Recreate strategy and the image + digest as its build identity; +- an idempotent provisioning hook from the development opening; +- `region-drill.sh` for rollout and forced-deletion evidence. +The in-cluster sync and drill wait on the maintainer. No director yet: assignment +is static and lives in Git. diff --git a/docs/STATUS.md b/docs/STATUS.md index a72f674..a0f8dc5 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1044,3 +1044,23 @@ M8.3 delivery moved to Argo CD, at the maintainer's request: - **Store Jobs:** they become sync hooks in each environment's path once their digests and runtime roles are supplied. - **Outside Argo:** Secrets and the store node label. + +## M8.4 region host (2026-09-29) + +The maintainer synced the dev, stage and platform applications through Argo CD, +and they are connected to PostgreSQL, with digests `661e433e…` (server) and +`9abb06ef…` (tools). + +ADR-0071 deploys `evo-region` as one replica per environment: +- the Recreate strategy, since a rolling update would deadlock on the lease; +- readiness `/readyz`, liveness `/livez`; +- no CPU limit; +- a 20 s drain inside a 45 s grace period. + +Each sync runs, in order: egress, migration, provisioning (idempotent, from the +new `development_opening`, which is byte-identical to the qualification fixture), +the host, then the store check. The image digest is also the build identity, and +the policy check enforces this. Dev is world 100 and stage world 200, both region 1. + +Next: apply `deploy/argocd/dev/project.yaml`, sync, and run +`tools/ops/region-drill.sh dev`. diff --git a/docs/adr/0071-region-host-deployment.md b/docs/adr/0071-region-host-deployment.md new file mode 100644 index 0000000..3c5f70c --- /dev/null +++ b/docs/adr/0071-region-host-deployment.md @@ -0,0 +1,92 @@ +# ADR-0071: Region host deployment in dev and stage + +## Status +Accepted for M8.4, 2026-09-29. It builds on: +- ADR-0064 to ADR-0066 (host, recovery and operations endpoints); +- ADR-0068 (build-bound authority); +- ADR-0069 and ADR-0070 (environments, stores and Argo delivery). + +It amends the M8.4 roadmap row: there is **no director** yet. + +## Context +M8.4 asks for: +- the host deployed through the existing GitOps path, with an immutable digest; +- measured resources, probes, a security context and a drain budget; +- evidence that a region is ready only after lease and recovery; +- a graceful rollout, and forced deletion of a worker. + +The named risk is a successful rollout hiding a failed region recovery. Regions +could so far only be provisioned from test code. + +M7.3 deferred the director because it would hold no authority, and would be a +placeholder. With one region per environment, the assignment is static and lives +in Git. + +## Decision +**Deployment `evo-region`** (`deploy/kubernetes/region`), one per environment. +- **Rollout.** `replicas: 1` with the **Recreate** strategy. A rolling update would + start a pod that cannot become ready while the old one holds the lease, and the + old one is never removed. Recreate drains the old pod (`--drain-ms 20000`, + inside a 45 s grace period) before the replacement starts. The replacement then + acquires, recovers, verifies the tail, and only then becomes ready. +- **Probes.** Readiness is `/readyz` (current lease and verified head). Liveness is + `/livez` only, so a process waiting out a lease is never restarted for being + unready. +- **Resources.** These come from the measured profile: a 64 Mi / 50m request and a + 256 Mi memory limit. There is **no CPU limit**, because throttling would delay + renewals. The production lease policy and the 1024 MiB object reserve apply. +- **Security.** Pod security `restricted`, a read-only root filesystem, no token, + and no capabilities. +- **Placement and access.** Pinned to the store node (`aevolution.io/stores=true`). + Its object claim is node-local, and the replacement lands where the data is. +- **Credentials.** The runtime login comes from its service file. The password file + is copied to private memory, and `exec` makes `evo-server` PID 1, so SIGTERM + reaches it. +- **Service.** A ClusterIP Service `evo-region-ops` on port 9100. Scraping is M8.5. +- **Identity.** Dev is world 100, region 1; stage is world 200, region 1. Both use + frame 9 and the development opening. + +**Build identity is the image digest.** Each overlay sets the server image digest +twice: as the image, and as `--build`, the identity recorded in every commit. The +policy check requires them to be equal. An image upgrade is therefore also a +visible build change, handled by the ADR-0068 barrier and `--retained`. + +**Provisioning.** `evo-world-clone provision` runs as a Sync hook (wave 0) with the +migrator login. The order is: +1. the database egress policy (wave −2); +2. the migration (wave −1); +3. provisioning (wave 0); +4. the Deployment and Service (wave 1); +5. the PostSync store check. + +Provisioning records the digest in `retained_builds`, then creates genesis from +`development-v1` only when the region does not exist. A region that exists, in any +state, is reported and never touched, so every sync can run it. The opening is +`evo::processes::development_opening`, byte-identical to the qualification fixture +(proved by `lineage-region-contracts`). World0's opening remains a separate, +reviewed decision. + +**Evidence tooling.** `tools/ops/region-drill.sh ` (World0 refused) +reads epoch and committed tick from the pod's own `/metrics`, then: +- performs `rollout restart`, and times the replacement to Ready; +- force-deletes the pod, and times the replacement to Ready. + +After each step, the replacement must report a newer epoch at a committed tick +that is no lower than before. + +## Consequences +- No inputs reach the region until a gateway exists. The host runs idle, holding + and renewing its lease, and tick debt accrues visibly. Commit-level fault + evidence under load is M8.6. +- A forced deletion's replacement waits out up to 20 s of lease. That is correct, + not a fault. +- Losing the store node leaves the region unavailable until the node returns + (ADR-0070); it is never re-created empty. +- Each environment's AppProject must allow Deployment and Service. + `deploy/argocd/dev/project.yaml` does, and supersedes the old example. + +## Alternatives rejected +- **RollingUpdate with maxSurge.** It deadlocks on the lease, as described above. +- **A StatefulSet.** It adds no guarantee the lease does not already provide. +- **A director.** With static single-region assignment, it would be a placeholder + with no authority. diff --git a/docs/adr/README.md b/docs/adr/README.md index b6ed52d..1a1f370 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -81,3 +81,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0068: Upgrade barrier and durability qualification](0068-upgrade-barrier.md) - accepted for M7.9b build-bound authority, retirement barrier fencing old writers, explicit refusal of rule/format changes and the measured recovery fixture. - [ADR-0069: Environment isolation and World0 protection](0069-environment-isolation.md) - accepted for M8.2 per-environment guardrails, native World0 admission protection, scoped Argo projects and verification. - [ADR-0070: Durable stores for dev and stage](0070-dev-stage-stores.md) - accepted for M8.3 external PostgreSQL per environment, Retain-class node-local object volumes, free-space reserve and in-cluster store qualification. +- [ADR-0071: Region host deployment in dev and stage](0071-region-host-deployment.md) - accepted for M8.4 single-replica Recreate host, digest-as-build identity, idempotent provisioning hook and the region drill; no director yet. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index c7c38d4..ab7cc7d 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2781,3 +2781,33 @@ admission policy and the stage and World0 AppProjects, then ran - The digests the maintainer pasted are local image IDs from the build step, not registry digests. The registry digest is printed by the Publish step. - Not executed: bootstrapping the platform Application, or any Argo sync. + +## M8.4 region host - 2026-09-29 + +- Windows MSVC Release CTest 59/59, including the new opening code. + `lineage-region-contracts` proves `development_opening(7)` is byte-identical + to the fixture, and that world 100 differs only by identity. +- WSL Linux `checks` set: 59/59 in each configuration. The format check passed. +- PostgreSQL suites: Release, analysis and ASan/UBSan passed. Analysis passed after + one optional-access fix in `provision`. +- Provisioning test: + - A runtime login cannot provision. + - The owner provisions `world=7 region=86 sequence=0 tick=0`, and a rerun reports + `already-provisioned`. + - A server then became ready and acknowledged ticks 1–5, after which the tool + reported `sequence=1 tick=5` and left the region untouched. +- Mutation: removing the already-provisioned early return was caught. +- `deploy-policy-check.py`: 50 objects, 0 failures. Four new mutations were each + caught: + - a build identity differing from the image digest; + - a RollingUpdate strategy; + - a CPU limit; + - a grace period shorter than the drain. + The checker first misread overlay patches as whole Deployments; it now checks + only the base. +- Kustomize renders dev and stage in order: egress policy (−2), migration hook + (−1), provisioning hook (0), Deployment and Service (1), then the store check + (PostSync). +- `region-drill.sh` passed `bash -n`, and refuses World0 before kubectl (also + enforced by the policy check). +- Not executed: the in-cluster region sync and the drill. diff --git a/docs/infrastructure/ENVIRONMENTS.md b/docs/infrastructure/ENVIRONMENTS.md index 10d7f75..34a4fad 100644 --- a/docs/infrastructure/ENVIRONMENTS.md +++ b/docs/infrastructure/ENVIRONMENTS.md @@ -116,6 +116,29 @@ this evidence in docs/engineering/VALIDATION.md: the Secret. The check must exit 3 with `database refused connect`. - Update the Secret's `pgpass` and sync again. It must exit 0. +## Region host (M8.4, ADR-0071) + +`deploy/kubernetes/` includes `stores/` and `region/`. A sync runs: +1. the database egress policy (wave −2); +2. the migration hook (wave −1); +3. the provisioning hook (wave 0, idempotent); +4. the `evo-region` Deployment and `evo-region-ops` Service (wave 1); +5. the store check (PostSync). + +Dev is world 100, region 1; stage is world 200, region 1. The server image digest +in `region/` is also the build identity. + +Before the first region sync, apply `deploy/argocd/dev/project.yaml` (it allows +Deployment and Service), and label the store node. + +Evidence: `bash tools/ops/region-drill.sh dev` (dev or stage only; World0 is +refused). It records the time to Ready after a rollout restart and after a forced +pod deletion, and checks that the new epoch resumed at the committed tick. Also +record: +- the Argo sync result; +- the first pod's `phase running epoch=... committed=...` line; +- that `/readyz` stays 503 while a replacement waits out the lease. + ## Break-glass How membership is granted is not decided yet; see docs/BACKLOG.md. Until it is, diff --git a/include/evo/processes/development_opening.hpp b/include/evo/processes/development_opening.hpp new file mode 100644 index 0000000..1962efd --- /dev/null +++ b/include/evo/processes/development_opening.hpp @@ -0,0 +1,18 @@ +#pragma once +#include "evo/processes/lineage_region.hpp" +#include +#include + +// M8.4 development opening (ADR-0071): the deterministic lineage region used to +// provision dev and stage regions, identical in content to the qualification fixture +// every M7 test runs (tests/fixtures/environmental_region.hpp). It is an engineering +// opening, not a world design: one habitat site, two bound organisms (producer 10, +// consumer 11), founder genome 1 with descendant 2 and one retained dead organism, +// rules 2, a 50 ms tick. Never provision World0 with it. +namespace evo::processes { +inline constexpr std::uint64_t development_opening_frame = 9; +enum class DevelopmentOpeningError : std::uint8_t { identity, build }; +// Built for `world` in frame development_opening_frame; the world must be nonzero. +[[nodiscard]] std::expected, DevelopmentOpeningError> +development_opening(std::uint64_t world); +} // namespace evo::processes diff --git a/src/processes/development_opening.cpp b/src/processes/development_opening.cpp new file mode 100644 index 0000000..7a9cc80 --- /dev/null +++ b/src/processes/development_opening.cpp @@ -0,0 +1,96 @@ +#include "evo/processes/development_opening.hpp" +#include +#include +#include + +namespace evo::processes { +namespace { +// Mirrors tests/fixtures/environmental_region.hpp exactly; the lineage-region +// contracts test proves byte equality with the fixture for world 7. +EnvironmentalConfig config(world::WorldIdentity id) { + EnvironmentalConfig c; + c.rules = 2; + c.world = id.world; + c.frame = id.frame; + c.producer = 1; + c.consumer = 2; + c.source = 3; + c.tick_microseconds = 50000; + c.area_um2 = 1; + c.irradiance_numerator = 11; + c.irradiance_denominator = 3; + c.efficiency = 600; + c.growth_numerator = 3; + c.growth_denominator = 2; + c.growth_cost = 1; + c.recipe = {1, 2, 1, 1, 1}; + c.intake.fill(100); + c.assimilation = 1000; + c.producer_min_mK = c.consumer_min_mK = 270000; + c.producer_max_mK = c.consumer_max_mK = 310000; + c.decay.fill(1); + return c; +} +std::unique_ptr community(world::WorldIdentity id) { + using namespace world; + constexpr auto maximum = std::numeric_limits::max(); + std::array environment{200, 6, 6, 400, 0, 0, 200, 0, 2000, 0}; + const FieldSite site{{0, 0, -16000}, {{0}, {0}, {0}, {10}, {20}, {3}}, {1}}; + std::array opening{}; + for (std::size_t k = 0; k < 10; ++k) + opening[k] = {static_cast(k), {&environment[k], 1}}; + const FieldDefaults defaults{ + {accounting::Femtograms{0}, accounting::Femtograms{maximum}, accounting::Femtograms{0}}}; + auto f = FieldStore::create(id, 1, defaults, {&site, 1}, opening, 2000000); + auto t = ChunkStore::create({}, 1, 2000000, {{VolumeUnits{100}, VolumeUnits{100}}}); + if (!f || !t || !(*t)->load({0, 0, -1000})) + return nullptr; + std::array accounts{}; + accounts[0].id = {1}; + accounts[1].id = {2}; + accounts[0].capacities.fill(100000000); + accounts[1].capacities.fill(100000000); + auto r = EnvironmentalTransfers::create(id, std::move(*f), std::move(*t), accounts, 4000000); + if (!r) + return nullptr; + auto o = EnvironmentalCommunity::create(config(id), std::move(*r), 8000000); + return o ? std::move(*o) : nullptr; +} +std::optional genome(std::uint64_t id, std::uint64_t parent) { + biology::GenomeDraft draft; + draft.id = id; + draft.origin_event = id; + draft.development_version = 1; + if (parent != 0) + draft.parents = {parent}; + draft.capabilities = {{biology::Capability::energy_conversion, 400, 0}, + {biology::Capability::membrane, 120, 0}, + {biology::Capability::target_size, 8'000'000, 0}}; + auto made = biology::Genome::create(draft); + if (!made) + return std::nullopt; + return *made; +} +} // namespace + +std::expected, DevelopmentOpeningError> +development_opening(std::uint64_t world) { + if (world == 0) + return std::unexpected{DevelopmentOpeningError::identity}; + const world::WorldIdentity id{world, development_opening_frame}; + auto founder = genome(1, 0); + auto descendant = genome(2, 1); + auto opening = community(id); + if (!founder || !descendant || !opening) + return std::unexpected{DevelopmentOpeningError::build}; + biology::LineageArchive archive; + if (!archive.admit(*founder) || !archive.admit(*descendant) || + !archive.record_birth(5, 1, Tick{0}) || !archive.record_death(5, Tick{0}) || + !archive.record_birth(10, 1, Tick{0}) || !archive.record_birth(11, 2, Tick{0})) + return std::unexpected{DevelopmentOpeningError::build}; + auto region = LineageRegion::create(std::move(opening), archive, {10, 11, 12, 3}); + if (!region) + return std::unexpected{DevelopmentOpeningError::build}; + return std::move(*region); +} +} // namespace evo::processes diff --git a/tests/lineage_region_tests.cpp b/tests/lineage_region_tests.cpp index 5a0fc5a..137b82b 100644 --- a/tests/lineage_region_tests.cpp +++ b/tests/lineage_region_tests.cpp @@ -1,3 +1,4 @@ +#include "evo/processes/development_opening.hpp" #include "evo/processes/lineage_region.hpp" #include "fixtures/environmental_region.hpp" @@ -164,6 +165,19 @@ void rebinding_changes_only_the_world() { CHECK(community->rebind_world(9) && community->rebind_world(region_identity.world)); CHECK(community->snapshot() == before); } + +// M8.4: the provisioning opening is byte-identical to the qualification fixture, and +// differs for another world only by the recorded identity. +void development_opening_matches_the_fixture() { + auto opening = development_opening(region_identity.world); + CHECK(opening && (*opening)->snapshot() == lineage_region()->snapshot()); + auto elsewhere = development_opening(100); + CHECK(elsewhere && (*elsewhere)->community().world() == 100); + CHECK((*elsewhere)->rebind_world(region_identity.world)); + CHECK((*elsewhere)->snapshot() == lineage_region()->snapshot()); + CHECK(!development_opening(0)); + CHECK(development_opening_frame == region_identity.frame); +} } // namespace int main() { @@ -172,6 +186,7 @@ int main() { deaths_are_recorded_in_the_same_tick(); snapshots_restore_and_continue(); rebinding_changes_only_the_world(); + development_opening_matches_the_fixture(); std::puts("lineage region contracts passed"); return 0; } catch (const std::exception& e) { diff --git a/tests/region_store_tests.cpp b/tests/region_store_tests.cpp index 023e962..49a35e2 100644 --- a/tests/region_store_tests.cpp +++ b/tests/region_store_tests.cpp @@ -199,9 +199,56 @@ void full_volume_server_acknowledges_nothing() { const auto after = head_of(r.key); CHECK(after.tick == before.tick && after.sequence == before.sequence); } +// M8.4: the provisioning tool creates a region once from the development opening; +// a rerun (every Argo sync) leaves it untouched, and a server then serves it. +void provisioning_is_idempotent() { + const ServerRegion r{{7, 86}, fresh("provision-86")}; + const auto run = [&](const std::string& user, int& code) { + Child tool{{env("EVO_WORLD_CLONE_BIN"), + "provision", + "--world", + "7", + "--region", + "86", + "--opening", + "development-v1", + "--pg-host", + env("PGHOST"), + "--pg-port", + env("PGPORT"), + "--pg-database", + env("PGDATABASE"), + "--pg-user", + user, + "--objects", + r.root.string(), + "--build", + "b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1", + "--retain-build", + "true"}}; + code = tool.wait_exit(30s); + return tool.lines(); + }; + int code = -1; + auto denied = run(env("EVO_PG_RUNTIME_USER"), code); + CHECK(code != 0 && !has(denied, "provisioned ")); + auto first = run(env("PGUSER"), code); + CHECK(code == 0 && has(first, "provisioned world=7 region=86 sequence=0 tick=0")); + auto second = run(env("PGUSER"), code); + CHECK(code == 0 && has(second, "already-provisioned world=7 region=86 sequence=0 tick=0")); + Child server{server_args(r)}; + CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); + server.send(input_lines(1, 5, false)); + CHECK(server.wait_line("ack " + request_hex(5) + " 5 ", 15s)); + server.signal(SIGTERM); + CHECK(server.wait_exit(20s) == 0); + auto after = run(env("PGUSER"), code); + CHECK(code == 0 && has(after, "already-provisioned world=7 region=86 sequence=1 tick=5")); +} } // namespace void run_region_store_tests() { + provisioning_is_idempotent(); full_volumes_publish_nothing(); failing_writes_leave_nothing(); check_store_qualifies_the_stores(); diff --git a/tools/ci/deploy-policy-check.py b/tools/ci/deploy-policy-check.py index f3c873a..34dde18 100644 --- a/tools/ci/deploy-policy-check.py +++ b/tools/ci/deploy-policy-check.py @@ -192,6 +192,36 @@ for path, doc in objects: check(doc['metadata'].get('namespace') != 'evolution-world0', f'{path}: store Jobs never target World0') +# M8.4 region host (ADR-0071): one owner at a time, readiness means authority. +for path, doc in objects: + # Overlay patches (region-values.yaml) are partial objects; the base is complete. + if (doc['kind'] == 'Deployment' and doc['metadata'].get('name') == 'evo-region' + and not path.endswith('region-values.yaml')): + spec = doc['spec'] + pod = spec['template']['spec'] + container = pod['containers'][0] + check(spec.get('replicas') == 1 and spec.get('strategy', {}).get('type') == 'Recreate', + f'{path}: the region host runs one replica with the Recreate strategy') + check(container.get('readinessProbe', {}).get('httpGet', {}).get('path') == '/readyz' + and container.get('livenessProbe', {}).get('httpGet', {}).get('path') == '/livez', + f'{path}: readiness must be /readyz and liveness /livez') + limits = container.get('resources', {}).get('limits', {}) + check('cpu' not in limits and 'memory' in limits, + f'{path}: memory limit required, no CPU limit (throttling delays renewals)') + script = ' '.join(container.get('args', [])) + drain = [int(x) for x in __import__('re').findall(r'--drain-ms (\d+)', script)] + check(len(drain) == 1 and pod.get('terminationGracePeriodSeconds', 30) * 1000 > drain[0], + f'{path}: the termination grace period must exceed --drain-ms') +for env in ('dev', 'stage'): + overlay = f'deploy/kubernetes/region/{env}/' + images = manifests.get(overlay + 'kustomization.yaml', [{}])[0].get('images', []) + server = [i.get('digest', '') for i in images if i.get('name') == 'aevolution-server'] + builds = [v.get('value') for d in manifests.get(overlay + 'region-values.yaml', []) + for c in d['spec']['template']['spec']['containers'] for v in c.get('env', []) + if v.get('name') == 'REGION_BUILD'] + check(len(server) == 1 and len(builds) == 2 and all('sha256:' + b == server[0] for b in builds), + f'{overlay}: the build identity must equal the server image digest') + # The platform bundle: every rendered object is protected from prune and deletion. bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0] check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false', @@ -243,12 +273,13 @@ for path, doc in objects: # The cleanup tool refuses World0 before kubectl runs (no kubectl on PATH here). with tempfile.TemporaryDirectory() as empty: env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable)) - for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''): - result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, 'tools/ops/cleanup-environment.sh'), bad], - capture_output=True, text=True, env=env) - check(result.returncode == 2 and ('refused' in result.stderr or 'usage' in result.stderr) - and 'kubectl' not in result.stderr, - f'cleanup must refuse {bad!r} before kubectl (rc={result.returncode})') + for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh'): + for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''): + result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad], + capture_output=True, text=True, env=env) + check(result.returncode == 2 and ('refused' in result.stderr or 'usage' in result.stderr) + and 'kubectl' not in result.stderr, + f'{tool} must refuse {bad!r} before kubectl (rc={result.returncode})') for message in failures: print('FAIL', message) diff --git a/tools/ops/region-drill.sh b/tools/ops/region-drill.sh new file mode 100644 index 0000000..9c925de --- /dev/null +++ b/tools/ops/region-drill.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# M8.4 region host drill (ADR-0071) for a disposable environment only. +# +# tools/ops/region-drill.sh [--context ] +# +# 1. Reads the ready region's authority from /metrics (epoch, committed tick). +# 2. Graceful rollout: `kubectl rollout restart`. With the Recreate strategy the old +# pod drains (commits, releases the lease) before the new one starts; measures the +# time until the new pod is Ready and checks it resumed at the committed tick with +# a newer epoch. +# 3. Forced worker deletion: deletes the pod with --grace-period=0 --force; the +# replacement must wait out the dead holder's lease (up to 20 s) and recover; the +# time to Ready is measured the same way. +# Refuses anything but dev or stage before touching the cluster; World0 is never +# drilled from here. Prints one line per measurement; exits nonzero on any failure. +set -euo pipefail +usage() { echo "usage: $0 [--context ]" >&2; exit 2; } +[ $# -ge 1 ] || usage +env="$1"; shift +case "$env" in + dev|stage) ;; + *) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2 + exit 2 ;; +esac +k=(kubectl --request-timeout=30s) +[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2") +ns="evolution-$env" +selector=app.kubernetes.io/name=evo-region + +pod_name() { "${k[@]}" get pods -n "$ns" -l "$selector" \ + -o jsonpath='{range .items[?(@.metadata.deletionTimestamp == "")]}{.metadata.name}{"\n"}{end}' | + head -1; } +ready_pod() { # Waits (up to 120 s) for a Ready pod other than $1; prints its name. + local old="$1" + for _ in $(seq 1 600); do + for p in $("${k[@]}" get pods -n "$ns" -l "$selector" -o name 2>/dev/null); do + p="${p#pod/}" + [ "$p" = "$old" ] && continue + ready="$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" + [ "$ready" = True ] && { echo "$p"; return 0; } + done + sleep 0.2 + done + return 1 +} +metric() { # metric : one value from the pod's own /metrics (bash /dev/tcp). + "${k[@]}" exec -n "$ns" "$1" -- /bin/bash -c \ + 'exec 3<>/dev/tcp/127.0.0.1/9100; printf "GET /metrics HTTP/1.1\r\nHost: drill\r\n\r\n" >&3; cat <&3' | + awk -v s="$2" '$1 == s {print $2}' +} +now_ms() { date +%s%3N; } + +first="$(pod_name)" +[ -n "$first" ] || { echo "FAIL no evo-region pod in $ns" >&2; exit 1; } +first="$(ready_pod "")" || { echo "FAIL no Ready evo-region pod in $ns" >&2; exit 1; } +epoch0="$(metric "$first" evo_region_lease_epoch)" +tick0="$(metric "$first" evo_region_committed_tick)" +echo "start pod=$first epoch=$epoch0 committed_tick=$tick0" + +start="$(now_ms)" +"${k[@]}" rollout restart -n "$ns" deployment/evo-region > /dev/null +second="$(ready_pod "$first")" || { echo "FAIL rollout: no Ready replacement" >&2; exit 1; } +rollout_ms=$(( $(now_ms) - start )) +epoch1="$(metric "$second" evo_region_lease_epoch)" +tick1="$(metric "$second" evo_region_committed_tick)" +echo "rollout ready_ms=$rollout_ms pod=$second epoch=$epoch1 committed_tick=$tick1" +[ "$epoch1" -gt "$epoch0" ] && [ "$tick1" -ge "$tick0" ] || + { echo "FAIL rollout did not resume with a newer epoch at the committed tick" >&2; exit 1; } + +start="$(now_ms)" +"${k[@]}" delete pod -n "$ns" "$second" --grace-period=0 --force > /dev/null 2>&1 +third="$(ready_pod "$second")" || { echo "FAIL forced deletion: no Ready replacement" >&2; exit 1; } +forced_ms=$(( $(now_ms) - start )) +epoch2="$(metric "$third" evo_region_lease_epoch)" +tick2="$(metric "$third" evo_region_committed_tick)" +echo "forced ready_ms=$forced_ms pod=$third epoch=$epoch2 committed_tick=$tick2" +[ "$epoch2" -gt "$epoch1" ] && [ "$tick2" -ge "$tick1" ] || + { echo "FAIL forced deletion did not resume with a newer epoch at the committed tick" >&2; exit 1; } +echo "PASS region drill in $ns"