From 4ef0fe81670c396d27cd421aa01681557cd67550 Mon Sep 17 00:00:00 2001 From: dingo Date: Wed, 30 Sep 2026 10:34:03 -0400 Subject: [PATCH] Keep earlier builds recoverable across region image upgrades Complete the maintainer's server image bump to edb39f by also setting the region build identity, and add REGION_RETAINED so the host keeps the builds that wrote its history on its recovery allowlist; without it the upgraded host would quarantine its own head as unsupported. Record the M8.4 drill (rollout 3.0 s, forced deletion 19.3 s to Ready, epochs 3 to 5) and the maintainer's M8.5 sync. Co-Authored-By: Claude Opus 5.5 --- deploy/kubernetes/region/base/deployment.yaml | 7 ++++++- deploy/kubernetes/region/dev/kustomization.yaml | 2 +- deploy/kubernetes/region/dev/region-values.yaml | 6 ++++-- .../kubernetes/region/stage/kustomization.yaml | 2 +- .../kubernetes/region/stage/region-values.yaml | 6 ++++-- docs/adr/0071-region-host-deployment.md | 11 +++++++++++ docs/engineering/VALIDATION.md | 16 ++++++++++++++++ tools/ci/deploy-policy-check.py | 8 ++++++++ 8 files changed, 51 insertions(+), 7 deletions(-) diff --git a/deploy/kubernetes/region/base/deployment.yaml b/deploy/kubernetes/region/base/deployment.yaml index 1809a33..370bd6a 100644 --- a/deploy/kubernetes/region/base/deployment.yaml +++ b/deploy/kubernetes/region/base/deployment.yaml @@ -52,7 +52,10 @@ spec: set -eu umask 077 cp /run/postgres/pgpass /tmp/pgpass - exec /usr/local/bin/evo-server --profile lineage \ + # Builds that wrote this region's history stay recoverable (ADR-0068). + retained=() + for build in $REGION_RETAINED; do retained+=(--retained "$build"); done + exec /usr/local/bin/evo-server --profile lineage "${retained[@]}" \ --world "$REGION_WORLD" --region "$REGION_ID" --frame "$REGION_FRAME" \ --pg-service /run/postgres/pg_service.conf --pg-passfile /tmp/pgpass \ --objects /objects --objects-reserve-mib 1024 \ @@ -63,6 +66,8 @@ spec: - {name: REGION_ID, value: set-by-overlay} - {name: REGION_FRAME, value: "9"} - {name: REGION_BUILD, value: set-by-overlay} + # Earlier builds whose commits this region may still hold, space-separated. + - {name: REGION_RETAINED, value: ""} ports: - {name: ops, containerPort: 9100, protocol: TCP} readinessProbe: diff --git a/deploy/kubernetes/region/dev/kustomization.yaml b/deploy/kubernetes/region/dev/kustomization.yaml index 8cc79f9..71cfe97 100644 --- a/deploy/kubernetes/region/dev/kustomization.yaml +++ b/deploy/kubernetes/region/dev/kustomization.yaml @@ -9,6 +9,6 @@ resources: images: - name: aevolution-server newName: registry.aeternalisgames.com/aeternalis/aevolution-server - digest: sha256:4af7209941643e0c6a100e2b4e2fef8a959b87fdbfac2173b7867cfc1808bd66 + digest: sha256:edb39f956df1f481d7ba724cac4ceb3a4fe494efc02f994d31d6c96dfae04a78 patches: - path: region-values.yaml diff --git a/deploy/kubernetes/region/dev/region-values.yaml b/deploy/kubernetes/region/dev/region-values.yaml index d1e7586..e29440d 100644 --- a/deploy/kubernetes/region/dev/region-values.yaml +++ b/deploy/kubernetes/region/dev/region-values.yaml @@ -11,7 +11,9 @@ spec: env: - {name: REGION_WORLD, value: "100"} - {name: REGION_ID, value: "1"} - - {name: REGION_BUILD, value: 4af7209941643e0c6a100e2b4e2fef8a959b87fdbfac2173b7867cfc1808bd66} + - {name: REGION_BUILD, value: edb39f956df1f481d7ba724cac4ceb3a4fe494efc02f994d31d6c96dfae04a78} + # Provisioned and committed under 4af72099: keep it recoverable. + - {name: REGION_RETAINED, value: "4af7209941643e0c6a100e2b4e2fef8a959b87fdbfac2173b7867cfc1808bd66"} --- apiVersion: batch/v1 kind: Job @@ -25,4 +27,4 @@ spec: env: - {name: REGION_WORLD, value: "100"} - {name: REGION_ID, value: "1"} - - {name: REGION_BUILD, value: 4af7209941643e0c6a100e2b4e2fef8a959b87fdbfac2173b7867cfc1808bd66} + - {name: REGION_BUILD, value: edb39f956df1f481d7ba724cac4ceb3a4fe494efc02f994d31d6c96dfae04a78} diff --git a/deploy/kubernetes/region/stage/kustomization.yaml b/deploy/kubernetes/region/stage/kustomization.yaml index 1ee5f70..84eaa6e 100644 --- a/deploy/kubernetes/region/stage/kustomization.yaml +++ b/deploy/kubernetes/region/stage/kustomization.yaml @@ -9,6 +9,6 @@ resources: images: - name: aevolution-server newName: registry.aeternalisgames.com/aeternalis/aevolution-server - digest: sha256:4af7209941643e0c6a100e2b4e2fef8a959b87fdbfac2173b7867cfc1808bd66 + digest: sha256:edb39f956df1f481d7ba724cac4ceb3a4fe494efc02f994d31d6c96dfae04a78 patches: - path: region-values.yaml diff --git a/deploy/kubernetes/region/stage/region-values.yaml b/deploy/kubernetes/region/stage/region-values.yaml index 7438596..683161e 100644 --- a/deploy/kubernetes/region/stage/region-values.yaml +++ b/deploy/kubernetes/region/stage/region-values.yaml @@ -11,7 +11,9 @@ spec: env: - {name: REGION_WORLD, value: "200"} - {name: REGION_ID, value: "1"} - - {name: REGION_BUILD, value: 4af7209941643e0c6a100e2b4e2fef8a959b87fdbfac2173b7867cfc1808bd66} + - {name: REGION_BUILD, value: edb39f956df1f481d7ba724cac4ceb3a4fe494efc02f994d31d6c96dfae04a78} + # Provisioned and committed under 4af72099: keep it recoverable. + - {name: REGION_RETAINED, value: "4af7209941643e0c6a100e2b4e2fef8a959b87fdbfac2173b7867cfc1808bd66"} --- apiVersion: batch/v1 kind: Job @@ -25,4 +27,4 @@ spec: env: - {name: REGION_WORLD, value: "200"} - {name: REGION_ID, value: "1"} - - {name: REGION_BUILD, value: 4af7209941643e0c6a100e2b4e2fef8a959b87fdbfac2173b7867cfc1808bd66} + - {name: REGION_BUILD, value: edb39f956df1f481d7ba724cac4ceb3a4fe494efc02f994d31d6c96dfae04a78} diff --git a/docs/adr/0071-region-host-deployment.md b/docs/adr/0071-region-host-deployment.md index 3c5f70c..0c72a3c 100644 --- a/docs/adr/0071-region-host-deployment.md +++ b/docs/adr/0071-region-host-deployment.md @@ -90,3 +90,14 @@ that is no lower than before. - **A StatefulSet.** It adds no guarantee the lease does not already provide. - **A director.** With static single-region assignment, it would be a placeholder with no authority. + +## Amendment: retained builds across image upgrades (2026-09-30) +Each commit records the build that wrote it, and recovery accepts only builds on +the host's allowlist. After an image upgrade, a region's existing head was +therefore written by the previous build. Without `--retained ` the new +host would refuse its own history as unsupported and quarantine. `region/` +now lists earlier builds in `REGION_RETAINED`, passed as `--retained` flags; the +policy check requires 64-hex entries distinct from the current build. The +provisioning hook records each new build in `retained_builds` before the host +starts. Retiring an old build from publication is a separate, deliberate ADR-0068 +barrier. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index b458ac9..cd46425 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2844,3 +2844,19 @@ admission policy and the stage and World0 AppProjects, then ran - Not executed: anything in the cluster (monitoring sync, target health, the `EvoRegionDown` injection), alert delivery (no receiver), and the node log rotation check. + +## M8.4 and M8.5 in-cluster results - 2026-09-30 + +- `tools/ops/region-drill.sh dev`, run by the maintainer (idle region, world 100): + - start: epoch 3, committed tick 0; + - rollout restart: the replacement was Ready in **3010 ms**, at epoch 4 with + committed tick 0 (Recreate drain released the lease); + - forced deletion: the replacement was Ready in **19268 ms**, at epoch 5 with + committed tick 0 (waiting out the 20 s lease); + - PASS. +- M8.5: the maintainer reports the new server digest pinned and the monitoring + project, platform, monitoring app and dev/stage applications synced. Target + health and the `EvoRegionDown` injection results were not recorded here. +- Image upgrade to `edb39f…`: completed with `REGION_RETAINED=4af72099…`, so the + existing dev and stage heads, written by the earlier build, stay recoverable + after the upgrade. diff --git a/tools/ci/deploy-policy-check.py b/tools/ci/deploy-policy-check.py index 2930d52..606e074 100644 --- a/tools/ci/deploy-policy-check.py +++ b/tools/ci/deploy-policy-check.py @@ -226,6 +226,14 @@ for env in ('dev', 'stage'): if v.get('name') == 'REGION_BUILD'] check(len(server) == 1 and len(builds) == 2 and all('sha256:' + b == server[0] for b in builds), f'{overlay}: the build identity must equal the server image digest') + retained = [v.get('value', '') for d in manifests.get(overlay + 'region-values.yaml', []) + for c in d['spec']['template']['spec']['containers'] for v in c.get('env', []) + if v.get('name') == 'REGION_RETAINED'] + for value in retained: + entries = value.split() + check(all(__import__('re').fullmatch(r'[0-9a-f]{64}', e) for e in entries) + and not set(entries) & set(builds), + f'{overlay}: REGION_RETAINED must list other builds as 64 hex digits') # M8.5 monitoring (ADR-0072): third-party images are pinned by digest, and Prometheus # needs no Kubernetes API access.