diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 4460cc2..841fcb4 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -134,6 +134,27 @@ exit 4 That is a build-retention failure, not a reason to reset the region. Re-run the supported sync hook using the current rendered manifest. +A sync that Argo records with `resources: [Deployment/evo-region]` runs no hooks. On +2026-10-08 syncs requested with the `aevolution-dev-agent` token were recorded that way +even when no resource filter was requested, so a promotion never retained its build. +Check the operation with: + +```bash +kubectl -n argocd get application aevolution-dev -o jsonpath='{.status.operationState.operation.sync}' +``` + +If it was limited to the Deployment, re-run the provision hook from the rendered manifest: +1. Render `deploy/kubernetes/dev` with the pinned kustomize and take + `Job/aevolution-region-provision`. Its image and `REGION_BUILD` must equal the + Deployment's. +2. Delete the old Job, create the rendered one and wait for `Complete`. The log shows + `already-provisioned`; the build was retained first. +3. `kubectl -n evolution-dev rollout restart deployment/evo-region`, then verify below. + +`REGION_SESSION_WINDOW` must name its UTC activation (`dev-YYYYMMDDTHHMMSS-...`) and end +at most 4 hours after it. evo-server refuses a longer window at startup +(`error m10-window-too-long`), and the deploy policy check refuses it before push. + ## 7. Verify the region All of these must be true: diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index ecfee6e..529d38f 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -5085,3 +5085,36 @@ Executed: - **Linux GCC 14:** `postgres-check.sh` with QUIC, all suites. - **Linux Clang 19:** clang-tidy 19 on the M10 process tests (clean); clang-format 19. - **Windows:** not applicable (the PostgreSQL process tests are Linux only). + +## 2026-10-08 — evolution-dev crashloop and recovery + +Incident: +- `b2f224d` set `REGION_SESSION_WINDOW` to end about 5.2 h after the pod started. evo-server + refuses windows longer than 4 h (`SessionWindow::max_duration_seconds`), so + `evo-region` exited with `error m10-window-too-long` in a loop. +- `4a72897` set a window of 3 h 50 min. The region then reached + `phase quarantined epoch=0 … exit 4`: the bot promotion `c3b40f4` (image `56647b49…`) + had never had its build retained. The provision hook still ran the previous image + (`2ccf6074…`, created 01:58Z). +- The two syncs requested afterwards with the agent token were both recorded by Argo + as limited to `Deployment/evo-region`, even though the second request carried no + resource filter. So no hooks ran. The cause is not visible with the agent's access + (possibly a guard on that account). + +Recovery, within the agent contract (no reset, no SQL): +- Rendered `Job/aevolution-region-provision` from `deploy/kubernetes/dev` at + `4a72897` (image and `REGION_BUILD` `56647b49…`), deleted the stale Job and created it. + It completed with `already-provisioned … sequence=578 tick=12084`, the build retained + first. +- `rollout restart deployment/evo-region`. The region came up `phase running epoch=54 + committed=12084`, `ready 1`, with history intact. Argo reports Synced and Healthy, and a + public-endpoint login attempt with an untrusted key was answered `unauthenticated` + (TLS and protocol fine). + +Prevention: +- The deploy policy now refuses a `REGION_SESSION_WINDOW` that does not name its UTC + activation or that ends more than 4 h after it (two new selftest cases, one with the + `b2f224d` value). +- DEPLOYMENT.md documents the Deployment-only sync symptom and the hook re-run. + +Executed: deploy policy check (0 failures) and selftest (46 cases, 0 failures). diff --git a/tools/ci/deploy-policy-check.py b/tools/ci/deploy-policy-check.py index 2091760..e56a127 100644 --- a/tools/ci/deploy-policy-check.py +++ b/tools/ci/deploy-policy-check.py @@ -687,6 +687,20 @@ for overlay, docs in renders.items(): check(text.count('--m10-listen') == 1 and '--m10-listen "$REGION_SESSION_LISTEN"' in text and '--session-window "$REGION_SESSION_WINDOW"' in text, f'{where}: one M10 listener and absolute session window, addressed only by GitOps env') + # The server refuses a window ending more than 4 h after it starts + # (SessionWindow::max_duration_seconds) and exits in a loop. The id carries + # its UTC activation (dev-YYYYMMDDTHHMMSS-...); the expiry must fall within + # 4 h of it. Checked here because the server sees it only at startup. + window = env.get('REGION_SESSION_WINDOW') or '' + if route_active and window: + import calendar, re, time + m = re.fullmatch(r'[0-9A-Za-z._-]*?(\d{8}T\d{6})Z?[0-9A-Za-z._-]*:(\d+)', window) + ok = False + if m: + start = calendar.timegm(time.strptime(m.group(1), '%Y%m%dT%H%M%S')) + ok = start < int(m.group(2)) <= start + 4 * 60 * 60 + check(ok, f'{where}: REGION_SESSION_WINDOW must name its UTC activation and end ' + 'within 4 h of it (the server maximum)') else: check('--m10-listen' not in text and '--synthetic-inputs' not in text, f'{where}: only evo-server may listen for sessions') diff --git a/tools/ci/deploy-policy-selftest.py b/tools/ci/deploy-policy-selftest.py index 7858986..cd231b0 100644 --- a/tools/ci/deploy-policy-selftest.py +++ b/tools/ci/deploy-policy-selftest.py @@ -106,6 +106,14 @@ CASES = [ ('non-RFC 1918 VIP without approval', lambda t: edit(t, ROUTE + 'service.yaml', '172.10.1.202', '172.11.1.210'), 'internal address'), # Review M9.8f P1: inline/JSON patches and namespace changes, seen only when rendered. + ('session window longer than the server maximum', lambda t: inline_patch( + t, 'deploy/kubernetes/dev/kustomization.yaml', + env_patch('REGION_SESSION_WINDOW', 'dev-20261008T111342-32432:1791477222')), + 'end within 4 h of it'), + ('session window without an activation time', lambda t: inline_patch( + t, 'deploy/kubernetes/dev/kustomization.yaml', + env_patch('REGION_SESSION_WINDOW', 'dev-manual:1791477222')), + 'end within 4 h of it'), ('inline patch: synthetic inputs on', lambda t: inline_patch( t, 'deploy/kubernetes/dev/kustomization.yaml', env_patch('REGION_INPUTS', 'development-v1')), 'sessions must be dev only with the route'),