From d917e8443df57bf3b848f1d849cbb45bbed76342 Mon Sep 17 00:00:00 2001 From: dingo Date: Mon, 5 Oct 2026 17:04:31 -0400 Subject: [PATCH] Route the dev session as QUIC over UDP 7400 The deployed M10 listener is QUIC (UDP) but the ADR-0087 route admitted only TCP 7400, so nothing reached it through the VIP. The route container port, Service and NetworkPolicy now carry UDP 7400 only (no TCP: nothing needs it). The policy check requires exactly UDP 7400 for the route and counts any protocol on 7400 as session exposure elsewhere; six new self-test cases. ADR-0087 amended (edge firewall and internal DNS as separate reviewed steps, end-to-end probe as acceptance); session protocol v1 recorded as frozen now that it is deployed. Co-Authored-By: Claude Opus 5.5 --- .../dev-session-route/kustomization.yaml | 2 +- .../region/dev-session-route/listener.yaml | 3 +- .../dev-session-route/networkpolicy.yaml | 4 +-- .../region/dev-session-route/service.yaml | 7 ++-- docs/STATUS.md | 5 +++ docs/adr/0087-dev-session-route.md | 34 ++++++++++++++++--- ...092-m10-intentions-and-durable-outcomes.md | 1 + docs/engineering/VALIDATION.md | 19 +++++++++++ schemas/network/session-v1.md | 7 ++-- tools/ci/deploy-policy-check.py | 26 +++++++------- tools/ci/deploy-policy-selftest.py | 25 ++++++++++++-- 11 files changed, 106 insertions(+), 27 deletions(-) diff --git a/deploy/kubernetes/region/dev-session-route/kustomization.yaml b/deploy/kubernetes/region/dev-session-route/kustomization.yaml index 5e32b8e..5e38009 100644 --- a/deploy/kubernetes/region/dev-session-route/kustomization.yaml +++ b/deploy/kubernetes/region/dev-session-route/kustomization.yaml @@ -2,7 +2,7 @@ # the reviewed absolute session window is active. The network inputs are reserved: # VIP 172.10.1.202 and client CIDR 172.10.1.0/24. # Route: void.aeternalisgames.com:7400 -> VIP -> Service -> evo-region pod:7400. -# Raw TCP only: TLS 1.3 terminates in evo-server; no HTTP ingress or nginx TLS. +# QUIC over UDP 7400 only: TLS 1.3 terminates in evo-server; no HTTP ingress, no TCP. apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component resources: diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index f3748bd..d1a1140 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -14,4 +14,5 @@ spec: - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - {name: REGION_SESSION_WINDOW, value: "dev-20261005-201514:1791233114"} ports: - - {name: session, containerPort: 7400, protocol: TCP} + # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. + - {name: session, containerPort: 7400, protocol: UDP} diff --git a/deploy/kubernetes/region/dev-session-route/networkpolicy.yaml b/deploy/kubernetes/region/dev-session-route/networkpolicy.yaml index 94efdd4..8c9e1a0 100644 --- a/deploy/kubernetes/region/dev-session-route/networkpolicy.yaml +++ b/deploy/kubernetes/region/dev-session-route/networkpolicy.yaml @@ -1,4 +1,4 @@ -# Session ingress to the region pod: TCP 7400 from the permitted development/test +# Session ingress to the region pod: UDP 7400 (QUIC) from the permitted development/test # client CIDR(s) only, which must equal the Service's loadBalancerSourceRanges. # Everything else stays denied by the platform default-deny policy. apiVersion: networking.k8s.io/v1 @@ -15,4 +15,4 @@ spec: - ipBlock: cidr: 172.10.1.0/24 ports: - - {protocol: TCP, port: 7400} + - {protocol: UDP, port: 7400} diff --git a/deploy/kubernetes/region/dev-session-route/service.yaml b/deploy/kubernetes/region/dev-session-route/service.yaml index c94227a..bc76ee8 100644 --- a/deploy/kubernetes/region/dev-session-route/service.yaml +++ b/deploy/kubernetes/region/dev-session-route/service.yaml @@ -8,7 +8,10 @@ # serves on every node; it carries no session traffic and must stay closed at the edge # firewall. Its allocation and quota accounting are verified before enabling (ADR-0087). # - loadBalancerSourceRanges duplicates the CIDR at the Service; the edge firewall is -# the third layer. TLS 1.3, the dev CA and single-use grants remain mandatory. +# the third layer. TLS 1.3 (inside QUIC), the dev CA and short-lived signed grants +# remain mandatory. +# - The M10 session is QUIC (ADR-0090): one UDP port, no TCP. A single-protocol Service +# needs no MetalLB mixed-protocol support. apiVersion: v1 kind: Service metadata: @@ -26,4 +29,4 @@ spec: selector: app.kubernetes.io/name: evo-region ports: - - {name: session, port: 7400, targetPort: session, protocol: TCP} + - {name: session, port: 7400, targetPort: session, protocol: UDP} diff --git a/docs/STATUS.md b/docs/STATUS.md index 934f3ae..6837365 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -37,6 +37,11 @@ evo-server QUIC listener (session control and action records under the host's le the headless `evo-session-probe`, with real-process tests including SIGKILL before and after an action's commit. M10.4 is complete; the GUI client still uses the M9 dev path. Next: M10.5 bounded snapshots and spatial interest. +Deployment (2026-10-05): the maintainer deployed the M10 QUIC listener to evolution-dev +behind the ADR-0087 route; session protocol v1 is therefore frozen. The route admitted +only TCP 7400 while QUIC is UDP, so it is changed to UDP 7400 (ADR-0087 amendment). The +route is not yet verified end to end: the edge firewall rule (UDP 7400 from +172.10.1.0/24), internal DNS to 172.10.1.202 and a probe run through the VIP remain. ## Previous gate - 2026-10-02 diff --git a/docs/adr/0087-dev-session-route.md b/docs/adr/0087-dev-session-route.md index d9a2924..d634d5d 100644 --- a/docs/adr/0087-dev-session-route.md +++ b/docs/adr/0087-dev-session-route.md @@ -13,8 +13,9 @@ Internal testers need the native client (M9.8e ZIP) to reach the dev region with `kubectl port-forward`. The cluster inventory (INVENTORY.md, 2026-09-29) shows RKE2's ingress-nginx 1.12.1 DaemonSet serving 80/443 on all four node addresses (public sites included), MetalLB 0.15.2 in FRR mode with one LoadBalancer (Gitea SSH), and an existing -HTTP Ingress for void.aeternalisgames.com in namespace `void`. The session protocol is -TLS 1.3 over raw TCP (ADR-0082), not HTTP. Sessions need synthetic carrier inputs, which +HTTP Ingress for void.aeternalisgames.com in namespace `void`. The session protocol was +TLS 1.3 over raw TCP (ADR-0082) when this was written; since 2026-10-05 the route carries +the M10 session, QUIC over UDP (ADR-0090/0091/0092; amendment below). Never HTTP. Sessions need synthetic carrier inputs, which ADR-0073 keeps drill-time only so dev history does not grow continuously. ## Decision (maintainer, 2026-10-02) @@ -35,8 +36,8 @@ reach only a ready pod; after a replacement, clients reconnect with a fresh gran baseline (ADR-0083). **Access.** Development workstation/test subnet only, in three layers: the edge -firewall, the Service source ranges and a NetworkPolicy admitting TCP 7400 to the region -pod from exactly those CIDRs. TLS with the dev CA and single-use, short-lived grants stay +firewall, the Service source ranges and a NetworkPolicy admitting UDP 7400 (QUIC; TCP 7400 +until 2026-10-05) to the region pod from exactly those CIDRs. TLS with the dev CA and single-use, short-lived grants stay mandatory; grants remain bearer tokens. **Listener.** The base keeps the opt-in listener on loopback through @@ -121,3 +122,28 @@ cannot enable listeners, sessions, inputs or session ports outside the route. One reviewed exception to "no LoadBalancer" for evolution-dev. The policy check and its fail-closed self-test (`tools/ci/deploy-policy-selftest.py`) encode the route's shape and the enabling preconditions. World0 and stage are untouched. + +## Amendment 2026-10-05: QUIC over UDP 7400 (maintainer) +The deployed listener is the M10 QUIC session (`--m10-listen`, ADR-0092), which is UDP. The +route still admitted only TCP 7400 (container port, Service, NetworkPolicy and the policy +check), so nothing reached the listener through the VIP, and nothing listened on TCP 7400; +the pod-side checks (ready, listener bound) could not show this. Decision: +- The route carries **UDP 7400 only**: container port, `evo-region-session` port and the + session NetworkPolicy. TCP 7400 is removed, not kept beside it: no supported protocol + needs it (the M9 TLS-over-TCP session is no longer wired into the deployment). The + Service has a single protocol, so MetalLB mixed-protocol support is not involved; the + VIP, `Local` traffic policy, no node ports, source ranges and the health-check note are + unchanged. +- `deploy-policy-check.py` requires exactly UDP 7400 for the route and a UDP container + port, and counts any protocol on 7400 (UDP, TCP, SCTP) as exposure of the session port + for every other policy and Service; the self-test refuses stale TCP route objects, TCP + beside UDP, and other policies admitting UDP or SCTP 7400. +- **Edge firewall: separate, explicitly reviewed change**, outside the repository: allow + UDP 7400 to 172.10.1.202 from 172.10.1.0/24 only, never a wider source range. Internal + DNS must resolve `void.aeternalisgames.com` to the VIP 172.10.1.202 (enabling step 4); + on 2026-10-05 it resolved to the control-plane address 172.10.1.160. +- **Acceptance** is end to end, not pod-side: `evo-session-probe` from 172.10.1.0/24 + succeeds through `void.aeternalisgames.com:7400 -> edge firewall -> MetalLB VIP -> + Service UDP 7400 -> NetworkPolicy UDP 7400 -> evo-region M10 listener`, with the dev CA + and issuer key read from the authorized cluster Secret without printing or persisting + them. Until that is recorded in VALIDATION.md the route is not declared working. diff --git a/docs/adr/0092-m10-intentions-and-durable-outcomes.md b/docs/adr/0092-m10-intentions-and-durable-outcomes.md index 8906300..11c9eba 100644 --- a/docs/adr/0092-m10-intentions-and-durable-outcomes.md +++ b/docs/adr/0092-m10-intentions-and-durable-outcomes.md @@ -50,6 +50,7 @@ duplicates and queries; older ones are resolved from the durable record (10.4b). **Protocol.** Session protocol v1 gains message types 6-12 and Welcome gains the server tick (schemas/network/session-v1.md). v1 has never been deployed, so it is amended in place, as ADR-0083 did for M9 v1; after its first deployment, changes need a new version. +v1 was deployed in evolution-dev on 2026-10-05 and is frozen from then on. The action kind is a raw u16 on the wire and validated by admission (an enum narrower than the wire field must never be cast from it). diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 05a05b8..dc0bd28 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -4309,3 +4309,22 @@ session and network tests; Windows MSVC 19.44 preset `quic` build (including the CTest 4/4 (binaries rebuilt, timestamps checked). Not done: the engineering GUI client on M10 sessions; packaging (the server image does not enable QUIC); a two-process cross-region move; ASan of the PostgreSQL suite. + +## 2026-10-05 — Dev session route moved to UDP 7400 for QUIC (ADR-0087 amendment) + +Finding: the deployed M10 QUIC listener (UDP) sat behind a route that admitted only TCP +7400 (container port, Service, NetworkPolicy, policy check). Pod-side checks (ready, +listener bound on 7400) could not detect it. Also found: `void.aeternalisgames.com` +resolves to 172.10.1.160 (control plane) from the workstation, not the VIP 172.10.1.202. +Changed: route container port, Service and NetworkPolicy to UDP 7400 only; policy check +(route exactly UDP 7400, UDP container port, any-protocol session exposure elsewhere) and +six new self-test cases; ADR-0087 amendment; session-v1 frozen note. +Executed: `tools/ci/deploy-policy-check.py` (71 objects in 69 files, 130 rendered objects +in 17 overlays, 0 failures) and `tools/ci/deploy-policy-selftest.py` (44 cases, all +refused) with pinned kustomize v5.8.1, before (38 cases) and after; rendered region/dev +inspected: Service `evo-region-session` UDP 7400 only on 172.10.1.202 with source range +172.10.1.0/24, `Local`, no node ports; NetworkPolicy UDP 7400 from 172.10.1.0/24; pod +port `session` UDP 7400; no other Service exposes 7400. +Not done (needs the maintainer): Argo sync of this change; the edge firewall rule; +internal DNS to 172.10.1.202; retrieving the dev CA and issuer key (no cluster credentials +on the workstation); the end-to-end probe through the VIP. The route is not verified. diff --git a/schemas/network/session-v1.md b/schemas/network/session-v1.md index 5874add..0a2ee29 100644 --- a/schemas/network/session-v1.md +++ b/schemas/network/session-v1.md @@ -8,9 +8,10 @@ each; types 6-8 travel as one QUIC DATAGRAM each, and only after Welcome. Integers little-endian. Every message has a fixed length for its type; any other length, magic, version, nonzero flags, unknown type or out-of-range value is refused, and the server then sends Close (`unsupported_version` for another version, otherwise -`protocol`) and closes the connection. v1 has not been deployed; it was amended in place -by M10.4 (types 6-12, Welcome server_tick). After its first deployment, changes need a -new version. +`protocol`) and closes the connection. v1 was amended in place once, before deployment, by +M10.4 (types 6-12, Welcome server_tick). **Frozen since 2026-10-05**: v1 is deployed in +evolution-dev (ADR-0087 route); any change to a message, value or rule needs a new +version. ## Header (8 bytes) | Offset | Size | Field | Rule | diff --git a/tools/ci/deploy-policy-check.py b/tools/ci/deploy-policy-check.py index a97b7f4..2091760 100644 --- a/tools/ci/deploy-policy-check.py +++ b/tools/ci/deploy-policy-check.py @@ -313,7 +313,8 @@ for path, doc in objects: if volume.get('secret', {}).get('secretName') == 'evolution-dev-session': check(volume['secret'].get('optional') is True and volume['secret'].get('defaultMode') == 0o440, f'{path}: the dev session Secret is optional and 0440') -ROUTE_PORT = [{'name': 'session', 'port': 7400, 'targetPort': 'session', 'protocol': 'TCP'}] +# The M10 session is QUIC (ADR-0090, ADR-0087 amendment 2026-10-05): UDP 7400 only. +ROUTE_PORT = [{'name': 'session', 'port': 7400, 'targetPort': 'session', 'protocol': 'UDP'}] REGION_SELECTOR = {'app.kubernetes.io/name': 'evo-region'} @@ -324,7 +325,7 @@ def validate_route_service(where, doc): check(spec.get('type') == 'LoadBalancer' and spec.get('externalTrafficPolicy') == 'Local' and spec.get('allocateLoadBalancerNodePorts') is False, f'{where}: the route is a LoadBalancer with Local traffic policy and no node ports') - check(spec.get('ports') == ROUTE_PORT, f'{where}: the route exposes exactly TCP 7400') + check(spec.get('ports') == ROUTE_PORT, f'{where}: the route exposes exactly UDP 7400 (QUIC)') check(spec.get('selector') == REGION_SELECTOR, f'{where}: the route selects only the region pod') annotations = doc['metadata'].get('annotations', {}) check(set(annotations) == {'metallb.io/loadBalancerIPs'}, @@ -351,13 +352,13 @@ def validate_route_service(where, doc): def validate_route_policy(where, doc, ranges): - """The route policy admits TCP 7400 to the region pod from exactly the Service ranges.""" + """The route policy admits UDP 7400 to the region pod from exactly the Service ranges.""" spec = doc['spec'] rules = spec.get('ingress', []) check(spec.get('podSelector') == {'matchLabels': REGION_SELECTOR} and spec.get('policyTypes') == ['Ingress'] and len(rules) == 1 - and rules[0].get('ports') == [{'protocol': 'TCP', 'port': 7400}], - f'{where}: the session policy admits only TCP 7400 to the region pod') + and rules[0].get('ports') == [{'protocol': 'UDP', 'port': 7400}], + f'{where}: the session policy admits only UDP 7400 to the region pod') peers = rules[0].get('from', []) if rules else [] check(peers and all(set(peer) == {'ipBlock'} and set(peer['ipBlock']) == {'cidr'} for peer in peers) and sorted(peer['ipBlock']['cidr'] for peer in peers) == sorted(ranges), @@ -380,7 +381,7 @@ for path, doc in objects: if doc['kind'] == 'Ingress': text = yaml.safe_dump(doc) check('7400' not in text and 'session' not in text, - f'{path}: the dev session is raw TCP, never an Ingress (ADR-0087)') + f'{path}: the dev session is QUIC over UDP, never an Ingress (ADR-0087)') policies = [(p, d) for p, d in objects if p.startswith(ROUTE) and d['kind'] == 'NetworkPolicy'] check(len(policies) == 1, f'{ROUTE}: exactly one session ingress policy') for path, doc in policies: @@ -639,14 +640,13 @@ def selects_region(selector): def admits_session(rule): - """True when an ingress rule admits TCP 7400 on the region pod: no or empty ports, a - TCP entry without a port, the 'session' name, 7400 itself or a range spanning it.""" + """True when an ingress rule admits 7400 on the region pod in any protocol (UDP for the + QUIC session, and TCP or SCTP, which nothing may expose there either): no or empty ports, + an entry without a port, the 'session' name, 7400 itself or a range spanning it.""" ports = rule.get('ports') if not ports: return True for port in ports: - if port.get('protocol', 'TCP') != 'TCP': - continue number = port.get('port') if number is None or number in (7400, 'session'): return True @@ -690,8 +690,10 @@ for overlay, docs in renders.items(): else: check('--m10-listen' not in text and '--synthetic-inputs' not in text, f'{where}: only evo-server may listen for sessions') - ports = sorted((p.get('name'), p.get('containerPort')) for p in container.get('ports', [])) - allowed = [('ops', 9100), ('session', 7400)] if route_active else [('ops', 9100)] + ports = sorted((p.get('name'), p.get('containerPort'), p.get('protocol', 'TCP')) + for p in container.get('ports', [])) + allowed = ([('ops', 9100, 'TCP'), ('session', 7400, 'UDP')] if route_active + else [('ops', 9100, 'TCP')]) if container.get('name') == 'evo-server': check(ports == sorted(allowed), f'{where}: container ports must be {allowed}') if kind == 'Service': diff --git a/tools/ci/deploy-policy-selftest.py b/tools/ci/deploy-policy-selftest.py index 99b2e7d..7858986 100644 --- a/tools/ci/deploy-policy-selftest.py +++ b/tools/ci/deploy-policy-selftest.py @@ -192,7 +192,28 @@ CASES = [ '172.10.1.0/24', '10.99.0.0/24'), 'exactly the Service source ranges'), ('policy admits another port', lambda t: edit(t, ROUTE + 'networkpolicy.yaml', 'port: 7400', 'port: 7401'), - 'only TCP 7400'), + 'only UDP 7400'), + # ADR-0087 amendment 2026-10-05: the route is QUIC over UDP; a stale TCP route fails. + ('route Service still TCP', lambda t: edit(t, ROUTE + 'service.yaml', 'protocol: UDP', 'protocol: TCP'), + 'exactly UDP 7400 (QUIC)'), + ('route policy still TCP', lambda t: edit(t, ROUTE + 'networkpolicy.yaml', 'protocol: UDP', 'protocol: TCP'), + 'only UDP 7400'), + ('route Service with TCP beside UDP', lambda t: append( + t, ROUTE + 'service.yaml', ' - {name: session-tcp, port: 7400, targetPort: 7400, protocol: TCP}\n'), + 'exactly UDP 7400 (QUIC)'), + ('route container port still TCP', lambda t: edit(t, ROUTE + 'listener.yaml', + 'containerPort: 7400, protocol: UDP', + 'containerPort: 7400, protocol: TCP'), + 'container ports must be'), + ('region policy admitting UDP 7400', lambda t: add_resource( + t, DEV, 'widened.yaml', region_policy('{matchLabels: {app.kubernetes.io/name: evo-region}}', + '{from: [{ipBlock: {cidr: 10.0.0.0/8}}], ' + 'ports: [{protocol: UDP, port: 7400}]}')), + 'admits the region session port'), + ('region policy admitting SCTP 7400', lambda t: add_resource( + t, DEV, 'widened.yaml', region_policy('{matchLabels: {app.kubernetes.io/name: evo-region}}', + '{ports: [{protocol: SCTP, port: 7400}]}')), + 'admits the region session port'), ('cluster traffic policy', lambda t: edit(t, ROUTE + 'service.yaml', 'externalTrafficPolicy: Local', 'externalTrafficPolicy: Cluster'), 'Local traffic policy'), ('node ports allocated', lambda t: edit(t, ROUTE + 'service.yaml', 'allocateLoadBalancerNodePorts: false', @@ -202,7 +223,7 @@ CASES = [ 'no PROXY protocol'), ('extra route port', lambda t: append(t, ROUTE + 'service.yaml', ' - {name: ops, port: 9100, targetPort: ops, protocol: TCP}\n'), - 'exactly TCP 7400'), + 'exactly UDP 7400'), ('another LoadBalancer', lambda t: edit(t, 'deploy/kubernetes/region/base/service.yaml', 'type: ClusterIP', 'type: LoadBalancer'), 'Services are forbidden'), ('session port on another Service', lambda t: append(t, 'deploy/kubernetes/region/base/service.yaml',