Add M9.8f dev session route design, disabled with fail-closed policy
ADR-0087 records the maintainer's route (void.aeternalisgames.com:7400 via a dedicated internal MetalLB VIP to the evolution-dev region pod, TLS end to end, CIDR-limited) and operator-opened windows enforced by evo-server. The route is a kustomize component included nowhere; the policy check encodes its shape and refuses enabling until VIP/CIDR, the dev quota and --session-window exist, with a 20-case self-test in the CI deploy stage. The base listener stays loopback. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -90,11 +90,12 @@ RUN apt-get update \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /source
|
||||
COPY deploy/ deploy/
|
||||
COPY tools/ci/deploy-policy-check.py tools/ci/
|
||||
COPY tools/ci/deploy-policy-check.py tools/ci/deploy-policy-selftest.py tools/ci/
|
||||
COPY tools/ops/ tools/ops/
|
||||
COPY tools/backup/ tools/backup/
|
||||
COPY tools/ci/ops-lib-check.sh tools/ci/
|
||||
RUN python3 tools/ci/deploy-policy-check.py && bash tools/ci/ops-lib-check.sh
|
||||
RUN python3 tools/ci/deploy-policy-check.py && python3 tools/ci/deploy-policy-selftest.py \
|
||||
&& bash tools/ci/ops-lib-check.sh
|
||||
# M8.5: monitoring configuration and alert tests with the pinned images' own tools.
|
||||
COPY --from=quay.io/prometheus/prometheus@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 /bin/promtool /usr/local/bin/promtool
|
||||
COPY --from=quay.io/prometheus/alertmanager@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a /bin/amtool /usr/local/bin/amtool
|
||||
|
||||
@@ -59,13 +59,14 @@ spec:
|
||||
inputs=()
|
||||
if [ -n "$REGION_INPUTS" ]; then inputs=(--synthetic-inputs "$REGION_INPUTS"); fi
|
||||
# Dev sessions are a drill-time override too (tools/ops/session-drill.sh,
|
||||
# ADR-0083): loopback only, reached by kubectl port-forward, never a Service.
|
||||
# ADR-0083): loopback by default, reached by kubectl port-forward. Only the
|
||||
# disabled dev-session-route component (ADR-0087) may bind the pod interface.
|
||||
# evo-server refuses any environment but dev and needs the synthetic carrier.
|
||||
# Consumed grants persist on the objects volume (outside the object
|
||||
# namespaces, ignored by backups) so a restarted pod refuses them.
|
||||
sessions=()
|
||||
if [ -n "$REGION_SESSION" ]; then
|
||||
sessions=(--session-listen 127.0.0.1:7400 --session-cert /run/session/server.pem
|
||||
sessions=(--session-listen "$REGION_SESSION_LISTEN" --session-cert /run/session/server.pem
|
||||
--session-key /run/session/server.key --session-grant-key /run/session/grant.key
|
||||
--session-grant-ledger /objects/dev-session-grants.v1
|
||||
--session-environment "$REGION_SESSION")
|
||||
@@ -89,6 +90,9 @@ spec:
|
||||
- {name: REGION_RESERVE_MIB, value: "1024"}
|
||||
# M9.7 drill-only: "dev" enables the session listener; Git keeps it empty.
|
||||
- {name: REGION_SESSION, value: ""}
|
||||
# Session listener address; the policy check allows only this loopback value
|
||||
# outside region/dev-session-route.
|
||||
- {name: REGION_SESSION_LISTEN, value: "127.0.0.1:7400"}
|
||||
ports:
|
||||
- {name: ops, containerPort: 9100, protocol: TCP}
|
||||
readinessProbe:
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# M9.8f dev session route for evolution-dev ONLY (ADR-0087). DISABLED: no
|
||||
# kustomization includes this component yet. Enabling it (components: in
|
||||
# region/dev/kustomization.yaml) is refused by tools/ci/deploy-policy-check.py until:
|
||||
# - the REPLACE markers carry the allocated internal MetalLB VIP and the permitted
|
||||
# development/test client CIDR(s);
|
||||
# - the platform dev quota allows exactly one LoadBalancer Service;
|
||||
# - evo-server enforces operator-opened session windows itself (--session-window).
|
||||
# Route: void.aeternalisgames.com:7400 -> VIP -> Service -> evo-region pod:7400 (TLS 1.3
|
||||
# terminates in the pod). Raw TCP only: no Ingress, no TLS at nginx, no PROXY protocol.
|
||||
apiVersion: kustomize.config.k8s.io/v1alpha1
|
||||
kind: Component
|
||||
resources:
|
||||
- service.yaml
|
||||
- networkpolicy.yaml
|
||||
patches:
|
||||
- path: listener.yaml
|
||||
@@ -0,0 +1,15 @@
|
||||
# With the route, the opt-in session listener binds the pod interface and names its
|
||||
# port; it still starts only when sessions are enabled for a window (ADR-0087).
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: evo-region
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: evo-server
|
||||
env:
|
||||
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
|
||||
ports:
|
||||
- {name: session, containerPort: 7400, protocol: TCP}
|
||||
@@ -0,0 +1,18 @@
|
||||
# Session ingress to the region pod: TCP 7400 from the permitted development/test
|
||||
# client CIDR(s) only, which must equal the Service's loadBalancerSourceRanges.
|
||||
# Everything else stays denied by the platform default-deny policy.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: evo-region-session-ingress
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: evo-region
|
||||
policyTypes: [Ingress]
|
||||
ingress:
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: REPLACE-dev-test-client-cidr
|
||||
ports:
|
||||
- {protocol: TCP, port: 7400}
|
||||
@@ -0,0 +1,26 @@
|
||||
# The one LoadBalancer Service AEvolution may own (ADR-0087), evolution-dev only.
|
||||
# - metallb.io/loadBalancerIPs pins the dedicated internal VIP that the internal DNS
|
||||
# name void.aeternalisgames.com resolves to (never the control-plane address).
|
||||
# - externalTrafficPolicy Local keeps client source addresses, so the NetworkPolicy's
|
||||
# ipBlock (same CIDRs) is enforceable; new connections reach only a ready pod.
|
||||
# - allocateLoadBalancerNodePorts false: no node ports (the quota forbids them).
|
||||
# - loadBalancerSourceRanges duplicates the CIDR at the Service; the edge firewall is
|
||||
# the third layer. TLS 1.3, the dev CA and single-use grants remain mandatory.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: evo-region-session
|
||||
annotations:
|
||||
metallb.io/loadBalancerIPs: REPLACE-dev-session-vip
|
||||
labels:
|
||||
app.kubernetes.io/name: evo-region
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
allocateLoadBalancerNodePorts: false
|
||||
externalTrafficPolicy: Local
|
||||
loadBalancerSourceRanges:
|
||||
- REPLACE-dev-test-client-cidr
|
||||
selector:
|
||||
app.kubernetes.io/name: evo-region
|
||||
ports:
|
||||
- {name: session, port: 7400, targetPort: session, protocol: TCP}
|
||||
+2
-1
@@ -458,7 +458,8 @@ terrain meshing or procedural creature animation is required.
|
||||
M9.8b adds bounded UI state, protected external input capture and bounded session
|
||||
event delivery. M9.8c integrates the launcher, legend, labelled inspector, click
|
||||
selection and session panel. M9.8d adds rebindable controls (ADR-0085). M9.8e
|
||||
packages a local Windows x64 ZIP (ADR-0086). ADR-0084 records the toolkit scopes.
|
||||
packages a local Windows x64 ZIP (ADR-0086). M9.8f designs the dev session route,
|
||||
committed disabled (ADR-0087). ADR-0084 records the toolkit scopes.
|
||||
|
||||
**M9.8 planning (2026-10-02):** user-confirmed Windows ZIP, temporary Gitea Actions
|
||||
artifacts, labelled engineering UI/launcher and control rebinding. See the
|
||||
|
||||
+5
-2
@@ -10,10 +10,13 @@ bounded errors, Retry/Cancel), legend, labelled inspector, click selection, sess
|
||||
panel with the intention outcome record, and visible notifications/dialog. M9.8d
|
||||
adds rebindable controls (ADR-0085) with conflict checks and a validated file. M9.8e
|
||||
builds a static-runtime Windows x64 ZIP locally with notices, identity, checksums and
|
||||
credential guards (ADR-0086); Actions publication needs a Windows runner. Tests,
|
||||
credential guards (ADR-0086); Actions publication needs a Windows runner. M9.8f
|
||||
designs the dev session route (ADR-0087), committed disabled with fail-closed policy
|
||||
tests; nothing was applied to the cluster. Tests,
|
||||
visual review and limits are recorded in engineering/VALIDATION.md.
|
||||
|
||||
Next: M9.8f dev TCP route design, then M9.8g qualification on the low-spec VM.
|
||||
Next: server-enforced session windows and the route inputs (VIP, CIDR, DNS) to enable
|
||||
ADR-0087, and M9.8g qualification on the low-spec VM.
|
||||
Windows packages will be built locally until a Windows runner exists. Testing is internal; 172.10.1.160 is the
|
||||
RKE2 control-plane address. VM qualification and the cluster restart drill remain
|
||||
outstanding, as agreed. See engineering/M9_8_DELIVERY_PLAN.md.
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# ADR-0069: Environment isolation and World0 protection
|
||||
|
||||
## Status
|
||||
ADR-0087 allows exactly one LoadBalancer Service, in evolution-dev only, for the dev
|
||||
session route once enabled; node ports stay forbidden and stage/World0 are unchanged.
|
||||
Accepted for M8.2, 2026-09-29. It builds on the M8.1 inventory
|
||||
(infrastructure/INVENTORY.md), ADR-0020 (delivery), and OPERATIONS.md (World0
|
||||
protection). Nothing here deploys a world; World0 launch stays gated by M8.8.
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# ADR-0083: Dev region sessions, no-effect intentions and the session drill
|
||||
|
||||
## Status
|
||||
Listener/Service rule narrowly superseded for evolution-dev by ADR-0087 (dev session
|
||||
route), effective only when that route is enabled; until then this ADR applies as is.
|
||||
Accepted for M9.7 (engineering client to one authoritative **dev** region). Decisions
|
||||
confirmed with the maintainer on 2026-10-01: durable no-effect intentions, an
|
||||
explicitly empty organism presentation, local real-process evidence plus a dev
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# ADR-0087: Dev session route and operator-opened session windows
|
||||
|
||||
## Status
|
||||
Accepted for M9.8f as design (2026-10-02), evolution-dev only. The route is committed
|
||||
**disabled**; nothing was applied to the cluster. Enabling it is blocked by policy until
|
||||
the inputs and server-side window enforcement below exist. Narrowly supersedes
|
||||
ADR-0083 (loopback-only listener, never a Service) and ADR-0069 (no LoadBalancer
|
||||
Services) for evolution-dev, and only once enabled.
|
||||
|
||||
## Context
|
||||
Internal testers need the native client (M9.8e ZIP) to reach the dev region without
|
||||
`kubectl port-forward`. The cluster inventory (INVENTORY.md, 2026-09-29) shows RKE2's
|
||||
ingress-nginx 1.12.1 DaemonSet serving 80/443 on all four node addresses (public sites
|
||||
included), MetalLB 0.15.2 in FRR mode with one LoadBalancer (Gitea SSH), and an existing
|
||||
HTTP Ingress for void.aeternalisgames.com in namespace `void`. The session protocol is
|
||||
TLS 1.3 over raw TCP (ADR-0082), not HTTP. Sessions need synthetic carrier inputs, which
|
||||
ADR-0073 keeps drill-time only so dev history does not grow continuously.
|
||||
|
||||
## Decision (maintainer, 2026-10-02)
|
||||
**Route.** `void.aeternalisgames.com:7400` resolves, in internal DNS, to a dedicated
|
||||
internal MetalLB VIP (never automatically the control-plane address 172.10.1.160) that
|
||||
fronts one LoadBalancer Service `evo-region-session` in evolution-dev, which forwards to
|
||||
the region pod's port 7400. TLS 1.3 terminates in the region pod; its dev-CA server
|
||||
certificate carries `void.aeternalisgames.com`, which the client verifies. No HTTP
|
||||
Ingress, no TLS termination at nginx, no PROXY-protocol prefix. The Service uses
|
||||
`externalTrafficPolicy: Local` (client addresses preserved), no node ports, a pinned
|
||||
`metallb.io/loadBalancerIPs` annotation and `loadBalancerSourceRanges`. New connections
|
||||
reach only a ready pod; after a replacement, clients reconnect with a fresh grant and
|
||||
baseline (ADR-0083).
|
||||
|
||||
**Access.** Development workstation/test subnet only, in three layers: the edge
|
||||
firewall, the Service source ranges and a NetworkPolicy admitting TCP 7400 to the region
|
||||
pod from exactly those CIDRs. TLS with the dev CA and single-use, short-lived grants stay
|
||||
mandatory; grants remain bearer tokens.
|
||||
|
||||
**Listener.** The base keeps the opt-in listener on loopback through
|
||||
`REGION_SESSION_LISTEN=127.0.0.1:7400`. Only the route component
|
||||
(`deploy/kubernetes/region/dev-session-route`) binds the pod interface and names the
|
||||
session port. The listener still starts only when sessions are enabled.
|
||||
|
||||
**Windows.** Sessions and synthetic inputs run only during an operator-opened test window
|
||||
with automatic expiry (default 30 minutes); always-on simulation waits for an explicit
|
||||
disposable-world lifecycle and retention. The endpoint (DNS, MetalLB Service) may stay
|
||||
configured permanently. A small dev-only GitOps configuration holds an activation id,
|
||||
the exact world/region identity and an absolute UTC expiry. Operations, via a manually
|
||||
triggered Gitea workflow: Open (default 30 min), Extend (new explicit expiry), Close
|
||||
(stop accepting sessions, close existing connections, stop synthetic inputs, drain
|
||||
pending commits) and Status (window, expiry, active sessions, committed tick).
|
||||
Expiry is enforced **inside evo-server**, not by a delayed cleanup job:
|
||||
- a restart keeps the original absolute deadline, never a fresh duration;
|
||||
- an expired configuration cannot reopen the listener;
|
||||
- expiry stops synthetic generation as well as connections;
|
||||
- a failed workflow cannot leave simulation running; cleanup may later restore the
|
||||
closed Git configuration. Grant expiry alone does not stop the generator.
|
||||
|
||||
## Enabling (all required; `tools/ci/deploy-policy-check.py` refuses otherwise)
|
||||
1. The VIP (private, from the MetalLB pool) and permitted client CIDR(s) (private, no
|
||||
wider than /16) replace the REPLACE markers; Service ranges equal the policy's.
|
||||
2. `deploy/kubernetes/platform/dev` raises `services.loadbalancers` to exactly 1;
|
||||
node ports stay 0; stage and World0 stay 0.
|
||||
3. evo-server implements the session window (`--session-window`) as above, with tests
|
||||
and an amendment to this ADR, and the base deployment passes it.
|
||||
4. Only `region/dev` includes the component. Then: internal DNS for void to the VIP, the
|
||||
edge firewall rule, and a server-side dry-run (quota accounting of the Local health
|
||||
check port is to be confirmed there), a new dev server certificate for
|
||||
void.aeternalisgames.com, and live connectivity recorded separately in VALIDATION.md.
|
||||
|
||||
## Open points for the maintainer
|
||||
- **Argo delivery.** The aevolution-dev Application has no automated sync, and the
|
||||
policy forbids auto-sync outside the platform app (ADR-0069). A workflow commit alone
|
||||
will not deploy; either a manual sync per window or a scoped dev exception must be
|
||||
decided before implementing the workflow. CI holds no cluster credentials (ADR-0020),
|
||||
so Status can report Git state; live fields need a defined read path.
|
||||
- **void namespace.** void.aeternalisgames.com already has an HTTP Ingress (namespace
|
||||
`void`). Pointing internal DNS at the VIP changes what internal clients reach for that
|
||||
name; confirm or retire the existing route.
|
||||
- Window limits: maximum window length and clock-skew tolerance.
|
||||
|
||||
## Alternatives considered
|
||||
- ingress-nginx tcp-services or SNI passthrough on 443: couples game sessions to the
|
||||
shared web ingress (443 likely internet-facing) and needs controller changes.
|
||||
- Port-forward only (ADR-0083): no tester-friendly endpoint.
|
||||
- NodePort: opens a port on every node; forbidden by ADR-0069.
|
||||
- Always-on sessions in dev: continuous synthetic history; rejected for now.
|
||||
|
||||
## Consequences
|
||||
One reviewed exception to "no LoadBalancer" for evolution-dev. The policy check and its
|
||||
fail-closed self-test (`tools/ci/deploy-policy-selftest.py`) encode the route's shape and
|
||||
the enabling preconditions. World0 and stage are untouched.
|
||||
@@ -97,5 +97,6 @@ design. No unadopted candidate is a link dependency of the proof.
|
||||
- [ADR-0083: Dev region sessions](0083-dev-region-sessions.md) - session server in evo-server, durable no-effect intentions on synthetic carriers, empty organism presentation, reconnect with fresh baselines, loopback-only drill-time dev deployment.
|
||||
|
||||
- [ADR-0084: Engineering UI toolkit](0084-engineering-ui-toolkit.md) - Dear ImGui 1.92.9b pinned (M9.8a) and integrated as the engineering client overlay with toolkit-first input routing (M9.8c).
|
||||
- [ADR-0087: Dev session route](0087-dev-session-route.md) - evolution-dev only: void.aeternalisgames.com:7400 via a dedicated internal MetalLB VIP to the region pod, TLS end to end, CIDR-limited, operator-opened windows enforced by evo-server; committed disabled with fail-closed policy tests.
|
||||
- [ADR-0086: Windows client package](0086-windows-client-package.md) - static-runtime Windows x64 ZIP with explicit allowlist, licence texts, build identity, SHA256 manifest, credential guards and deterministic layout; local packaging until a Windows runner exists.
|
||||
- [ADR-0085: Rebindable engineering controls](0085-rebindable-engineering-controls.md) - SDL-free action/HID-key bindings, conflict validation, versioned all-or-nothing controls file, binding capture before toolkit/world.
|
||||
|
||||
@@ -4,7 +4,9 @@ Status: M9.8a probe, M9.8b owned state/capture/event bounds, M9.8c launcher,
|
||||
legend, labelled inspector and click selection, and M9.8d rebindable controls
|
||||
complete, 2026-10-02. M9.8e local Windows ZIP complete (ADR-0086); its Gitea
|
||||
Actions publication waits for a Windows runner and its clean-VM run for M9.8g.
|
||||
Remaining segments planned.
|
||||
M9.8f route design, disabled manifests and fail-closed policy tests complete
|
||||
(ADR-0087); enabling waits for the VIP, client CIDR, DNS, dev quota and server-enforced
|
||||
session windows. M9.8g remains.
|
||||
No application UI or deployment completion is claimed. See ADR-0084 for the toolkit boundary.
|
||||
|
||||
## Confirmed scope
|
||||
@@ -64,9 +66,10 @@ All selected biological data remains a read-only presentation copy.
|
||||
- No Windows runner exists yet (confirmed 2026-10-02). Local builds are the initial path. Do not silently route a Windows
|
||||
package job to the existing Linux runner or assume an untested cross-compile.
|
||||
The packaging script can run on the existing Windows development host meanwhile.
|
||||
- Confirmed control plane: 172.10.1.160. Still needed for direct service routing:
|
||||
nginx implementation/configuration, exposed TCP port,
|
||||
source restrictions and certificate name. The protocol is TLS/TCP, not HTTP;
|
||||
- Confirmed control plane: 172.10.1.160. Direct routing decided in ADR-0087: dedicated
|
||||
internal MetalLB VIP, TCP 7400, void.aeternalisgames.com, CIDR-limited. Still needed:
|
||||
the VIP from the MetalLB pool, the permitted client CIDR, internal DNS, and decisions
|
||||
on Argo delivery of session windows and the existing void HTTP Ingress. The protocol is TLS/TCP, not HTTP;
|
||||
a standard HTTP Ingress rule does not implement it. Port-forward remains supported.
|
||||
- VM specification and whether graphics acceleration is available. Hardware-specific
|
||||
performance acceptance cannot be declared before these are known.
|
||||
|
||||
@@ -3896,3 +3896,37 @@ Executed on the Windows development host (MSVC 19.44.35217, CMake 3.31.6):
|
||||
|
||||
Not done: Gitea Actions publication (no Windows runner), a run on a clean Windows
|
||||
VM (M9.8g), code signing, hardware/DPI qualification, legal review of notices.
|
||||
|
||||
|
||||
## 2026-10-02 — M9.8f dev session route design (ADR-0087)
|
||||
|
||||
Maintainer decisions: dedicated internal MetalLB VIP, TCP 7400,
|
||||
void.aeternalisgames.com, TLS ending in the region pod, development/test CIDR only
|
||||
(firewall + Service source ranges + NetworkPolicy), no Ingress/nginx TLS/PROXY
|
||||
protocol; operator-opened windows with server-enforced absolute expiry; scope limited
|
||||
to design, manifests and policy tests. Nothing was applied to the cluster.
|
||||
|
||||
Implemented: `REGION_SESSION_LISTEN` (base stays 127.0.0.1:7400); the disabled
|
||||
component `deploy/kubernetes/region/dev-session-route` (LoadBalancer Service with
|
||||
Local policy, no node ports, pinned VIP annotation and source ranges; CIDR-only
|
||||
ingress policy on TCP 7400; listener patch to the pod interface) carrying REPLACE
|
||||
markers; policy rules for the route's exact shape, LoadBalancer/NodePort and
|
||||
session-port exclusivity, no session Ingress, evolution-dev-only placement, quota
|
||||
limits, and enabling preconditions (no markers, dev quota of exactly one
|
||||
LoadBalancer, server `--session-window`); patch files are now recognized generally.
|
||||
`tools/ci/deploy-policy-selftest.py` copies the manifests and applies 20 unsafe changes.
|
||||
|
||||
Executed (WSL Ubuntu, Python 3 with PyYAML 6.0.1): deploy policy check on the
|
||||
repository, 70 objects in 68 files, 0 failures; self-test 20/20 refused, baseline
|
||||
copy passes; ops-lib and ci-path fixtures pass; session-drill usage unchanged.
|
||||
kustomize v5.4.3 (official release, SHA-256 checked against its checksums.txt, in a
|
||||
scratch directory, not a project dependency) rendered region/dev as committed (listener
|
||||
127.0.0.1:7400, no session Service) and a temporary enabled copy with example private
|
||||
values (LoadBalancer evo-region-session and the CIDR policy in evolution-dev, listener
|
||||
0.0.0.0:7400, session container port merged).
|
||||
|
||||
Not run: the ci-deploy Docker stage (no Docker in this WSL distribution; its commands
|
||||
were run directly), any cluster/server dry-run, MetalLB allocation, DNS, firewall or live
|
||||
connectivity. Server-enforced session windows and the Gitea window workflow are not
|
||||
implemented; Argo delivery for windows and the existing void HTTP Ingress are open
|
||||
maintainer decisions recorded in ADR-0087.
|
||||
|
||||
@@ -82,3 +82,12 @@ exposed through a Service, NodePort or Ingress. A grant bundle lasts at most one
|
||||
and each connection consumes one grant. WASD sends intentions; the small chip under
|
||||
the intention dot turns white (admitted), green (durable, no effect) or red/grey
|
||||
(rejected or outcome unknown). Console lines report every session event.
|
||||
|
||||
## 6. Planned direct route (M9.8f, disabled)
|
||||
|
||||
ADR-0087 designs `void.aeternalisgames.com:7400` through a dedicated internal MetalLB
|
||||
VIP to the region pod (TLS end to end, development/test CIDR only), opened only in
|
||||
operator windows that evo-server itself expires. The manifests are in
|
||||
`deploy/kubernetes/region/dev-session-route` and are not included anywhere; the policy
|
||||
check refuses to enable them until the VIP, client CIDR, dev quota and the server-side
|
||||
`--session-window` exist. Until then use the port-forward procedures above.
|
||||
|
||||
@@ -46,7 +46,7 @@ case "$suite" in
|
||||
pattern="$common|$native|tests/|$postgres_tools|third_party/mbedtls/|tools/prepare-mbedtls\.cmake|tools/mbedtls-pin\.cmake|deploy/docker/Dockerfile"
|
||||
;;
|
||||
deploy)
|
||||
pattern="$common|deploy/|tools/ops/|tools/backup/|tools/ci/deploy-policy-check\.py|tools/ci/monitoring-check\.sh|tools/ci/ops-lib-check\.sh"
|
||||
pattern="$common|deploy/|tools/ops/|tools/backup/|tools/ci/deploy-policy-(check|selftest)\.py|tools/ci/monitoring-check\.sh|tools/ci/ops-lib-check\.sh"
|
||||
;;
|
||||
server-image)
|
||||
pattern="$common|$native|migrations/|tools/prepare-postgres\.sh|tools/ci/server-image-smoke\.sh|deploy/docker/Dockerfile"
|
||||
|
||||
+120
-12
@@ -24,8 +24,14 @@ environment-isolation or World0-protection invariant is broken:
|
||||
required; upgrade-rehearsal flags never appear in a deployed workload;
|
||||
- the Pods the backup and barrier scripts generate (rendered with --print-manifest)
|
||||
keep the database-tool security contract: non-root 10001 with fsGroup 10001 so
|
||||
the 0440 Secret files are readable, no token, read-only root, no escalation.
|
||||
the 0440 Secret files are readable, no token, read-only root, no escalation;
|
||||
- the dev session route (ADR-0087): loopback listener everywhere except the
|
||||
evolution-dev route component; that component is the only LoadBalancer and the
|
||||
only Service on the session port, narrowly shaped, and it cannot be enabled while
|
||||
markers are unresolved or before server-enforced session windows exist.
|
||||
"""
|
||||
import ipaddress
|
||||
import re
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -55,7 +61,15 @@ for base, _, files in os.walk(os.path.join(ROOT, 'deploy')):
|
||||
path = os.path.join(base, name)
|
||||
manifests[os.path.relpath(path, ROOT).replace(os.sep, '/')] = documents(path)
|
||||
objects = [(p, d) for p, ds in manifests.items() for d in ds
|
||||
if 'kind' in d and d['kind'] != 'Kustomization']
|
||||
if 'kind' in d and d['kind'] not in ('Kustomization', 'Component')]
|
||||
# Files used as kustomize patches hold partial objects (checked only for what they set).
|
||||
patch_files = set()
|
||||
for p, ds in manifests.items():
|
||||
if p.endswith('kustomization.yaml') and ds:
|
||||
for entry in ds[0].get('patches', []):
|
||||
target = entry.get('path') if isinstance(entry, dict) else entry
|
||||
if target:
|
||||
patch_files.add(os.path.normpath(os.path.join(os.path.dirname(p), target)).replace(os.sep, '/'))
|
||||
|
||||
# Secrets and service-account tokens.
|
||||
for path, doc in objects:
|
||||
@@ -84,6 +98,21 @@ check(dns.get('policyTypes') == ['Egress'] and set(dns_ports) == {53} and len(dn
|
||||
quota = base.get('ResourceQuota/aevolution-quota', {}).get('spec', {}).get('hard', {})
|
||||
check(quota.get('services.loadbalancers') == '0' and quota.get('services.nodeports') == '0',
|
||||
'quota must forbid LoadBalancer and NodePort services')
|
||||
# ADR-0087: only the dev overlay may raise LoadBalancers, to exactly one, and only
|
||||
# together with the session route; node ports stay forbidden everywhere.
|
||||
ROUTE = 'deploy/kubernetes/region/dev-session-route/'
|
||||
route_users = sorted(p for p, ds in manifests.items() if p.endswith('kustomization.yaml') and ds
|
||||
and any('dev-session-route' in str(e) for e in ds[0].get('resources', []) + ds[0].get('components', [])))
|
||||
route_enabled = route_users == ['deploy/kubernetes/region/dev/kustomization.yaml']
|
||||
check(route_users in ([], ['deploy/kubernetes/region/dev/kustomization.yaml']),
|
||||
f'only region/dev may include the dev session route: {route_users}')
|
||||
for path, doc in objects:
|
||||
if doc['kind'] == 'ResourceQuota' and path != f'{platform}/base/quota.yaml':
|
||||
hard = doc.get('spec', {}).get('hard', {})
|
||||
check(hard.get('services.nodeports', '0') == '0', f'{path}: node ports stay forbidden')
|
||||
allowed = ('0', '1') if path.startswith(f'{platform}/dev/') and route_enabled else ('0',)
|
||||
check(hard.get('services.loadbalancers', '0') in allowed,
|
||||
f'{path}: LoadBalancers only in evolution-dev, at most one, with the session route')
|
||||
for env in ('dev', 'stage', 'world0'):
|
||||
k = f'{platform}/{env}/kustomization.yaml'
|
||||
ns_path = f'{platform}/{env}/namespace.yaml'
|
||||
@@ -212,7 +241,7 @@ for path, doc in objects:
|
||||
for path, doc in objects:
|
||||
# Overlay patches (region-values.yaml) are partial objects; the base is complete.
|
||||
if (doc['kind'] == 'Deployment' and doc['metadata'].get('name') == 'evo-region'
|
||||
and not path.endswith('region-values.yaml')):
|
||||
and path not in patch_files):
|
||||
spec = doc['spec']
|
||||
pod = spec['template']['spec']
|
||||
container = pod['containers'][0]
|
||||
@@ -238,22 +267,101 @@ for path, doc in objects:
|
||||
# M9.7 (ADR-0083): dev sessions are a drill-time override as well.
|
||||
if variable.get('name') == 'REGION_SESSION':
|
||||
check(variable.get('value') == '', f'{path}: dev sessions are never enabled in Git')
|
||||
# M9.7 (ADR-0083): the session listener is loopback-only (kubectl port-forward), and its
|
||||
# Secret is optional, 0440 and never exposed through a Service port.
|
||||
# M9.7 (ADR-0083) and M9.8f (ADR-0087): the session listener is loopback (kubectl
|
||||
# port-forward) everywhere except the evolution-dev route component; its Secret is
|
||||
# optional and 0440; only that component's Service exposes the session port.
|
||||
for path, doc in objects:
|
||||
if path.startswith('deploy/kubernetes/region/') and doc['kind'] == 'Deployment' and not path.endswith('region-values.yaml'):
|
||||
if path.startswith('deploy/kubernetes/region/') and doc['kind'] == 'Deployment':
|
||||
pod = doc['spec']['template']['spec']
|
||||
for container in pod.get('containers', []):
|
||||
for variable in container.get('env', []):
|
||||
if variable.get('name') == 'REGION_SESSION_LISTEN':
|
||||
expected = '0.0.0.0:7400' if path.startswith(ROUTE) else '127.0.0.1:7400'
|
||||
check(variable.get('value') == expected,
|
||||
f'{path}: REGION_SESSION_LISTEN must be {expected} here')
|
||||
if path in patch_files:
|
||||
continue
|
||||
script = ' '.join(pod['containers'][0].get('args', []))
|
||||
check('--session-listen 127.0.0.1:' in script and '0.0.0.0:7400' not in script,
|
||||
f'{path}: the session listener binds loopback only')
|
||||
check('--session-listen "$REGION_SESSION_LISTEN"' in script and '0.0.0.0' not in script.replace('--ops-listen 0.0.0.0:9100', ''),
|
||||
f'{path}: the session listener address comes only from REGION_SESSION_LISTEN')
|
||||
for volume in pod.get('volumes', []):
|
||||
if volume.get('secret', {}).get('secretName') == 'evolution-dev-session':
|
||||
check(volume['secret'].get('optional') is True and volume['secret'].get('defaultMode') == 0o440,
|
||||
f'{path}: the dev session Secret is optional and 0440')
|
||||
route_ranges = []
|
||||
for path, doc in objects:
|
||||
if doc['kind'] == 'Service':
|
||||
for port in doc['spec'].get('ports', []):
|
||||
check(port.get('port') != 7400 and port.get('targetPort') not in (7400, 'session'),
|
||||
f'{path}: the dev session port is never exposed by a Service')
|
||||
spec = doc['spec']
|
||||
session_port = any(port.get('port') == 7400 or port.get('targetPort') in (7400, 'session')
|
||||
for port in spec.get('ports', []))
|
||||
is_route = path == ROUTE + 'service.yaml' and doc['metadata'].get('name') == 'evo-region-session'
|
||||
if not is_route:
|
||||
check(not session_port, f'{path}: only the dev session route Service may expose the session port')
|
||||
check(spec.get('type', 'ClusterIP') == 'ClusterIP',
|
||||
f'{path}: {spec.get("type")} Services are forbidden (ADR-0069)')
|
||||
continue
|
||||
route_ranges = spec.get('loadBalancerSourceRanges', [])
|
||||
check(spec.get('type') == 'LoadBalancer' and spec.get('externalTrafficPolicy') == 'Local'
|
||||
and spec.get('allocateLoadBalancerNodePorts') is False,
|
||||
f'{path}: the route is a LoadBalancer with Local traffic policy and no node ports')
|
||||
check(spec.get('ports') == [{'name': 'session', 'port': 7400, 'targetPort': 'session', 'protocol': 'TCP'}],
|
||||
f'{path}: the route exposes exactly TCP 7400')
|
||||
check(spec.get('selector') == {'app.kubernetes.io/name': 'evo-region'},
|
||||
f'{path}: the route selects only the region pod')
|
||||
annotations = doc['metadata'].get('annotations', {})
|
||||
check(set(annotations) == {'metallb.io/loadBalancerIPs'},
|
||||
f'{path}: only a pinned MetalLB address annotation (no PROXY protocol or sharing)')
|
||||
vip = annotations.get('metallb.io/loadBalancerIPs', '')
|
||||
check(len(route_ranges) >= 1, f'{path}: loadBalancerSourceRanges must name the permitted CIDRs')
|
||||
if not vip.startswith('REPLACE'):
|
||||
try:
|
||||
check(ipaddress.ip_address(vip).is_private, f'{path}: the VIP must be an internal address')
|
||||
except ValueError:
|
||||
check(False, f'{path}: the VIP must be one IP address')
|
||||
for cidr in route_ranges:
|
||||
if cidr.startswith('REPLACE'):
|
||||
continue
|
||||
try:
|
||||
network = ipaddress.ip_network(cidr)
|
||||
check(network.is_private and network.prefixlen >= (16 if network.version == 4 else 48),
|
||||
f'{path}: {cidr} must be a private range no wider than /16 (IPv6 /48)')
|
||||
except ValueError:
|
||||
check(False, f'{path}: {cidr} is not a CIDR')
|
||||
if doc['kind'] == 'Ingress':
|
||||
text = yaml.safe_dump(doc)
|
||||
check('7400' not in text and 'session' not in text,
|
||||
f'{path}: the dev session is raw TCP, never an Ingress (ADR-0087)')
|
||||
policies = [(p, d) for p, d in objects if p.startswith(ROUTE) and d['kind'] == 'NetworkPolicy']
|
||||
check(len(policies) == 1, f'{ROUTE}: exactly one session ingress policy')
|
||||
for path, doc in policies:
|
||||
spec = doc['spec']
|
||||
rules = spec.get('ingress', [])
|
||||
blocks = [peer.get('ipBlock', {}) for rule in rules for peer in rule.get('from', [])]
|
||||
check(spec.get('podSelector') == {'matchLabels': {'app.kubernetes.io/name': 'evo-region'}}
|
||||
and spec.get('policyTypes') == ['Ingress'] and len(rules) == 1
|
||||
and rules[0].get('ports') == [{'protocol': 'TCP', 'port': 7400}],
|
||||
f'{path}: the session policy admits only TCP 7400 to the region pod')
|
||||
check(all(set(b) == {'cidr'} for b in blocks) and len(blocks) == len(rules[0].get('from', []))
|
||||
and sorted(b.get('cidr') for b in blocks) == sorted(route_ranges),
|
||||
f'{path}: the session policy admits exactly the Service source ranges')
|
||||
for path, doc in objects:
|
||||
if path.startswith(ROUTE):
|
||||
check(doc.get('metadata', {}).get('namespace') in (None, 'evolution-dev'),
|
||||
f'{path}: the session route belongs to evolution-dev only')
|
||||
# Enabling the route fails closed until its values are real and the server enforces
|
||||
# operator-opened windows itself (expiry survives restarts; ADR-0087).
|
||||
if route_enabled:
|
||||
for name in os.listdir(os.path.join(ROOT, ROUTE)):
|
||||
with open(os.path.join(ROOT, ROUTE, name), encoding='utf-8') as handle:
|
||||
check('REPLACE' not in handle.read(), f'{ROUTE}{name}: unresolved REPLACE marker with the route enabled')
|
||||
base_script = ' '.join(' '.join(c.get('args', [])) for p, d in objects
|
||||
if p == 'deploy/kubernetes/region/base/deployment.yaml' and d['kind'] == 'Deployment'
|
||||
for c in d['spec']['template']['spec']['containers'])
|
||||
check('--session-window' in base_script,
|
||||
'the dev session route needs server-enforced session windows (--session-window, ADR-0087)')
|
||||
dev_quota = [d.get('spec', {}).get('hard', {}).get('services.loadbalancers') for p, d in objects
|
||||
if d['kind'] == 'ResourceQuota' and p.startswith(f'{platform}/dev/')]
|
||||
check(dev_quota == ['1'], 'the dev quota must allow exactly one LoadBalancer for the route')
|
||||
for env in ('dev', 'stage'):
|
||||
overlay = f'deploy/kubernetes/region/{env}/'
|
||||
images = manifests.get(overlay + 'kustomization.yaml', [{}])[0].get('images', [])
|
||||
@@ -311,7 +419,7 @@ for directory, _, files in os.walk(os.path.join(ROOT, 'deploy', 'kubernetes')):
|
||||
# M8.9 scheduling (ADR-0076). Node-local volumes pin their pods: added CPU does not
|
||||
# make the objects volume portable. Compute is a preference so one node still works.
|
||||
for path, doc in objects:
|
||||
if doc['kind'] not in workload_kinds or path.endswith('region-values.yaml'):
|
||||
if doc['kind'] not in workload_kinds or path in patch_files:
|
||||
continue
|
||||
spec = doc.get('spec', {})
|
||||
pod = spec.get('jobTemplate', {}).get('spec', {}).get('template', {}).get('spec') \
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""M9.8f fail-closed tests for deploy-policy-check.py (ADR-0087). Needs Python 3 and
|
||||
PyYAML only. Copies the manifests and the tools the check runs into a temporary tree,
|
||||
applies one unsafe change at a time and requires the check to fail with the expected
|
||||
message; the unmodified copy must pass.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
|
||||
ROUTE = 'deploy/kubernetes/region/dev-session-route/'
|
||||
DEV = 'deploy/kubernetes/region/dev/kustomization.yaml'
|
||||
REAL = [(ROUTE + 'service.yaml', 'REPLACE-dev-session-vip', '10.20.30.40'),
|
||||
(ROUTE + 'service.yaml', 'REPLACE-dev-test-client-cidr', '10.20.0.0/24'),
|
||||
(ROUTE + 'networkpolicy.yaml', 'REPLACE-dev-test-client-cidr', '10.20.0.0/24')]
|
||||
|
||||
|
||||
def edit(tree, path, old, new, count=1):
|
||||
full = os.path.join(tree, path)
|
||||
with open(full, encoding='utf-8') as f:
|
||||
text = f.read()
|
||||
if old not in text:
|
||||
raise SystemExit(f'selftest setup: {old!r} not in {path}')
|
||||
with open(full, 'w', encoding='utf-8') as f:
|
||||
f.write(text.replace(old, new, count))
|
||||
|
||||
|
||||
def append(tree, path, text):
|
||||
with open(os.path.join(tree, path), 'a', encoding='utf-8') as f:
|
||||
f.write(text)
|
||||
|
||||
|
||||
def write(tree, path, text):
|
||||
full = os.path.join(tree, path)
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
with open(full, 'w', encoding='utf-8') as f:
|
||||
f.write(text)
|
||||
|
||||
|
||||
def enable(tree):
|
||||
append(tree, DEV, 'components:\n - ../dev-session-route\n')
|
||||
|
||||
|
||||
def resolve(tree):
|
||||
for path, old, new in REAL:
|
||||
edit(tree, path, old, new)
|
||||
|
||||
|
||||
def quota_one(tree):
|
||||
write(tree, 'deploy/kubernetes/platform/dev/quota-session.yaml',
|
||||
'apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: aevolution-quota\n'
|
||||
'spec:\n hard:\n services.loadbalancers: "1"\n services.nodeports: "0"\n')
|
||||
|
||||
|
||||
CASES = [
|
||||
('enabling with unresolved markers', lambda t: enable(t), 'unresolved REPLACE marker'),
|
||||
('enabling before server-enforced windows', lambda t: (enable(t), resolve(t), quota_one(t)),
|
||||
'server-enforced session windows'),
|
||||
('enabling without the dev quota', lambda t: (enable(t), resolve(t)), 'exactly one LoadBalancer'),
|
||||
('route included by stage', lambda t: append(t, 'deploy/kubernetes/region/stage/kustomization.yaml',
|
||||
'components:\n - ../dev-session-route\n'),
|
||||
'only region/dev may include'),
|
||||
('stage quota allowing a LoadBalancer', lambda t: write(
|
||||
t, 'deploy/kubernetes/platform/stage/quota-lb.yaml',
|
||||
'apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: aevolution-quota\n'
|
||||
'spec:\n hard:\n services.loadbalancers: "1"\n'), 'LoadBalancers only in evolution-dev'),
|
||||
('world-open source range', lambda t: (edit(t, ROUTE + 'service.yaml', 'REPLACE-dev-test-client-cidr', '0.0.0.0/0'),
|
||||
edit(t, ROUTE + 'networkpolicy.yaml', 'REPLACE-dev-test-client-cidr', '0.0.0.0/0')),
|
||||
'private range'),
|
||||
('public VIP', lambda t: edit(t, ROUTE + 'service.yaml', 'REPLACE-dev-session-vip', '8.8.8.8'), 'internal address'),
|
||||
('policy CIDR differs from the Service', lambda t: edit(t, ROUTE + 'networkpolicy.yaml',
|
||||
'REPLACE-dev-test-client-cidr', '10.99.0.0/24'),
|
||||
'exactly the Service source ranges'),
|
||||
('policy admits another port', lambda t: edit(t, ROUTE + 'networkpolicy.yaml', 'port: 7400', 'port: 7401'),
|
||||
'only TCP 7400'),
|
||||
('cluster traffic policy', lambda t: edit(t, ROUTE + 'service.yaml', 'externalTrafficPolicy: Local',
|
||||
'externalTrafficPolicy: Cluster'), 'Local traffic policy'),
|
||||
('node ports allocated', lambda t: edit(t, ROUTE + 'service.yaml', 'allocateLoadBalancerNodePorts: false',
|
||||
'allocateLoadBalancerNodePorts: true'), 'no node ports'),
|
||||
('PROXY protocol annotation', lambda t: edit(t, ROUTE + 'service.yaml', ' metallb.io/loadBalancerIPs:',
|
||||
' example.io/proxy-protocol: "true"\n metallb.io/loadBalancerIPs:'),
|
||||
'no PROXY protocol'),
|
||||
('extra route port', lambda t: append(t, ROUTE + 'service.yaml',
|
||||
' - {name: ops, port: 9100, targetPort: ops, protocol: TCP}\n'),
|
||||
'exactly TCP 7400'),
|
||||
('another LoadBalancer', lambda t: edit(t, 'deploy/kubernetes/region/base/service.yaml', 'type: ClusterIP',
|
||||
'type: LoadBalancer'), 'Services are forbidden'),
|
||||
('session port on another Service', lambda t: append(t, 'deploy/kubernetes/region/base/service.yaml',
|
||||
' - {name: session, port: 7400, targetPort: 7400, protocol: TCP}\n'),
|
||||
'only the dev session route Service'),
|
||||
('session Ingress', lambda t: write(
|
||||
t, 'deploy/kubernetes/region/dev/ingress.yaml',
|
||||
'apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: session\nspec:\n rules:\n'
|
||||
' - host: void.aeternalisgames.com\n http:\n paths:\n - path: /\n'
|
||||
' pathType: Prefix\n backend:\n service:\n'
|
||||
' name: evo-region-session\n port: {number: 7400}\n'), 'never an Ingress'),
|
||||
('base listener on the pod interface', lambda t: edit(
|
||||
t, 'deploy/kubernetes/region/base/deployment.yaml', 'value: "127.0.0.1:7400"', 'value: "0.0.0.0:7400"'),
|
||||
'REGION_SESSION_LISTEN must be 127.0.0.1:7400'),
|
||||
('hard-coded listener address', lambda t: edit(
|
||||
t, 'deploy/kubernetes/region/base/deployment.yaml', '--session-listen "$REGION_SESSION_LISTEN"',
|
||||
'--session-listen 0.0.0.0:7400'), 'only from REGION_SESSION_LISTEN'),
|
||||
('route listener patch elsewhere', lambda t: edit(
|
||||
t, 'deploy/kubernetes/region/dev/region-values.yaml', '- {name: REGION_ID, value: "1"}',
|
||||
'- {name: REGION_ID, value: "1"}\n - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}'),
|
||||
'REGION_SESSION_LISTEN must be 127.0.0.1:7400'),
|
||||
('route object in another namespace', lambda t: edit(
|
||||
t, ROUTE + 'networkpolicy.yaml', ' name: evo-region-session-ingress',
|
||||
' name: evo-region-session-ingress\n namespace: evolution-stage'), 'evolution-dev only'),
|
||||
]
|
||||
|
||||
|
||||
def copy_tree(target):
|
||||
for directory in ('deploy', 'tools/ops', 'tools/backup'):
|
||||
shutil.copytree(os.path.join(ROOT, directory), os.path.join(target, directory))
|
||||
os.makedirs(os.path.join(target, 'tools', 'ci'))
|
||||
shutil.copy(os.path.join(ROOT, 'tools', 'ci', 'deploy-policy-check.py'), os.path.join(target, 'tools', 'ci'))
|
||||
|
||||
|
||||
def run(tree):
|
||||
result = subprocess.run([sys.executable, os.path.join(tree, 'tools', 'ci', 'deploy-policy-check.py')],
|
||||
capture_output=True, text=True)
|
||||
return result.returncode, result.stdout + result.stderr
|
||||
|
||||
|
||||
failures = []
|
||||
with tempfile.TemporaryDirectory() as clean:
|
||||
copy_tree(clean)
|
||||
code, output = run(clean)
|
||||
if code != 0:
|
||||
failures.append(f'baseline copy must pass:\n{output}')
|
||||
for name, mutate, expected in CASES:
|
||||
with tempfile.TemporaryDirectory() as tree:
|
||||
copy_tree(tree)
|
||||
mutate(tree)
|
||||
code, output = run(tree)
|
||||
refused = code != 0 and re.search(re.escape(expected), output)
|
||||
print(f'{"refused" if refused else "NOT REFUSED"}: {name}')
|
||||
if not refused:
|
||||
failures.append(f'{name}: expected failure mentioning {expected!r}, got rc={code}\n{output}')
|
||||
for message in failures:
|
||||
print('FAIL', message)
|
||||
print(f'deploy policy selftest: {len(CASES)} cases, {len(failures)} failures')
|
||||
sys.exit(1 if failures else 0)
|
||||
Reference in New Issue
Block a user