From 4440c6c981727237e1d84b9addea2bb1fbe7da48 Mon Sep 17 00:00:00 2001 From: dingo Date: Fri, 27 Feb 2026 14:44:08 -0500 Subject: [PATCH] Split sor-host into dedicated gateway and workers deployments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before: 3 pods ran ENABLED_SERVICES=all — gateway, auth, and every business-logic service co-hosted. This defeated the purpose of the gateway pattern and prevented independent scaling. After: sor-gateway (2 replicas) — ENABLED_SERVICES=gateway,auth • WebSocket edge, accepts client connections • Auth co-located (JWT validation is a localhost gRPC call, no network hop) • Lighter resource footprint: 150m/128Mi request, 500m/512Mi limit • Ingress at hub.aeternalisgames.com → sor-gateway:80 sor-workers (3 replicas) — ENABLED_SERVICES=chat,collection,bazaar, content,versioning,matchmaking,game • Subscribes to NATS, handles all business-logic messages • Holds DB connection pools for all schemas • Internal only — no ingress • Heavier resources: 300m/384Mi request, 1000m/1Gi limit configmap.yaml: removed ENABLED_SERVICES=all and Services__Auth (each deployment now sets these directly in its own env block) ingress.yaml: hub.aeternalisgames.com backend sor-host → sor-gateway ArgoCD prune=true will delete the old sor-host Deployment + Service from the cluster on next sync. Generated with [Claude Code](https://claude.ai/code) via [Happy](https://happy.engineering) Co-Authored-By: Claude Co-Authored-By: Happy --- deploy/k8s/configmap.yaml | 15 +-- deploy/k8s/deployment-gateway.yaml | 111 ++++++++++++++++++ ...ment-host.yaml => deployment-workers.yaml} | 35 +++--- deploy/k8s/ingress.yaml | 2 +- ...service-host.yaml => service-gateway.yaml} | 4 +- deploy/k8s/service-workers.yaml | 14 +++ 6 files changed, 153 insertions(+), 28 deletions(-) create mode 100644 deploy/k8s/deployment-gateway.yaml rename deploy/k8s/{deployment-host.yaml => deployment-workers.yaml} (72%) rename deploy/k8s/{service-host.yaml => service-gateway.yaml} (81%) create mode 100644 deploy/k8s/service-workers.yaml diff --git a/deploy/k8s/configmap.yaml b/deploy/k8s/configmap.yaml index f3a332c..1df1c20 100644 --- a/deploy/k8s/configmap.yaml +++ b/deploy/k8s/configmap.yaml @@ -4,22 +4,19 @@ metadata: name: sor-config namespace: stone-server data: - # ── SOR.Host ──────────────────────────────────────────────────────────────── - ENABLED_SERVICES: "all" + # ── Shared config (all pods) ───────────────────────────────────────────────── + # ENABLED_SERVICES and Services__Auth are NOT set here — each deployment sets + # them directly in its own env block so gateway and workers are independently + # configured without a shared ConfigMap override. ASPNETCORE_URLS: "http://+:8080" ASPNETCORE_ENVIRONMENT: "Production" - # WebSocket gateway settings + # WebSocket / gateway settings (consumed only by pods running the gateway service) WebSocket__Path: "/ws" Gateway__AuthTimeoutMs: "10000" Gateway__MaxMessagesPerSecond: "20" - # gRPC: gateway calls auth at this address. - # In single-pod (ENABLED_SERVICES=all) mode both run in the same process on the same port. - Services__Auth: "http://localhost:8080" - - # NATS message bus — update to your NATS service address. - # TODO: replace with your actual NATS service URL. + # NATS message bus Nats__Url: "nats://nats.stone-server.svc.cluster.local:4222" # ── SOR.Admin ─────────────────────────────────────────────────────────────── diff --git a/deploy/k8s/deployment-gateway.yaml b/deploy/k8s/deployment-gateway.yaml new file mode 100644 index 0000000..74b264e --- /dev/null +++ b/deploy/k8s/deployment-gateway.yaml @@ -0,0 +1,111 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sor-gateway + namespace: stone-server + labels: + app: sor-gateway +spec: + replicas: 2 + selector: + matchLabels: + app: sor-gateway + template: + metadata: + labels: + app: sor-gateway + spec: + imagePullSecrets: + - name: harbor-credentials + + containers: + - name: sor-gateway + image: registry.aeternalisgames.com/sor/sor-server:latest + ports: + - containerPort: 8080 + + # ── Shared non-secret config ─────────────────────────────────────── + envFrom: + - configMapRef: + name: sor-config + + # ── Gateway-specific overrides + secrets ─────────────────────────── + env: + # This pod runs only the WebSocket gateway and the auth gRPC service. + # Auth is co-located so JWT validation has no extra network hop. + - name: ENABLED_SERVICES + value: "gateway,auth" + + # Auth gRPC is in the same process — call ourselves. + - name: Services__Auth + value: "http://localhost:8080" + + # Postgres — Identity schema only (auth needs it for account lookups). + - name: ConnectionStrings__Default + valueFrom: + secretKeyRef: + name: sor-app-secrets + key: ConnectionStrings__Default + + # Redis — used by gateway for client push delivery. + - name: ConnectionStrings__Redis + valueFrom: + secretKeyRef: + name: redis-credentials + key: connection-string + + # CDN — not used by gateway/auth, but included to keep secret binding uniform. + - name: Cdn__PublicBaseUrl + valueFrom: + secretKeyRef: + name: sor-app-secrets + key: Cdn__PublicBaseUrl + - name: Cdn__EndpointUrl + valueFrom: + secretKeyRef: + name: sor-app-secrets + key: Cdn__EndpointUrl + - name: Cdn__AccessKey + valueFrom: + secretKeyRef: + name: sor-app-secrets + key: Cdn__AccessKey + - name: Cdn__SecretKey + valueFrom: + secretKeyRef: + name: sor-app-secrets + key: Cdn__SecretKey + - name: Cdn__BucketName + valueFrom: + secretKeyRef: + name: sor-app-secrets + key: Cdn__BucketName + + # ── Health checks ────────────────────────────────────────────────── + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + + readinessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + + # ── Resources ───────────────────────────────────────────────────── + # Lighter than workers: no business-logic DB connections. + resources: + requests: + cpu: "150m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" diff --git a/deploy/k8s/deployment-host.yaml b/deploy/k8s/deployment-workers.yaml similarity index 72% rename from deploy/k8s/deployment-host.yaml rename to deploy/k8s/deployment-workers.yaml index b24336a..1ba492a 100644 --- a/deploy/k8s/deployment-host.yaml +++ b/deploy/k8s/deployment-workers.yaml @@ -1,54 +1,56 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: sor-host + name: sor-workers namespace: stone-server labels: - app: sor-host + app: sor-workers spec: - replicas: 1 + replicas: 3 selector: matchLabels: - app: sor-host + app: sor-workers template: metadata: labels: - app: sor-host + app: sor-workers spec: - # Pull images from Harbor using the harbor-credentials secret. imagePullSecrets: - name: harbor-credentials containers: - - name: sor-host - # CI updates this image tag on every push to master. + - name: sor-workers image: registry.aeternalisgames.com/sor/sor-server:latest ports: - containerPort: 8080 - # ── Non-secret config from ConfigMap ────────────────────────────── + # ── Shared non-secret config ─────────────────────────────────────── envFrom: - configMapRef: name: sor-config - # ── Secrets injected as individual env vars ──────────────────────── + # ── Workers-specific overrides + secrets ─────────────────────────── env: - # Postgres — from the sor-app-secrets Secret (see deploy/secrets/). + # Business-logic services only — no WebSocket, no auth gRPC. + # These pods subscribe to NATS and handle all game/store/social messages. + - name: ENABLED_SERVICES + value: "chat,collection,bazaar,content,versioning,matchmaking,game" + + # Postgres — all schemas (chat, collection, bazaar, catalog, game, identity). - name: ConnectionStrings__Default valueFrom: secretKeyRef: name: sor-app-secrets key: ConnectionStrings__Default - # Redis — from the pre-existing redis-credentials Secret. - # TODO: verify the key name matches your redis-credentials secret. + # Redis — used by services for client push delivery (IClientPush). - name: ConnectionStrings__Redis valueFrom: secretKeyRef: name: redis-credentials key: connection-string - # CDN (Ceph RadosGW S3-compatible) + # CDN — used by content and bazaar services for asset URLs. - name: Cdn__PublicBaseUrl valueFrom: secretKeyRef: @@ -95,10 +97,11 @@ spec: failureThreshold: 3 # ── Resources ───────────────────────────────────────────────────── + # Heavier than gateway: holds DB connection pools for all service schemas. resources: requests: - cpu: "250m" - memory: "256Mi" + cpu: "300m" + memory: "384Mi" limits: cpu: "1000m" memory: "1Gi" diff --git a/deploy/k8s/ingress.yaml b/deploy/k8s/ingress.yaml index 5bafe30..9f9e5a3 100644 --- a/deploy/k8s/ingress.yaml +++ b/deploy/k8s/ingress.yaml @@ -43,7 +43,7 @@ spec: pathType: Prefix backend: service: - name: sor-host + name: sor-gateway port: number: 80 diff --git a/deploy/k8s/service-host.yaml b/deploy/k8s/service-gateway.yaml similarity index 81% rename from deploy/k8s/service-host.yaml rename to deploy/k8s/service-gateway.yaml index 93dbc93..89d3e67 100644 --- a/deploy/k8s/service-host.yaml +++ b/deploy/k8s/service-gateway.yaml @@ -1,11 +1,11 @@ apiVersion: v1 kind: Service metadata: - name: sor-host + name: sor-gateway namespace: stone-server spec: selector: - app: sor-host + app: sor-gateway ports: - name: http protocol: TCP diff --git a/deploy/k8s/service-workers.yaml b/deploy/k8s/service-workers.yaml new file mode 100644 index 0000000..72339d8 --- /dev/null +++ b/deploy/k8s/service-workers.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: sor-workers + namespace: stone-server +spec: + selector: + app: sor-workers + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 8080 + type: ClusterIP