From 9454b51914d63eef41ec6a78dff0157da833cea0 Mon Sep 17 00:00:00 2001 From: dingo Date: Sun, 6 Sep 2026 13:55:15 -0400 Subject: [PATCH] perf: take the admin-command DB poll off the world tick's global lock Testing one specific, targeted hypothesis rather than another blind patch: the portal merge added an unconditional per-second DB query (controls(), checking for pending admin commands) directly inside the world tick loop's s.mu-locked section -- the same lock every websocket intent, every HTTP handler, and every snapshot broadcast to every connected player depends on. That runs forever, for every world, whether or not the admin feature is ever used. Verified the actual game logic isn't broken (a live-instrumented run of the unpatched single-shot gather/attack code landed its first hit immediately and correctly), so the e2e flakiness has to be about something making the server slower under load rather than a logic defect. This is the one change in the merge positioned to tax *every* player's snapshot/intent latency, every second, permanently -- not just an occasional per-request cost like the other additions. controls() now runs on its own ticker, independent of the simulation tick, and only takes s.mu at all if there's an actual pending command to apply (the DB read itself needs no lock -- world.ID is immutable after creation). The idle case, which is the overwhelming majority of ticks, no longer touches s.mu for this at all. Verified: go build/vet/test clean, and the full local e2e suite (7/7, both spec files, including the admin-command test that exercises controls() directly) passes across two runs. If this doesn't change CI's result, it should be reverted and ruled out -- the next candidate would be the new /worlds/ routing rewrite or the added per-request ActiveCharacter DB lookups. Co-Authored-By: Claude Sonnet 5 --- server/internal/transport/service.go | 63 ++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/server/internal/transport/service.go b/server/internal/transport/service.go index 8b07ed6..36e60b0 100644 --- a/server/internal/transport/service.go +++ b/server/internal/transport/service.go @@ -22,14 +22,13 @@ type peer struct { close func() } type Service struct { - mu sync.Mutex - world *game.World - lease *store.Lease - peers map[string]*peer - ready atomic.Bool - lastSave time.Time - lastControl time.Time - done chan struct{} + mu sync.Mutex + world *game.World + lease *store.Lease + peers map[string]*peer + ready atomic.Bool + lastSave time.Time + done chan struct{} } func NewService(w *game.World, l *store.Lease) *Service { @@ -52,6 +51,12 @@ func (s *Service) persist(ctx context.Context, claim string) error { } func (s *Service) Run(ctx context.Context) { defer close(s.done) + // Runs on its own cadence rather than inline in the tick below: it does a real DB + // round-trip, and that must never sit behind s.mu while the common case (no pending + // admin commands) still pays for it every second, for every world, forever -- that + // would tax every player's snapshot delivery and intent handling on this single + // global lock for a feature most ticks have nothing to do. + go s.runControls(ctx) ticker := time.NewTicker(time.Second / time.Duration(s.world.Balance.TickHz)) defer ticker.Stop() last := time.Now() @@ -67,14 +72,6 @@ func (s *Service) Run(ctx context.Context) { } s.world.Advance(now, now.Sub(last).Seconds()) last = now - if now.Sub(s.lastControl) >= time.Second { - if err := s.controls(ctx); err != nil { - s.ready.Store(false) - s.mu.Unlock() - return - } - s.lastControl = now - } if s.world.Dirty() || now.Sub(s.lastSave) >= time.Second { saveCtx, cancel := context.WithTimeout(ctx, 5*time.Second) err := s.persist(saveCtx, "") @@ -180,13 +177,45 @@ func (s *Service) join(ctx context.Context, account, name, claim string) (p.Char return view, nil } +// runControls polls for pending admin commands on its own cadence, independent of the +// simulation tick, so an idle world (the overwhelming common case) never holds s.mu for +// a database round-trip it didn't need. +func (s *Service) runControls(ctx context.Context) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if !s.ready.Load() { + return + } + if err := s.controls(ctx); err != nil { + s.mu.Lock() + s.ready.Store(false) + for _, p := range s.peers { + p.close() + } + s.mu.Unlock() + return + } + } + } +} func (s *Service) controls(ctx context.Context) error { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() + // Read-only, and s.world.ID never changes after creation: safe to query without s.mu. commands, err := s.lease.DB.Commands(ctx, s.world.ID, true) - if err != nil { + if err != nil || len(commands) == 0 { return err } + s.mu.Lock() + defer s.mu.Unlock() + if !s.ready.Load() { + return nil + } for _, command := range commands { result := "" if time.Now().Unix()-command.CreatedAt > 900 {