Files
sor/Server/src/SOR.Game/GameStartupCleanupService.cs
T
530b56c33c Fix: ghost-match bug — session stays Active after game ends
Two root causes fixed:

1. Race condition (primary cause): SaveStateAsync committed the
   transaction with Phase=GameOver in StateJson, but session.Status
   was not set to Completed until a separate SaveChangesAsync in
   FinalizeGameAsync. If the server crashed or any exception was
   thrown between those two calls, the session was left Active with
   a GameOver state — causing the "active game" prompt on next login.

   Fix: each game-over handler (concede, force_concede, attack) now
   sets session.Status/WinnerAccountId/WinCondition/EndedAt BEFORE
   calling SaveStateAsync, so all fields commit atomically in the
   same transaction.

2. Wrong save order in FinalizeGameAsync: identityDb.SaveChangesAsync
   (ELO update) ran before gameDb.SaveChangesAsync (Status=Completed).
   Any transient identity-DB error left the session permanently Active.

   Fix: gameDb is saved first; ELO runs after, so a failed ELO update
   never causes a ghost match.

3. Defensive startup heal (GameStartupCleanupService): on server
   start, Active sessions whose StateJson contains Phase=GameOver are
   now detected and marked Completed. game.over is pushed to any
   still-connected clients so they can exit cleanly.

Generated with [Claude Code](https://claude.ai/code)
via [Happy](https://happy.engineering)

Co-Authored-By: Claude <noreply@anthropic.com>
Co-Authored-By: Happy <yesreply@happy.engineering>
2026-03-18 20:02:10 -04:00

174 lines
7.8 KiB
C#

namespace SOR.Game;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using SOR.Database.Contexts;
using SOR.Database.Enums;
using SOR.Networking.Abstractions.Push;
using SOR.Shared.Networking;
/// <summary>
/// Runs once at startup before the service begins accepting WebSocket messages.
///
/// Any GameSession stuck in the Initializing state means the previous Game process
/// died (redeploy, crash) before both players sent game.ready. The clients on
/// those sessions are stuck on a loading screen and will never recover on their own
/// because they already sent game.ready to the old process.
///
/// This service abandons Initializing sessions that are clearly stale (older than
/// StaleThresholdSeconds) and pushes game.session_abandoned to both players so
/// their clients can return to the main menu.
///
/// The staleness threshold exists to avoid a race condition: MatchmakerService is
/// registered before GameStartupCleanupService and its BackgroundService.ExecuteAsync
/// fires immediately (no leading delay) on a background thread. If queued players
/// were waiting when the process restarted, the matchmaker can create a brand-new
/// Initializing session within the first few milliseconds of startup — before this
/// service's query runs. Excluding sessions younger than 60 s ensures those fresh
/// sessions are left alone so game.ready can proceed normally.
/// </summary>
internal sealed class GameStartupCleanupService : IHostedService
{
private const int StaleThresholdSeconds = 60;
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<GameStartupCleanupService> _logger;
public GameStartupCleanupService(
IServiceScopeFactory scopeFactory,
ILogger<GameStartupCleanupService> logger)
{
_scopeFactory = scopeFactory;
_logger = logger;
}
public async Task StartAsync(CancellationToken cancellationToken)
{
await using var scope = _scopeFactory.CreateAsyncScope();
var gameDb = scope.ServiceProvider.GetRequiredService<GameDbContext>();
var clientPush = scope.ServiceProvider.GetRequiredService<IClientPush>();
// Only target sessions old enough to be definitely stuck.
// Sessions created in the last 60 s may still be mid-handshake.
var staleThreshold = DateTimeOffset.UtcNow.AddSeconds(-StaleThresholdSeconds);
// ── 1. Abandon stale Initializing sessions ────────────────────────────
var stuck = await gameDb.GameSessions
.Where(s => s.Status == GameSessionStatus.Initializing
&& s.CreatedAt <= staleThreshold)
.ToListAsync(cancellationToken);
if (stuck.Count > 0)
{
_logger.LogWarning(
"Startup cleanup: abandoning {Count} Initializing session(s) left over from previous process.",
stuck.Count);
var now = DateTimeOffset.UtcNow;
var push = new GameSessionAbandonedPush(Guid.Empty, "server_restart");
foreach (var session in stuck)
{
session.Status = GameSessionStatus.Abandoned;
session.EndedAt = now;
session.WinCondition = "server_restart";
// Best-effort push — clients on the loading screen will get a signal to
// return to menu. Ignore errors so a bad Redis connection doesn't block
// the rest of the cleanup or prevent the service from starting.
try
{
var sessionPush = push with { SessionId = session.Id };
await Task.WhenAll(
clientPush.PushAsync(
session.PlayerOneAccountId.ToString(),
ServerMessage.Push("game.session_abandoned", sessionPush),
cancellationToken),
clientPush.PushAsync(
session.PlayerTwoAccountId.ToString(),
ServerMessage.Push("game.session_abandoned", sessionPush),
cancellationToken));
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Could not push session_abandoned for session {SessionId}; clients will time out naturally.",
session.Id);
}
}
}
// ── 2. Heal ghost-Active sessions (StateJson has Phase=GameOver but Status=Active) ──
//
// These are left behind when the server crashes in the narrow window between
// SaveStateAsync committing the state and FinalizeGameAsync saving the status.
// They show up as "active game" on the player's next login even though the game
// is truly over. We can't award ELO retroactively, so we just mark Completed
// and push game.over with zeroed deltas so both clients can return to the menu.
var ghostActive = await gameDb.GameSessions
.Where(s => s.Status == GameSessionStatus.Active && s.StateJson != null)
.ToListAsync(cancellationToken);
var ghostCount = 0;
foreach (var session in ghostActive)
{
try
{
var state = GameEngine.Deserialize(session.StateJson!);
if (state.Phase != GamePhase.GameOver) continue;
_logger.LogWarning(
"Startup cleanup: session {SessionId} is Active but StateJson has Phase=GameOver — marking Completed.",
session.Id);
session.Status = GameSessionStatus.Completed;
session.EndedAt ??= DateTimeOffset.UtcNow;
// WinnerAccountId may already be set if the crash was after setting it; leave it.
ghostCount++;
// Push game.over so any still-connected clients can exit cleanly.
try
{
var ghostPush = new GameOverPush(
SessionId : session.Id,
WinnerAccountId : session.WinnerAccountId ?? Guid.Empty,
LoserAccountId : Guid.Empty,
WinCondition : session.WinCondition ?? "server_restart",
WinnerRankDelta : 0,
LoserRankDelta : 0,
WinnerOpDelta : 0,
LoserOpDelta : 0);
await Task.WhenAll(
clientPush.PushAsync(session.PlayerOneAccountId.ToString(),
ServerMessage.Push("game.over", ghostPush), cancellationToken),
clientPush.PushAsync(session.PlayerTwoAccountId.ToString(),
ServerMessage.Push("game.over", ghostPush), cancellationToken));
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Could not push game.over for ghost session {SessionId}.", session.Id);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Could not process ghost session {SessionId} during startup cleanup.", session.Id);
}
}
await gameDb.SaveChangesAsync(cancellationToken);
_logger.LogInformation(
"Startup cleanup complete: {InitCount} Initializing session(s) abandoned, {GhostCount} ghost Active session(s) healed.",
stuck.Count, ghostCount);
}
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
}