Add Redis, Wal, Services

This commit is contained in:
dingo
2025-11-10 07:42:03 -05:00
parent 7339d537f9
commit 72bc097a71
8 changed files with 386 additions and 25 deletions
+5
View File
@@ -0,0 +1,5 @@
namespace Server.Persistence;
public interface IWalPublisher
{
Task PublishAsync(string type, ulong charId, ulong zoneId, object payload, long tickId);
}
+28
View File
@@ -0,0 +1,28 @@
using System.Text.Json;
using Server.Redis;
namespace Server.Persistence;
public sealed class WalPublisher : IWalPublisher
{
private readonly IRedisManager _redis;
public WalPublisher(IRedisManager redis) { _redis = redis; }
public async Task PublishAsync(string type, ulong charId, ulong zoneId, object payload, long tickId)
{
var ev = new
{
eventId = Guid.NewGuid().ToString("N"),
type,
charId,
zoneId,
tickId,
ts = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
payload = payload
};
var json = JsonSerializer.Serialize(ev);
await _redis.Db.StreamAddAsync("wal:game", new StackExchange.Redis.NameValueEntry[] {
new("data", json)
}, maxLength: 1_000_000, useApproximateMaxLength: true);
}
}
+61 -25
View File
@@ -4,24 +4,26 @@ using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Hosting;
using Server.Data;
using Server.Networking;
using Server.Persistence;
using Server.Redis;
using Server.Services;
var builder = WebApplication.CreateBuilder(args);
//Test2
// ---- Build connection string from env (K8s secrets env vars) ----
// ---- MySQL conn string from env ----
string mysqlConnectionString = ConnectionStringFactory.BuildMySqlFromEnv();
// ---- Service registrations (MUST be before Build) ----
// ---- Services (before Build) ----
builder.Services.AddSingleton<MessageRouter>();
builder.Services.AddSingleton<NetworkManager>();
// EF Core (single registration)
builder.Services.AddDbContext<GameDbContext>(options =>
options.UseMySql(mysqlConnectionString, ServerVersion.AutoDetect(mysqlConnectionString)));
// DatabaseOptions (used by DatabaseManager if you wired it)
builder.Services.AddOptions<DatabaseOptions>()
.Configure(o =>
{
@@ -33,22 +35,39 @@ builder.Services.AddOptions<DatabaseOptions>()
.Validate(o => !string.IsNullOrWhiteSpace(o.ConnectionString), "DB connection string required")
.ValidateOnStart();
// Redis + zone leases + WAL
builder.Services.AddSingleton<IRedisManager, RedisManager>();
builder.Services.AddSingleton<IZoneLeaseService, ZoneLeaseService>();
builder.Services.AddHostedService(sp => (ZoneLeaseService)sp.GetRequiredService<IZoneLeaseService>());
builder.Services.AddSingleton<IWalPublisher, WalPublisher>();
// Health checks (optional DB/Redis implementations)
builder.Services.AddHealthChecks()
.AddCheck<DatabaseHealthCheck>("mysql", tags: new[] { "ready" })
//.AddCheck<RedisHealthCheck>("redis", tags: new[] { "ready" })
;
.AddCheck<DatabaseHealthCheck>("mysql", tags: new[] { "ready" });
// .AddCheck<RedisHealthCheck>("redis", tags: new[] { "ready" });
// ---- Build the app AFTER all service registrations ----
var app = builder.Build();
// ---- Health endpoints ----
app.MapGet("/health/live", () => Results.Ok(new { status = "live" }));
app.MapGet("/health/ready", () => Results.Ok(new { status = "ready" }));
// ---- WebSockets endpoint ----
app.UseWebSockets(new WebSocketOptions
app.MapGet("/health/ready", async (IRedisManager rm, HealthCheckService hc) =>
{
KeepAliveInterval = TimeSpan.FromSeconds(20),
ReceiveBufferSize = 64 * 1024
// Ping Redis and run registered health checks tagged "ready"
try
{
var pong = await rm.Db.PingAsync();
var report = await hc.CheckHealthAsync(r => r.Tags.Contains("ready"));
var ok = report.Status == HealthStatus.Healthy;
return ok
? Results.Ok(new { status = "ready", redisLatencyMs = pong.TotalMilliseconds })
: Results.StatusCode(503);
}
catch { return Results.StatusCode(503); }
});
// ---- WebSockets endpoint ----
app.UseWebSockets(new WebSocketOptions { KeepAliveInterval = TimeSpan.FromSeconds(20) });
app.Map("/ws", async (HttpContext httpContext, NetworkManager networkManager) =>
{
if (!httpContext.WebSockets.IsWebSocketRequest)
@@ -56,21 +75,38 @@ app.Map("/ws", async (HttpContext httpContext, NetworkManager networkManager) =>
httpContext.Response.StatusCode = StatusCodes.Status400BadRequest;
return;
}
var acceptedWebSocket = await httpContext.WebSockets.AcceptWebSocketAsync();
await networkManager.AcceptWebSocketAsync(httpContext, acceptedWebSocket);
var ws = await httpContext.WebSockets.AcceptWebSocketAsync();
await networkManager.AcceptWebSocketAsync(httpContext, ws);
});
// ---- Start TCP listener (after Build) ----
// ---- Start TCP listener ----
int tcpPort = int.TryParse(Environment.GetEnvironmentVariable("TCP_PORT"), out var parsed) ? parsed : 40000;
var tcpListener = new TcpListener(IPAddress.Any, tcpPort);
tcpListener.Start();
app.Logger.LogInformation("TCP listening on {Port}", tcpPort);
TcpListener? tcpListener = null;
try
{
tcpListener = new TcpListener(IPAddress.Any, tcpPort);
tcpListener.Start();
app.Logger.LogInformation("TCP listening on {Port}", tcpPort);
}
catch (Exception ex)
{
app.Logger.LogCritical(ex, "Failed to start TCP listener on port {Port}", tcpPort);
throw; // fail fast so k8s restarts the pod
}
var networkManager = app.Services.GetRequiredService<NetworkManager>();
_ = networkManager.AcceptTcpLoopAsync(tcpListener, app.Lifetime.ApplicationStopping);
// ---- Tick loop ----
_ = networkManager.AcceptTcpLoopAsync(tcpListener!, app.Lifetime.ApplicationStopping);
_ = networkManager.RunTickLoopAsync(app.Lifetime.ApplicationStopping);
// ---- Run ----
// Graceful shutdown for TCP
app.Lifetime.ApplicationStopping.Register(() =>
{
try { tcpListener?.Stop(); } catch { }
});
// ---- (Optional) acquire a dev zone on startup ----
// var leases = app.Services.GetRequiredService<IZoneLeaseService>();
// _ = leases.TryAcquireAsync(zoneId: 1);
await app.RunAsync();
+178
View File
@@ -0,0 +1,178 @@
using System;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StackExchange.Redis;
using System.Linq;
namespace Server.Redis;
public interface IRedisManager : IAsyncDisposable
{
IDatabase Db { get; }
ISubscriber Bus { get; }
Task<bool> AcquireLeaseAsync(string key, string value, TimeSpan ttl);
Task<bool> RenewLeaseAsync(string key, string value, TimeSpan ttl); // compare-and-set
Task<bool> ReleaseLeaseAsync(string key, string value);
Task<bool> SetJsonAsync<T>(string key, T value, TimeSpan? ttl = null);
Task<T?> GetJsonAsync<T>(string key);
Task<long> StreamAddAsync(string stream, string json, string? id = "*");
Task<RedisValue[]> StreamReadGroupAsync(string stream, string group, string consumer, int count = 100, int blockMs = 0);
Task AcknowledgeAsync(string stream, string group, params string[] ids);
}
public sealed class RedisManager : IRedisManager
{
private readonly ILogger<RedisManager> _logger;
private readonly ConnectionMultiplexer _mux;
public IDatabase Db => _mux.GetDatabase();
public ISubscriber Bus => _mux.GetSubscriber();
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = false
};
public RedisManager(ILogger<RedisManager> logger)
{
_logger = logger;
var connString = BuildRedisConnectionStringFromEnv();
_mux = ConnectionMultiplexer.Connect(connString);
_logger.LogInformation("Connected to Redis: {Endpoints}",
string.Join(",", _mux.GetEndPoints().Select(ep => ep.ToString())));
}
public async ValueTask DisposeAsync()
{
try { await _mux.CloseAsync(); } catch { }
_mux.Dispose();
}
// ---------- Leases (simple Redlock-ish) ----------
public async Task<bool> AcquireLeaseAsync(string key, string value, TimeSpan ttl)
{
return await Db.StringSetAsync(key, value, ttl, When.NotExists);
}
public async Task<bool> RenewLeaseAsync(string key, string value, TimeSpan ttl)
{
// Lua script: if value matches, update TTL (PEXPIRE) and keep value
const string script = @"
if redis.call('GET', KEYS[1]) == ARGV[1] then
redis.call('PEXPIRE', KEYS[1], ARGV[2])
return 1
else
return 0
end";
var res = (int)(long)await Db.ScriptEvaluateAsync(script,
new RedisKey[] { key },
new RedisValue[] { value, (long)ttl.TotalMilliseconds });
return res == 1;
}
public async Task<bool> ReleaseLeaseAsync(string key, string value)
{
// Lua script: delete only if owned by 'value'
const string script = @"
if redis.call('GET', KEYS[1]) == ARGV[1] then
return redis.call('DEL', KEYS[1])
else
return 0
end";
var res = (int)(long)await Db.ScriptEvaluateAsync(script,
new RedisKey[] { key },
new RedisValue[] { value });
return res == 1;
}
// ---------- JSON helpers ----------
public async Task<bool> SetJsonAsync<T>(string key, T value, TimeSpan? ttl = null)
{
var bytes = JsonSerializer.SerializeToUtf8Bytes(value, JsonOptions);
return await Db.StringSetAsync(key, bytes, ttl);
}
public async Task<T?> GetJsonAsync<T>(string key)
{
var bytes = await Db.StringGetAsync(key);
if (bytes.IsNullOrEmpty) return default;
return JsonSerializer.Deserialize<T>((byte[])bytes!, JsonOptions);
}
// ---------- Streams (write-ahead) ----------
public async Task<long> StreamAddAsync(string stream, string json, string? id = "*")
{
var entryId = await Db.StreamAddAsync(stream, "data", json,
maxLength: null, useApproximateMaxLength: false, messageId: id ?? "*");
return entryId.HasValue ? 1 : 0;
}
public async Task<RedisValue[]> StreamReadGroupAsync(
string stream, string group, string consumer, int count = 100, int blockMs = 0)
{
// Ensure group exists (create if not present)
try
{
await Db.StreamCreateConsumerGroupAsync(stream, group, "$", createStream: true);
}
catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP"))
{
// group already exists
}
var deadline = blockMs > 0 ? DateTime.UtcNow.AddMilliseconds(blockMs) : (DateTime?)null;
StackExchange.Redis.StreamEntry[] entries;
// Poll until we have entries or we hit the deadline
while (true)
{
entries = await Db.StreamReadGroupAsync(stream, group, consumer, count: count, noAck: false);
if (entries.Length > 0) break;
if (deadline == null) break;
var remainingMs = (int)(deadline.Value - DateTime.UtcNow).TotalMilliseconds;
if (remainingMs <= 0) break;
// small sleep to avoid hot loop
await Task.Delay(Math.Min(remainingMs, 100));
}
// Return the "data" field only (matches your current interface)
var result = new RedisValue[entries.Length];
for (int i = 0; i < entries.Length; i++)
result[i] = entries[i].Values[0].Value; // field "data"
return result;
}
public async Task AcknowledgeAsync(string stream, string group, params string[] ids)
{
if (ids.Length == 0) return;
await Db.StreamAcknowledgeAsync(stream, group, Array.ConvertAll(ids, x => (RedisValue)x));
}
// ---------- Conn string from env ----------
private static string BuildRedisConnectionStringFromEnv()
{
var full = Environment.GetEnvironmentVariable("REDIS_CONNECTION_STRING");
if (!string.IsNullOrWhiteSpace(full)) return full;
var host = Environment.GetEnvironmentVariable("REDIS_HOST") ?? "localhost";
var port = int.TryParse(Environment.GetEnvironmentVariable("REDIS_PORT"), out var p) ? p : 6379;
var password = Environment.GetEnvironmentVariable("REDIS_PASSWORD");
var db = int.TryParse(Environment.GetEnvironmentVariable("REDIS_DB"), out var d) ? d : 0;
var cs = $"{host}:{port},defaultDatabase={db},abortConnect=false,syncTimeout=10000,connectRetry=3,connectTimeout=5000";
if (!string.IsNullOrWhiteSpace(password)) cs += $",password={password}";
return cs;
}
}
+9
View File
@@ -0,0 +1,9 @@
namespace Server.Services;
public interface IZoneLeaseService
{
ValueTask<bool> TryAcquireAsync(ulong zoneId, CancellationToken ct = default);
ValueTask<bool> RenewAsync(ulong zoneId, CancellationToken ct = default);
ValueTask ReleaseAsync(ulong zoneId, CancellationToken ct = default);
bool IsOwner(ulong zoneId);
long CurrentFence(ulong zoneId);
}
+67
View File
@@ -0,0 +1,67 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using Server.Redis;
namespace Server.Services;
public sealed class ZoneLeaseService : IZoneLeaseService, IHostedService, IDisposable
{
private readonly IRedisManager _redis;
private readonly ILogger<ZoneLeaseService> _log;
private readonly string _podId = Environment.GetEnvironmentVariable("POD_ID") ?? Guid.NewGuid().ToString("N");
private readonly TimeSpan _ttl = TimeSpan.FromSeconds(5);
private readonly TimeSpan _renew = TimeSpan.FromSeconds(2);
private readonly ConcurrentDictionary<ulong, long> _owned = new();
private Timer? _timer;
public ZoneLeaseService(IRedisManager redis, ILogger<ZoneLeaseService> log)
{ _redis = redis; _log = log; }
public bool IsOwner(ulong zoneId) => _owned.ContainsKey(zoneId);
public long CurrentFence(ulong zoneId) => _owned.TryGetValue(zoneId, out var f) ? f : -1;
public async ValueTask<bool> TryAcquireAsync(ulong zoneId, CancellationToken ct = default)
{
var fence = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
var val = $"{_podId}:{fence}";
var key = $"zone:owner:{{{zoneId}}}"; // hash tag for cluster
var ok = await _redis.Db.StringSetAsync(key, val, _ttl, when: StackExchange.Redis.When.NotExists);
if (ok) { _owned[zoneId] = fence; _log.LogInformation("Zone {Zone} acquired, fence={Fence}", zoneId, fence); }
return ok;
}
public async ValueTask<bool> RenewAsync(ulong zoneId, CancellationToken ct = default)
{
if (!_owned.TryGetValue(zoneId, out var fence)) return false;
var key = $"zone:owner:{{{zoneId}}}";
// renew if matches (Lua CAS)
const string lua = @"
if redis.call('GET', KEYS[1]) == ARGV[1] then
redis.call('PEXPIRE', KEYS[1], ARGV[2]); return 1
else return 0 end";
var res = (int)(long)await _redis.Db.ScriptEvaluateAsync(lua,
new StackExchange.Redis.RedisKey[] { key },
new StackExchange.Redis.RedisValue[] { $"{_podId}:{fence}", (long)_ttl.TotalMilliseconds });
return res == 1;
}
public async ValueTask ReleaseAsync(ulong zoneId, CancellationToken ct = default)
{
if (!_owned.TryRemove(zoneId, out var fence)) return;
var key = $"zone:owner:{{{zoneId}}}";
const string lua = @"if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end";
await _redis.Db.ScriptEvaluateAsync(lua,
new StackExchange.Redis.RedisKey[] { key },
new StackExchange.Redis.RedisValue[] { $"{_podId}:{fence}" });
_log.LogInformation("Zone {Zone} released", zoneId);
}
// Background renew loop
public Task StartAsync(CancellationToken _) { _timer = new Timer(async _ => await RenewAll(), null, _renew, _renew); return Task.CompletedTask; }
public Task StopAsync(CancellationToken _) { _timer?.Dispose(); return Task.CompletedTask; }
private async Task RenewAll()
{
foreach (var z in _owned.Keys) { if (!await RenewAsync(z)) _owned.TryRemove(z, out _); }
}
public void Dispose() => _timer?.Dispose();
}
+1
View File
@@ -20,6 +20,7 @@
<PackageReference Include="MySqlConnector" Version="2.4.0" />
<PackageReference Include="Polly" Version="8.6.4" />
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="9.0.0" />
<PackageReference Include="StackExchange.Redis" Version="2.9.32" />
</ItemGroup>
<ItemGroup>
+37
View File
@@ -0,0 +1,37 @@
using StackExchange.Redis;
namespace Server.World;
public static class SpatialIndex
{
public const int CellSize = 50; // units
public static (int cx, int cy, int cz) Cell(double x, double y, double z) =>
((int)Math.Floor(x / CellSize), (int)Math.Floor(y / CellSize), (int)Math.Floor(z / CellSize));
public static async Task UpdateAsync(IDatabase db, ulong zoneId, ulong charId,
(int cx, int cy, int cz) oldCell, (int cx, int cy, int cz) newCell, TimeSpan ttl)
{
var oldKey = $"z:{{{zoneId}}}:c:{oldCell.cx}:{oldCell.cy}:{oldCell.cz}";
var newKey = $"z:{{{zoneId}}}:c:{newCell.cx}:{newCell.cy}:{newCell.cz}";
var batch = db.CreateBatch();
var tasks = new List<Task>();
if (oldCell != newCell)
tasks.Add(batch.SetRemoveAsync(oldKey, charId));
tasks.Add(batch.SetAddAsync(newKey, charId));
tasks.Add(batch.KeyExpireAsync(newKey, ttl));
batch.Execute();
await Task.WhenAll(tasks);
}
public static IEnumerable<string> NeighborCellKeys(ulong zoneId, (int cx, int cy, int cz) c, int r = 1)
{
for (int dx = -r; dx <= r; dx++)
for (int dy = -r; dy <= r; dy++)
yield return $"z:{{{zoneId}}}:c:{c.cx + dx}:{c.cy + dy}:{c.cz}";
}
}