Add Redis, Wal, Services
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
namespace Server.Persistence;
|
||||
public interface IWalPublisher
|
||||
{
|
||||
Task PublishAsync(string type, ulong charId, ulong zoneId, object payload, long tickId);
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
using System.Text.Json;
|
||||
using Server.Redis;
|
||||
|
||||
namespace Server.Persistence;
|
||||
|
||||
public sealed class WalPublisher : IWalPublisher
|
||||
{
|
||||
private readonly IRedisManager _redis;
|
||||
public WalPublisher(IRedisManager redis) { _redis = redis; }
|
||||
|
||||
public async Task PublishAsync(string type, ulong charId, ulong zoneId, object payload, long tickId)
|
||||
{
|
||||
var ev = new
|
||||
{
|
||||
eventId = Guid.NewGuid().ToString("N"),
|
||||
type,
|
||||
charId,
|
||||
zoneId,
|
||||
tickId,
|
||||
ts = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
|
||||
payload = payload
|
||||
};
|
||||
var json = JsonSerializer.Serialize(ev);
|
||||
await _redis.Db.StreamAddAsync("wal:game", new StackExchange.Redis.NameValueEntry[] {
|
||||
new("data", json)
|
||||
}, maxLength: 1_000_000, useApproximateMaxLength: true);
|
||||
}
|
||||
}
|
||||
+61
-25
@@ -4,24 +4,26 @@ using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Server.Data;
|
||||
using Server.Networking;
|
||||
using Server.Persistence;
|
||||
using Server.Redis;
|
||||
using Server.Services;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
//Test2
|
||||
// ---- Build connection string from env (K8s secrets env vars) ----
|
||||
|
||||
// ---- MySQL conn string from env ----
|
||||
string mysqlConnectionString = ConnectionStringFactory.BuildMySqlFromEnv();
|
||||
|
||||
// ---- Service registrations (MUST be before Build) ----
|
||||
// ---- Services (before Build) ----
|
||||
builder.Services.AddSingleton<MessageRouter>();
|
||||
builder.Services.AddSingleton<NetworkManager>();
|
||||
|
||||
// EF Core (single registration)
|
||||
builder.Services.AddDbContext<GameDbContext>(options =>
|
||||
options.UseMySql(mysqlConnectionString, ServerVersion.AutoDetect(mysqlConnectionString)));
|
||||
|
||||
// DatabaseOptions (used by DatabaseManager if you wired it)
|
||||
builder.Services.AddOptions<DatabaseOptions>()
|
||||
.Configure(o =>
|
||||
{
|
||||
@@ -33,22 +35,39 @@ builder.Services.AddOptions<DatabaseOptions>()
|
||||
.Validate(o => !string.IsNullOrWhiteSpace(o.ConnectionString), "DB connection string required")
|
||||
.ValidateOnStart();
|
||||
|
||||
// Redis + zone leases + WAL
|
||||
builder.Services.AddSingleton<IRedisManager, RedisManager>();
|
||||
builder.Services.AddSingleton<IZoneLeaseService, ZoneLeaseService>();
|
||||
builder.Services.AddHostedService(sp => (ZoneLeaseService)sp.GetRequiredService<IZoneLeaseService>());
|
||||
builder.Services.AddSingleton<IWalPublisher, WalPublisher>();
|
||||
|
||||
// Health checks (optional DB/Redis implementations)
|
||||
builder.Services.AddHealthChecks()
|
||||
.AddCheck<DatabaseHealthCheck>("mysql", tags: new[] { "ready" })
|
||||
//.AddCheck<RedisHealthCheck>("redis", tags: new[] { "ready" })
|
||||
;
|
||||
.AddCheck<DatabaseHealthCheck>("mysql", tags: new[] { "ready" });
|
||||
// .AddCheck<RedisHealthCheck>("redis", tags: new[] { "ready" });
|
||||
|
||||
// ---- Build the app AFTER all service registrations ----
|
||||
var app = builder.Build();
|
||||
|
||||
// ---- Health endpoints ----
|
||||
app.MapGet("/health/live", () => Results.Ok(new { status = "live" }));
|
||||
app.MapGet("/health/ready", () => Results.Ok(new { status = "ready" }));
|
||||
// ---- WebSockets endpoint ----
|
||||
app.UseWebSockets(new WebSocketOptions
|
||||
app.MapGet("/health/ready", async (IRedisManager rm, HealthCheckService hc) =>
|
||||
{
|
||||
KeepAliveInterval = TimeSpan.FromSeconds(20),
|
||||
ReceiveBufferSize = 64 * 1024
|
||||
// Ping Redis and run registered health checks tagged "ready"
|
||||
try
|
||||
{
|
||||
var pong = await rm.Db.PingAsync();
|
||||
var report = await hc.CheckHealthAsync(r => r.Tags.Contains("ready"));
|
||||
var ok = report.Status == HealthStatus.Healthy;
|
||||
return ok
|
||||
? Results.Ok(new { status = "ready", redisLatencyMs = pong.TotalMilliseconds })
|
||||
: Results.StatusCode(503);
|
||||
}
|
||||
catch { return Results.StatusCode(503); }
|
||||
});
|
||||
|
||||
// ---- WebSockets endpoint ----
|
||||
app.UseWebSockets(new WebSocketOptions { KeepAliveInterval = TimeSpan.FromSeconds(20) });
|
||||
|
||||
app.Map("/ws", async (HttpContext httpContext, NetworkManager networkManager) =>
|
||||
{
|
||||
if (!httpContext.WebSockets.IsWebSocketRequest)
|
||||
@@ -56,21 +75,38 @@ app.Map("/ws", async (HttpContext httpContext, NetworkManager networkManager) =>
|
||||
httpContext.Response.StatusCode = StatusCodes.Status400BadRequest;
|
||||
return;
|
||||
}
|
||||
|
||||
var acceptedWebSocket = await httpContext.WebSockets.AcceptWebSocketAsync();
|
||||
await networkManager.AcceptWebSocketAsync(httpContext, acceptedWebSocket);
|
||||
var ws = await httpContext.WebSockets.AcceptWebSocketAsync();
|
||||
await networkManager.AcceptWebSocketAsync(httpContext, ws);
|
||||
});
|
||||
// ---- Start TCP listener (after Build) ----
|
||||
|
||||
// ---- Start TCP listener ----
|
||||
int tcpPort = int.TryParse(Environment.GetEnvironmentVariable("TCP_PORT"), out var parsed) ? parsed : 40000;
|
||||
var tcpListener = new TcpListener(IPAddress.Any, tcpPort);
|
||||
tcpListener.Start();
|
||||
app.Logger.LogInformation("TCP listening on {Port}", tcpPort);
|
||||
TcpListener? tcpListener = null;
|
||||
|
||||
try
|
||||
{
|
||||
tcpListener = new TcpListener(IPAddress.Any, tcpPort);
|
||||
tcpListener.Start();
|
||||
app.Logger.LogInformation("TCP listening on {Port}", tcpPort);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
app.Logger.LogCritical(ex, "Failed to start TCP listener on port {Port}", tcpPort);
|
||||
throw; // fail fast so k8s restarts the pod
|
||||
}
|
||||
|
||||
var networkManager = app.Services.GetRequiredService<NetworkManager>();
|
||||
_ = networkManager.AcceptTcpLoopAsync(tcpListener, app.Lifetime.ApplicationStopping);
|
||||
|
||||
// ---- Tick loop ----
|
||||
_ = networkManager.AcceptTcpLoopAsync(tcpListener!, app.Lifetime.ApplicationStopping);
|
||||
_ = networkManager.RunTickLoopAsync(app.Lifetime.ApplicationStopping);
|
||||
|
||||
// ---- Run ----
|
||||
// Graceful shutdown for TCP
|
||||
app.Lifetime.ApplicationStopping.Register(() =>
|
||||
{
|
||||
try { tcpListener?.Stop(); } catch { }
|
||||
});
|
||||
|
||||
// ---- (Optional) acquire a dev zone on startup ----
|
||||
// var leases = app.Services.GetRequiredService<IZoneLeaseService>();
|
||||
// _ = leases.TryAcquireAsync(zoneId: 1);
|
||||
|
||||
await app.RunAsync();
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
using System;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StackExchange.Redis;
|
||||
using System.Linq;
|
||||
|
||||
namespace Server.Redis;
|
||||
|
||||
public interface IRedisManager : IAsyncDisposable
|
||||
{
|
||||
IDatabase Db { get; }
|
||||
ISubscriber Bus { get; }
|
||||
|
||||
Task<bool> AcquireLeaseAsync(string key, string value, TimeSpan ttl);
|
||||
Task<bool> RenewLeaseAsync(string key, string value, TimeSpan ttl); // compare-and-set
|
||||
Task<bool> ReleaseLeaseAsync(string key, string value);
|
||||
|
||||
Task<bool> SetJsonAsync<T>(string key, T value, TimeSpan? ttl = null);
|
||||
Task<T?> GetJsonAsync<T>(string key);
|
||||
|
||||
Task<long> StreamAddAsync(string stream, string json, string? id = "*");
|
||||
Task<RedisValue[]> StreamReadGroupAsync(string stream, string group, string consumer, int count = 100, int blockMs = 0);
|
||||
Task AcknowledgeAsync(string stream, string group, params string[] ids);
|
||||
}
|
||||
|
||||
public sealed class RedisManager : IRedisManager
|
||||
{
|
||||
private readonly ILogger<RedisManager> _logger;
|
||||
private readonly ConnectionMultiplexer _mux;
|
||||
|
||||
public IDatabase Db => _mux.GetDatabase();
|
||||
public ISubscriber Bus => _mux.GetSubscriber();
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
public RedisManager(ILogger<RedisManager> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
|
||||
var connString = BuildRedisConnectionStringFromEnv();
|
||||
_mux = ConnectionMultiplexer.Connect(connString);
|
||||
_logger.LogInformation("Connected to Redis: {Endpoints}",
|
||||
string.Join(",", _mux.GetEndPoints().Select(ep => ep.ToString())));
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
try { await _mux.CloseAsync(); } catch { }
|
||||
_mux.Dispose();
|
||||
}
|
||||
|
||||
// ---------- Leases (simple Redlock-ish) ----------
|
||||
public async Task<bool> AcquireLeaseAsync(string key, string value, TimeSpan ttl)
|
||||
{
|
||||
return await Db.StringSetAsync(key, value, ttl, When.NotExists);
|
||||
}
|
||||
|
||||
public async Task<bool> RenewLeaseAsync(string key, string value, TimeSpan ttl)
|
||||
{
|
||||
// Lua script: if value matches, update TTL (PEXPIRE) and keep value
|
||||
const string script = @"
|
||||
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
||||
redis.call('PEXPIRE', KEYS[1], ARGV[2])
|
||||
return 1
|
||||
else
|
||||
return 0
|
||||
end";
|
||||
var res = (int)(long)await Db.ScriptEvaluateAsync(script,
|
||||
new RedisKey[] { key },
|
||||
new RedisValue[] { value, (long)ttl.TotalMilliseconds });
|
||||
return res == 1;
|
||||
}
|
||||
|
||||
public async Task<bool> ReleaseLeaseAsync(string key, string value)
|
||||
{
|
||||
// Lua script: delete only if owned by 'value'
|
||||
const string script = @"
|
||||
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
||||
return redis.call('DEL', KEYS[1])
|
||||
else
|
||||
return 0
|
||||
end";
|
||||
var res = (int)(long)await Db.ScriptEvaluateAsync(script,
|
||||
new RedisKey[] { key },
|
||||
new RedisValue[] { value });
|
||||
return res == 1;
|
||||
}
|
||||
|
||||
// ---------- JSON helpers ----------
|
||||
public async Task<bool> SetJsonAsync<T>(string key, T value, TimeSpan? ttl = null)
|
||||
{
|
||||
var bytes = JsonSerializer.SerializeToUtf8Bytes(value, JsonOptions);
|
||||
return await Db.StringSetAsync(key, bytes, ttl);
|
||||
}
|
||||
|
||||
public async Task<T?> GetJsonAsync<T>(string key)
|
||||
{
|
||||
var bytes = await Db.StringGetAsync(key);
|
||||
if (bytes.IsNullOrEmpty) return default;
|
||||
return JsonSerializer.Deserialize<T>((byte[])bytes!, JsonOptions);
|
||||
}
|
||||
|
||||
// ---------- Streams (write-ahead) ----------
|
||||
public async Task<long> StreamAddAsync(string stream, string json, string? id = "*")
|
||||
{
|
||||
var entryId = await Db.StreamAddAsync(stream, "data", json,
|
||||
maxLength: null, useApproximateMaxLength: false, messageId: id ?? "*");
|
||||
return entryId.HasValue ? 1 : 0;
|
||||
}
|
||||
|
||||
|
||||
public async Task<RedisValue[]> StreamReadGroupAsync(
|
||||
string stream, string group, string consumer, int count = 100, int blockMs = 0)
|
||||
{
|
||||
// Ensure group exists (create if not present)
|
||||
try
|
||||
{
|
||||
await Db.StreamCreateConsumerGroupAsync(stream, group, "$", createStream: true);
|
||||
}
|
||||
catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP"))
|
||||
{
|
||||
// group already exists
|
||||
}
|
||||
|
||||
var deadline = blockMs > 0 ? DateTime.UtcNow.AddMilliseconds(blockMs) : (DateTime?)null;
|
||||
StackExchange.Redis.StreamEntry[] entries;
|
||||
|
||||
// Poll until we have entries or we hit the deadline
|
||||
while (true)
|
||||
{
|
||||
entries = await Db.StreamReadGroupAsync(stream, group, consumer, count: count, noAck: false);
|
||||
if (entries.Length > 0) break;
|
||||
if (deadline == null) break;
|
||||
|
||||
var remainingMs = (int)(deadline.Value - DateTime.UtcNow).TotalMilliseconds;
|
||||
if (remainingMs <= 0) break;
|
||||
|
||||
// small sleep to avoid hot loop
|
||||
await Task.Delay(Math.Min(remainingMs, 100));
|
||||
}
|
||||
|
||||
// Return the "data" field only (matches your current interface)
|
||||
var result = new RedisValue[entries.Length];
|
||||
for (int i = 0; i < entries.Length; i++)
|
||||
result[i] = entries[i].Values[0].Value; // field "data"
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public async Task AcknowledgeAsync(string stream, string group, params string[] ids)
|
||||
{
|
||||
if (ids.Length == 0) return;
|
||||
await Db.StreamAcknowledgeAsync(stream, group, Array.ConvertAll(ids, x => (RedisValue)x));
|
||||
}
|
||||
|
||||
// ---------- Conn string from env ----------
|
||||
private static string BuildRedisConnectionStringFromEnv()
|
||||
{
|
||||
var full = Environment.GetEnvironmentVariable("REDIS_CONNECTION_STRING");
|
||||
if (!string.IsNullOrWhiteSpace(full)) return full;
|
||||
|
||||
var host = Environment.GetEnvironmentVariable("REDIS_HOST") ?? "localhost";
|
||||
var port = int.TryParse(Environment.GetEnvironmentVariable("REDIS_PORT"), out var p) ? p : 6379;
|
||||
var password = Environment.GetEnvironmentVariable("REDIS_PASSWORD");
|
||||
var db = int.TryParse(Environment.GetEnvironmentVariable("REDIS_DB"), out var d) ? d : 0;
|
||||
|
||||
var cs = $"{host}:{port},defaultDatabase={db},abortConnect=false,syncTimeout=10000,connectRetry=3,connectTimeout=5000";
|
||||
if (!string.IsNullOrWhiteSpace(password)) cs += $",password={password}";
|
||||
return cs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace Server.Services;
|
||||
public interface IZoneLeaseService
|
||||
{
|
||||
ValueTask<bool> TryAcquireAsync(ulong zoneId, CancellationToken ct = default);
|
||||
ValueTask<bool> RenewAsync(ulong zoneId, CancellationToken ct = default);
|
||||
ValueTask ReleaseAsync(ulong zoneId, CancellationToken ct = default);
|
||||
bool IsOwner(ulong zoneId);
|
||||
long CurrentFence(ulong zoneId);
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Server.Redis;
|
||||
|
||||
namespace Server.Services;
|
||||
|
||||
public sealed class ZoneLeaseService : IZoneLeaseService, IHostedService, IDisposable
|
||||
{
|
||||
private readonly IRedisManager _redis;
|
||||
private readonly ILogger<ZoneLeaseService> _log;
|
||||
private readonly string _podId = Environment.GetEnvironmentVariable("POD_ID") ?? Guid.NewGuid().ToString("N");
|
||||
private readonly TimeSpan _ttl = TimeSpan.FromSeconds(5);
|
||||
private readonly TimeSpan _renew = TimeSpan.FromSeconds(2);
|
||||
private readonly ConcurrentDictionary<ulong, long> _owned = new();
|
||||
private Timer? _timer;
|
||||
|
||||
public ZoneLeaseService(IRedisManager redis, ILogger<ZoneLeaseService> log)
|
||||
{ _redis = redis; _log = log; }
|
||||
|
||||
public bool IsOwner(ulong zoneId) => _owned.ContainsKey(zoneId);
|
||||
public long CurrentFence(ulong zoneId) => _owned.TryGetValue(zoneId, out var f) ? f : -1;
|
||||
|
||||
public async ValueTask<bool> TryAcquireAsync(ulong zoneId, CancellationToken ct = default)
|
||||
{
|
||||
var fence = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
var val = $"{_podId}:{fence}";
|
||||
var key = $"zone:owner:{{{zoneId}}}"; // hash tag for cluster
|
||||
var ok = await _redis.Db.StringSetAsync(key, val, _ttl, when: StackExchange.Redis.When.NotExists);
|
||||
if (ok) { _owned[zoneId] = fence; _log.LogInformation("Zone {Zone} acquired, fence={Fence}", zoneId, fence); }
|
||||
return ok;
|
||||
}
|
||||
|
||||
public async ValueTask<bool> RenewAsync(ulong zoneId, CancellationToken ct = default)
|
||||
{
|
||||
if (!_owned.TryGetValue(zoneId, out var fence)) return false;
|
||||
var key = $"zone:owner:{{{zoneId}}}";
|
||||
// renew if matches (Lua CAS)
|
||||
const string lua = @"
|
||||
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
||||
redis.call('PEXPIRE', KEYS[1], ARGV[2]); return 1
|
||||
else return 0 end";
|
||||
var res = (int)(long)await _redis.Db.ScriptEvaluateAsync(lua,
|
||||
new StackExchange.Redis.RedisKey[] { key },
|
||||
new StackExchange.Redis.RedisValue[] { $"{_podId}:{fence}", (long)_ttl.TotalMilliseconds });
|
||||
return res == 1;
|
||||
}
|
||||
|
||||
public async ValueTask ReleaseAsync(ulong zoneId, CancellationToken ct = default)
|
||||
{
|
||||
if (!_owned.TryRemove(zoneId, out var fence)) return;
|
||||
var key = $"zone:owner:{{{zoneId}}}";
|
||||
const string lua = @"if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end";
|
||||
await _redis.Db.ScriptEvaluateAsync(lua,
|
||||
new StackExchange.Redis.RedisKey[] { key },
|
||||
new StackExchange.Redis.RedisValue[] { $"{_podId}:{fence}" });
|
||||
_log.LogInformation("Zone {Zone} released", zoneId);
|
||||
}
|
||||
|
||||
// Background renew loop
|
||||
public Task StartAsync(CancellationToken _) { _timer = new Timer(async _ => await RenewAll(), null, _renew, _renew); return Task.CompletedTask; }
|
||||
public Task StopAsync(CancellationToken _) { _timer?.Dispose(); return Task.CompletedTask; }
|
||||
private async Task RenewAll()
|
||||
{
|
||||
foreach (var z in _owned.Keys) { if (!await RenewAsync(z)) _owned.TryRemove(z, out _); }
|
||||
}
|
||||
public void Dispose() => _timer?.Dispose();
|
||||
}
|
||||
@@ -20,6 +20,7 @@
|
||||
<PackageReference Include="MySqlConnector" Version="2.4.0" />
|
||||
<PackageReference Include="Polly" Version="8.6.4" />
|
||||
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="9.0.0" />
|
||||
<PackageReference Include="StackExchange.Redis" Version="2.9.32" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace Server.World;
|
||||
|
||||
public static class SpatialIndex
|
||||
{
|
||||
public const int CellSize = 50; // units
|
||||
public static (int cx, int cy, int cz) Cell(double x, double y, double z) =>
|
||||
((int)Math.Floor(x / CellSize), (int)Math.Floor(y / CellSize), (int)Math.Floor(z / CellSize));
|
||||
|
||||
public static async Task UpdateAsync(IDatabase db, ulong zoneId, ulong charId,
|
||||
(int cx, int cy, int cz) oldCell, (int cx, int cy, int cz) newCell, TimeSpan ttl)
|
||||
{
|
||||
var oldKey = $"z:{{{zoneId}}}:c:{oldCell.cx}:{oldCell.cy}:{oldCell.cz}";
|
||||
var newKey = $"z:{{{zoneId}}}:c:{newCell.cx}:{newCell.cy}:{newCell.cz}";
|
||||
|
||||
var batch = db.CreateBatch();
|
||||
var tasks = new List<Task>();
|
||||
|
||||
if (oldCell != newCell)
|
||||
tasks.Add(batch.SetRemoveAsync(oldKey, charId));
|
||||
|
||||
tasks.Add(batch.SetAddAsync(newKey, charId));
|
||||
tasks.Add(batch.KeyExpireAsync(newKey, ttl));
|
||||
|
||||
batch.Execute();
|
||||
await Task.WhenAll(tasks);
|
||||
}
|
||||
|
||||
|
||||
public static IEnumerable<string> NeighborCellKeys(ulong zoneId, (int cx, int cy, int cz) c, int r = 1)
|
||||
{
|
||||
for (int dx = -r; dx <= r; dx++)
|
||||
for (int dy = -r; dy <= r; dy++)
|
||||
yield return $"z:{{{zoneId}}}:c:{c.cx + dx}:{c.cy + dy}:{c.cz}";
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user