diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 0000000..e041b20 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,145 @@ +# SOR Neural AI Training Tools + +This directory contains the Python training pipeline for the SOR neural decision engine +(tiers 9-10). The workflow is: + +``` +self-play (admin) -> JSONL export (admin) -> train_neural.py -> model.onnx -> hot-swap (admin) +``` + +--- + +## Prerequisites + +Python 3.11+ with pip: + +```bash +pip install -r tools/requirements.txt +``` + +GPU training is supported via CUDA. Use `--cpu` to force CPU-only mode. + +--- + +## Step 1 — Generate self-play data (Admin portal) + +1. Navigate to **Admin -> AI -> Calibration**. +2. Click **Run All Pairs** (or a specific difficulty pair) to trigger in-process self-play games. +3. Wait for the run to complete — check the calibration table for results. + +Alternatively, trigger a full calibration run via the API: +``` +POST /api/ai/calibration/run-all +``` + +--- + +## Step 2 — Export training data (Admin portal) + +1. Navigate to **Admin -> AI -> Calibration**. +2. Click **Export Training Data** and enter an output path on the server + (e.g. `/var/sor/training/data.jsonl`). +3. The exporter replays completed game sessions and writes one JSON line per action: + +```json +{ + "GameSessionId": "3fa85f64...", + "TurnNumber": 5, + "ActorAccountId": "...", + "Features": [0.0, 1.0, ...], + "ActionType": "AiPlayCardAction", + "ActionJson": "{}", + "Outcome": 1.0 +} +``` + +`Outcome` is `+1.0` if the acting player won the game, `-1.0` if they lost. + +--- + +## Step 3 — Train the model + +```bash +cd /path/to/SOR +python tools/train_neural.py --data /var/sor/training/data.jsonl --out /var/sor/models/model.onnx --epochs 50 --batch-size 512 --lr 1e-3 +``` + +### Options + +| Flag | Default | Description | +|------|---------|-------------| +| `--data` | *(required)* | Path to the JSONL export file | +| `--out` | `model.onnx` | Output ONNX file path | +| `--epochs` | `50` | Number of training epochs | +| `--batch-size` | `512` | Mini-batch size | +| `--lr` | `0.001` | AdamW initial learning rate | +| `--val-split` | `0.1` | Fraction of data held out for validation | +| `--dropout` | `0.2` | Dropout probability in the MLP | +| `--seed` | `42` | Random seed | +| `--cpu` | off | Force CPU training (ignores CUDA) | + +The script prints per-epoch train/val loss and marks the best checkpoint with `*`. +It restores the best weights before exporting. + +### Model architecture + +``` +Input [1, 1024] float32 (FeatureExtractor output) +Linear 1024 -> 512 + LayerNorm + ReLU + Dropout +Linear 512 -> 256 + LayerNorm + ReLU + Dropout +Linear 256 -> 128 + LayerNorm + ReLU +Linear 128 -> 1 + Tanh +Output [1, 1] float32 (score in [-1, 1]) +``` + +Total parameters: ~920k + +--- + +## Step 4 — Deploy the model + +### Option A: Hot-swap via admin portal (no restart) + +1. Copy the `model.onnx` file to the path configured in `appsettings`: + ```json + "Ai": { "NeuralModelPath": "/var/sor/models/model.onnx" } + ``` +2. Navigate to **Admin -> AI -> Neural Model**. +3. Click **Hot-swap Model**. The old session is disposed atomically and the new + model is loaded from disk. All subsequent tier 9-10 decisions use the new model. + +### Option B: Restart the service + +The model is loaded at startup by `AiNeuralStartupService`. If the file exists at the +configured path, it will be loaded automatically on the next deploy. + +--- + +## ONNX contract + +The exported model must satisfy: + +| Property | Value | +|----------|-------| +| Input tensor name | `features` | +| Input shape | `[1, 1024]` (or `[batch, 1024]` with dynamic batch) | +| Input dtype | `float32` | +| Output tensor name | `score` | +| Output shape | `[1, 1]` (or `[batch, 1]`) | +| Output dtype | `float32` | +| Output range | `[-1, 1]` (tanh) | + +These names are defined as constants in `NeuralModelRegistry.cs` (`InputName`, `OutputName`). +The Python script and the C# inference code must agree on these names exactly. + +--- + +## Iterative training + +For best results, run multiple rounds: + +1. Start with random-engine games (all difficulties) to generate diverse data. +2. Train an initial model and deploy it. +3. Run additional self-play with the neural engine enabled (tiers 9-10 play each other). +4. Append new exports to the existing JSONL and retrain with more epochs. +5. Monitor calibration win rates in the admin portal after each deployment. diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 0000000..ca11f43 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,4 @@ +torch==2.3.1 +onnx==1.16.1 +onnxruntime==1.18.0 +numpy==1.26.4 diff --git a/tools/train_neural.py b/tools/train_neural.py new file mode 100644 index 0000000..3be97b3 --- /dev/null +++ b/tools/train_neural.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# SOR Neural AI Training Script +# Trains a MLP on GameReplayExporter JSONL, exports ONNX for NeuralDecisionEngine. +# ONNX contract: input='features' [1,1024] float32, output='score' [1,1] float32 +# Usage: python train_neural.py --data training.jsonl --out model.onnx +from __future__ import annotations +import argparse, json, sys, time +import numpy as np +import torch, torch.nn as nn, torch.optim as optim +from torch.utils.data import DataLoader, Dataset, random_split +FEATURE_COUNT=1024 +ONNX_INPUT='features' +ONNX_OUTPUT='score' + +class _R: + __slots__=('f','o') + def __init__(self,f,o): self.f=f; self.o=o + +def _load(path): + recs=[] + with open(path,encoding='utf-8') as fh: + for no,raw in enumerate(fh,1): + raw=raw.strip() + if not raw: continue + try: + d=json.loads(raw) + recs.append(_R(d['Features'],float(d['Outcome']))) + except Exception as e: print(f'[warn] line {no}: {e}',file=sys.stderr) + return recs + +class DS(Dataset): + def __init__(self,recs): + n=len(recs) + self.X=np.empty((n,FEATURE_COUNT),dtype=np.float32) + self.y=np.empty((n,1),dtype=np.float32) + for i,r in enumerate(recs): self.X[i]=r.f; self.y[i]=r.o + def __len__(self): return len(self.y) + def __getitem__(self,i): return torch.from_numpy(self.X[i]),torch.from_numpy(self.y[i]) + +class Net(nn.Module): + # MLP: 1024->512->256->128->1 (tanh). Output in [-1,1], +1=win, -1=loss. + def __init__(self,d=0.2): + super().__init__() + self.net=nn.Sequential( + nn.Linear(1024,512),nn.LayerNorm(512),nn.ReLU(),nn.Dropout(d), + nn.Linear(512,256),nn.LayerNorm(256),nn.ReLU(),nn.Dropout(d), + nn.Linear(256,128),nn.LayerNorm(128),nn.ReLU(), + nn.Linear(128,1),nn.Tanh()) + def forward(self,x): return self.net(x) + +def _train(m,tl,vl,epochs,lr,dev): + m.to(dev) + opt=optim.AdamW(m.parameters(),lr=lr,weight_decay=1e-4) + sch=optim.lr_scheduler.CosineAnnealingLR(opt,T_max=epochs) + crit=nn.MSELoss() + bv=float('inf'); bs=None + for ep in range(1,epochs+1): + t0=time.time(); m.train(); tls=0.0 + for X,y in tl: + X,y=X.to(dev),y.to(dev); opt.zero_grad() + loss=crit(m(X),y); loss.backward() + nn.utils.clip_grad_norm_(m.parameters(),1.0); opt.step() + tls+=loss.item()*len(X) + tls/=len(tl.dataset); m.eval(); vls=0.0 + with torch.no_grad(): + for X,y in vl: + X,y=X.to(dev),y.to(dev); vls+=crit(m(X),y).item()*len(X) + vls/=len(vl.dataset); sch.step() + star=' *' if vls {out}') + try: + import onnxruntime as ort + s=ort.InferenceSession(out,providers=['CPUExecutionProvider']) + r=s.run([ONNX_OUTPUT],{ONNX_INPUT:np.zeros((1,FEATURE_COUNT),dtype=np.float32)}) + print(f'ONNX ok score={r[0][0][0]:.4f}') + except Exception as e: print(f'[warn] onnx check: {e}',file=sys.stderr) + +def main(): + p=argparse.ArgumentParser() + p.add_argument('--data',required=True) + p.add_argument('--out',default='model.onnx') + p.add_argument('--epochs',type=int,default=50) + p.add_argument('--batch-size',type=int,default=512) + p.add_argument('--lr',type=float,default=1e-3) + p.add_argument('--val-split',type=float,default=0.1) + p.add_argument('--dropout',type=float,default=0.2) + p.add_argument('--seed',type=int,default=42) + p.add_argument('--cpu',action='store_true') + a=p.parse_args() + torch.manual_seed(a.seed); np.random.seed(a.seed) + dev=torch.device('cpu' if a.cpu or not torch.cuda.is_available() else 'cuda') + print(f'device={dev}') + recs=_load(a.data) + if not recs: sys.exit('no records') + print(f'{len(recs):,} records loaded') + bad={i for i,r in enumerate(recs) if len(r.f)!=FEATURE_COUNT} + if bad: print(f'dropping {len(bad)} bad-length recs'); recs=[r for i,r in enumerate(recs) if i not in bad] + ds=DS(recs); vn=max(1,int(len(ds)*a.val_split)); tn=len(ds)-vn + tr,vr=random_split(ds,[tn,vn],generator=torch.Generator().manual_seed(a.seed)) + tl=DataLoader(tr,batch_size=a.batch_size,shuffle=True,num_workers=0) + vl=DataLoader(vr,batch_size=a.batch_size,shuffle=False,num_workers=0) + print(f'train={tn:,} val={vn:,}') + m=Net(a.dropout) + print(f'params={sum(p.numel() for p in m.parameters()):,}') + _train(m,tl,vl,a.epochs,a.lr,dev) + _export(m,a.out) + print(f'Deploy: copy {a.out} to Ai:NeuralModelPath, hot-swap via admin.') + +if __name__=='__main__': main()