From 9971f83e8bd3bb70799ee0fedb66167b874d23dd Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Tue, 11 Aug 2026 01:45:41 -0700
Subject: [PATCH] perf: frank-default-small ctx 200000/stream, --no-mmap, batch
 3072/1536

Lower each stream to 200000 ctx (pool 600000) to free KV VRAM for the larger
batch. Add --no-mmap so the CPU-resident MoE experts load eagerly into RAM
(avoids decode page faults). Keep KV on GPU. Experts already fully on CPU.
---
 scripts/frank-default-small.env    | 8 ++++++--
 scripts/serve-llama-frank-small.sh | 2 ++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env
index 2257cb0..affe1d7 100644
--- a/scripts/frank-default-small.env
+++ b/scripts/frank-default-small.env
@@ -4,8 +4,9 @@ MODEL_DIR=/home/dank/models
 ALIAS=frank-default-small
 HOST=0.0.0.0
 PORT=7770
-# 3 parallel slots x 262144 ctx each = 786432 total context pool.
-CTX_SIZE=786432
+# 3 parallel slots x 200000 ctx each = 600000 total context pool.
+# (Reduced from 262144/stream per operator: allows the larger batch to fit VRAM.)
+CTX_SIZE=600000
 PARALLEL=3
 SPEC_TYPE=draft-dflash
 SPEC_N_MAX=7
@@ -21,5 +22,8 @@ N_GPU_LAYERS=all
 BATCH_SIZE=3072
 UBATCH_SIZE=1536
 CTX_CHECKPOINTS=4
+# Load the model into RAM rather than lazy-mapping it. The MoE experts live on
+# CPU, so eager RAM load avoids page faults during decode (uses ~31GB RAM).
+NO_MMAP=1
 THREADS=8
 THREADS_HTTP=8
\ No newline at end of file
diff --git a/scripts/serve-llama-frank-small.sh b/scripts/serve-llama-frank-small.sh
index 4dcbe9b..0fa72f0 100755
--- a/scripts/serve-llama-frank-small.sh
+++ b/scripts/serve-llama-frank-small.sh
@@ -35,6 +35,7 @@ fi
 : "${BATCH_SIZE:=2048}"
 : "${UBATCH_SIZE:=512}"
 : "${CTX_CHECKPOINTS:=0}"
+: "${NO_MMAP:=0}"
 : "${THREADS:=8}"        # 7700X physical cores
 : "${THREADS_HTTP:=8}"
 
@@ -66,6 +67,7 @@ exec "$LLAMA_SERVER" \
     --batch-size "$BATCH_SIZE" \
     --ubatch-size "$UBATCH_SIZE" \
     --ctx-checkpoints "$CTX_CHECKPOINTS" \
+    $([ "$NO_MMAP" = "1" ] && printf -- "--no-mmap") \
     --threads "$THREADS" \
     --threads-http "$THREADS_HTTP" \
     --alias "$ALIAS" \