frank-default-small: ctx=786432 max, parallel=3, batch=6144

scripts/frank-default-small.env

@@ -20,10 +20,10 @@ ALIAS=frank-default-small
 BIND_ADDR=0.0.0.0
 LISTEN_PORT=7770
 
-# 4 parallel slots x ~175104 ctx each = ~700000 total context pool.
-# Dual-f16 KV at 700000 cells leaves ~2.3 GiB free on the 3090.
-CTX_SIZE=700000
-PARALLEL=4
+# 3 parallel slots x 262144 ctx each = ~786432 total context pool (max per-slot).
+# Dual-f16 KV at 786432 cells leaves ~15 GiB free on the 3090.
+CTX_SIZE=786432
+PARALLEL=3
 
 # Spec: ngram-mod (n_min=48, n_max=64, n_match=24).
 SPEC_TYPE=ngram-mod
@@ -44,7 +44,7 @@ VERBOSITY=4
 CTX_CHECKPOINTS=4
 
 # Big prefill batches (ubatch 3072 is the top prefill tier).
-BATCH_SIZE=8192
+BATCH_SIZE=6144
 UBATCH_SIZE=3072
 
 # Eager RAM load (MoE experts on CPU; ~31 GB RAM).

scripts/serve-frank-small.sh

@@ -27,8 +27,8 @@ fi
 : "${ALIAS:=frank-default-small}"
 : "${BIND_ADDR:=0.0.0.0}"
 : "${LISTEN_PORT:=7770}"
-: "${CTX_SIZE:=700000}"
-: "${PARALLEL:=4}"
+: "${CTX_SIZE:=786432}"
+: "${PARALLEL:=3}"
 : "${SPEC_TYPE:=ngram-mod}"
 : "${NGRAM_MOD_N_MIN:=48}"
 : "${NGRAM_MOD_N_MAX:=64}"
@@ -38,7 +38,7 @@ fi
 : "${VERBOSITY:=4}"
 : "${N_CPU_MOE:=36}"
 : "${N_GPU_LAYERS:=all}"
-: "${BATCH_SIZE:=8192}"
+: "${BATCH_SIZE:=6144}"
 : "${UBATCH_SIZE:=3072}"
 : "${CTX_CHECKPOINTS:=4}"
 : "${NO_MMAP:=1}"