diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 7528922..7d91383 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -20,10 +20,10 @@ ALIAS=frank-default-small BIND_ADDR=0.0.0.0 LISTEN_PORT=7770 -# 4 parallel slots x ~175104 ctx each = ~700000 total context pool. -# Dual-f16 KV at 700000 cells leaves ~2.3 GiB free on the 3090. -CTX_SIZE=700000 -PARALLEL=4 +# 3 parallel slots x 262144 ctx each = ~786432 total context pool (max per-slot). +# Dual-f16 KV at 786432 cells leaves ~15 GiB free on the 3090. +CTX_SIZE=786432 +PARALLEL=3 # Spec: ngram-mod (n_min=48, n_max=64, n_match=24). SPEC_TYPE=ngram-mod @@ -44,7 +44,7 @@ VERBOSITY=4 CTX_CHECKPOINTS=4 # Big prefill batches (ubatch 3072 is the top prefill tier). -BATCH_SIZE=8192 +BATCH_SIZE=6144 UBATCH_SIZE=3072 # Eager RAM load (MoE experts on CPU; ~31 GB RAM). diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 6bb9f19..7e28332 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -27,8 +27,8 @@ fi : "${ALIAS:=frank-default-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7770}" -: "${CTX_SIZE:=700000}" -: "${PARALLEL:=4}" +: "${CTX_SIZE:=786432}" +: "${PARALLEL:=3}" : "${SPEC_TYPE:=ngram-mod}" : "${NGRAM_MOD_N_MIN:=48}" : "${NGRAM_MOD_N_MAX:=64}" @@ -38,7 +38,7 @@ fi : "${VERBOSITY:=4}" : "${N_CPU_MOE:=36}" : "${N_GPU_LAYERS:=all}" -: "${BATCH_SIZE:=8192}" +: "${BATCH_SIZE:=6144}" : "${UBATCH_SIZE:=3072}" : "${CTX_CHECKPOINTS:=4}" : "${NO_MMAP:=1}"