diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 2257cb0..affe1d7 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -4,8 +4,9 @@ MODEL_DIR=/home/dank/models ALIAS=frank-default-small HOST=0.0.0.0 PORT=7770 -# 3 parallel slots x 262144 ctx each = 786432 total context pool. -CTX_SIZE=786432 +# 3 parallel slots x 200000 ctx each = 600000 total context pool. +# (Reduced from 262144/stream per operator: allows the larger batch to fit VRAM.) +CTX_SIZE=600000 PARALLEL=3 SPEC_TYPE=draft-dflash SPEC_N_MAX=7 @@ -21,5 +22,8 @@ N_GPU_LAYERS=all BATCH_SIZE=3072 UBATCH_SIZE=1536 CTX_CHECKPOINTS=4 +# Load the model into RAM rather than lazy-mapping it. The MoE experts live on +# CPU, so eager RAM load avoids page faults during decode (uses ~31GB RAM). +NO_MMAP=1 THREADS=8 THREADS_HTTP=8 \ No newline at end of file diff --git a/scripts/serve-llama-frank-small.sh b/scripts/serve-llama-frank-small.sh index 4dcbe9b..0fa72f0 100755 --- a/scripts/serve-llama-frank-small.sh +++ b/scripts/serve-llama-frank-small.sh @@ -35,6 +35,7 @@ fi : "${BATCH_SIZE:=2048}" : "${UBATCH_SIZE:=512}" : "${CTX_CHECKPOINTS:=0}" +: "${NO_MMAP:=0}" : "${THREADS:=8}" # 7700X physical cores : "${THREADS_HTTP:=8}" @@ -66,6 +67,7 @@ exec "$LLAMA_SERVER" \ --batch-size "$BATCH_SIZE" \ --ubatch-size "$UBATCH_SIZE" \ --ctx-checkpoints "$CTX_CHECKPOINTS" \ + $([ "$NO_MMAP" = "1" ] && printf -- "--no-mmap") \ --threads "$THREADS" \ --threads-http "$THREADS_HTTP" \ --alias "$ALIAS" \