diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 652000f..2257cb0 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -15,5 +15,11 @@ CACHE_TYPE_V=f16 N_CPU_MOE=40 # Every non-expert tensor (attention, norms, embedding, LM head) goes to GPU. N_GPU_LAYERS=all +# Larger batches for prefill throughput. The bigger physical batch grows GPU +# activation buffers; ctx-checkpoints parks attention checkpoints in RAM to +# hold peak GPU memory down so the larger ubatch fits without OOM. +BATCH_SIZE=3072 +UBATCH_SIZE=1536 +CTX_CHECKPOINTS=4 THREADS=8 THREADS_HTTP=8 \ No newline at end of file diff --git a/scripts/serve-llama-frank-small.sh b/scripts/serve-llama-frank-small.sh index 5fe8969..4dcbe9b 100755 --- a/scripts/serve-llama-frank-small.sh +++ b/scripts/serve-llama-frank-small.sh @@ -32,6 +32,9 @@ fi : "${CACHE_TYPE_V:=f16}" : "${N_CPU_MOE:=40}" # all 40 layers: MoE expert weights stay on CPU : "${N_GPU_LAYERS:=all}" # every non-expert tensor goes to GPU +: "${BATCH_SIZE:=2048}" +: "${UBATCH_SIZE:=512}" +: "${CTX_CHECKPOINTS:=0}" : "${THREADS:=8}" # 7700X physical cores : "${THREADS_HTTP:=8}" @@ -60,6 +63,9 @@ exec "$LLAMA_SERVER" \ --parallel "$PARALLEL" \ --n-cpu-moe "$N_CPU_MOE" \ --gpu-layers "$N_GPU_LAYERS" \ + --batch-size "$BATCH_SIZE" \ + --ubatch-size "$UBATCH_SIZE" \ + --ctx-checkpoints "$CTX_CHECKPOINTS" \ --threads "$THREADS" \ --threads-http "$THREADS_HTTP" \ --alias "$ALIAS" \