diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 56b22fd..cb117d7 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -26,13 +26,14 @@ N_GPU_LAYERS=all # long-context decode to ~1 t/s at 200k ctx. q8_0 V halves V memory too. CACHE_TYPE_K=q8_0 CACHE_TYPE_V=q8_0 -# Batch sizing constrained by ROCm memory at 602100 ctx (3x200700): -# KV ~50GB + model 29.77GB leave only ~16GB for compute graphs. The compute -# graph scales with UBATCH, so ubatch must stay ~512; ubatch 2048 OOMs even -# at batch 2048. Proven-fits config: batch 2048 / ubatch 512. -# User-specified ubatch 4096 / batch 12288 is not feasible here (see bench). -BATCH_SIZE=2048 -UBATCH_SIZE=512 +# Batch sizing: empirically verified 2026-08-13 with caches dropped. +# batch 8192 / ubatch 2048 loads and serves (61 GiB VRAM peak, ctx 602100). +# The earlier "ubatch 2048 OOMs" notes in frank-default-large-benchmark.md +# were confounded by page-cache pressure: with the model's pages fully +# cached, the KV/graph allocation stalls (VRAM stuck at 28 GiB). +# Drop caches before starting a load: sync && echo 3 > /proc/sys/vm/drop_caches. +BATCH_SIZE=8192 +UBATCH_SIZE=2048 # Sampling settings — per HauhauCS Qwen3.6-27B model card. # Thinking-general: temp=1.0, top_p=0.95, top_k=20, min_p=0. # Clients may override per-request; these are the server-side defaults. diff --git a/scripts/serve-llama-frank-large.sh b/scripts/serve-llama-frank-large.sh index b7645d0..ab459bc 100755 --- a/scripts/serve-llama-frank-large.sh +++ b/scripts/serve-llama-frank-large.sh @@ -35,8 +35,8 @@ fi : "${N_GPU_LAYERS:=all}" : "${CACHE_TYPE_K:=q8_0}" : "${CACHE_TYPE_V:=q8_0}" -: "${BATCH_SIZE:=2048}" -: "${UBATCH_SIZE:=512}" +: "${BATCH_SIZE:=8192}" +: "${UBATCH_SIZE:=2048}" : "${TEMPERATURE:=1.0}" : "${TOP_K:=20}" : "${TOP_P:=0.95}"