diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 59b936a..6de99fd 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -23,7 +23,11 @@ N_GPU_LAYERS=all # KV on GPU, quantized (K q8_0, V f16). CACHE_TYPE_K=q8_0 CACHE_TYPE_V=f16 -BATCH_SIZE=12288 -UBATCH_SIZE=4096 +# Batch sizing constrained by ROCm memory at 602100 ctx (3x200700): +# KV cache alone ~50GB, model 29.77GB; only ~16GB left for compute graphs. +# Batch 12288 (80.7GB buffer) and 6144 (40.9GB) both OOM; 2048 fits. +# Tried per request (ubatch 4096 / batch 12288) -> OOM. See bench notes. +BATCH_SIZE=2048 +UBATCH_SIZE=2048 THREADS=8 THREADS_HTTP=8 diff --git a/scripts/serve-llama-frank-large.sh b/scripts/serve-llama-frank-large.sh index 48a3bf2..f447653 100755 --- a/scripts/serve-llama-frank-large.sh +++ b/scripts/serve-llama-frank-large.sh @@ -35,8 +35,8 @@ fi : "${N_GPU_LAYERS:=all}" : "${CACHE_TYPE_K:=q8_0}" : "${CACHE_TYPE_V:=f16}" -: "${BATCH_SIZE:=12288}" -: "${UBATCH_SIZE:=4096}" +: "${BATCH_SIZE:=2048}" +: "${UBATCH_SIZE:=2048}" : "${THREADS:=8}" : "${THREADS_HTTP:=8}"