diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index c2c3b1c..227c9fa 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -20,9 +20,12 @@ SPEC_NGRAM_MOD_N_MATCH=24 DEVICE=ROCm0 # Dense model (Qwen3.6-27B, not MoE): no MoE CPU offload. Full offload. N_GPU_LAYERS=all -# KV on GPU, quantized (K q8_0, V f16). +# KV on GPU, quantized (K q8_0, V q8_0). V MUST match K: this llama.cpp +# build has GGML_CUDA_FA_ALL_QUANTS=OFF, and Flash Attention requires +# K type == V type. Mixed q8_0/f16 silently disables FA, collapsing +# long-context decode to ~1 t/s at 200k ctx. q8_0 V halves V memory too. CACHE_TYPE_K=q8_0 -CACHE_TYPE_V=f16 +CACHE_TYPE_V=q8_0 # Batch sizing constrained by ROCm memory at 602100 ctx (3x200700): # KV ~50GB + model 29.77GB leave only ~16GB for compute graphs. The compute # graph scales with UBATCH, so ubatch must stay ~512; ubatch 2048 OOMs even @@ -30,5 +33,5 @@ CACHE_TYPE_V=f16 # User-specified ubatch 4096 / batch 12288 is not feasible here (see bench). BATCH_SIZE=2048 UBATCH_SIZE=512 -THREADS=8 -THREADS_HTTP=8 +THREADS=12 +THREADS_HTTP=1 diff --git a/scripts/serve-llama-frank-large.sh b/scripts/serve-llama-frank-large.sh index a008861..359d8ec 100755 --- a/scripts/serve-llama-frank-large.sh +++ b/scripts/serve-llama-frank-large.sh @@ -34,11 +34,11 @@ fi : "${DEVICE:=ROCm0}" : "${N_GPU_LAYERS:=all}" : "${CACHE_TYPE_K:=q8_0}" -: "${CACHE_TYPE_V:=f16}" +: "${CACHE_TYPE_V:=q8_0}" : "${BATCH_SIZE:=2048}" : "${UBATCH_SIZE:=512}" -: "${THREADS:=8}" -: "${THREADS_HTTP:=8}" +: "${THREADS:=12}" +: "${THREADS_HTTP:=1}" LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp/build/bin/llama-server}" MODEL="$MODEL_DIR/$MODEL_FILE"