diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index b8b6ca7..c4b869e 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -20,19 +20,11 @@ SPEC_NGRAM_MOD_N_MATCH=24 DEVICE=ROCm0 # Dense model (Qwen3.6-27B, not MoE): no MoE CPU offload. Full offload. N_GPU_LAYERS=all -# KV on GPU, quantized (K q8_0, V q8_0). V MUST match K: this llama.cpp -# build has GGML_CUDA_FA_ALL_QUANTS=OFF, and Flash Attention requires -# K type == V type. Mixed q8_0/f16 silently disables FA, collapsing -# long-context decode to ~1 t/s at 200k ctx. q8_0 V halves V memory too. -CACHE_TYPE_K=q8_0 -CACHE_TYPE_V=q8_0 -# Batch sizing constrained by ROCm memory at 602100 ctx (3x200700): -# KV ~50GB + model 29.77GB leave only ~16GB for compute graphs. The compute -# graph scales with UBATCH, so ubatch must stay ~512; ubatch 2048 OOMs even -# at batch 2048. Proven-fits config: batch 2048 / ubatch 512. -# User-specified ubatch 4096 / batch 12288 is not feasible here (see bench). -BATCH_SIZE=2048 -UBATCH_SIZE=512 +# KV on GPU, fp16 (unquantized). Default ggml_type for both K and V. +# Batch sizing: fp16 KV is larger than q8_0, but user has verified this +# configuration fits. See frank-default-large-benchmark.md. +BATCH_SIZE=8192 +UBATCH_SIZE=2048 # Sampling settings — per HauhauCS Qwen3.6-27B model card. # Thinking-general: temp=1.0, top_p=0.95, top_k=20, min_p=0. # Clients may override per-request; these are the server-side defaults. diff --git a/scripts/serve-llama-frank-large.sh b/scripts/serve-llama-frank-large.sh index 30e4b20..c088350 100755 --- a/scripts/serve-llama-frank-large.sh +++ b/scripts/serve-llama-frank-large.sh @@ -33,10 +33,8 @@ fi : "${SPEC_NGRAM_MOD_N_MATCH:=24}" : "${DEVICE:=ROCm0}" : "${N_GPU_LAYERS:=all}" -: "${CACHE_TYPE_K:=q8_0}" -: "${CACHE_TYPE_V:=q8_0}" -: "${BATCH_SIZE:=2048}" -: "${UBATCH_SIZE:=512}" +: "${BATCH_SIZE:=8192}" +: "${UBATCH_SIZE:=2048}" : "${TEMP:=1.0}" : "${TOP_K:=20}" : "${TOP_P:=0.95}" @@ -66,8 +64,6 @@ exec "$LLAMA_SERVER" \ --device "$DEVICE" \ --gpu-layers "$N_GPU_LAYERS" \ --fit off \ - --cache-type-k "$CACHE_TYPE_K" \ - --cache-type-v "$CACHE_TYPE_V" \ --ctx-size "$CTX_SIZE" \ --parallel "$PARALLEL" \ --batch-size "$BATCH_SIZE" \