diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index abb3fb7..9ca3eea 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -4,25 +4,23 @@ MODEL_DIR=/home/dank/models ALIAS=frank-default-large HOST=0.0.0.0 PORT=7770 -# 6 parallel slots x 65536 ctx each = 393216 total context pool. -CTX_SIZE=393216 -PARALLEL=6 -# Spec: DFlash draft model layered with ngram-mod. --spec-type accepts a -# comma-separated list; llama.cpp chains the implementations (verified in -# common/speculative.cpp). --spec-draft-n-max must stay below the DFlash -# block size (BF16 sidecar = 8 token block), so 8 is the ceiling. -SPEC_TYPE=draft-dflash,ngram-mod +# 3 parallel slots x 200700 ctx each = 602100 total context pool. +CTX_SIZE=602100 +PARALLEL=3 +# Spec: DFlash draft model. Bench (4k, ROCm) showed draft-dflash fastest +# generation (20.2 t/s): DFlash+ngram 16.9, ngram-only 8.7. ngram layering +# reduced throughput, so it is not layered in production. +SPEC_TYPE=draft-dflash SPEC_DRAFT_N_MAX=8 SPEC_NGRAM_MOD_N_MIN=48 SPEC_NGRAM_MOD_N_MAX=64 SPEC_NGRAM_MOD_N_MATCH=24 -# Backend device for offload, from --list-devices. Pick the faster of: -# ROCm0 (HIP / ROCm backend) -# Vulkan0 (RADV Vulkan backend) +# Backend device from --list-devices. ROCm wins on generation; Vulkan renders +# DFlash spec unusable (NONE-op abort) and gen is slower on ngram (6.9 t/s). DEVICE=ROCm0 # Dense model (Qwen3.6-27B, not MoE): no MoE CPU offload. Full offload. N_GPU_LAYERS=all -# KV on GPU, quantized (K q8_0, V f16); mirrors the small service profile. +# KV on GPU, quantized (K q8_0, V f16). CACHE_TYPE_K=q8_0 CACHE_TYPE_V=f16 BATCH_SIZE=2048 diff --git a/scripts/serve-llama-frank-large.sh b/scripts/serve-llama-frank-large.sh index 8fa6806..937ef94 100755 --- a/scripts/serve-llama-frank-large.sh +++ b/scripts/serve-llama-frank-large.sh @@ -24,9 +24,9 @@ fi : "${ALIAS:=frank-default-large}" : "${HOST:=0.0.0.0}" : "${PORT:=7770}" -: "${CTX_SIZE:=393216}" -: "${PARALLEL:=6}" -: "${SPEC_TYPE:=draft-dflash,ngram-mod}" +: "${CTX_SIZE:=602100}" +: "${PARALLEL:=3}" +: "${SPEC_TYPE:=draft-dflash}" : "${SPEC_DRAFT_N_MAX:=8}" : "${SPEC_NGRAM_MOD_N_MIN:=48}" : "${SPEC_NGRAM_MOD_N_MAX:=64}"