ling trial: layer ngram-mod under draft-mtp; add ngram args to serve script

This commit is contained in:
Frank committed 2026-08-14 01:40:46 -07:00
1 parent c275cbaf57
commit 3204b4b29e
2 files changed
+11 -1

No files matched your search

+6
View File
@@ -34,6 +34,9 @@ fi
: "${PARALLEL:=2}"
: "${SPEC_TYPE:=draft-mtp}" # MTP block is bundled in the model GGUF
: "${SPEC_DRAFT_N_MAX:=3}"
: "${SPEC_NGRAM_MOD_N_MIN:=48}"
: "${SPEC_NGRAM_MOD_N_MAX:=64}"
: "${SPEC_NGRAM_MOD_N_MATCH:=24}"
: "${DEVICE:=Vulkan0}"
: "${N_GPU_LAYERS:=all}"
: "${BATCH_SIZE:=512}"
@@ -62,6 +65,9 @@ exec "$LLAMA_SERVER" \
--model "$MODEL" \
--spec-type "$SPEC_TYPE" \
--spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
--spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN" \
--spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX" \
--spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH" \
--device "$DEVICE" \
--gpu-layers "$N_GPU_LAYERS" \
--flash-attn auto \