ling trial: layer ngram-mod under draft-mtp; add ngram args to serve script
This commit is contained in:
1 parent
c275cbaf57
commit
3204b4b29e
2 files changed
+11
-1
No files matched your search
@@ -34,6 +34,9 @@ fi
|
||||
: "${PARALLEL:=2}"
|
||||
: "${SPEC_TYPE:=draft-mtp}" # MTP block is bundled in the model GGUF
|
||||
: "${SPEC_DRAFT_N_MAX:=3}"
|
||||
: "${SPEC_NGRAM_MOD_N_MIN:=48}"
|
||||
: "${SPEC_NGRAM_MOD_N_MAX:=64}"
|
||||
: "${SPEC_NGRAM_MOD_N_MATCH:=24}"
|
||||
: "${DEVICE:=Vulkan0}"
|
||||
: "${N_GPU_LAYERS:=all}"
|
||||
: "${BATCH_SIZE:=512}"
|
||||
@@ -62,6 +65,9 @@ exec "$LLAMA_SERVER" \
|
||||
--model "$MODEL" \
|
||||
--spec-type "$SPEC_TYPE" \
|
||||
--spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
|
||||
--spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN" \
|
||||
--spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX" \
|
||||
--spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH" \
|
||||
--device "$DEVICE" \
|
||||
--gpu-layers "$N_GPU_LAYERS" \
|
||||
--flash-attn auto \
|
||||
|
||||
Reference in new issue
Block a user