dank/serverAppConfigs
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

Add ngram-mod to spec pipeline: -1.2 tok/s regression

Browse Source 
- DFlash+ngram-mod: 17.11 tok/s vs DFlash alone: 18.34 tok/s
- ngram overhead not offset by acceptance rate at current settings
- Keeping in place for now, may tune ngram-mod params later
This commit is contained in:
Frank
2026-08-07 16:25:49 -07:00
parent 764a65080f
commit 2426ca563a
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/llama-server.service
+1 -1
View File
@@ -6,7 +6,7 @@ After=network.target
Type=simple
User=dank
Group=dank
ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-type draft-dflash --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --load-mode none
ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-type draft-dflash,ngram-mod --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --load-mode none
Restart=on-failure
RestartSec=10
scripts/serve-llamacpp-dflash.sh
+1 -1
View File
@@ -26,7 +26,7 @@ echo ""
exec "${SERVER_BIN}" \
--hf-repo "${TARGET_REPO}" \
--spec-draft-hf "${DRAFT_REPO}" \
--spec-type draft-dflash \
--spec-type draft-dflash,ngram-mod \
--spec-draft-n-max "${DRAFT_N_MAX}" \
--device ROCm0 \
--spec-draft-device ROCm0 \