Enable DFlash speculative decoding, remove mlock
- Add --spec-type draft-dflash (was missing, DFlash was inactive) - Change --load-mode from mlock to none (RLIMIT_MEMLOCK too small for 2.7GB) - Increase validation max_tokens from 256 to 1024 with complex ML prompt - Baseline: 18.76 tok/s (was 7.62 tok/s without DFlash)
This commit is contained in:
1 parent
1949f0651b
commit
25d57184a6
3 files changed
+5
-4
No files matched your search
@@ -6,7 +6,7 @@ After=network.target
|
|||||||
Type=simple
|
Type=simple
|
||||||
User=dank
|
User=dank
|
||||||
Group=dank
|
Group=dank
|
||||||
ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --load-mode mlock
|
ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-type draft-dflash --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --load-mode none
|
||||||
Restart=on-failure
|
Restart=on-failure
|
||||||
RestartSec=10
|
RestartSec=10
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ echo ""
|
|||||||
exec "${SERVER_BIN}" \
|
exec "${SERVER_BIN}" \
|
||||||
--hf-repo "${TARGET_REPO}" \
|
--hf-repo "${TARGET_REPO}" \
|
||||||
--spec-draft-hf "${DRAFT_REPO}" \
|
--spec-draft-hf "${DRAFT_REPO}" \
|
||||||
|
--spec-type draft-dflash \
|
||||||
--spec-draft-n-max "${DRAFT_N_MAX}" \
|
--spec-draft-n-max "${DRAFT_N_MAX}" \
|
||||||
--device ROCm0 \
|
--device ROCm0 \
|
||||||
--spec-draft-device ROCm0 \
|
--spec-draft-device ROCm0 \
|
||||||
@@ -38,4 +39,4 @@ exec "${SERVER_BIN}" \
|
|||||||
--port "${PORT}" \
|
--port "${PORT}" \
|
||||||
--host "0.0.0.0" \
|
--host "0.0.0.0" \
|
||||||
--threads 16 \
|
--threads 16 \
|
||||||
--load-mode mlock
|
--load-mode none
|
||||||
+2
-2
@@ -5,7 +5,7 @@ set -euo pipefail
|
|||||||
|
|
||||||
PORT="${LLAMA_PORT:-8080}"
|
PORT="${LLAMA_PORT:-8080}"
|
||||||
HOST="${LLAMA_HOST:-127.0.0.1}"
|
HOST="${LLAMA_HOST:-127.0.0.1}"
|
||||||
MAX_TOKENS="${MAX_TOKENS:-256}"
|
MAX_TOKENS="${MAX_TOKENS:-1024}"
|
||||||
|
|
||||||
echo "=== llama.cpp Validation Test ==="
|
echo "=== llama.cpp Validation Test ==="
|
||||||
echo "Target: http://${HOST}:${PORT}"
|
echo "Target: http://${HOST}:${PORT}"
|
||||||
@@ -36,7 +36,7 @@ RESPONSE=$(curl -s \
|
|||||||
-H "Content-Type: application/json" \
|
-H "Content-Type: application/json" \
|
||||||
-d "{
|
-d "{
|
||||||
\"messages\": [
|
\"messages\": [
|
||||||
{\"role\": \"user\", \"content\": \"What is 2+2? Answer in one sentence.\"}
|
{\"role\": \"user\", \"content\": \"Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone.\"}
|
||||||
],
|
],
|
||||||
\"max_tokens\": ${MAX_TOKENS},
|
\"max_tokens\": ${MAX_TOKENS},
|
||||||
\"temperature\": 0.7,
|
\"temperature\": 0.7,
|
||||||
|
|||||||
Reference in new issue
Block a user