diff --git a/scripts/llama-server.service b/scripts/llama-server.service index 4f049f0..2a76b96 100644 --- a/scripts/llama-server.service +++ b/scripts/llama-server.service @@ -6,7 +6,7 @@ After=network.target Type=simple User=dank Group=dank -ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --load-mode mlock +ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-type draft-dflash --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --load-mode none Restart=on-failure RestartSec=10 diff --git a/scripts/serve-llamacpp-dflash.sh b/scripts/serve-llamacpp-dflash.sh index 845498a..70b33bd 100755 --- a/scripts/serve-llamacpp-dflash.sh +++ b/scripts/serve-llamacpp-dflash.sh @@ -26,6 +26,7 @@ echo "" exec "${SERVER_BIN}" \ --hf-repo "${TARGET_REPO}" \ --spec-draft-hf "${DRAFT_REPO}" \ + --spec-type draft-dflash \ --spec-draft-n-max "${DRAFT_N_MAX}" \ --device ROCm0 \ --spec-draft-device ROCm0 \ @@ -38,4 +39,4 @@ exec "${SERVER_BIN}" \ --port "${PORT}" \ --host "0.0.0.0" \ --threads 16 \ - --load-mode mlock + --load-mode none diff --git a/scripts/validate.sh b/scripts/validate.sh index 7b006de..b3fccf1 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -5,7 +5,7 @@ set -euo pipefail PORT="${LLAMA_PORT:-8080}" HOST="${LLAMA_HOST:-127.0.0.1}" -MAX_TOKENS="${MAX_TOKENS:-256}" +MAX_TOKENS="${MAX_TOKENS:-1024}" echo "=== llama.cpp Validation Test ===" echo "Target: http://${HOST}:${PORT}" @@ -36,7 +36,7 @@ RESPONSE=$(curl -s \ -H "Content-Type: application/json" \ -d "{ \"messages\": [ - {\"role\": \"user\", \"content\": \"What is 2+2? Answer in one sentence.\"} + {\"role\": \"user\", \"content\": \"Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone.\"} ], \"max_tokens\": ${MAX_TOKENS}, \"temperature\": 0.7,