Frank
1949f0651b
- Abandon vLLM/FP8 (Triton compilation hangs, ~0.5 tok/s on Strix Halo) - llama.cpp v10133 with ROCm0 offload, DFlash draft model - Main: unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 (~22GB) - Draft: Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 (~1.8GB) - 3 parallel streams, 131072 context/stream - systemd service with minimal sandboxing (ROCm requires /dev/kfd access) - Baseline: 7.62 tok/s (threshold 5.0 tok/s)
42 lines
1.2 KiB
Bash
Executable File
42 lines
1.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve Qwen3.6-27B with DFlash + MTP speculative decoding via llama.cpp on AMD Strix Halo (gfx1151)
|
|
# Uses llama.cpp built with ROCm support
|
|
# 3 parallel streams, 131072 context each
|
|
set -euo pipefail
|
|
|
|
SERVER_BIN="/home/dank/llama.cpp/build/bin/llama-server"
|
|
PORT="${LLAMA_PORT:-8080}"
|
|
N_PARALLEL="${LLAMA_N_PARALLEL:-3}"
|
|
CTX_SIZE="${LLAMA_CTX_SIZE:-131072}"
|
|
BATCH_SIZE="${LLAMA_BATCH_SIZE:-4096}"
|
|
DRAFT_N_MAX="${LLAMA_DRAFT_N_MAX:-16}"
|
|
|
|
# Models via Hugging Face
|
|
TARGET_REPO="unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0"
|
|
DRAFT_REPO="Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0"
|
|
|
|
echo "Starting llama.cpp server..."
|
|
echo " Target model: ${TARGET_REPO}"
|
|
echo " Draft model: ${DRAFT_REPO}"
|
|
echo " Parallel streams: ${N_PARALLEL}"
|
|
echo " Context/stream: ${CTX_SIZE}"
|
|
echo " Draft tokens: ${DRAFT_N_MAX}"
|
|
echo ""
|
|
|
|
exec "${SERVER_BIN}" \
|
|
--hf-repo "${TARGET_REPO}" \
|
|
--spec-draft-hf "${DRAFT_REPO}" \
|
|
--spec-draft-n-max "${DRAFT_N_MAX}" \
|
|
--device ROCm0 \
|
|
--spec-draft-device ROCm0 \
|
|
--gpu-layers all \
|
|
--fit on \
|
|
--fit-target 512 \
|
|
--parallel "${N_PARALLEL}" \
|
|
--ctx-size "${CTX_SIZE}" \
|
|
--batch-size "${BATCH_SIZE}" \
|
|
--port "${PORT}" \
|
|
--host "0.0.0.0" \
|
|
--threads 16 \
|
|
--load-mode mlock
|