Pivot to llama.cpp with DFlash speculative decoding on ROCm
- Abandon vLLM/FP8 (Triton compilation hangs, ~0.5 tok/s on Strix Halo) - llama.cpp v10133 with ROCm0 offload, DFlash draft model - Main: unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 (~22GB) - Draft: Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 (~1.8GB) - 3 parallel streams, 131072 context/stream - systemd service with minimal sandboxing (ROCm requires /dev/kfd access) - Baseline: 7.62 tok/s (threshold 5.0 tok/s)
This commit is contained in:
Frank
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=llama.cpp DFlash+MTP Inference Server (Qwen3.6-27B on ROCm)
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --load-mode mlock
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve Qwen3.6-27B with DFlash + MTP speculative decoding via llama.cpp on AMD Strix Halo (gfx1151)
|
||||
# Uses llama.cpp built with ROCm support
|
||||
# 3 parallel streams, 131072 context each
|
||||
set -euo pipefail
|
||||
|
||||
SERVER_BIN="/home/dank/llama.cpp/build/bin/llama-server"
|
||||
PORT="${LLAMA_PORT:-8080}"
|
||||
N_PARALLEL="${LLAMA_N_PARALLEL:-3}"
|
||||
CTX_SIZE="${LLAMA_CTX_SIZE:-131072}"
|
||||
BATCH_SIZE="${LLAMA_BATCH_SIZE:-4096}"
|
||||
DRAFT_N_MAX="${LLAMA_DRAFT_N_MAX:-16}"
|
||||
|
||||
# Models via Hugging Face
|
||||
TARGET_REPO="unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0"
|
||||
DRAFT_REPO="Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0"
|
||||
|
||||
echo "Starting llama.cpp server..."
|
||||
echo " Target model: ${TARGET_REPO}"
|
||||
echo " Draft model: ${DRAFT_REPO}"
|
||||
echo " Parallel streams: ${N_PARALLEL}"
|
||||
echo " Context/stream: ${CTX_SIZE}"
|
||||
echo " Draft tokens: ${DRAFT_N_MAX}"
|
||||
echo ""
|
||||
|
||||
exec "${SERVER_BIN}" \
|
||||
--hf-repo "${TARGET_REPO}" \
|
||||
--spec-draft-hf "${DRAFT_REPO}" \
|
||||
--spec-draft-n-max "${DRAFT_N_MAX}" \
|
||||
--device ROCm0 \
|
||||
--spec-draft-device ROCm0 \
|
||||
--gpu-layers all \
|
||||
--fit on \
|
||||
--fit-target 512 \
|
||||
--parallel "${N_PARALLEL}" \
|
||||
--ctx-size "${CTX_SIZE}" \
|
||||
--batch-size "${BATCH_SIZE}" \
|
||||
--port "${PORT}" \
|
||||
--host "0.0.0.0" \
|
||||
--threads 16 \
|
||||
--load-mode mlock
|
||||
@@ -1,30 +0,0 @@
|
||||
#!/bin/bash
|
||||
# vLLM serve script for ThinkingCap-Qwen3.6-27B-FP8 on dankclaw (ROCm)
|
||||
# Run via systemd service: vllm-thinkingcap.service
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Environment
|
||||
export HSA_OVERRIDE_GFX_VERSION=11.5.1
|
||||
export HF_HOME="/home/dank/.cache/huggingface"
|
||||
|
||||
# vLLM virtual environment — use direct paths, no activate script
|
||||
VENV="/home/dank/vllm-env"
|
||||
export PATH="${VENV}/bin:${PATH}"
|
||||
|
||||
# Model
|
||||
MODEL="bottlecapai/ThinkingCap-Qwen3.6-27B-FP8"
|
||||
|
||||
# Serve
|
||||
exec "${VENV}/bin/vllm" serve "${MODEL}" \
|
||||
--port 8000 \
|
||||
--host 0.0.0.0 \
|
||||
--kv-cache-dtype fp8 \
|
||||
--gpu-memory-utilization 0.80 \
|
||||
--enforce-eager \
|
||||
--max-model-len 32768 \
|
||||
--max-num-batched-tokens 2048 \
|
||||
--max-num-seqs 32 \
|
||||
--reasoning-parser qwen3 \
|
||||
--language-model-only \
|
||||
--dtype auto
|
||||
+56
-33
@@ -1,55 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# Validation end-to-end test for vLLM on dankclaw
|
||||
# Sends a simple prompt and measures total response time
|
||||
# Usage: ./scripts/validate.sh [host] [port]
|
||||
# Validation test for llama.cpp DFlash+MTP server
|
||||
# Sends a simple prompt and measures generation speed (tokens/sec)
|
||||
set -euo pipefail
|
||||
|
||||
HOST="${1:-dankclaw}"
|
||||
PORT="${2:-8000}"
|
||||
MODEL="bottlecapai/ThinkingCap-Qwen3.6-27B-FP8"
|
||||
PORT="${LLAMA_PORT:-8080}"
|
||||
HOST="${LLAMA_HOST:-127.0.0.1}"
|
||||
MAX_TOKENS="${MAX_TOKENS:-256}"
|
||||
|
||||
echo "=== vLLM Validation Test ==="
|
||||
echo "Target: ${HOST}:${PORT}"
|
||||
echo "Model: ${MODEL}"
|
||||
echo "=== llama.cpp Validation Test ==="
|
||||
echo "Target: http://${HOST}:${PORT}"
|
||||
echo "Max tokens: ${MAX_TOKENS}"
|
||||
echo ""
|
||||
|
||||
# Wait for server to be ready
|
||||
echo "Waiting for server..."
|
||||
for i in $(seq 1 300); do
|
||||
if curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
|
||||
echo "Server ready after $((i * 2))s"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 300 ]; then
|
||||
echo "FAIL: Server did not become ready in 600s"
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Send the request and capture timing
|
||||
echo ""
|
||||
echo "Sending validation prompt..."
|
||||
START_TIME=$(date +%s%N)
|
||||
|
||||
RESPONSE=$(curl -s --max-time 60 -w "\n%{http_code}" \
|
||||
"http://${HOST}:${PORT}/v1/chat/completions" \
|
||||
RESPONSE=$(curl -s \
|
||||
-X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"model\": \"${MODEL}\",
|
||||
\"messages\": [
|
||||
{\"role\": \"user\", \"content\": \"What is 2 plus 2? Answer in one word.\"}
|
||||
{\"role\": \"user\", \"content\": \"What is 2+2? Answer in one sentence.\"}
|
||||
],
|
||||
\"max_tokens\": 128,
|
||||
\"temperature\": 0.1
|
||||
\"max_tokens\": ${MAX_TOKENS},
|
||||
\"temperature\": 0.7,
|
||||
\"top_p\": 0.95
|
||||
}")
|
||||
|
||||
END_TIME=$(date +%s%N)
|
||||
ELAPSED_MS=$(( (END_TIME - START_TIME) / 1000000 ))
|
||||
|
||||
HTTP_CODE=$(echo "$RESPONSE" | tail -1)
|
||||
BODY=$(echo "$RESPONSE" | sed '$d')
|
||||
# Extract token counts and timing from response
|
||||
COMPLETION_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0")
|
||||
TOTAL_TIME_MS=$(( (END_TIME - START_TIME) / 1000000 ))
|
||||
|
||||
echo "HTTP Status: ${HTTP_CODE}"
|
||||
echo "Response Time: ${ELAPSED_MS}ms"
|
||||
if [ "$COMPLETION_TOKENS" -eq 0 ]; then
|
||||
echo "FAIL: No tokens generated or response parse error"
|
||||
echo "Response: $RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Calculate tokens/sec
|
||||
TOK_PER_SEC=$(python3 -c "print(f'{${COMPLETION_TOKENS} * 1000 / ${TOTAL_TIME_MS}:.2f}')")
|
||||
|
||||
echo ""
|
||||
echo "=== Results ==="
|
||||
echo "Completion tokens: ${COMPLETION_TOKENS}"
|
||||
echo "Total time: ${TOTAL_TIME_MS}ms"
|
||||
echo "Speed: ${TOK_PER_SEC} tok/s"
|
||||
echo ""
|
||||
|
||||
# Check minimum speed threshold
|
||||
MIN_SPEED="${MIN_SPEED:-5.0}"
|
||||
PASS=$(python3 -c "print('yes' if ${TOK_PER_SEC} >= ${MIN_SPEED} else 'no')")
|
||||
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
if [ "$ELAPSED_MS" -gt 60000 ]; then
|
||||
echo "FAIL — Response time ${ELAPSED_MS}ms exceeds 60s limit"
|
||||
echo "TIME=${ELAPSED_MS}"
|
||||
exit 1
|
||||
fi
|
||||
CONTENT=$(echo "$BODY" | python3 -c "import sys,json; print(json.load(sys.stdin)['choices'][0]['message']['content'].strip())" 2>/dev/null)
|
||||
echo "Model Output: ${CONTENT}"
|
||||
echo ""
|
||||
echo "PASS — Server responded successfully"
|
||||
echo "TIME=${ELAPSED_MS}"
|
||||
if [ "$PASS" = "yes" ]; then
|
||||
echo "PASS: Speed ${TOK_PER_SEC} tok/s >= ${MIN_SPEED} tok/s threshold"
|
||||
exit 0
|
||||
else
|
||||
echo "FAIL — Server returned HTTP ${HTTP_CODE}"
|
||||
echo "Response: ${BODY}"
|
||||
echo "FAIL: Speed ${TOK_PER_SEC} tok/s < ${MIN_SPEED} tok/s threshold"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user