From 1949f0651b97041a2398a4ae4eccb5be93403cdc Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 7 Aug 2026 15:41:27 -0700 Subject: [PATCH] Pivot to llama.cpp with DFlash speculative decoding on ROCm - Abandon vLLM/FP8 (Triton compilation hangs, ~0.5 tok/s on Strix Halo) - llama.cpp v10133 with ROCm0 offload, DFlash draft model - Main: unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 (~22GB) - Draft: Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 (~1.8GB) - 3 parallel streams, 131072 context/stream - systemd service with minimal sandboxing (ROCm requires /dev/kfd access) - Baseline: 7.62 tok/s (threshold 5.0 tok/s) --- scripts/llama-server.service | 14 +++++ scripts/serve-llamacpp-dflash.sh | 41 +++++++++++++++ scripts/serve-thinkingcap.sh | 30 ----------- scripts/validate.sh | 89 ++++++++++++++++++++------------ 4 files changed, 111 insertions(+), 63 deletions(-) create mode 100644 scripts/llama-server.service create mode 100755 scripts/serve-llamacpp-dflash.sh delete mode 100644 scripts/serve-thinkingcap.sh diff --git a/scripts/llama-server.service b/scripts/llama-server.service new file mode 100644 index 0000000..4f049f0 --- /dev/null +++ b/scripts/llama-server.service @@ -0,0 +1,14 @@ +[Unit] +Description=llama.cpp DFlash+MTP Inference Server (Qwen3.6-27B on ROCm) +After=network.target + +[Service] +Type=simple +User=dank +Group=dank +ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --load-mode mlock +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/serve-llamacpp-dflash.sh b/scripts/serve-llamacpp-dflash.sh new file mode 100755 index 0000000..845498a --- /dev/null +++ b/scripts/serve-llamacpp-dflash.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Serve Qwen3.6-27B with DFlash + MTP speculative decoding via llama.cpp on AMD Strix Halo (gfx1151) +# Uses llama.cpp built with ROCm support +# 3 parallel streams, 131072 context each +set -euo pipefail + +SERVER_BIN="/home/dank/llama.cpp/build/bin/llama-server" +PORT="${LLAMA_PORT:-8080}" +N_PARALLEL="${LLAMA_N_PARALLEL:-3}" +CTX_SIZE="${LLAMA_CTX_SIZE:-131072}" +BATCH_SIZE="${LLAMA_BATCH_SIZE:-4096}" +DRAFT_N_MAX="${LLAMA_DRAFT_N_MAX:-16}" + +# Models via Hugging Face +TARGET_REPO="unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0" +DRAFT_REPO="Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0" + +echo "Starting llama.cpp server..." +echo " Target model: ${TARGET_REPO}" +echo " Draft model: ${DRAFT_REPO}" +echo " Parallel streams: ${N_PARALLEL}" +echo " Context/stream: ${CTX_SIZE}" +echo " Draft tokens: ${DRAFT_N_MAX}" +echo "" + +exec "${SERVER_BIN}" \ + --hf-repo "${TARGET_REPO}" \ + --spec-draft-hf "${DRAFT_REPO}" \ + --spec-draft-n-max "${DRAFT_N_MAX}" \ + --device ROCm0 \ + --spec-draft-device ROCm0 \ + --gpu-layers all \ + --fit on \ + --fit-target 512 \ + --parallel "${N_PARALLEL}" \ + --ctx-size "${CTX_SIZE}" \ + --batch-size "${BATCH_SIZE}" \ + --port "${PORT}" \ + --host "0.0.0.0" \ + --threads 16 \ + --load-mode mlock diff --git a/scripts/serve-thinkingcap.sh b/scripts/serve-thinkingcap.sh deleted file mode 100644 index 630521b..0000000 --- a/scripts/serve-thinkingcap.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# vLLM serve script for ThinkingCap-Qwen3.6-27B-FP8 on dankclaw (ROCm) -# Run via systemd service: vllm-thinkingcap.service - -set -euo pipefail - -# Environment -export HSA_OVERRIDE_GFX_VERSION=11.5.1 -export HF_HOME="/home/dank/.cache/huggingface" - -# vLLM virtual environment — use direct paths, no activate script -VENV="/home/dank/vllm-env" -export PATH="${VENV}/bin:${PATH}" - -# Model -MODEL="bottlecapai/ThinkingCap-Qwen3.6-27B-FP8" - -# Serve -exec "${VENV}/bin/vllm" serve "${MODEL}" \ - --port 8000 \ - --host 0.0.0.0 \ - --kv-cache-dtype fp8 \ - --gpu-memory-utilization 0.80 \ - --enforce-eager \ - --max-model-len 32768 \ - --max-num-batched-tokens 2048 \ - --max-num-seqs 32 \ - --reasoning-parser qwen3 \ - --language-model-only \ - --dtype auto diff --git a/scripts/validate.sh b/scripts/validate.sh index 4f43533..7b006de 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -1,55 +1,78 @@ #!/usr/bin/env bash -# Validation end-to-end test for vLLM on dankclaw -# Sends a simple prompt and measures total response time -# Usage: ./scripts/validate.sh [host] [port] +# Validation test for llama.cpp DFlash+MTP server +# Sends a simple prompt and measures generation speed (tokens/sec) +set -euo pipefail -HOST="${1:-dankclaw}" -PORT="${2:-8000}" -MODEL="bottlecapai/ThinkingCap-Qwen3.6-27B-FP8" +PORT="${LLAMA_PORT:-8080}" +HOST="${LLAMA_HOST:-127.0.0.1}" +MAX_TOKENS="${MAX_TOKENS:-256}" -echo "=== vLLM Validation Test ===" -echo "Target: ${HOST}:${PORT}" -echo "Model: ${MODEL}" +echo "=== llama.cpp Validation Test ===" +echo "Target: http://${HOST}:${PORT}" +echo "Max tokens: ${MAX_TOKENS}" echo "" +# Wait for server to be ready +echo "Waiting for server..." +for i in $(seq 1 300); do + if curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then + echo "Server ready after $((i * 2))s" + break + fi + if [ "$i" -eq 300 ]; then + echo "FAIL: Server did not become ready in 600s" + exit 1 + fi + sleep 2 +done + +# Send the request and capture timing +echo "" +echo "Sending validation prompt..." START_TIME=$(date +%s%N) -RESPONSE=$(curl -s --max-time 60 -w "\n%{http_code}" \ - "http://${HOST}:${PORT}/v1/chat/completions" \ +RESPONSE=$(curl -s \ + -X POST "http://${HOST}:${PORT}/v1/chat/completions" \ -H "Content-Type: application/json" \ -d "{ - \"model\": \"${MODEL}\", \"messages\": [ - {\"role\": \"user\", \"content\": \"What is 2 plus 2? Answer in one word.\"} + {\"role\": \"user\", \"content\": \"What is 2+2? Answer in one sentence.\"} ], - \"max_tokens\": 128, - \"temperature\": 0.1 + \"max_tokens\": ${MAX_TOKENS}, + \"temperature\": 0.7, + \"top_p\": 0.95 }") END_TIME=$(date +%s%N) -ELAPSED_MS=$(( (END_TIME - START_TIME) / 1000000 )) -HTTP_CODE=$(echo "$RESPONSE" | tail -1) -BODY=$(echo "$RESPONSE" | sed '$d') +# Extract token counts and timing from response +COMPLETION_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0") +TOTAL_TIME_MS=$(( (END_TIME - START_TIME) / 1000000 )) -echo "HTTP Status: ${HTTP_CODE}" -echo "Response Time: ${ELAPSED_MS}ms" +if [ "$COMPLETION_TOKENS" -eq 0 ]; then + echo "FAIL: No tokens generated or response parse error" + echo "Response: $RESPONSE" + exit 1 +fi + +# Calculate tokens/sec +TOK_PER_SEC=$(python3 -c "print(f'{${COMPLETION_TOKENS} * 1000 / ${TOTAL_TIME_MS}:.2f}')") + echo "" +echo "=== Results ===" +echo "Completion tokens: ${COMPLETION_TOKENS}" +echo "Total time: ${TOTAL_TIME_MS}ms" +echo "Speed: ${TOK_PER_SEC} tok/s" +echo "" + +# Check minimum speed threshold +MIN_SPEED="${MIN_SPEED:-5.0}" +PASS=$(python3 -c "print('yes' if ${TOK_PER_SEC} >= ${MIN_SPEED} else 'no')") -if [ "$HTTP_CODE" = "200" ]; then - if [ "$ELAPSED_MS" -gt 60000 ]; then - echo "FAIL — Response time ${ELAPSED_MS}ms exceeds 60s limit" - echo "TIME=${ELAPSED_MS}" - exit 1 - fi - CONTENT=$(echo "$BODY" | python3 -c "import sys,json; print(json.load(sys.stdin)['choices'][0]['message']['content'].strip())" 2>/dev/null) - echo "Model Output: ${CONTENT}" - echo "" - echo "PASS — Server responded successfully" - echo "TIME=${ELAPSED_MS}" +if [ "$PASS" = "yes" ]; then + echo "PASS: Speed ${TOK_PER_SEC} tok/s >= ${MIN_SPEED} tok/s threshold" exit 0 else - echo "FAIL — Server returned HTTP ${HTTP_CODE}" - echo "Response: ${BODY}" + echo "FAIL: Speed ${TOK_PER_SEC} tok/s < ${MIN_SPEED} tok/s threshold" exit 1 fi