Pivot to llama.cpp with DFlash speculative decoding on ROCm

- Abandon vLLM/FP8 (Triton compilation hangs, ~0.5 tok/s on Strix Halo)
- llama.cpp v10133 with ROCm0 offload, DFlash draft model
- Main: unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 (~22GB)
- Draft: Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 (~1.8GB)
- 3 parallel streams, 131072 context/stream
- systemd service with minimal sandboxing (ROCm requires /dev/kfd access)
- Baseline: 7.62 tok/s (threshold 5.0 tok/s)

scripts/llama-server.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=llama.cpp DFlash+MTP Inference Server (Qwen3.6-27B on ROCm)
+After=network.target
+
+[Service]
+Type=simple
+User=dank
+Group=dank
+ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --load-mode mlock
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

scripts/serve-llamacpp-dflash.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Serve Qwen3.6-27B with DFlash + MTP speculative decoding via llama.cpp on AMD Strix Halo (gfx1151)
+# Uses llama.cpp built with ROCm support
+# 3 parallel streams, 131072 context each
+set -euo pipefail
+
+SERVER_BIN="/home/dank/llama.cpp/build/bin/llama-server"
+PORT="${LLAMA_PORT:-8080}"
+N_PARALLEL="${LLAMA_N_PARALLEL:-3}"
+CTX_SIZE="${LLAMA_CTX_SIZE:-131072}"
+BATCH_SIZE="${LLAMA_BATCH_SIZE:-4096}"
+DRAFT_N_MAX="${LLAMA_DRAFT_N_MAX:-16}"
+
+# Models via Hugging Face
+TARGET_REPO="unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0"
+DRAFT_REPO="Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0"
+
+echo "Starting llama.cpp server..."
+echo "  Target model: ${TARGET_REPO}"
+echo "  Draft model:  ${DRAFT_REPO}"
+echo "  Parallel streams: ${N_PARALLEL}"
+echo "  Context/stream: ${CTX_SIZE}"
+echo "  Draft tokens: ${DRAFT_N_MAX}"
+echo ""
+
+exec "${SERVER_BIN}" \
+    --hf-repo "${TARGET_REPO}" \
+    --spec-draft-hf "${DRAFT_REPO}" \
+    --spec-draft-n-max "${DRAFT_N_MAX}" \
+    --device ROCm0 \
+    --spec-draft-device ROCm0 \
+    --gpu-layers all \
+    --fit on \
+    --fit-target 512 \
+    --parallel "${N_PARALLEL}" \
+    --ctx-size "${CTX_SIZE}" \
+    --batch-size "${BATCH_SIZE}" \
+    --port "${PORT}" \
+    --host "0.0.0.0" \
+    --threads 16 \
+    --load-mode mlock

scripts/serve-thinkingcap.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-# vLLM serve script for ThinkingCap-Qwen3.6-27B-FP8 on dankclaw (ROCm)
-# Run via systemd service: vllm-thinkingcap.service
-
-set -euo pipefail
-
-# Environment
-export HSA_OVERRIDE_GFX_VERSION=11.5.1
-export HF_HOME="/home/dank/.cache/huggingface"
-
-# vLLM virtual environment — use direct paths, no activate script
-VENV="/home/dank/vllm-env"
-export PATH="${VENV}/bin:${PATH}"
-
-# Model
-MODEL="bottlecapai/ThinkingCap-Qwen3.6-27B-FP8"
-
-# Serve
-exec "${VENV}/bin/vllm" serve "${MODEL}" \
-    --port 8000 \
-    --host 0.0.0.0 \
-    --kv-cache-dtype fp8 \
-    --gpu-memory-utilization 0.80 \
-    --enforce-eager \
-    --max-model-len 32768 \
-    --max-num-batched-tokens 2048 \
-    --max-num-seqs 32 \
-    --reasoning-parser qwen3 \
-    --language-model-only \
-    --dtype auto

scripts/validate.sh

@@ -1,55 +1,78 @@
 #!/usr/bin/env bash
-# Validation end-to-end test for vLLM on dankclaw
+# Validation test for llama.cpp DFlash+MTP server
-# Sends a simple prompt and measures total response time
+# Sends a simple prompt and measures generation speed (tokens/sec)
-# Usage: ./scripts/validate.sh [host] [port]
+set -euo pipefail
 
-HOST="${1:-dankclaw}"
+PORT="${LLAMA_PORT:-8080}"
-PORT="${2:-8000}"
+HOST="${LLAMA_HOST:-127.0.0.1}"
-MODEL="bottlecapai/ThinkingCap-Qwen3.6-27B-FP8"
+MAX_TOKENS="${MAX_TOKENS:-256}"
 
-echo "=== vLLM Validation Test ==="
+echo "=== llama.cpp Validation Test ==="
-echo "Target: ${HOST}:${PORT}"
+echo "Target: http://${HOST}:${PORT}"
-echo "Model: ${MODEL}"
+echo "Max tokens: ${MAX_TOKENS}"
 echo ""
 
+# Wait for server to be ready
+echo "Waiting for server..."
+for i in $(seq 1 300); do
+    if curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
+        echo "Server ready after $((i * 2))s"
+        break
+    fi
+    if [ "$i" -eq 300 ]; then
+        echo "FAIL: Server did not become ready in 600s"
+        exit 1
+    fi
+    sleep 2
+done
+
+# Send the request and capture timing
+echo ""
+echo "Sending validation prompt..."
 START_TIME=$(date +%s%N)
 
-RESPONSE=$(curl -s --max-time 60 -w "\n%{http_code}" \
+RESPONSE=$(curl -s \
-    "http://${HOST}:${PORT}/v1/chat/completions" \
+    -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
     -H "Content-Type: application/json" \
     -d "{
-        \"model\": \"${MODEL}\",
         \"messages\": [
-            {\"role\": \"user\", \"content\": \"What is 2 plus 2? Answer in one word.\"}
+            {\"role\": \"user\", \"content\": \"What is 2+2? Answer in one sentence.\"}
         ],
-        \"max_tokens\": 128,
+        \"max_tokens\": ${MAX_TOKENS},
-        \"temperature\": 0.1
+        \"temperature\": 0.7,
+        \"top_p\": 0.95
     }")
 
 END_TIME=$(date +%s%N)
-ELAPSED_MS=$(( (END_TIME - START_TIME) / 1000000 ))
 
-HTTP_CODE=$(echo "$RESPONSE" | tail -1)
+# Extract token counts and timing from response
-BODY=$(echo "$RESPONSE" | sed '$d')
+COMPLETION_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0")
+TOTAL_TIME_MS=$(( (END_TIME - START_TIME) / 1000000 ))
 
-echo "HTTP Status: ${HTTP_CODE}"
+if [ "$COMPLETION_TOKENS" -eq 0 ]; then
-echo "Response Time: ${ELAPSED_MS}ms"
+    echo "FAIL: No tokens generated or response parse error"
+    echo "Response: $RESPONSE"
+    exit 1
+fi
+
+# Calculate tokens/sec
+TOK_PER_SEC=$(python3 -c "print(f'{${COMPLETION_TOKENS} * 1000 / ${TOTAL_TIME_MS}:.2f}')")
+
 echo ""
+echo "=== Results ==="
+echo "Completion tokens: ${COMPLETION_TOKENS}"
+echo "Total time: ${TOTAL_TIME_MS}ms"
+echo "Speed: ${TOK_PER_SEC} tok/s"
+echo ""
+
+# Check minimum speed threshold
+MIN_SPEED="${MIN_SPEED:-5.0}"
+PASS=$(python3 -c "print('yes' if ${TOK_PER_SEC} >= ${MIN_SPEED} else 'no')")
 
-if [ "$HTTP_CODE" = "200" ]; then
+if [ "$PASS" = "yes" ]; then
-    if [ "$ELAPSED_MS" -gt 60000 ]; then
+    echo "PASS: Speed ${TOK_PER_SEC} tok/s >= ${MIN_SPEED} tok/s threshold"
-        echo "FAIL — Response time ${ELAPSED_MS}ms exceeds 60s limit"
-        echo "TIME=${ELAPSED_MS}"
-        exit 1
-    fi
-    CONTENT=$(echo "$BODY" | python3 -c "import sys,json; print(json.load(sys.stdin)['choices'][0]['message']['content'].strip())" 2>/dev/null)
-    echo "Model Output: ${CONTENT}"
-    echo ""
-    echo "PASS — Server responded successfully"
-    echo "TIME=${ELAPSED_MS}"
     exit 0
 else
-    echo "FAIL — Server returned HTTP ${HTTP_CODE}"
+    echo "FAIL: Speed ${TOK_PER_SEC} tok/s < ${MIN_SPEED} tok/s threshold"
-    echo "Response: ${BODY}"
     exit 1
 fi