Pivot to llama.cpp with DFlash speculative decoding on ROCm

- Abandon vLLM/FP8 (Triton compilation hangs, ~0.5 tok/s on Strix Halo)
- llama.cpp v10133 with ROCm0 offload, DFlash draft model
- Main: unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 (~22GB)
- Draft: Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 (~1.8GB)
- 3 parallel streams, 131072 context/stream
- systemd service with minimal sandboxing (ROCm requires /dev/kfd access)
- Baseline: 7.62 tok/s (threshold 5.0 tok/s)

scripts/validate.sh

@@ -1,55 +1,78 @@
 #!/usr/bin/env bash
-# Validation end-to-end test for vLLM on dankclaw
-# Sends a simple prompt and measures total response time
-# Usage: ./scripts/validate.sh [host] [port]
+# Validation test for llama.cpp DFlash+MTP server
+# Sends a simple prompt and measures generation speed (tokens/sec)
+set -euo pipefail
 
-HOST="${1:-dankclaw}"
-PORT="${2:-8000}"
-MODEL="bottlecapai/ThinkingCap-Qwen3.6-27B-FP8"
+PORT="${LLAMA_PORT:-8080}"
+HOST="${LLAMA_HOST:-127.0.0.1}"
+MAX_TOKENS="${MAX_TOKENS:-256}"
 
-echo "=== vLLM Validation Test ==="
-echo "Target: ${HOST}:${PORT}"
-echo "Model: ${MODEL}"
+echo "=== llama.cpp Validation Test ==="
+echo "Target: http://${HOST}:${PORT}"
+echo "Max tokens: ${MAX_TOKENS}"
 echo ""
 
+# Wait for server to be ready
+echo "Waiting for server..."
+for i in $(seq 1 300); do
+    if curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
+        echo "Server ready after $((i * 2))s"
+        break
+    fi
+    if [ "$i" -eq 300 ]; then
+        echo "FAIL: Server did not become ready in 600s"
+        exit 1
+    fi
+    sleep 2
+done
+
+# Send the request and capture timing
+echo ""
+echo "Sending validation prompt..."
 START_TIME=$(date +%s%N)
 
-RESPONSE=$(curl -s --max-time 60 -w "\n%{http_code}" \
-    "http://${HOST}:${PORT}/v1/chat/completions" \
+RESPONSE=$(curl -s \
+    -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
     -H "Content-Type: application/json" \
     -d "{
-        \"model\": \"${MODEL}\",
         \"messages\": [
-            {\"role\": \"user\", \"content\": \"What is 2 plus 2? Answer in one word.\"}
+            {\"role\": \"user\", \"content\": \"What is 2+2? Answer in one sentence.\"}
         ],
-        \"max_tokens\": 128,
-        \"temperature\": 0.1
+        \"max_tokens\": ${MAX_TOKENS},
+        \"temperature\": 0.7,
+        \"top_p\": 0.95
     }")
 
 END_TIME=$(date +%s%N)
-ELAPSED_MS=$(( (END_TIME - START_TIME) / 1000000 ))
 
-HTTP_CODE=$(echo "$RESPONSE" | tail -1)
-BODY=$(echo "$RESPONSE" | sed '$d')
+# Extract token counts and timing from response
+COMPLETION_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0")
+TOTAL_TIME_MS=$(( (END_TIME - START_TIME) / 1000000 ))
 
-echo "HTTP Status: ${HTTP_CODE}"
-echo "Response Time: ${ELAPSED_MS}ms"
+if [ "$COMPLETION_TOKENS" -eq 0 ]; then
+    echo "FAIL: No tokens generated or response parse error"
+    echo "Response: $RESPONSE"
+    exit 1
+fi
+
+# Calculate tokens/sec
+TOK_PER_SEC=$(python3 -c "print(f'{${COMPLETION_TOKENS} * 1000 / ${TOTAL_TIME_MS}:.2f}')")
+
 echo ""
+echo "=== Results ==="
+echo "Completion tokens: ${COMPLETION_TOKENS}"
+echo "Total time: ${TOTAL_TIME_MS}ms"
+echo "Speed: ${TOK_PER_SEC} tok/s"
+echo ""
+
+# Check minimum speed threshold
+MIN_SPEED="${MIN_SPEED:-5.0}"
+PASS=$(python3 -c "print('yes' if ${TOK_PER_SEC} >= ${MIN_SPEED} else 'no')")
 
-if [ "$HTTP_CODE" = "200" ]; then
-    if [ "$ELAPSED_MS" -gt 60000 ]; then
-        echo "FAIL — Response time ${ELAPSED_MS}ms exceeds 60s limit"
-        echo "TIME=${ELAPSED_MS}"
-        exit 1
-    fi
-    CONTENT=$(echo "$BODY" | python3 -c "import sys,json; print(json.load(sys.stdin)['choices'][0]['message']['content'].strip())" 2>/dev/null)
-    echo "Model Output: ${CONTENT}"
-    echo ""
-    echo "PASS — Server responded successfully"
-    echo "TIME=${ELAPSED_MS}"
+if [ "$PASS" = "yes" ]; then
+    echo "PASS: Speed ${TOK_PER_SEC} tok/s >= ${MIN_SPEED} tok/s threshold"
     exit 0
 else
-    echo "FAIL — Server returned HTTP ${HTTP_CODE}"
-    echo "Response: ${BODY}"
+    echo "FAIL: Speed ${TOK_PER_SEC} tok/s < ${MIN_SPEED} tok/s threshold"
     exit 1
 fi