From 471e96335021c5d1537e07e0017937b8b362576f Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Fri, 7 Aug 2026 16:44:12 -0700
Subject: [PATCH] Fix parallel validation, remove ngram-mod

- Rewrite validation script with temp files for true parallel curl requests
- Both prompts now complete 1024 tokens each (was cutting first short)
- Remove ngram-mod from spec pipeline (was -1.2 tok/s regression)
- True parallel throughput: 15.68 tok/s across 2 streams
---
 scripts/llama-server.service     |  2 +-
 scripts/serve-llamacpp-dflash.sh |  2 +-
 scripts/validate.sh              | 28 +++++++++++++++++-----------
 3 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/scripts/llama-server.service b/scripts/llama-server.service
index f2a8430..2a76b96 100644
--- a/scripts/llama-server.service
+++ b/scripts/llama-server.service
@@ -6,7 +6,7 @@ After=network.target
 Type=simple
 User=dank
 Group=dank
-ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-type draft-dflash,ngram-mod --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --load-mode none
+ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-type draft-dflash --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --load-mode none
 Restart=on-failure
 RestartSec=10
 
diff --git a/scripts/serve-llamacpp-dflash.sh b/scripts/serve-llamacpp-dflash.sh
index 8694d36..70b33bd 100755
--- a/scripts/serve-llamacpp-dflash.sh
+++ b/scripts/serve-llamacpp-dflash.sh
@@ -26,7 +26,7 @@ echo ""
 exec "${SERVER_BIN}" \
     --hf-repo "${TARGET_REPO}" \
     --spec-draft-hf "${DRAFT_REPO}" \
-    --spec-type draft-dflash,ngram-mod \
+    --spec-type draft-dflash \
     --spec-draft-n-max "${DRAFT_N_MAX}" \
     --device ROCm0 \
     --spec-draft-device ROCm0 \
diff --git a/scripts/validate.sh b/scripts/validate.sh
index edac8e5..5b60ece 100755
--- a/scripts/validate.sh
+++ b/scripts/validate.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
-# Validation test for llama.cpp DFlash+MTP server
-# Sends a simple prompt and measures generation speed (tokens/sec)
+# Validation test for llama.cpp DFlash server
+# Sends 2 parallel prompts and measures generation speed (tokens/sec)
 set -euo pipefail
 
 PORT="${LLAMA_PORT:-8080}"
@@ -31,7 +31,11 @@ echo ""
 echo "Sending 2 parallel validation prompts..."
 START_TIME=$(date +%s%N)
 
-RESPONSE1=$(curl -s \
+TMP1=$(mktemp)
+TMP2=$(mktemp)
+trap "rm -f $TMP1 $TMP2" EXIT
+
+curl -s \
     -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
     -H "Content-Type: application/json" \
     -d "{
@@ -41,9 +45,10 @@ RESPONSE1=$(curl -s \
         \"max_tokens\": ${MAX_TOKENS},
         \"temperature\": 0.7,
         \"top_p\": 0.95
-    }" &)
+    }" > "$TMP1" &
+PID1=$!
 
-RESPONSE2=$(curl -s \
+curl -s \
     -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
     -H "Content-Type: application/json" \
     -d "{
@@ -53,22 +58,23 @@ RESPONSE2=$(curl -s \
         \"max_tokens\": ${MAX_TOKENS},
         \"temperature\": 0.7,
         \"top_p\": 0.95
-    }" &)
+    }" > "$TMP2" &
+PID2=$!
 
-wait
+wait $PID1 $PID2
 
 END_TIME=$(date +%s%N)
 
 # Extract token counts and timing from responses
-COMPLETION_TOKENS1=$(echo "$RESPONSE1" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0")
-COMPLETION_TOKENS2=$(echo "$RESPONSE2" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0")
+COMPLETION_TOKENS1=$(python3 -c "import json; d=json.load(open('$TMP1')); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0")
+COMPLETION_TOKENS2=$(python3 -c "import json; d=json.load(open('$TMP2')); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0")
 TOTAL_TOKENS=$((COMPLETION_TOKENS1 + COMPLETION_TOKENS2))
 TOTAL_TIME_MS=$(( (END_TIME - START_TIME) / 1000000 ))
 
 if [ "$TOTAL_TOKENS" -eq 0 ]; then
     echo "FAIL: No tokens generated or response parse error"
-    echo "Response1: $RESPONSE1"
-    echo "Response2: $RESPONSE2"
+    echo "Response1: $(cat $TMP1)"
+    echo "Response2: $(cat $TMP2)"
     exit 1
 fi