From d0c80b5b1a8cd3c503a991ca59126fa7066b860c Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Tue, 11 Aug 2026 16:14:44 -0700
Subject: [PATCH] bench: 4k pipeline harness w/ early crash check; parametrized
 spec-type

---
 scripts/bench-pipeline-64k.sh | 69 ++++++++++++++++++++++++-----------
 1 file changed, 48 insertions(+), 21 deletions(-)

diff --git a/scripts/bench-pipeline-64k.sh b/scripts/bench-pipeline-64k.sh
index 255e0f1..23e8cf9 100644
--- a/scripts/bench-pipeline-64k.sh
+++ b/scripts/bench-pipeline-64k.sh
@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
-# bench-pipeline-64k.sh — 64k-context benchmark through the FULL speculative
-# pipeline (draft-dflash + ngram-mod) on a single backend.
+# bench-pipeline-64k.sh — 4k-context benchmark through a speculative pipeline on
+# a single backend.
 #
-# Starts llama-server with the DFlash+ngram chain, sends a ~64k-token prompt,
-# and reads the authoritative prefill / generation tok/s and draft acceptance
-# from the server's own `slot print_timing` lines.
+# Default spec chain is draft-dflash + ngram-mod. Pass a second argument to
+# override --spec-type (e.g. "ngram-mod" to bench ngram only, or "none" for no
+# speculation). The DFlash draft model is only referenced when the spec type
+# includes "dflash".
+#
+# Reads authoritative prefill / generation tok/s and draft acceptance from the
+# server's own `slot print_timing` lines.
 #
 # Usage:
-#   ./bench-pipeline-64k.sh <ROCm0|Vulkan0>
+#   ./bench-pipeline-64k.sh <ROCm0|Vulkan0> [spec-type]
 set -euo pipefail
 
-DEVICE="${1:?usage: bench-pipeline-64k.sh <ROCm0|Vulkan0>}"
+DEVICE="${1:?usage: bench-pipeline-64k.sh <ROCm0|Vulkan0> [spec-type]}"
+SPEC_TYPE="${2:-draft-dflash,ngram-mod}"
 MODEL_DIR=/home/dank/models
 MODEL_FILE=Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf
 DRAFT_FILE=Qwen3.6-27B-DFlash-Q8_0.gguf
@@ -19,7 +24,7 @@ MODEL="$MODEL_DIR/$MODEL_FILE"
 DRAFT="$MODEL_DIR/$DRAFT_FILE"
 
 PORT=7891
-CTX=65536
+CTX=8192        # 8k ctx: room for a ~4k prompt + generation
 PARALLEL=1
 BATCH=2048
 UBATCH=1024
@@ -31,12 +36,18 @@ for f in "$MODEL" "$DRAFT"; do
 done
 
 : > "$LOG"
-echo "=== 64k full-pipeline bench: device=$DEVICE ==="
+echo "=== 4k pipeline bench: device=$DEVICE spec=$SPEC_TYPE ==="
+
+# Only add the draft-model argument when the spec type references dflash.
+DRAFT_ARGS=()
+if [[ "$SPEC_TYPE" == *dflash* ]]; then
+    DRAFT_ARGS+=(--spec-draft-model "$DRAFT")
+fi
 
 "$LLAMA_SERVER" \
     --model "$MODEL" \
-    --spec-draft-model "$DRAFT" \
-    --spec-type "draft-dflash,ngram-mod" \
+    "${DRAFT_ARGS[@]}" \
+    --spec-type "$SPEC_TYPE" \
     --spec-draft-n-max 8 \
     --spec-ngram-mod-n-min 48 \
     --spec-ngram-mod-n-max 64 \
@@ -52,6 +63,7 @@ echo "=== 64k full-pipeline bench: device=$DEVICE ==="
     --alias bench \
     --host 127.0.0.1 \
     --port "$PORT" \
+    --ignore-eos \
     >> "$LOG" 2>&1 &
 SERVER_PID=$!
 trap 'kill $SERVER_PID 2>/dev/null; wait $SERVER_PID 2>/dev/null' EXIT
@@ -59,7 +71,13 @@ trap 'kill $SERVER_PID 2>/dev/null; wait $SERVER_PID 2>/dev/null' EXIT
 echo "server pid $SERVER_PID on 127.0.0.1:$PORT"
 echo -n "waiting for server"
 ready=0
-for i in $(seq 1 400); do
+crashed=0
+for i in $(seq 1 200); do
+    if ! kill -0 "$SERVER_PID" 2>/dev/null; then
+        # server process died -> backend abort (e.g. Vulkan NONE-op crash)
+        crashed=1
+        break
+    fi
     if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
         echo " [ready]"
         ready=1
@@ -68,19 +86,27 @@ for i in $(seq 1 400); do
     echo -n "."
     sleep 2
 done
-[ "$ready" = "1" ] || { echo " FAIL"; tail -25 "$LOG"; exit 1; }
+if [ "$crashed" = "1" ]; then
+    echo " BACKEND_CRASHED (server died before ready)"
+    tail -30 "$LOG"
+    exit 2
+fi
+[ "$ready" = "1" ] || { echo " FAIL (timeout)"; tail -25 "$LOG"; exit 1; }
 
-# --- drive a ~64k-token generation, then pull timings from the server log ---
-python3 - "$DEVICE" "$PORT" "$OUT_TOKENS" "$LOG" <<'PY'
+# --- drive a ~4k-token generation, then pull timings from the server log ---
+python3 - "$DEVICE" "$SPEC_TYPE" "$PORT" "$OUT_TOKENS" "$LOG" <<'PY'
 import sys, time, json, urllib.request, os
-device, port, out_tokens, log = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4]
+device, spec, port, out_tokens, log = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]), sys.argv[5]
 
-# Build a prompt that actually occupies ~64k tokens. Tokenizer demo data:
-# this model averages ~0.25 tokens/char; ~250k chars -> ~62k tokens.
+# Build a ~4k filler prompt, then append an incomplete continuation so the
+# model produces real generation (pure repetitive filler stops early/EOS).
 unit = ("The careful study of natural language reveals patterns of meaning that "
         "emerge from the deliberate arrangement of words and phrases, and the "
         "patient reader soon learns to recognize the structure beneath the surface. ")
-prompt = unit * 1900  # ~250k chars
+prompt = (unit * 120)  # ~16k chars -> ~4k tokens
+# Append a clear continuation target that forces real generation.
+prompt += ("\n\nNow, in a detailed and flowing paragraph, continue this essay about "
+           "the philosophy of language and its influence on modern computation. ")
 
 body = json.dumps({
     "prompt": prompt,
@@ -108,8 +134,9 @@ def grab(lines, label):
     if lines:
         print(label + ": " + lines[-1].strip())
 
-grab(prompt_evals, "PREFILL(timing)")
-grab(evals, "GENERATION(timing)")
+print(f"== RESULT spec={spec} device={device} ==")
+grab(prompt_evals, "PREFILL")
+grab(evals, "GENERATION")
 grab(accepts, "DRAFT_ACCEPTANCE")
 print(f"prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s")
 PY