dankclaw: switch backend to Vulkan0 (48% faster decode at 262k ctx); add bench-high-ctx.sh

scripts/bench-high-ctx.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+# bench-high-ctx.sh — high-context prefill/generation bench for
+# frank-default-large (Qwen3.8-27B UD-Q8_K_XL, --spec-default) on a scratch
+# port.  Compares backends at very high context.
+#
+# Reads authoritative prefill / generation tok/s from the server's own
+# `slot print_timing` lines.
+#
+# Usage:
+#   ./bench-high-ctx.sh <ROCm0|Vulkan0> <ctx> [out_tokens]
+# Example:
+#   ./bench-high-ctx.sh ROCm0 262144
+set -euo pipefail
+
+DEVICE="${1:?usage: bench-high-ctx.sh <ROCm0|Vulkan0> <ctx> [out_tokens]}"
+CTX="${2:?usage: bench-high-ctx.sh <ROCm0|Vulkan0> <ctx> [out_tokens]}"
+OUT_TOKENS="${3:-256}"
+
+MODEL_DIR=/home/dank/models
+MODEL_FILE=Qwen3.8-27B-UD-Q8_K_XL.gguf
+LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
+MODEL="$MODEL_DIR/$MODEL_FILE"
+
+PORT=7892
+PARALLEL=1
+BATCH=8192
+UBATCH=256
+LOG="/tmp/highctx-$DEVICE.log"
+
+[[ -f "$MODEL" ]] || { echo "missing: $MODEL" >&2; exit 1; }
+: > "$LOG"
+
+echo "=== high-ctx bench: device=$DEVICE ctx=$CTX spec=--spec-default ==="
+
+"$LLAMA_SERVER" \
+    --model "$MODEL" \
+    --spec-default \
+    --device "$DEVICE" \
+    --gpu-layers all \
+    --fit off \
+    --cache-type-k q8_0 \
+    --cache-type-v q8_0 \
+    --ctx-size "$CTX" \
+    --parallel "$PARALLEL" \
+    --batch-size "$BATCH" \
+    --ubatch-size "$UBATCH" \
+    --alias bench \
+    --host 127.0.0.1 \
+    --port "$PORT" \
+    >> "$LOG" 2>&1 &
+SERVER_PID=$!
+trap 'kill $SERVER_PID 2>/dev/null; wait $SERVER_PID 2>/dev/null' EXIT
+
+echo "server pid $SERVER_PID on 127.0.0.1:$PORT"
+echo -n "waiting for server"
+ready=0
+crashed=0
+for i in $(seq 1 600); do
+    if ! kill -0 "$SERVER_PID" 2>/dev/null; then
+        crashed=1
+        break
+    fi
+    if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
+        echo " [ready]"
+        ready=1
+        break
+    fi
+    echo -n "."
+    sleep 2
+done
+if [ "$crashed" = "1" ]; then
+    echo " BACKEND_CRASHED (server died before ready)"
+    tail -30 "$LOG"
+    exit 2
+fi
+[ "$ready" = "1" ] || { echo " FAIL (timeout)"; tail -25 "$LOG"; exit 1; }
+
+python3 - "$DEVICE" "$CTX" "$PORT" "$OUT_TOKENS" "$LOG" <<'PY'
+import sys, time, json, urllib.request
+device, ctx, port, out_tokens, log = sys.argv[1], int(sys.argv[2]), sys.argv[3], int(sys.argv[4]), sys.argv[5]
+
+# Flowing, varied literary prose (no repeated walls of identical text, so the
+# model does NOT legitimately EOS early and ngram acceptance is realistic).
+VAR = (
+    "narrative nonfiction blends observation with reflection, ",
+    "and a careful writer lets each clause earn its place ",
+    "in the sentence's rising line, so meaning accumulates slowly. ",
+    "The reader is not hurried; the reader is led. Loose and periodic, ",
+    "the style turns on concrete nouns and verbs that carry weight, ",
+    "and it resists the abstract noun that hides a vague claim. ",
+    "A paragraph is not a list; it is a single turn of thought that ",
+    "opens, develops, and lands. This is the craft of prose: choose, ",
+    "arrange, and above all omit. Omission is the discipline that ",
+    "keeps the whole clear, and clarity is the courtesy we owe the ",
+    "willing reader."
+)
+# ~76 tokens per outer loop (measured on this model); target ~75% of ctx.
+n = len(VAR)
+parts = []
+t = 0
+while t * 76 < (ctx * 3) // 4:
+    for k in range(5):
+        parts.append(VAR[(t + k) % n])
+    parts.append(f" This is the {device} subject, cued by note {t}. ")
+    t += 1
+prompt = "".join(parts)
+# Explicit continuation request forces real generation.
+prompt += ("\n\nNow, in a detailed and flowing paragraph, continue this essay about "
+           "the philosophy of language and its influence on modern computation. ")
+
+body = json.dumps({
+    "prompt": prompt,
+    "max_tokens": out_tokens,
+    "temperature": 0.0,
+    "stream": False,
+}).encode()
+req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",
+                             data=body, headers={"Content-Type": "application/json"})
+t0 = time.time()
+with urllib.request.urlopen(req, timeout=7200) as resp:
+    out = json.load(resp)
+dt = time.time() - t0
+pt = out.get("usage", {}).get("prompt_tokens", 0)
+ct = out.get("usage", {}).get("completion_tokens", 0)
+print(f"request done: prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s", flush=True)
+
+lines = open(log, encoding="utf-8", errors="ignore").read().splitlines()
+prompt_evals = [l for l in lines if "prompt eval time" in l]
+evals       = [l for l in lines if "      eval time" in l and "prompt eval" not in l]
+
+def grab(lines, label):
+    if lines:
+        print(label + ": " + lines[-1].strip())
+
+print(f"== RESULT device={device} ctx={ctx} spec=--spec-default ==")
+grab(prompt_evals, "PREFILL")
+grab(evals, "GENERATION")
+print(f"prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s")
+PY

scripts/frank-default-large.env

@@ -28,8 +28,12 @@ SPEC_DRAFT_P_MIN=
 SPEC_NGRAM_MOD_N_MIN=48
 SPEC_NGRAM_MOD_N_MAX=64
 SPEC_NGRAM_MOD_N_MATCH=24
-# Backend device from --list-devices. ROCm0 is the production backend.
+# Backend device from --list-devices. Vulkan0 selected 2026-08-15 for high
-DEVICE=ROCm0
+# context: at 262144 ctx (~200k prompt) Vulkan generates ~5.7 t/s vs ROCm
+# ~3.8 t/s (48% faster decode), while ROCm prefills ~12% faster (119 vs 106
+# t/s).  Generation is the interactive cost, so Vulkan0 wins at high ctx.
+# See docs/bench-high-ctx results and [[qwen3.8-spec-matrix]].
+DEVICE=Vulkan0
 # Dense model: full offload. (Not MoE.)
 N_GPU_LAYERS=all
 # KV on GPU, quantized (K q8_0, V q8_0). Keep K == V for Flash Attention;