From 5296cf11d7ccd78f349832b53a18a00f397bfdab Mon Sep 17 00:00:00 2001 From: Frank Date: Sat, 15 Aug 2026 17:03:01 -0700 Subject: [PATCH] dankclaw: switch backend to Vulkan0 (48% faster decode at 262k ctx); add bench-high-ctx.sh --- scripts/bench-high-ctx.sh | 139 ++++++++++++++++++++++++++++++++ scripts/frank-default-large.env | 8 +- 2 files changed, 145 insertions(+), 2 deletions(-) create mode 100755 scripts/bench-high-ctx.sh diff --git a/scripts/bench-high-ctx.sh b/scripts/bench-high-ctx.sh new file mode 100755 index 0000000..090e138 --- /dev/null +++ b/scripts/bench-high-ctx.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# bench-high-ctx.sh — high-context prefill/generation bench for +# frank-default-large (Qwen3.8-27B UD-Q8_K_XL, --spec-default) on a scratch +# port. Compares backends at very high context. +# +# Reads authoritative prefill / generation tok/s from the server's own +# `slot print_timing` lines. +# +# Usage: +# ./bench-high-ctx.sh [out_tokens] +# Example: +# ./bench-high-ctx.sh ROCm0 262144 +set -euo pipefail + +DEVICE="${1:?usage: bench-high-ctx.sh [out_tokens]}" +CTX="${2:?usage: bench-high-ctx.sh [out_tokens]}" +OUT_TOKENS="${3:-256}" + +MODEL_DIR=/home/dank/models +MODEL_FILE=Qwen3.8-27B-UD-Q8_K_XL.gguf +LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server +MODEL="$MODEL_DIR/$MODEL_FILE" + +PORT=7892 +PARALLEL=1 +BATCH=8192 +UBATCH=256 +LOG="/tmp/highctx-$DEVICE.log" + +[[ -f "$MODEL" ]] || { echo "missing: $MODEL" >&2; exit 1; } +: > "$LOG" + +echo "=== high-ctx bench: device=$DEVICE ctx=$CTX spec=--spec-default ===" + +"$LLAMA_SERVER" \ + --model "$MODEL" \ + --spec-default \ + --device "$DEVICE" \ + --gpu-layers all \ + --fit off \ + --cache-type-k q8_0 \ + --cache-type-v q8_0 \ + --ctx-size "$CTX" \ + --parallel "$PARALLEL" \ + --batch-size "$BATCH" \ + --ubatch-size "$UBATCH" \ + --alias bench \ + --host 127.0.0.1 \ + --port "$PORT" \ + >> "$LOG" 2>&1 & +SERVER_PID=$! +trap 'kill $SERVER_PID 2>/dev/null; wait $SERVER_PID 2>/dev/null' EXIT + +echo "server pid $SERVER_PID on 127.0.0.1:$PORT" +echo -n "waiting for server" +ready=0 +crashed=0 +for i in $(seq 1 600); do + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + crashed=1 + break + fi + if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then + echo " [ready]" + ready=1 + break + fi + echo -n "." + sleep 2 +done +if [ "$crashed" = "1" ]; then + echo " BACKEND_CRASHED (server died before ready)" + tail -30 "$LOG" + exit 2 +fi +[ "$ready" = "1" ] || { echo " FAIL (timeout)"; tail -25 "$LOG"; exit 1; } + +python3 - "$DEVICE" "$CTX" "$PORT" "$OUT_TOKENS" "$LOG" <<'PY' +import sys, time, json, urllib.request +device, ctx, port, out_tokens, log = sys.argv[1], int(sys.argv[2]), sys.argv[3], int(sys.argv[4]), sys.argv[5] + +# Flowing, varied literary prose (no repeated walls of identical text, so the +# model does NOT legitimately EOS early and ngram acceptance is realistic). +VAR = ( + "narrative nonfiction blends observation with reflection, ", + "and a careful writer lets each clause earn its place ", + "in the sentence's rising line, so meaning accumulates slowly. ", + "The reader is not hurried; the reader is led. Loose and periodic, ", + "the style turns on concrete nouns and verbs that carry weight, ", + "and it resists the abstract noun that hides a vague claim. ", + "A paragraph is not a list; it is a single turn of thought that ", + "opens, develops, and lands. This is the craft of prose: choose, ", + "arrange, and above all omit. Omission is the discipline that ", + "keeps the whole clear, and clarity is the courtesy we owe the ", + "willing reader." +) +# ~76 tokens per outer loop (measured on this model); target ~75% of ctx. +n = len(VAR) +parts = [] +t = 0 +while t * 76 < (ctx * 3) // 4: + for k in range(5): + parts.append(VAR[(t + k) % n]) + parts.append(f" This is the {device} subject, cued by note {t}. ") + t += 1 +prompt = "".join(parts) +# Explicit continuation request forces real generation. +prompt += ("\n\nNow, in a detailed and flowing paragraph, continue this essay about " + "the philosophy of language and its influence on modern computation. ") + +body = json.dumps({ + "prompt": prompt, + "max_tokens": out_tokens, + "temperature": 0.0, + "stream": False, +}).encode() +req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions", + data=body, headers={"Content-Type": "application/json"}) +t0 = time.time() +with urllib.request.urlopen(req, timeout=7200) as resp: + out = json.load(resp) +dt = time.time() - t0 +pt = out.get("usage", {}).get("prompt_tokens", 0) +ct = out.get("usage", {}).get("completion_tokens", 0) +print(f"request done: prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s", flush=True) + +lines = open(log, encoding="utf-8", errors="ignore").read().splitlines() +prompt_evals = [l for l in lines if "prompt eval time" in l] +evals = [l for l in lines if " eval time" in l and "prompt eval" not in l] + +def grab(lines, label): + if lines: + print(label + ": " + lines[-1].strip()) + +print(f"== RESULT device={device} ctx={ctx} spec=--spec-default ==") +grab(prompt_evals, "PREFILL") +grab(evals, "GENERATION") +print(f"prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s") +PY diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 31cb629..150ee0c 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -28,8 +28,12 @@ SPEC_DRAFT_P_MIN= SPEC_NGRAM_MOD_N_MIN=48 SPEC_NGRAM_MOD_N_MAX=64 SPEC_NGRAM_MOD_N_MATCH=24 -# Backend device from --list-devices. ROCm0 is the production backend. -DEVICE=ROCm0 +# Backend device from --list-devices. Vulkan0 selected 2026-08-15 for high +# context: at 262144 ctx (~200k prompt) Vulkan generates ~5.7 t/s vs ROCm +# ~3.8 t/s (48% faster decode), while ROCm prefills ~12% faster (119 vs 106 +# t/s). Generation is the interactive cost, so Vulkan0 wins at high ctx. +# See docs/bench-high-ctx results and [[qwen3.8-spec-matrix]]. +DEVICE=Vulkan0 # Dense model: full offload. (Not MoE.) N_GPU_LAYERS=all # KV on GPU, quantized (K q8_0, V q8_0). Keep K == V for Flash Attention;