140 lines
4.6 KiB
Bash
Executable File
140 lines
4.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# bench-high-ctx.sh — high-context prefill/generation bench for
|
|
# frank-default-large (Qwen3.8-27B UD-Q8_K_XL, --spec-default) on a scratch
|
|
# port. Compares backends at very high context.
|
|
#
|
|
# Reads authoritative prefill / generation tok/s from the server's own
|
|
# `slot print_timing` lines.
|
|
#
|
|
# Usage:
|
|
# ./bench-high-ctx.sh <ROCm0|Vulkan0> <ctx> [out_tokens]
|
|
# Example:
|
|
# ./bench-high-ctx.sh ROCm0 262144
|
|
set -euo pipefail
|
|
|
|
DEVICE="${1:?usage: bench-high-ctx.sh <ROCm0|Vulkan0> <ctx> [out_tokens]}"
|
|
CTX="${2:?usage: bench-high-ctx.sh <ROCm0|Vulkan0> <ctx> [out_tokens]}"
|
|
OUT_TOKENS="${3:-256}"
|
|
|
|
MODEL_DIR=/home/dank/models
|
|
MODEL_FILE=Qwen3.8-27B-UD-Q8_K_XL.gguf
|
|
LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
|
|
MODEL="$MODEL_DIR/$MODEL_FILE"
|
|
|
|
PORT=7892
|
|
PARALLEL=1
|
|
BATCH=8192
|
|
UBATCH=256
|
|
LOG="/tmp/highctx-$DEVICE.log"
|
|
|
|
[[ -f "$MODEL" ]] || { echo "missing: $MODEL" >&2; exit 1; }
|
|
: > "$LOG"
|
|
|
|
echo "=== high-ctx bench: device=$DEVICE ctx=$CTX spec=--spec-default ==="
|
|
|
|
"$LLAMA_SERVER" \
|
|
--model "$MODEL" \
|
|
--spec-default \
|
|
--device "$DEVICE" \
|
|
--gpu-layers all \
|
|
--fit off \
|
|
--cache-type-k q8_0 \
|
|
--cache-type-v q8_0 \
|
|
--ctx-size "$CTX" \
|
|
--parallel "$PARALLEL" \
|
|
--batch-size "$BATCH" \
|
|
--ubatch-size "$UBATCH" \
|
|
--alias bench \
|
|
--host 127.0.0.1 \
|
|
--port "$PORT" \
|
|
>> "$LOG" 2>&1 &
|
|
SERVER_PID=$!
|
|
trap 'kill $SERVER_PID 2>/dev/null; wait $SERVER_PID 2>/dev/null' EXIT
|
|
|
|
echo "server pid $SERVER_PID on 127.0.0.1:$PORT"
|
|
echo -n "waiting for server"
|
|
ready=0
|
|
crashed=0
|
|
for i in $(seq 1 600); do
|
|
if ! kill -0 "$SERVER_PID" 2>/dev/null; then
|
|
crashed=1
|
|
break
|
|
fi
|
|
if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
|
|
echo " [ready]"
|
|
ready=1
|
|
break
|
|
fi
|
|
echo -n "."
|
|
sleep 2
|
|
done
|
|
if [ "$crashed" = "1" ]; then
|
|
echo " BACKEND_CRASHED (server died before ready)"
|
|
tail -30 "$LOG"
|
|
exit 2
|
|
fi
|
|
[ "$ready" = "1" ] || { echo " FAIL (timeout)"; tail -25 "$LOG"; exit 1; }
|
|
|
|
python3 - "$DEVICE" "$CTX" "$PORT" "$OUT_TOKENS" "$LOG" <<'PY'
|
|
import sys, time, json, urllib.request
|
|
device, ctx, port, out_tokens, log = sys.argv[1], int(sys.argv[2]), sys.argv[3], int(sys.argv[4]), sys.argv[5]
|
|
|
|
# Flowing, varied literary prose (no repeated walls of identical text, so the
|
|
# model does NOT legitimately EOS early and ngram acceptance is realistic).
|
|
VAR = (
|
|
"narrative nonfiction blends observation with reflection, ",
|
|
"and a careful writer lets each clause earn its place ",
|
|
"in the sentence's rising line, so meaning accumulates slowly. ",
|
|
"The reader is not hurried; the reader is led. Loose and periodic, ",
|
|
"the style turns on concrete nouns and verbs that carry weight, ",
|
|
"and it resists the abstract noun that hides a vague claim. ",
|
|
"A paragraph is not a list; it is a single turn of thought that ",
|
|
"opens, develops, and lands. This is the craft of prose: choose, ",
|
|
"arrange, and above all omit. Omission is the discipline that ",
|
|
"keeps the whole clear, and clarity is the courtesy we owe the ",
|
|
"willing reader."
|
|
)
|
|
# ~76 tokens per outer loop (measured on this model); target ~75% of ctx.
|
|
n = len(VAR)
|
|
parts = []
|
|
t = 0
|
|
while t * 76 < (ctx * 3) // 4:
|
|
for k in range(5):
|
|
parts.append(VAR[(t + k) % n])
|
|
parts.append(f" This is the {device} subject, cued by note {t}. ")
|
|
t += 1
|
|
prompt = "".join(parts)
|
|
# Explicit continuation request forces real generation.
|
|
prompt += ("\n\nNow, in a detailed and flowing paragraph, continue this essay about "
|
|
"the philosophy of language and its influence on modern computation. ")
|
|
|
|
body = json.dumps({
|
|
"prompt": prompt,
|
|
"max_tokens": out_tokens,
|
|
"temperature": 0.0,
|
|
"stream": False,
|
|
}).encode()
|
|
req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",
|
|
data=body, headers={"Content-Type": "application/json"})
|
|
t0 = time.time()
|
|
with urllib.request.urlopen(req, timeout=7200) as resp:
|
|
out = json.load(resp)
|
|
dt = time.time() - t0
|
|
pt = out.get("usage", {}).get("prompt_tokens", 0)
|
|
ct = out.get("usage", {}).get("completion_tokens", 0)
|
|
print(f"request done: prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s", flush=True)
|
|
|
|
lines = open(log, encoding="utf-8", errors="ignore").read().splitlines()
|
|
prompt_evals = [l for l in lines if "prompt eval time" in l]
|
|
evals = [l for l in lines if " eval time" in l and "prompt eval" not in l]
|
|
|
|
def grab(lines, label):
|
|
if lines:
|
|
print(label + ": " + lines[-1].strip())
|
|
|
|
print(f"== RESULT device={device} ctx={ctx} spec=--spec-default ==")
|
|
grab(prompt_evals, "PREFILL")
|
|
grab(evals, "GENERATION")
|
|
print(f"prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s")
|
|
PY
|