Clarify llama-bench harness scope
This commit is contained in:
1 parent
dc69e2b4b4
commit
863897f92a
10 files changed
+78
-718
No files matched your search
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
# Benchmark ubatch with llama-bench for frank-default-large.
|
||||
#
|
||||
# This is intentionally a ubatch-only benchmark. It uses one llama-bench
|
||||
# process, a fixed 16K prompt workload, and a fixed batch size large enough to
|
||||
# hold that workload. No server process, HTTP requests, or log scraping are used.
|
||||
#
|
||||
# Run on Dankclaw after stopping frank-default-large:
|
||||
# sudo systemctl stop frank-default-large
|
||||
# ./scripts/bench-ubatch.sh
|
||||
# sudo systemctl start frank-default-large
|
||||
set -euo pipefail
|
||||
|
||||
: "${LLAMA_BENCH:=/home/dank/GitPackages/ROCmFPX/build-strix-rocmfp4/bin/llama-bench}"
|
||||
: "${MODEL:=/home/dank/models/kingjones-Qwen3.8-27B-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf}"
|
||||
: "${UBATCH_VALUES:=256,512,768,1024,1280,1536,1792,2048,3072,4096,5120,6144}"
|
||||
: "${CONTEXT:=16384}"
|
||||
: "${GEN_TOKENS:=128}"
|
||||
: "${REPETITIONS:=3}"
|
||||
: "${DEVICE:=ROCm0}"
|
||||
: "${OUTPUT:=jsonl}"
|
||||
|
||||
if [[ ! -x "$LLAMA_BENCH" ]]; then
|
||||
printf 'missing executable llama-bench: %s\n' "$LLAMA_BENCH" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -r "$MODEL" ]]; then
|
||||
printf 'missing or unreadable model: %s\n' "$MODEL" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exec "$LLAMA_BENCH" \
|
||||
--model "$MODEL" \
|
||||
--n-prompt "$CONTEXT" \
|
||||
--n-gen "$GEN_TOKENS" \
|
||||
--batch-size "$CONTEXT" \
|
||||
--ubatch-size "$UBATCH_VALUES" \
|
||||
--device "$DEVICE" \
|
||||
--n-gpu-layers 999 \
|
||||
--cache-type-k f16 \
|
||||
--cache-type-v f16 \
|
||||
--flash-attn on \
|
||||
--mmap 0 \
|
||||
--repetitions "$REPETITIONS" \
|
||||
--output "$OUTPUT"
|
||||
Reference in new issue
Block a user