diff --git a/scripts/bench-batch-ubatch.sh b/scripts/bench-batch-ubatch.sh new file mode 100755 index 0000000..47129da --- /dev/null +++ b/scripts/bench-batch-ubatch.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# +# bench-batch-ubatch.sh — how large may the logical batch and the micro-batch be? +# +# Sweeps llama.cpp's --batch-size against --ubatch-size with llama-bench and records, per +# cell, the prefill rate, the decode rate and the PEAK VRAM of each card. The VRAM half is +# not decoration: batch/ubatch is the knob that sizes the activation buffers, so a cell +# that looks fast because it silently fell back to a smaller allocation would otherwise be +# indistinguishable from a cell that is genuinely fast. +# +# ONE llama-bench INVOCATION PER CELL, on purpose. llama-bench calls exit(1) the moment a +# test cannot run, so a single infeasible ubatch would take every later cell with it — +# and "infeasible" is one of the things this sweep is looking for. +# +# RUN IT FROM A TRANSIENT UNIT, NOT FROM A SHELL. The deployment's unit sets +# LimitMEMLOCK=infinity because the ROCm backend pins staging buffers; an SSH shell on +# these hosts gets 8 MiB, and a load that fails on that proves nothing about the model: +# +# sudo systemd-run --unit=fm-bench --collect --no-block --property=Type=oneshot \ +# --property=LimitMEMLOCK=infinity --property=User=dank \ +# /home/dank/serverAppConfigs/scripts/bench-batch-ubatch.sh +# +# It stops $SERVICE for the duration and starts it again on exit (trap), because a bench +# beside a live service starves the same GPU. Results land in $OUT: one directory per cell +# (bench.json, bench.log, vram.tsv) plus cells.tsv, appended as the sweep proceeds, so +# progress can be read without waiting for the end. A re-run skips every cell that already +# has a bench.json, so an interrupted sweep resumes. +# +set -uo pipefail + +BIN=${BIN:-/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin} +MODEL=${MODEL:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf} +OUT=${OUT:-/home/dank/models/logs/bench-bub-$(date +%Y%m%d-%H%M%S)} + +# what a cell measures: a prefill and a decode, both with a 64k KV already resident, so the +# numbers describe the agent-loop shape (prompt landing in a long conversation) rather than +# a cold start +N_PROMPT=${N_PROMPT:-65536} +N_GEN=${N_GEN:-128} +N_DEPTH=${N_DEPTH:-65536} +REPS=${REPS:-3} +BATCHES=${BATCHES:-"2048 4096 6144 8192"} +UBATCHES=${UBATCHES:-"512 768 1024 1280 1536 1792 2048"} +SEED_CELLS=${SEED_CELLS:-"6144/1536"} # the deployed cell first: it is the reference + +# how the model is loaded — must match the deployment, or the numbers describe a service +# that does not exist +NGL=${NGL:-999} +DEVICES=${DEVICES:-ROCm0/ROCm1} # llama-bench separates with '/'; ROCm2 is the iGPU +SPLIT_MODE=${SPLIT_MODE:-layer} +TENSOR_SPLIT=${TENSOR_SPLIT:-36/29} # slash here — the llama-server unit takes commas +FLASH_ATTN=${FLASH_ATTN:-on} +KV_TYPE=${KV_TYPE:-f16} +THREADS=${THREADS:-8} + +SERVICE=${SERVICE:-frank-medium} +BOOTED= + +service_stop() { + if systemctl is-active --quiet "$SERVICE"; then + echo "stopping $SERVICE for the sweep" >&2 + sudo systemctl stop "$SERVICE" + BOOTED=1 + fi +} + +service_start() { + if [ -n "$BOOTED" ]; then + echo "starting $SERVICE again" >&2 + sudo systemctl start "$SERVICE" + fi +} + +trap service_start EXIT + +run_cell() { + local b=$1 ub=$2 dir="$OUT/b${b}_ub${ub}" sampler start rc wall + mkdir -p "$dir" + if [ -s "$dir/bench.json" ]; then + echo "b=$b ub=$ub: bench.json present, skipping" >&2 + return 0 + fi + echo "b=$b ub=$ub: running" >&2 + + # 2 s sampling for the cell's whole life, load included + ( + while :; do + rocm-smi --showmeminfo vram --csv 2>/dev/null | + awk -F, -v t="$(date +%s)" '/^card[01],/ { print t, $1, $2, $3 }' + sleep 2 + done + ) > "$dir/vram.tsv" 2>/dev/null & + sampler=$! + + start=$(date +%s) + "$BIN/llama-bench" -m "$MODEL" \ + -p "$N_PROMPT" -n "$N_GEN" -d "$N_DEPTH" \ + -b "$b" -ub "$ub" \ + -ngl "$NGL" -dev "$DEVICES" -sm "$SPLIT_MODE" -ts "$TENSOR_SPLIT" \ + -fa "$FLASH_ATTN" -ctk "$KV_TYPE" -ctv "$KV_TYPE" -t "$THREADS" \ + -r "$REPS" --no-warmup --progress -o json \ + > "$dir/bench.json" 2> "$dir/bench.log" + rc=$? + + kill "$sampler" 2>/dev/null + wait "$sampler" 2>/dev/null + wall=$(( $(date +%s) - start )) + + if [ "$rc" != 0 ]; then + grep -iE "error|failed|abort" "$dir/bench.log" | tail -3 >> "$OUT/failures.txt" + echo "b=$b ub=$ub: exit $rc" >&2 + fi + + python3 - "$dir/bench.json" "$dir/vram.tsv" "$b" "$ub" "$rc" "$wall" <<'PY' +import json, sys + +json_path, vram_path, b, ub, rc, wall = sys.argv[1:7] + +def rates(tests, key): + hit = [t for t in tests if t.get(key)] + if not hit: + return "-\t-" + return "%.2f\t%.2f" % (hit[0]["avg_ts"], hit[0]["stddev_ts"]) + +try: + tests = json.load(open(json_path)) +except Exception: + tests = [] + +peak = {} +for line in open(vram_path): + parts = line.split() + if len(parts) == 4: + _, card, total, used = parts + if card not in peak or int(used) > peak[card][1]: + peak[card] = (int(total), int(used)) + +def free_gib(card): + if card not in peak: + return "-" + total, used = peak[card] + return "%.2f" % ((total - used) / 2**30) + +row = [b, ub, rc, wall, rates(tests, "n_prompt"), rates(tests, "n_gen"), + free_gib("card0"), free_gib("card1"), len(tests)] +print("\t".join(str(f) for f in row)) +PY +} + +mkdir -p "$OUT" +echo "output directory: $OUT" >&2 +if [ ! -s "$OUT/cells.tsv" ]; then + printf 'batch\tubatch\trc\twall_s\tpp_tok_s\tpp_sd\ttg_tok_s\ttg_sd\tgpu0_free_GiB\tgpu1_free_GiB\ttests\n' \ + > "$OUT/cells.tsv" +fi + +service_stop + +for cell in $SEED_CELLS; do + run_cell "${cell%%/*}" "${cell##*/}" >> "$OUT/cells.tsv" +done + +for b in $BATCHES; do + for ub in $UBATCHES; do + run_cell "$b" "$ub" >> "$OUT/cells.tsv" + done +done + +echo "sweep complete: $OUT/cells.tsv" >&2 +cat "$OUT/cells.tsv" >&2