diff --git a/docs/LLM/ubatch-benchmark-large.md b/docs/LLM/ubatch-benchmark-large.md index b8ad1e3..68b08ed 100644 --- a/docs/LLM/ubatch-benchmark-large.md +++ b/docs/LLM/ubatch-benchmark-large.md @@ -13,7 +13,7 @@ timestamp: 2026-09-01 # frank-default-large ubatch sweep This benchmark sweeps the active `frank-default-large` model on Dankclaw using -scratch ports and an 8K-context workload. The production service is stopped +scratch ports and a 16K-context workload. The production service is stopped before the sweep and restarted by the benchmark's exit handler. ## Sweep @@ -29,7 +29,8 @@ Run on Dankclaw from the repository checkout: STOPSVC=1 ./scripts/bench-ubatch-rocmi4.sh ``` -The script uses the active Q6 AGENT model, standalone MTP draft, tracked Sharp +The script uses the active Q6 AGENT model, standalone MTP draft, a fixed +16K batch, and the tracked Sharp template, and `--reasoning-format deepseek`. It reports server timing lines and per-run generation throughput. Run only when the production service can be interrupted; the GPU is shared with live workloads. diff --git a/scripts/bench-ubatch-rocmi4.sh b/scripts/bench-ubatch-rocmi4.sh index 70026fe..1f1f199 100755 --- a/scripts/bench-ubatch-rocmi4.sh +++ b/scripts/bench-ubatch-rocmi4.sh @@ -3,7 +3,7 @@ # frank-default-large ROCmFPX standalone-MTP stack (dankclaw, gfx1151). # # Boots one llama-server instance per ubatch on a scratch port with a fixed -# -b and -c 8192, runs a standardized ~75%-of-ctx prose workload, and reads +# -b and -c 16384, runs a standardized ~75%-of-ctx prose workload, and reads # authoritative prefill/generation tok/s from the server's own print_timing # lines. Steps from UBATCH_MIN to UBATCH_MAX inclusive by UBATCH_STEP. # @@ -30,8 +30,8 @@ fi : "${UBATCH_MAX:=6144}" : "${UBATCH_STEP:=256}" : "${UBATCH_VALUES:=256 512 768 1024 1280 1536 2048 3072 4096 5120 6144}" -: "${CTX:=8192}" # 8k contexts -: "${BENCH_BATCH:=$CTX}" # fixed batch = ctx, so the full 8k prompt fits +: "${CTX:=16384}" # 16k contexts +: "${BENCH_BATCH:=16384}" # batch is fixed to the 16k context size : "${OUT_TOKENS:=256}" : "${RUNS:=3}" # decode runs per ubatch : "${PORT_BASE:=7894}" # scratch ports 7894.. @@ -118,7 +118,7 @@ run_one() { local log="/tmp/ubatch-${ubatch}-p${port}.log" : > "$log" - echo "=== ubatch=${ubatch} ctx=${CTX} port=${port} runs=${runs} ===" + echo "=== ubatch=${ubatch} ctx=${CTX} batch=${BENCH_BATCH} port=${port} runs=${runs} ===" "$LLAMA_BIN" \ --model "$MODEL" \ "${MMPROJ_ARGS[@]}" \ @@ -142,13 +142,19 @@ run_one() { -lv 4 >>"$log" 2>&1 & local pid=$! SCRATCH_PID="$pid" - wait_health "$pid" "$port" || { tail -30 "$log"; return 2; } + if ! wait_health "$pid" "$port"; then + tail -30 "$log" + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + SCRATCH_PID="" + return 2 + fi # one python pass per run; reuse the same boot across runs local r for r in $(seq 1 "$runs"); do echo "--- run ${r}/${runs} ---" - python3 - "$ubatch" "$port" "$out_tokens" "$CTX" "$log" "$r" <<'PY' + if ! python3 - "$ubatch" "$port" "$out_tokens" "$CTX" "$log" "$r" <<'PY' import sys, time, json, urllib.request ubatch, port, out_tokens, ctx, log, run_number = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), sys.argv[5], sys.argv[6] @@ -214,6 +220,12 @@ print(f"== RESULT ubatch={ubatch} run={run_number} " f"PREFILL={grab(prefill)} GEN_TPS={g_tps} " f"pt={pt} ct={ct} wall={dt:.1f}s") PY + then + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + SCRATCH_PID="" + return 2 + fi done kill "$pid" 2>/dev/null || true wait "$pid" 2>/dev/null || true diff --git a/tests/frank_large_config.bats b/tests/frank_large_config.bats index 2d774bd..727713e 100644 --- a/tests/frank_large_config.bats +++ b/tests/frank_large_config.bats @@ -69,6 +69,8 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" grep -q ': "${UBATCH_MAX:=6144}"' "$bench" grep -q 'UBATCH_VALUES.*256 512 768 1024 1280 1536 2048 3072 4096 5120 6144' "$bench" grep -q ': "${STOPSVC:=1}"' "$bench" + grep -q ': "${CTX:=16384}"' "$bench" + grep -q ': "${BENCH_BATCH:=16384}"' "$bench" grep -q 'MODEL_FILE' "$bench" grep -q 'REASONING_FORMAT' "$bench" }