#!/bin/bash
# bench-vllm-serve.sh — concurrency sweep against a vLLM endpoint.
#
# Usage: bench-vllm-serve.sh <tag> <port> [concurrency ...]
#   e.g. bench-vllm-serve.sh baseline 7770 1 4 8
#
# Runs `vllm bench serve` from inside the serving image (so the client matches
# the server build) through an OpenAI-compatible endpoint, on the code corpus
# scripts/bench-vllm-corpus.py writes.
#
# Two traps this script exists to encode, both of which produced silent garbage
# during the 2026-09-16 frank-medium work:
#
#   1. The tokenizer must be passed explicitly and /models mounted. With only
#      `--model frank-medium` the client tries to resolve that alias on
#      HuggingFace and dies before sending a request — a run that looks like a
#      benchmark and reports nothing.
#   2. `--ignore-eos` is deliberate: it measures decode throughput at a fixed
#      output length. Drop it when you care about real stop behaviour, but then
#      do not compare the numbers with anything measured under it.
#
# Also note the corpora are code-shaped on purpose (see the generator's header):
# synthetic random prompts misreport speculative acceptance badly.
#
# Methodology, and the numbers these runs produced on 2x R9700, are in
# serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md.
set -u

TAG=${1:?usage: bench-vllm-serve.sh <tag> <port> [concurrency ...]}
PORT=${2:?usage: bench-vllm-serve.sh <tag> <port> [concurrency ...]}
shift 2
CONCS=("$@")
[ "${#CONCS[@]}" -eq 0 ] && CONCS=(1 4 8)

IMG=${IMG:-localhost/radiance-vllm:0.9.3}
MODELS_DIR=${MODELS_DIR:-/home/dank/models}
TOK=${TOK:-/models/Qwen3.8-27B-Uncensored-MXFP4-awq}
BENCH_DIR=${BENCH_DIR:-/home/dank/bench}
CORPUS=${CORPUS:-$BENCH_DIR/corpus-code.jsonl}
OUT=${OUT:-$BENCH_DIR/$TAG}
mkdir -p "$OUT"

if [ ! -f "$CORPUS" ]; then
    echo "corpus $CORPUS is missing; run scripts/bench-vllm-corpus.py first" >&2
    exit 1
fi

for c in "${CONCS[@]}"; do
    echo "== $TAG conc=$c (port $PORT)"
    podman run --rm --network=host -v "$MODELS_DIR":/models:ro -v "$BENCH_DIR":/bench:ro \
        -e HF_HUB_OFFLINE=1 --entrypoint bash "$IMG" -lc "
        vllm bench serve --backend openai-chat \
          --base-url http://127.0.0.1:$PORT --endpoint /v1/chat/completions \
          --model frank-medium --tokenizer $TOK \
          --dataset-name custom --dataset-path /bench/$(basename "$CORPUS") \
          --custom-output-len 512 --num-prompts 16 --max-concurrency $c \
          --disable-shuffle --seed 0 --ignore-eos \
          --percentile-metrics ttft,tpot --metric-percentiles 50,99 \
          --disable-tqdm 2>&1 | grep -E 'Successful requests|Benchmark duration|Output token throughput|Total Token throughput|Mean TTFT|P50 TTFT|P50 TPOT|Acceptance rate|Acceptance length'
    " 2>&1 | sed 's/^/  /'
    echo
done
echo "results for $TAG written under $OUT"