From d104eef4e0f65771307cba5eb12db7801fb9810e Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 16 Sep 2026 06:59:44 -0700 Subject: [PATCH] bench: commit the harnesses the frank-medium numbers came from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every measurement behind the frank-medium runbook was taken with scripts that lived in /tmp on two hosts — reproducible only until the next reboot, and invisible to anyone reading the repository. This moves the useful four into scripts/, where this repo already keeps its bench harnesses: bench-vllm-corpus.py code-shaped corpus from the repo's own sources, because synthetic random prompts misreport draft acceptance badly (2.9-3.6 tok/update against ~5.8 on code) bench-vllm-serve.sh concurrency sweep; encodes the two traps that produced silent garbage — the tokenizer argument, without which the client dies resolving "frank-medium" on HF and reports no numbers at all, and the --ignore-eos semantics that must not be compared across bench-turn-latency.py per-turn TTFT for an agent-loop shape; the instrument that decided the routing policy, and the one that shows cold-vs-warm prefill directly bench-llm-soak.py sustained multi-family load; what found the capacity wall (three 30k-token families, 75% cold re-prefills, zero failures) Each header carries its traps and the interpretation; the runbook's Benchmarks section now points at them instead of describing methodology with no tooling. Also adds the missing half of the script seam: scripts/*.py are compiled, not just scripts/*.sh syntax-checked — a harness with a syntax error could otherwise ship unnoticed until someone needed it during an incident. Both python harnesses were smoke-tested against the live fleet before commit: the corpus builder wrote 15 prompts, and the turn harness measured 12.5 s cold against 1.4 s warm on the same prompt — the cache behaviour the runbook claims. --- scripts/bench-llm-soak.py | 128 ++++++++++++++++++ scripts/bench-turn-latency.py | 114 ++++++++++++++++ scripts/bench-vllm-corpus.py | 85 ++++++++++++ scripts/bench-vllm-serve.sh | 63 +++++++++ .../LLM/frank-medium-radiance-mxfp4.md | 9 ++ tests/scripts_syntax.bats | 13 ++ 6 files changed, 412 insertions(+) create mode 100755 scripts/bench-llm-soak.py create mode 100755 scripts/bench-turn-latency.py create mode 100755 scripts/bench-vllm-corpus.py create mode 100755 scripts/bench-vllm-serve.sh diff --git a/scripts/bench-llm-soak.py b/scripts/bench-llm-soak.py new file mode 100755 index 0000000..f0b590a --- /dev/null +++ b/scripts/bench-llm-soak.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""bench-llm-soak.py — sustained multi-family load, and what it reveals. + +Purpose is not a headline number. It is to catch what single measurements miss: +a pinned replica that degrades, KV pressure that appears only after many large +prefills, a routing table that grows, a proxy failover storm. Each family keeps +its own large shared prefix (the agent-loop shape) and runs turn after turn for +the requested duration. + +What it found on frank-medium (2026-09-16): with three concurrent families at +~30k tokens each, ~75 % of turns were cold re-prefills at a 13.9 s median, +because two families plus their cached prefixes exceed one card's 122,636-token +KV pool — a capacity wall that placement and large-request admission were both +measured against and neither moved. Zero requests failed: the cost of the wall +is latency, not errors. + +Usage: + bench-llm-soak.py [salt] + +The salt rides at the front of every prompt so a run never inherits a warm +prefix cache, and so the router's identity window sees a new conversation. +""" + +from __future__ import annotations + +import json +import statistics +import sys +import threading +import time +import urllib.request + +BASE = sys.argv[1].rstrip("/") if len(sys.argv) > 1 else "http://127.0.0.1:7770" +MINUTES = float(sys.argv[2]) if len(sys.argv) > 2 else 10 +FAMILIES = int(sys.argv[3]) if len(sys.argv) > 3 else 3 +SALT = sys.argv[4] if len(sys.argv) > 4 else "SOAK" + +DOCS = [ + "/home/dank/bench/bigdoc-a.txt", + "/home/dank/bench/bigdoc-b.txt", + "/home/dank/serverAppConfigs/scripts/deploy-remote.sh", +] + +QUESTIONS = [ + "Which parts of this document would break if the host rebooted?", + "Summarise the ordering guarantees this document relies on.", + "Which commands here need root, and what fails without it?", + "Point out the two riskiest assumptions in this document.", + "What would you change first if you had ten minutes?", +] + +deadline = time.time() + MINUTES * 60 +lock = threading.Lock() +rows: list[tuple[str, float, bool]] = [] + + +def ask(kind: str, prompt: str) -> None: + body = { + "model": "frank-medium", + "messages": [{"role": "user", "content": prompt}], + "max_tokens": 48, + "stream": True, + } + req = urllib.request.Request( + BASE + "/v1/chat/completions", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + ) + start = time.time() + ok = False + try: + with urllib.request.urlopen(req, timeout=900) as resp: + for line in resp: + if line.startswith(b"data: ") and b'"content"' in line: + ok = True + break + except Exception as exc: + print(f" {kind} error: {type(exc).__name__}: {str(exc)[:70]}", flush=True) + elapsed = (time.time() - start) * 1000.0 + with lock: + rows.append((kind, elapsed, ok)) + print(f" {kind}: {elapsed:9.1f} ms {'ok' if ok else 'FAIL'}", flush=True) + + +def family(index: int) -> None: + try: + with open(DOCS[index % len(DOCS)], errors="replace") as fh: + doc = (fh.read() * 3)[:120_000] + except OSError: + # Fall back to any available source so the soak can run on a host that + # has not had the big documents generated. + with open(DOCS[-1], errors="replace") as fh: + doc = (fh.read() * 3)[:120_000] + history: list[dict] = [] + turn = 0 + while time.time() < deadline: + question = QUESTIONS[turn % len(QUESTIONS)] + ask(f"fam{index}", f"{SALT}-{index}.\n{doc}\n\nQuestion: {question}") + history += [{"role": "assistant", "content": "Acknowledged."}, + {"role": "user", "content": question}] + turn += 1 + + +def main() -> int: + threads = [threading.Thread(target=family, args=(i,), daemon=True) + for i in range(FAMILIES)] + for t in threads: + t.start() + for t in threads: + t.join() + + with lock: + snapshot = list(rows) + ok = [r for r in snapshot if r[2]] + print(f"\nSOAK salt={SALT} families={FAMILIES} minutes={MINUTES} " + f"requests={len(snapshot)} ok={len(ok)} failed={len(snapshot) - len(ok)}") + for fam in sorted({r[0] for r in snapshot}): + times = [r[1] for r in ok if r[0] == fam] + if not times: + continue + cold = [t for t in times if t >= 5000] + print(f" {fam}: n={len(times)} median={statistics.median(times):.0f} ms " + f"re-prefills={len(cold)}/{len(times)}") + return 0 if len(ok) == len(snapshot) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/bench-turn-latency.py b/scripts/bench-turn-latency.py new file mode 100755 index 0000000..6df45a6 --- /dev/null +++ b/scripts/bench-turn-latency.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""bench-turn-latency.py — per-turn TTFT for an agent-loop shape. + +Every other harness in this directory measures a *batch* of independent +requests. That answers throughput, and it is the wrong instrument for the number +a user feels: an agent loop re-sends the same large prefix turn after turn, and +the deployed routing pins such a conversation to one replica so its prefix cache +stays warm. This script drives that shape and prints TTFT per turn. + +It is also the measurement that produced the routing decision on 2026-09-16 +(round-robin 2676 ms mean warm turn against pinned 1096 ms on 90k-token +contexts), and the one that shows a cold-vs-warm prefill directly: pump the same +prompt twice and the second is a cache lookup. + +Usage: + bench-turn-latency.py [families] [salt] + bench-turn-latency.py http://127.0.0.1:7770 5 3 SALT1 # 3 families, 5 turns each + +Prompts carry the salt as their FIRST bytes: that changes both the router's +identity window and vLLM's prefix-cache key, which is what keeps a run from +silently inheriting a warm cache from the previous one. + +Two cards behind a proxy: send against the proxy (:7770) to measure the fleet, +or a replica directly (:7771, :7772) to measure one card. +""" + +from __future__ import annotations + +import json +import statistics +import sys +import threading +import time +import urllib.request + +BASE = sys.argv[1].rstrip("/") if len(sys.argv) > 1 else "http://127.0.0.1:7770" +TURNS = int(sys.argv[2]) if len(sys.argv) > 2 else 5 +FAMILIES = int(sys.argv[3]) if len(sys.argv) > 3 else 3 +SALT = sys.argv[4] if len(sys.argv) > 4 else f"BENCH{int(time.time())}" + +DOC = "/home/dank/serverAppConfigs/scripts/deploy-remote.sh" +MAX_CHARS = 120_000 + +lock = threading.Lock() +rows: list[tuple[str, float, bool]] = [] + + +def ttft_ms(prompt: str) -> tuple[float, bool]: + body = { + "model": "frank-medium", + "messages": [{"role": "user", "content": prompt}], + "max_tokens": 32, + "stream": True, + } + req = urllib.request.Request( + BASE + "/v1/chat/completions", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + ) + start = time.time() + try: + with urllib.request.urlopen(req, timeout=900) as resp: + for line in resp: + if line.startswith(b"data: ") and b'"content"' in line: + return (time.time() - start) * 1000.0, True + except Exception as exc: + print(f" error: {type(exc).__name__}: {str(exc)[:70]}", flush=True) + return (time.time() - start) * 1000.0, False + + +def family(index: int) -> None: + with open(DOC, errors="replace") as fh: + doc = (fh.read() * 3)[:MAX_CHARS] + # History grows the way a real conversation does, so later turns are + # continuations — the requests the router pins and the cache can serve. + history: list[dict] = [] + for turn in range(TURNS): + prompt = (f"{SALT}-{index} conversation.\n{doc}\n" + f"Turn {turn}: which part of the above needs root?") + ms, ok = ttft_ms(prompt) + with lock: + rows.append((f"fam{index}", ms, ok)) + print(f" fam{index} turn{turn + 1}: {ms:9.1f} ms {'ok' if ok else 'FAIL'}", + flush=True) + history += [{"role": "assistant", "content": "Acknowledged."}, + {"role": "user", "content": f"turn {turn}"}] + + +def main() -> int: + threads = [threading.Thread(target=family, args=(i,), daemon=True) + for i in range(FAMILIES)] + for t in threads: + t.start() + for t in threads: + t.join() + + with lock: + snapshot = list(rows) + ok = [r for r in snapshot if r[2]] + print(f"\nTURNS salt={SALT} families={FAMILIES} requests={len(snapshot)} " + f"ok={len(ok)} failed={len(snapshot) - len(ok)}") + for fam in sorted({r[0] for r in snapshot}): + times = [r[1] for r in ok if r[0] == fam] + if not times: + continue + warm = [t for t in times[1:] if t < 5000] + print(f" {fam}: first={times[0]:.0f} ms " + f"median={statistics.median(times):.0f} ms " + f"warm_turns={len(warm)}/{max(len(times) - 1, 1)}") + return 0 if len(ok) == len(snapshot) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/bench-vllm-corpus.py b/scripts/bench-vllm-corpus.py new file mode 100755 index 0000000..df0e7cf --- /dev/null +++ b/scripts/bench-vllm-corpus.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""bench-vllm-corpus.py — build a code-shaped benchmark corpus from the repo. + +Why this exists: `vllm bench serve --dataset-name random` produces synthetic +token soup, and speculative decoding accepts it nothing like real code — measured +on frank-medium, random prompts accepted 2.9-3.6 tokens/update against the ~5.8 +the upstream reference reports on a code mix, which makes the random corpus +under-report throughput by a wide margin and would pick the wrong SPEC depth. + +The prompts are the shape agents actually send: a repository artifact plus an +instruction that requires reading it. Output length is fixed at bench time +(`--custom-output-len`), so the corpus carries prompts only. + +Usage: + bench-vllm-corpus.py [repo-root] [out.jsonl] + +Then feed it to the endpoint (see scripts/bench-vllm-serve.sh, which passes the +tokenizer the client needs and salts prompts so no run inherits a warm prefix +cache). Methodology and the numbers these corpora produced are in +serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md. +""" + +from __future__ import annotations + +import json +import pathlib +import sys + +REPO = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else "/home/dank/serverAppConfigs") +OUT = pathlib.Path(sys.argv[2] if len(sys.argv) > 2 else "/home/dank/bench/corpus-code.jsonl") + +TARGETS = [ + "scripts/deploy-remote.sh", + "scripts/deploy-plan.py", + "scripts/deploy.sh", + "dankserver/frank-medium/radiance-entrypoint.sh", + "dankserver/frank-medium/quadlet/frank-medium-a.container", + "dankserver/frank-medium/proxy/affinity-proxy.py", + "dankserver/frank-medium/proxy/routing.py", + "tests/frank_medium_quadlet.bats", + "tests/deployment_scoping.bats", + "dankclaw/hermes/quadlet/hermes.container", + "dankclaw/anubis/quadlet/anubis.container", + "shared/prometheus/prometheus.yml", + "shared/prometheus/rules/llm-fleet.yml", + "shared/loki/loki-config.yaml", + "dankclaw/compose.yaml", + "dankserver/copyparty/quadlet/copyparty.container", +] + +PROMPTS = [ + "Read this repository artifact and summarise what it does, then list every " + "assumption it makes about the host it runs on.\n\n```\n{body}\n```", + "Here is a file from our homelab repository. Identify any bug, race or " + "missing error path, and show the minimal patch that fixes it.\n\n```\n{body}\n```", + "Explain this configuration to a new operator, then rewrite it with " + "comments that state the constraint each line protects.\n\n```\n{body}\n```", + "This file changed in a recent deploy. Write the changelog entry and a test " + "that would fail if the change regressed.\n\n```\n{body}\n```", +] + +MAX_LINES = 220 + + +def main() -> int: + samples = [] + for i, rel in enumerate(TARGETS): + path = REPO / rel + if not path.is_file(): + continue + body = "\n".join(path.read_text(errors="replace").splitlines()[:MAX_LINES]) + samples.append({"prompt": PROMPTS[i % len(PROMPTS)].format(body=body)}) + if not samples: + print(f"no corpus sources found under {REPO}", file=sys.stderr) + return 1 + OUT.parent.mkdir(parents=True, exist_ok=True) + with OUT.open("w") as fh: + for sample in samples: + fh.write(json.dumps(sample) + "\n") + print(f"wrote {len(samples)} prompts to {OUT}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/bench-vllm-serve.sh b/scripts/bench-vllm-serve.sh new file mode 100755 index 0000000..97cae60 --- /dev/null +++ b/scripts/bench-vllm-serve.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# bench-vllm-serve.sh — concurrency sweep against a vLLM endpoint. +# +# Usage: bench-vllm-serve.sh [concurrency ...] +# e.g. bench-vllm-serve.sh baseline 7770 1 4 8 +# +# Runs `vllm bench serve` from inside the serving image (so the client matches +# the server build) through an OpenAI-compatible endpoint, on the code corpus +# scripts/bench-vllm-corpus.py writes. +# +# Two traps this script exists to encode, both of which produced silent garbage +# during the 2026-09-16 frank-medium work: +# +# 1. The tokenizer must be passed explicitly and /models mounted. With only +# `--model frank-medium` the client tries to resolve that alias on +# HuggingFace and dies before sending a request — a run that looks like a +# benchmark and reports nothing. +# 2. `--ignore-eos` is deliberate: it measures decode throughput at a fixed +# output length. Drop it when you care about real stop behaviour, but then +# do not compare the numbers with anything measured under it. +# +# Also note the corpora are code-shaped on purpose (see the generator's header): +# synthetic random prompts misreport speculative acceptance badly. +# +# Methodology, and the numbers these runs produced on 2x R9700, are in +# serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md. +set -u + +TAG=${1:?usage: bench-vllm-serve.sh [concurrency ...]} +PORT=${2:?usage: bench-vllm-serve.sh [concurrency ...]} +shift 2 +CONCS=("$@") +[ "${#CONCS[@]}" -eq 0 ] && CONCS=(1 4 8) + +IMG=${IMG:-localhost/radiance-vllm:0.9.3} +MODELS_DIR=${MODELS_DIR:-/home/dank/models} +TOK=${TOK:-/models/Qwen3.8-27B-Uncensored-MXFP4-awq} +BENCH_DIR=${BENCH_DIR:-/home/dank/bench} +CORPUS=${CORPUS:-$BENCH_DIR/corpus-code.jsonl} +OUT=${OUT:-$BENCH_DIR/$TAG} +mkdir -p "$OUT" + +if [ ! -f "$CORPUS" ]; then + echo "corpus $CORPUS is missing; run scripts/bench-vllm-corpus.py first" >&2 + exit 1 +fi + +for c in "${CONCS[@]}"; do + echo "== $TAG conc=$c (port $PORT)" + podman run --rm --network=host -v "$MODELS_DIR":/models:ro -v "$BENCH_DIR":/bench:ro \ + -e HF_HUB_OFFLINE=1 --entrypoint bash "$IMG" -lc " + vllm bench serve --backend openai-chat \ + --base-url http://127.0.0.1:$PORT --endpoint /v1/chat/completions \ + --model frank-medium --tokenizer $TOK \ + --dataset-name custom --dataset-path /bench/$(basename "$CORPUS") \ + --custom-output-len 512 --num-prompts 16 --max-concurrency $c \ + --disable-shuffle --seed 0 --ignore-eos \ + --percentile-metrics ttft,tpot --metric-percentiles 50,99 \ + --disable-tqdm 2>&1 | grep -E 'Successful requests|Benchmark duration|Output token throughput|Total Token throughput|Mean TTFT|P50 TTFT|P50 TPOT|Acceptance rate|Acceptance length' + " 2>&1 | sed 's/^/ /' + echo +done +echo "results for $TAG written under $OUT" diff --git a/serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md b/serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md index 38cbd5b..48507cf 100644 --- a/serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md +++ b/serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md @@ -268,6 +268,15 @@ Restarting a stopped replica takes about 150 s (warm compile cache). ## Benchmarks +The harnesses live in the repository — the numbers below are reproducible with +`scripts/bench-vllm-corpus.py` (build the code corpus), `scripts/bench-vllm-serve.sh` +(concurrency sweep), `scripts/bench-turn-latency.py` (per-turn TTFT for an +agent-loop shape, which is what the routing decision optimizes) and +`scripts/bench-llm-soak.py` (sustained multi-family load). Each carries the +traps in its header: the tokenizer argument that makes a `vllm bench` run report +nothing, the salted prompts that keep a run from inheriting a warm cache, and +why the corpus is code-shaped rather than `random`. + `vllm bench serve` against the proxy (both cards serving), random dataset, `--ignore-eos`, `2026-09-16`: diff --git a/tests/scripts_syntax.bats b/tests/scripts_syntax.bats index 555948b..162f23c 100644 --- a/tests/scripts_syntax.bats +++ b/tests/scripts_syntax.bats @@ -29,3 +29,16 @@ load helpers skip "shellcheck not installed locally" fi } + +@test "every python script in scripts/ compiles" { + # The shell half of this seam covered scripts/*.sh; the python half was only + # covered where a test happened to reference a specific file, so a syntax + # error in a committed harness could ship unnoticed until someone ran it + # during an incident. + local any=0 + while IFS= read -r p; do + any=1 + python3 -m py_compile "$p" || { echo "syntax error in $p"; return 1; } + done < <(find "$REPO_ROOT/scripts" -name '*.py' -type f 2>/dev/null | sort) + [ "$any" -eq 1 ] || skip "no python scripts yet" +}