From bcdd898750f9c0c94c673e23b1ed25080f67b07a Mon Sep 17 00:00:00 2001 From: Frank Date: Sun, 20 Sep 2026 17:51:27 -0700 Subject: [PATCH] test(bench): drop the pre-probe from the conversation battery The warm-up request was a warm prefix-cache hit on the next run, so it hid exactly what the arms were meant to differ on (prefill). Token counts now come from the streamed usage of each job itself, and --salt defeats cross-arm inheritance. --- scripts/bench-conversation.py | 70 +++++++++++++++++------------------ 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/scripts/bench-conversation.py b/scripts/bench-conversation.py index 10574da..213f817 100755 --- a/scripts/bench-conversation.py +++ b/scripts/bench-conversation.py @@ -17,10 +17,10 @@ prefix-cache key and vLLM's APC key, so no arm silently inherits a warm cache from the one before it. Run one arm, restart the engine, run the next with a new salt. -Answers are generated at temperature 0 so the arms can be compared for -identity as well as speed. Identity is only meaningful across configurations -of the SAME model — different runtimes serving different quants legitimately -answer differently, which is why the first line of every answer is printed. +Answers are generated at temperature 0 so arms serving the SAME model can be +compared for identity as well as speed; different runtimes serving different +quants legitimately answer differently, which is why the first line of every +answer is printed. Usage: bench-conversation.py [--model NAME] [--salt TAG] @@ -36,7 +36,6 @@ stopped while anything measures. import argparse import json -import math import statistics import sys import threading @@ -78,7 +77,7 @@ BATTERY = [ ] # The long-context job reads real repo files so prefill is large and identical -# across arms (~30k characters, roughly 8k tokens). +# across arms (~30k characters). LONG_FILES = [ "/home/dank/serverAppConfigs/AGENTS.md", "/home/dank/serverAppConfigs/serverAppConfigs-docs/LLM/" @@ -103,9 +102,15 @@ def long_prompt() -> str: def stream_chat(base: str, model: str, messages, max_tokens: int, timeout: int): - """One streamed chat completion -> (ttft, total, completion_tokens, text). + """One streamed chat completion -> (ttft, total, usage, delta_count, text). - Returns ttft=None when the engine produced no content delta. + usage is the engine's own {prompt_tokens, completion_tokens}, asked for with + stream_options.include_usage. There is deliberately NO cheap non-streamed + pre-probe to learn the prompt length: that probe would populate the prefix + cache and turn the measured run's prefill into a cache hit, which is exactly + the difference between runtimes this harness is trying to see. + + ttft is None when the engine produced no content delta. """ body = { "model": model, @@ -123,7 +128,7 @@ def stream_chat(base: str, model: str, messages, max_tokens: int, timeout: int): start = time.perf_counter() ttft = None deltas = 0 - usage_tokens = None + usage = {} text = [] with urllib.request.urlopen(request, timeout=timeout) as response: for raw in response: @@ -138,35 +143,24 @@ def stream_chat(base: str, model: str, messages, max_tokens: int, timeout: int): except json.JSONDecodeError: continue if chunk.get("usage"): - usage_tokens = chunk["usage"].get("completion_tokens") + usage.update(chunk["usage"]) for choice in chunk.get("choices", []): - piece = (choice.get("delta") or {}) - content = piece.get("content") or piece.get("reasoning_content") or "" + piece = choice.get("delta") or {} + # llama.cpp streams thinking as reasoning_content; vLLM's + # reasoning parser streams it as reasoning. Either one is a + # generated token, so TTFT must be the first token of ANY kind — + # counting only `content` made a thinking-heavy arm look like it + # answered instantly (measured 2026-09-20: radiance's geomean + # read 994 tok/s with 6 of 10 jobs recorded as "(empty)"). + content = next((piece[k] for k in ("content", "reasoning", + "reasoning_content") if piece.get(k)), "") if content: if ttft is None: ttft = time.perf_counter() - start deltas += 1 text.append(content) total = time.perf_counter() - start - return ttft, total, (usage_tokens or deltas), "".join(text) - - -def usage_prompt(base: str, model: str, messages, max_tokens: int): - """prompt_tokens for a job, from one non-streamed call with max_tokens=1.""" - body = { - "model": model, - "messages": messages, - "max_tokens": 1, - "temperature": 0, - "stream": False, - } - request = urllib.request.Request( - base + "/v1/chat/completions", - data=json.dumps(body).encode(), - headers={"Content-Type": "application/json"}, - ) - with urllib.request.urlopen(request, timeout=300) as response: - return json.load(response).get("usage", {}).get("prompt_tokens") + return ttft, total, usage, deltas, "".join(text) def main() -> int: @@ -196,22 +190,23 @@ def main() -> int: ] limit = args.max_tokens or cap try: - ptok = usage_prompt(base, args.model, messages, limit) - ttft, total, ctok, text = stream_chat( + ttft, total, usage, deltas, text = stream_chat( base, args.model, messages, limit, args.timeout) except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as exc: print(f"{name:<10} FAILED {exc}") rows.append({"job": name, "error": str(exc)}) continue + ptok = usage.get("prompt_tokens") + ctok = usage.get("completion_tokens") or deltas decode = None if ttft is not None and ctok and total > ttft: decode = (ctok - 1) / (total - ttft) - prefill_rate = (ptok / ttft) if (ptok and ttft) else None + prefill = (ptok / ttft) if (ptok and ttft) else None first = text.strip().splitlines()[0][:48] if text.strip() else "(empty)" print(f"{name:<10} {ptok or 0:>7} {ctok:>6} " f"{(ttft or 0):>8.2f} {(decode or 0):>8.1f} {total:>8.2f} {first}") rows.append({"job": name, "prompt_tokens": ptok, "completion_tokens": ctok, - "ttft_s": ttft, "decode_tps": decode, "prefill_tps": prefill_rate, + "ttft_s": ttft, "decode_tps": decode, "prefill_tps": prefill, "total_s": total, "answer_head": first}) good = [r for r in rows if r.get("decode_tps")] @@ -236,8 +231,9 @@ def main() -> int: def worker(): try: - ttft, total, ctok, _ = stream_chat( + ttft, total, usage, deltas, _ = stream_chat( base, args.model, messages, 640, args.timeout) + ctok = usage.get("completion_tokens") or deltas with lock: results.append((ttft, total, ctok)) except Exception as exc: # noqa: BLE001 - report, do not abort the burst @@ -253,7 +249,7 @@ def main() -> int: wall = time.perf_counter() - t0 ok = [r for r in results if r[0] is not None] if ok: - tokens = sum(r[2] for r in ok) + tokens = sum(r[2] for r in ok if isinstance(r[2], int)) print(f"burst {args.burst}x plan: wall {wall:.1f}s aggregate " f"{tokens / wall:.1f} tok/s " f"per-stream {statistics.mean([r[2] / r[1] for r in ok]):.1f} tok/s"