diff --git a/scripts/bench-conversation.py b/scripts/bench-conversation.py index 10574da..213f817 100755 --- a/scripts/bench-conversation.py +++ b/scripts/bench-conversation.py @@ -17,10 +17,10 @@ prefix-cache key and vLLM's APC key, so no arm silently inherits a warm cache from the one before it. Run one arm, restart the engine, run the next with a new salt. -Answers are generated at temperature 0 so the arms can be compared for -identity as well as speed. Identity is only meaningful across configurations -of the SAME model — different runtimes serving different quants legitimately -answer differently, which is why the first line of every answer is printed. +Answers are generated at temperature 0 so arms serving the SAME model can be +compared for identity as well as speed; different runtimes serving different +quants legitimately answer differently, which is why the first line of every +answer is printed. Usage: bench-conversation.py [--model NAME] [--salt TAG] @@ -36,7 +36,6 @@ stopped while anything measures. import argparse import json -import math import statistics import sys import threading @@ -78,7 +77,7 @@ BATTERY = [ ] # The long-context job reads real repo files so prefill is large and identical -# across arms (~30k characters, roughly 8k tokens). +# across arms (~30k characters). LONG_FILES = [ "/home/dank/serverAppConfigs/AGENTS.md", "/home/dank/serverAppConfigs/serverAppConfigs-docs/LLM/" @@ -103,9 +102,15 @@ def long_prompt() -> str: def stream_chat(base: str, model: str, messages, max_tokens: int, timeout: int): - """One streamed chat completion -> (ttft, total, completion_tokens, text). + """One streamed chat completion -> (ttft, total, usage, delta_count, text). - Returns ttft=None when the engine produced no content delta. + usage is the engine's own {prompt_tokens, completion_tokens}, asked for with + stream_options.include_usage. There is deliberately NO cheap non-streamed + pre-probe to learn the prompt length: that probe would populate the prefix + cache and turn the measured run's prefill into a cache hit, which is exactly + the difference between runtimes this harness is trying to see. + + ttft is None when the engine produced no content delta. """ body = { "model": model, @@ -123,7 +128,7 @@ def stream_chat(base: str, model: str, messages, max_tokens: int, timeout: int): start = time.perf_counter() ttft = None deltas = 0 - usage_tokens = None + usage = {} text = [] with urllib.request.urlopen(request, timeout=timeout) as response: for raw in response: @@ -138,35 +143,24 @@ def stream_chat(base: str, model: str, messages, max_tokens: int, timeout: int): except json.JSONDecodeError: continue if chunk.get("usage"): - usage_tokens = chunk["usage"].get("completion_tokens") + usage.update(chunk["usage"]) for choice in chunk.get("choices", []): - piece = (choice.get("delta") or {}) - content = piece.get("content") or piece.get("reasoning_content") or "" + piece = choice.get("delta") or {} + # llama.cpp streams thinking as reasoning_content; vLLM's + # reasoning parser streams it as reasoning. Either one is a + # generated token, so TTFT must be the first token of ANY kind — + # counting only `content` made a thinking-heavy arm look like it + # answered instantly (measured 2026-09-20: radiance's geomean + # read 994 tok/s with 6 of 10 jobs recorded as "(empty)"). + content = next((piece[k] for k in ("content", "reasoning", + "reasoning_content") if piece.get(k)), "") if content: if ttft is None: ttft = time.perf_counter() - start deltas += 1 text.append(content) total = time.perf_counter() - start - return ttft, total, (usage_tokens or deltas), "".join(text) - - -def usage_prompt(base: str, model: str, messages, max_tokens: int): - """prompt_tokens for a job, from one non-streamed call with max_tokens=1.""" - body = { - "model": model, - "messages": messages, - "max_tokens": 1, - "temperature": 0, - "stream": False, - } - request = urllib.request.Request( - base + "/v1/chat/completions", - data=json.dumps(body).encode(), - headers={"Content-Type": "application/json"}, - ) - with urllib.request.urlopen(request, timeout=300) as response: - return json.load(response).get("usage", {}).get("prompt_tokens") + return ttft, total, usage, deltas, "".join(text) def main() -> int: @@ -196,22 +190,23 @@ def main() -> int: ] limit = args.max_tokens or cap try: - ptok = usage_prompt(base, args.model, messages, limit) - ttft, total, ctok, text = stream_chat( + ttft, total, usage, deltas, text = stream_chat( base, args.model, messages, limit, args.timeout) except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as exc: print(f"{name:<10} FAILED {exc}") rows.append({"job": name, "error": str(exc)}) continue + ptok = usage.get("prompt_tokens") + ctok = usage.get("completion_tokens") or deltas decode = None if ttft is not None and ctok and total > ttft: decode = (ctok - 1) / (total - ttft) - prefill_rate = (ptok / ttft) if (ptok and ttft) else None + prefill = (ptok / ttft) if (ptok and ttft) else None first = text.strip().splitlines()[0][:48] if text.strip() else "(empty)" print(f"{name:<10} {ptok or 0:>7} {ctok:>6} " f"{(ttft or 0):>8.2f} {(decode or 0):>8.1f} {total:>8.2f} {first}") rows.append({"job": name, "prompt_tokens": ptok, "completion_tokens": ctok, - "ttft_s": ttft, "decode_tps": decode, "prefill_tps": prefill_rate, + "ttft_s": ttft, "decode_tps": decode, "prefill_tps": prefill, "total_s": total, "answer_head": first}) good = [r for r in rows if r.get("decode_tps")] @@ -236,8 +231,9 @@ def main() -> int: def worker(): try: - ttft, total, ctok, _ = stream_chat( + ttft, total, usage, deltas, _ = stream_chat( base, args.model, messages, 640, args.timeout) + ctok = usage.get("completion_tokens") or deltas with lock: results.append((ttft, total, ctok)) except Exception as exc: # noqa: BLE001 - report, do not abort the burst @@ -253,7 +249,7 @@ def main() -> int: wall = time.perf_counter() - t0 ok = [r for r in results if r[0] is not None] if ok: - tokens = sum(r[2] for r in ok) + tokens = sum(r[2] for r in ok if isinstance(r[2], int)) print(f"burst {args.burst}x plan: wall {wall:.1f}s aggregate " f"{tokens / wall:.1f} tok/s " f"per-stream {statistics.mean([r[2] / r[1] for r in ok]):.1f} tok/s"