autoresearch: harness setup
Benchmark entrypoint: bash autoresearch.sh Goal: Maximize frank-large decode throughput on the Strix Halo iGPU (gfx1151) via quantization >=q6/fp6, engine flags, spec-decode shape, or runtime swap — context limits unchanged, KV stays bf16.
This commit is contained in:
Frank
Executable
+72
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
# autoresearch.sh — canonical speed benchmark for frank-large (Dankclaw iGPU).
|
||||
#
|
||||
# Window semantics per the repo Benchmarking Policy: the deployed unit is
|
||||
# STOPPED, the candidate (this worktree's scripts/frank-large.env +
|
||||
# scripts/serve-frank-large.sh, rsynced to ~/jobs/autoresearch so the host's
|
||||
# git checkout stays clean) boots standalone on :7770, the fixed greedy
|
||||
# battery runs against it ON the host, and a trap restores the deployed
|
||||
# baseline unit whatever happens. The candidate never touches the host
|
||||
# checkout; "keep" in the loop means a real deploy later.
|
||||
#
|
||||
# Emits: METRIC decode_tokens_per_sec (primary, higher better),
|
||||
# METRIC prefill_tokens_per_sec, METRIC ttft_ms, ASI identity lines.
|
||||
set -euo pipefail
|
||||
|
||||
REPO="$(cd "$(dirname "$0")" && pwd)"
|
||||
HOST=dankclaw
|
||||
RDIR=/home/dank/jobs/autoresearch
|
||||
HEALTH_TRIES=200 # x5s = ~17 min ceiling (BF16 boots ~230s; INT8/dequant may differ)
|
||||
|
||||
restore() {
|
||||
local rc=$?
|
||||
trap - EXIT
|
||||
echo "[harness] tearing down candidate, restoring deployed baseline..." >&2
|
||||
ssh "$HOST" 'sudo -n systemctl stop frank-large.service 2>/dev/null;
|
||||
podman rm -f frank-large-mimo 2>/dev/null;
|
||||
true' >/dev/null 2>&1 || true
|
||||
ssh "$HOST" 'sudo -n systemctl start frank-large.service' >/dev/null 2>&1 || true
|
||||
exit $rc
|
||||
}
|
||||
trap restore EXIT
|
||||
|
||||
# 1. Ship the candidate (env + serve + battery) outside the git checkout.
|
||||
ssh "$HOST" "mkdir -p $RDIR"
|
||||
rsync -a "$REPO/scripts/frank-large.env" "$REPO/scripts/serve-frank-large.sh" \
|
||||
"$REPO/scripts/autoresearch-bench.py" "$HOST:$RDIR/"
|
||||
|
||||
# 2. Free the GPU and the port: stop the deployed unit (mask is unnecessary—
|
||||
# the unit only starts via systemd, and we never enable more of it).
|
||||
ssh "$HOST" 'sudo -n systemctl stop frank-large.service;
|
||||
podman rm -f frank-large-mimo 2>/dev/null; true' >/dev/null
|
||||
|
||||
# 3. Boot the candidate detached. The serve script's own gates run first;
|
||||
# a gate failure exits here with the reason.
|
||||
echo "[harness] starting candidate..." >&2
|
||||
if ! ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \
|
||||
bash $RDIR/serve-frank-large.sh >$RDIR/serve.log 2>&1 </dev/null & echo ok" \
|
||||
| grep -q ok; then
|
||||
echo "ERROR: could not launch candidate" >&2; exit 1
|
||||
fi
|
||||
|
||||
# 4. Wait for health (503 while loading; anything else = boot failure).
|
||||
echo "[harness] waiting for health..." >&2
|
||||
if ! ssh "$HOST" "for i in \$(seq 1 $HEALTH_TRIES); do
|
||||
code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 \
|
||||
http://127.0.0.1:7770/health 2>/dev/null || echo 000)
|
||||
[ \"\$code\" = 200 ] && echo UP && exit 0
|
||||
sleep 5
|
||||
done; exit 1" | grep -q UP; then
|
||||
echo "ERROR: candidate never became healthy; serve.log tail:" >&2
|
||||
ssh "$HOST" "tail -30 $RDIR/serve.log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 5. Identity evidence for the loop's records (exact engine argv).
|
||||
ASI_ARGS="$(ssh "$HOST" 'podman inspect frank-large-mimo \
|
||||
--format "{{join .Config.Cmd \" \"}}" 2>/dev/null' | tr -d "\n" | head -c 400)"
|
||||
echo "ASI engine_args=${ASI_ARGS:0:400}"
|
||||
|
||||
# 6. The battery, on the host, medians over 3 reps x 2 greedy prompts.
|
||||
ssh "$HOST" "python3 $RDIR/autoresearch-bench.py \
|
||||
--url http://127.0.0.1:7770 --model frank-large"
|
||||
Executable
+140
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
"""autoresearch-bench.py — deterministic greedy battery for frank-large.
|
||||
|
||||
Runs ON Dankclaw (no LAN jitter) against the candidate container on :7770.
|
||||
Fixed embedded corpus, temperature 0, streaming so decode speed is measured
|
||||
between real token arrivals (not request totals). Median of REPS samples per
|
||||
metric is the reported value; medians are far more stable than means under
|
||||
GPU thermal drift.
|
||||
|
||||
Metrics emitted (stdout, one per line):
|
||||
METRIC decode_tokens_per_sec=<float> primary, higher better
|
||||
METRIC prefill_tokens_per_sec=<float> secondary
|
||||
METRIC ttft_ms=<float> secondary, lower better
|
||||
Exit non-zero if the endpoint is unhealthy or the served alias is wrong.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
# Fixed corpus: deterministic, ~28k tokens after tokenization (same scale as
|
||||
# the 2026-10-01 bench window). Built by repeating a fixed paragraph block a
|
||||
# fixed number of times - NO random, NO clock, NO network.
|
||||
_BLOCK = (
|
||||
"The rendering pipeline walks the scene graph once per frame, resolving "
|
||||
"material bindings into descriptor sets before any draw call is recorded. "
|
||||
"Because the allocator recycles command buffers only after their fence "
|
||||
"signals, a submission that outlives its frame must pin the pool entry; "
|
||||
"otherwise the next frame overwrites recorded commands in flight. The "
|
||||
"validation layer flags exactly this class of lifetime bug, but only for "
|
||||
"buffers it can see - GPU-side timestamps and queue-family ownership "
|
||||
"transfers hide in the driver's own bookkeeping, which is why release "
|
||||
"builds must lean on explicit barriers and why a frame-time regression "
|
||||
"often shows up as corruption one frame later than its cause.\n"
|
||||
)
|
||||
LONG_CTX = _BLOCK * 220 # ~28k-token prefill, matching the deployed workload
|
||||
_LONG = LONG_CTX + "\nNow, in one focused paragraph (80-150 words), explain " \
|
||||
"which lifetime bugs this pipeline can still suffer and what explicit " \
|
||||
"mitigations the text implies."
|
||||
_SHORT = "Design a rate limiter for a homelab reverse proxy that must not " \
|
||||
"lock out the LAN. Think it through, then give the algorithm."
|
||||
|
||||
REPS = 3
|
||||
MAX_TOKENS = 256
|
||||
REQ_TIMEOUT = 900
|
||||
|
||||
|
||||
def stream_completion(url: str, model: str, prompt: str):
|
||||
"""Greedy streaming completion.
|
||||
|
||||
Returns (ttft_s, decode_tps, prefill_tps, prompt_tokens, n_tokens).
|
||||
ttft is RELATIVE to request send: time.perf_counter()'s epoch is host
|
||||
boot, and returning the raw stamp read as an 11-day first token.
|
||||
"""
|
||||
body = json.dumps({
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": MAX_TOKENS,
|
||||
"temperature": 0,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
}).encode()
|
||||
req = urllib.request.Request(
|
||||
url.rstrip("/") + "/v1/chat/completions", data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
t0 = time.perf_counter()
|
||||
first = None
|
||||
stamps = []
|
||||
usage = {}
|
||||
with urllib.request.urlopen(req, timeout=REQ_TIMEOUT) as resp:
|
||||
for raw in resp:
|
||||
line = raw.decode(errors="replace").strip()
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
payload = line[5:].strip()
|
||||
if payload == "[DONE]":
|
||||
break
|
||||
chunk = json.loads(payload)
|
||||
if chunk.get("usage"):
|
||||
usage = chunk["usage"]
|
||||
for ch in chunk.get("choices", []):
|
||||
d = ch.get("delta") or {}
|
||||
# reasoning AND content deltas both arrive token by token;
|
||||
# count either so greedy text length drives the rate.
|
||||
if d.get("content") or d.get("reasoning"):
|
||||
now = time.perf_counter()
|
||||
if first is None:
|
||||
first = now
|
||||
stamps.append(now)
|
||||
if first is None or len(stamps) < 2:
|
||||
# A zero-argmax degenerate answer would poison the median; raise.
|
||||
raise RuntimeError(f"stream produced {len(stamps)} tokens")
|
||||
ttft_s = first - t0
|
||||
decode_s = stamps[-1] - stamps[0]
|
||||
decode_tps = (len(stamps) - 1) / decode_s if decode_s > 0 else 0.0
|
||||
prompt_tokens = usage.get("prompt_tokens") or 0
|
||||
prefill_tps = prompt_tokens / ttft_s if prompt_tokens else 0.0
|
||||
return ttft_s, decode_tps, prefill_tps, prompt_tokens, len(stamps)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--url", default="http://127.0.0.1:7770")
|
||||
ap.add_argument("--model", default="frank-large")
|
||||
args = ap.parse_args()
|
||||
|
||||
with urllib.request.urlopen(args.url.rstrip("/") + "/v1/models",
|
||||
timeout=10) as r:
|
||||
ids = [m["id"] for m in json.load(r)["data"]]
|
||||
if args.model not in ids:
|
||||
print(f"FATAL: served ids {ids}, expected {args.model}",
|
||||
file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# Warmup (compile/graph warm + allocator settle), NOT measured.
|
||||
stream_completion(args.url, args.model, _SHORT)
|
||||
|
||||
decodes, prefills, ttfts = [], [], []
|
||||
for rep in range(REPS):
|
||||
for name, prompt in (("long", _LONG), ("short", _SHORT)):
|
||||
ttft, dt, pt, ptk, ntok = stream_completion(
|
||||
args.url, args.model, prompt)
|
||||
decodes.append(dt)
|
||||
ttfts.append(ttft * 1000.0)
|
||||
if name == "long":
|
||||
prefills.append(pt)
|
||||
print(f" rep{rep} {name:5s} ttft={ttft*1000:8.1f}ms "
|
||||
f"prefill={pt:8.1f} t/s decode={dt:6.2f} t/s "
|
||||
f"({ntok} tok, prompt {ptk})", file=sys.stderr)
|
||||
|
||||
med = lambda xs: statistics.median(xs)
|
||||
print(f"METRIC decode_tokens_per_sec={med(decodes):.2f}")
|
||||
print(f"METRIC prefill_tokens_per_sec={med(prefills):.1f}")
|
||||
print(f"METRIC ttft_ms={med(ttfts):.1f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user