From 5df6ef3166b44217823a01bf7deeca9a629eeb73 Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 1 Oct 2026 20:41:51 -0700 Subject: [PATCH] autoresearch: harness setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Benchmark entrypoint: bash autoresearch.sh Goal: Maximize frank-large decode throughput on the Strix Halo iGPU (gfx1151) via quantization >=q6/fp6, engine flags, spec-decode shape, or runtime swap — context limits unchanged, KV stays bf16. --- autoresearch.sh | 72 +++++++++++++++++ scripts/autoresearch-bench.py | 140 ++++++++++++++++++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100755 autoresearch.sh create mode 100755 scripts/autoresearch-bench.py diff --git a/autoresearch.sh b/autoresearch.sh new file mode 100755 index 0000000..a52bc36 --- /dev/null +++ b/autoresearch.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# autoresearch.sh — canonical speed benchmark for frank-large (Dankclaw iGPU). +# +# Window semantics per the repo Benchmarking Policy: the deployed unit is +# STOPPED, the candidate (this worktree's scripts/frank-large.env + +# scripts/serve-frank-large.sh, rsynced to ~/jobs/autoresearch so the host's +# git checkout stays clean) boots standalone on :7770, the fixed greedy +# battery runs against it ON the host, and a trap restores the deployed +# baseline unit whatever happens. The candidate never touches the host +# checkout; "keep" in the loop means a real deploy later. +# +# Emits: METRIC decode_tokens_per_sec (primary, higher better), +# METRIC prefill_tokens_per_sec, METRIC ttft_ms, ASI identity lines. +set -euo pipefail + +REPO="$(cd "$(dirname "$0")" && pwd)" +HOST=dankclaw +RDIR=/home/dank/jobs/autoresearch +HEALTH_TRIES=200 # x5s = ~17 min ceiling (BF16 boots ~230s; INT8/dequant may differ) + +restore() { + local rc=$? + trap - EXIT + echo "[harness] tearing down candidate, restoring deployed baseline..." >&2 + ssh "$HOST" 'sudo -n systemctl stop frank-large.service 2>/dev/null; + podman rm -f frank-large-mimo 2>/dev/null; + true' >/dev/null 2>&1 || true + ssh "$HOST" 'sudo -n systemctl start frank-large.service' >/dev/null 2>&1 || true + exit $rc +} +trap restore EXIT + +# 1. Ship the candidate (env + serve + battery) outside the git checkout. +ssh "$HOST" "mkdir -p $RDIR" +rsync -a "$REPO/scripts/frank-large.env" "$REPO/scripts/serve-frank-large.sh" \ + "$REPO/scripts/autoresearch-bench.py" "$HOST:$RDIR/" + +# 2. Free the GPU and the port: stop the deployed unit (mask is unnecessary— +# the unit only starts via systemd, and we never enable more of it). +ssh "$HOST" 'sudo -n systemctl stop frank-large.service; + podman rm -f frank-large-mimo 2>/dev/null; true' >/dev/null + +# 3. Boot the candidate detached. The serve script's own gates run first; +# a gate failure exits here with the reason. +echo "[harness] starting candidate..." >&2 +if ! ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \ + bash $RDIR/serve-frank-large.sh >$RDIR/serve.log 2>&1 &2; exit 1 +fi + +# 4. Wait for health (503 while loading; anything else = boot failure). +echo "[harness] waiting for health..." >&2 +if ! ssh "$HOST" "for i in \$(seq 1 $HEALTH_TRIES); do + code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 \ + http://127.0.0.1:7770/health 2>/dev/null || echo 000) + [ \"\$code\" = 200 ] && echo UP && exit 0 + sleep 5 + done; exit 1" | grep -q UP; then + echo "ERROR: candidate never became healthy; serve.log tail:" >&2 + ssh "$HOST" "tail -30 $RDIR/serve.log" >&2 || true + exit 1 +fi + +# 5. Identity evidence for the loop's records (exact engine argv). +ASI_ARGS="$(ssh "$HOST" 'podman inspect frank-large-mimo \ + --format "{{join .Config.Cmd \" \"}}" 2>/dev/null' | tr -d "\n" | head -c 400)" +echo "ASI engine_args=${ASI_ARGS:0:400}" + +# 6. The battery, on the host, medians over 3 reps x 2 greedy prompts. +ssh "$HOST" "python3 $RDIR/autoresearch-bench.py \ + --url http://127.0.0.1:7770 --model frank-large" diff --git a/scripts/autoresearch-bench.py b/scripts/autoresearch-bench.py new file mode 100755 index 0000000..de9468e --- /dev/null +++ b/scripts/autoresearch-bench.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""autoresearch-bench.py — deterministic greedy battery for frank-large. + +Runs ON Dankclaw (no LAN jitter) against the candidate container on :7770. +Fixed embedded corpus, temperature 0, streaming so decode speed is measured +between real token arrivals (not request totals). Median of REPS samples per +metric is the reported value; medians are far more stable than means under +GPU thermal drift. + +Metrics emitted (stdout, one per line): + METRIC decode_tokens_per_sec= primary, higher better + METRIC prefill_tokens_per_sec= secondary + METRIC ttft_ms= secondary, lower better +Exit non-zero if the endpoint is unhealthy or the served alias is wrong. +""" +import argparse +import json +import statistics +import sys +import time +import urllib.request + +# Fixed corpus: deterministic, ~28k tokens after tokenization (same scale as +# the 2026-10-01 bench window). Built by repeating a fixed paragraph block a +# fixed number of times - NO random, NO clock, NO network. +_BLOCK = ( + "The rendering pipeline walks the scene graph once per frame, resolving " + "material bindings into descriptor sets before any draw call is recorded. " + "Because the allocator recycles command buffers only after their fence " + "signals, a submission that outlives its frame must pin the pool entry; " + "otherwise the next frame overwrites recorded commands in flight. The " + "validation layer flags exactly this class of lifetime bug, but only for " + "buffers it can see - GPU-side timestamps and queue-family ownership " + "transfers hide in the driver's own bookkeeping, which is why release " + "builds must lean on explicit barriers and why a frame-time regression " + "often shows up as corruption one frame later than its cause.\n" +) +LONG_CTX = _BLOCK * 220 # ~28k-token prefill, matching the deployed workload +_LONG = LONG_CTX + "\nNow, in one focused paragraph (80-150 words), explain " \ + "which lifetime bugs this pipeline can still suffer and what explicit " \ + "mitigations the text implies." +_SHORT = "Design a rate limiter for a homelab reverse proxy that must not " \ + "lock out the LAN. Think it through, then give the algorithm." + +REPS = 3 +MAX_TOKENS = 256 +REQ_TIMEOUT = 900 + + +def stream_completion(url: str, model: str, prompt: str): + """Greedy streaming completion. + + Returns (ttft_s, decode_tps, prefill_tps, prompt_tokens, n_tokens). + ttft is RELATIVE to request send: time.perf_counter()'s epoch is host + boot, and returning the raw stamp read as an 11-day first token. + """ + body = json.dumps({ + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": MAX_TOKENS, + "temperature": 0, + "stream": True, + "stream_options": {"include_usage": True}, + }).encode() + req = urllib.request.Request( + url.rstrip("/") + "/v1/chat/completions", data=body, + headers={"Content-Type": "application/json"}) + t0 = time.perf_counter() + first = None + stamps = [] + usage = {} + with urllib.request.urlopen(req, timeout=REQ_TIMEOUT) as resp: + for raw in resp: + line = raw.decode(errors="replace").strip() + if not line.startswith("data:"): + continue + payload = line[5:].strip() + if payload == "[DONE]": + break + chunk = json.loads(payload) + if chunk.get("usage"): + usage = chunk["usage"] + for ch in chunk.get("choices", []): + d = ch.get("delta") or {} + # reasoning AND content deltas both arrive token by token; + # count either so greedy text length drives the rate. + if d.get("content") or d.get("reasoning"): + now = time.perf_counter() + if first is None: + first = now + stamps.append(now) + if first is None or len(stamps) < 2: + # A zero-argmax degenerate answer would poison the median; raise. + raise RuntimeError(f"stream produced {len(stamps)} tokens") + ttft_s = first - t0 + decode_s = stamps[-1] - stamps[0] + decode_tps = (len(stamps) - 1) / decode_s if decode_s > 0 else 0.0 + prompt_tokens = usage.get("prompt_tokens") or 0 + prefill_tps = prompt_tokens / ttft_s if prompt_tokens else 0.0 + return ttft_s, decode_tps, prefill_tps, prompt_tokens, len(stamps) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--url", default="http://127.0.0.1:7770") + ap.add_argument("--model", default="frank-large") + args = ap.parse_args() + + with urllib.request.urlopen(args.url.rstrip("/") + "/v1/models", + timeout=10) as r: + ids = [m["id"] for m in json.load(r)["data"]] + if args.model not in ids: + print(f"FATAL: served ids {ids}, expected {args.model}", + file=sys.stderr) + sys.exit(2) + + # Warmup (compile/graph warm + allocator settle), NOT measured. + stream_completion(args.url, args.model, _SHORT) + + decodes, prefills, ttfts = [], [], [] + for rep in range(REPS): + for name, prompt in (("long", _LONG), ("short", _SHORT)): + ttft, dt, pt, ptk, ntok = stream_completion( + args.url, args.model, prompt) + decodes.append(dt) + ttfts.append(ttft * 1000.0) + if name == "long": + prefills.append(pt) + print(f" rep{rep} {name:5s} ttft={ttft*1000:8.1f}ms " + f"prefill={pt:8.1f} t/s decode={dt:6.2f} t/s " + f"({ntok} tok, prompt {ptk})", file=sys.stderr) + + med = lambda xs: statistics.median(xs) + print(f"METRIC decode_tokens_per_sec={med(decodes):.2f}") + print(f"METRIC prefill_tokens_per_sec={med(prefills):.1f}") + print(f"METRIC ttft_ms={med(ttfts):.1f}") + + +if __name__ == "__main__": + main()