autoresearch: harness setup
Benchmark entrypoint: bash autoresearch.sh Goal: Make frank-primary faster: raise decode tok/s on the tcclaviger vLLM endpoint (dankserver :7770, Qwen3.8-Flash-Next-Uncensored-FP8, 2x R9700, automatic tiered-expert offload). Context size may be altered (MAXLEN/SEQS/KV_BYTES and friends); quantization may NOT be altered (the 131-shard FP8 payload and the digest-pinned image are frozen).
This commit is contained in:
Frank
+192
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
"""frank-primary decode measurement — runs INSIDE the engine container.
|
||||
|
||||
The method is the fleet's established one (the 2026-09-29 / 2026-10-01
|
||||
tuning campaigns, /home/dank/mtp_measure.py): SSE chunk counting is invalid
|
||||
under speculative decoding because rejected drafts change the accounting, so
|
||||
decode is measured from the server's own usage counters:
|
||||
|
||||
decode_tok_s = completion_tokens / (wall - ttft)
|
||||
|
||||
with ttft = wall time to the FIRST content-bearing chunk (thinking arrives in
|
||||
`reasoning`/`delta.reasoning` on this stack and is excluded from the decode
|
||||
window — and is deterministic per prompt at temperature 0, so the exclusion
|
||||
cancels in any A/B on the same workload).
|
||||
|
||||
Workload (fixed, deterministic, offline):
|
||||
* prompts built from a large fixed-seed shuffled word list so tokens rarely
|
||||
repeat (no n-gram/prefix predictability of the text itself), sized with
|
||||
the server's own /v1/messages/count_tokens counter
|
||||
* depths 115 / 1056 / 4096 target prompt tokens
|
||||
* one 64-token warm pass per depth (warms the expert arena), then
|
||||
max_tokens=256 at temperature 0, 3 runs, median
|
||||
Everything is greedy with a fixed seed — the same workload every run, no
|
||||
network, no time-of-day dependence.
|
||||
|
||||
Env:
|
||||
AR_BASE endpoint (default http://127.0.0.1:7770)
|
||||
AR_MODEL served name (default frank-primary)
|
||||
AR_PROMPT_CEILING hard prompt ceiling in tokens (default 100000)
|
||||
|
||||
Output: progress on stderr, ONE json object on stdout (the last line).
|
||||
Exit codes: 0 ok, 2 endpoint unhealthy / no usable completion, 3 bad usage.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
BASE = os.environ.get("AR_BASE", "http://127.0.0.1:7770")
|
||||
MODEL = os.environ.get("AR_MODEL", "frank-primary")
|
||||
PROMPT_CEILING = int(os.environ.get("AR_PROMPT_CEILING", "100000"))
|
||||
DEPTHS = (115, 1056, 4096)
|
||||
WARM_TOKENS = 64
|
||||
MAX_TOKENS = 256
|
||||
RUNS = 3
|
||||
|
||||
# 43-word x 12 = 516-word vocabulary, seed 1234 — the campaign's exact list
|
||||
# and seed, so prompts are byte-identical to the historical measurements.
|
||||
_VOCAB = ("alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima "
|
||||
"mike november oscar papa quebec romeo sierra tango uniform victor whiskey "
|
||||
"xray yankee zulu cardinal ordinal topology kernel scheduler resident "
|
||||
"unified aperture expert arena slab offload prefetch gemm triton quant "
|
||||
"tensor shard checkpoint manifest embed router cache page table window "
|
||||
"gradient sparse dense moe attention head dim block stream decode prefill "
|
||||
"bandwidth latency throughput residency eviction tier swap pin memlock").split()
|
||||
_R = random.Random(1234)
|
||||
_ORDER = list(range(len(_VOCAB)))
|
||||
_R.shuffle(_ORDER)
|
||||
|
||||
|
||||
def log(msg):
|
||||
print("[measure] %s" % msg, file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def post_json(path, body, timeout):
|
||||
req = urllib.request.Request(
|
||||
BASE + path, data=json.dumps(body).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return json.load(r)
|
||||
|
||||
|
||||
def count_tokens(text):
|
||||
"""Token count without a prefill, from the server's own counter."""
|
||||
body = {"model": MODEL,
|
||||
"messages": [{"role": "user", "content": text}]}
|
||||
r = post_json("/v1/messages/count_tokens", body, timeout=300)
|
||||
n = r.get("input_tokens") or r.get("count") or r.get("token_count")
|
||||
if not n:
|
||||
raise RuntimeError("count_tokens returned no count: %r" % (r,))
|
||||
return int(n)
|
||||
|
||||
|
||||
def words(n):
|
||||
return " ".join(_VOCAB[_ORDER[i % len(_VOCAB)]] for i in range(n))
|
||||
|
||||
|
||||
def calibrate():
|
||||
"""Measure words->tokens from the server's own counter."""
|
||||
return 2000.0 / count_tokens(words(2000))
|
||||
|
||||
|
||||
def prompt_for(target_tokens, wpt):
|
||||
n = max(1, int(target_tokens * wpt))
|
||||
text = words(n)
|
||||
pt = count_tokens(text)
|
||||
while pt > PROMPT_CEILING:
|
||||
n = max(1, int(n * (PROMPT_CEILING / pt) * 0.98))
|
||||
text = words(n)
|
||||
pt = count_tokens(text)
|
||||
return text, pt
|
||||
|
||||
|
||||
def measured(prompt, max_tokens):
|
||||
"""Return (ttft_s, decode_tok_s, completion_tokens) via usage counters."""
|
||||
body = {"model": MODEL,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens, "temperature": 0,
|
||||
"stream": True, "stream_options": {"include_usage": True}}
|
||||
req = urllib.request.Request(
|
||||
BASE + "/v1/chat/completions", data=json.dumps(body).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
t0 = time.time()
|
||||
ttft = None
|
||||
usage = {}
|
||||
with urllib.request.urlopen(req, timeout=1800) as r:
|
||||
for raw in r:
|
||||
line = raw.decode("utf-8", "replace").strip()
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
s = line[5:].strip()
|
||||
if s == "[DONE]":
|
||||
break
|
||||
try:
|
||||
ev = json.loads(s)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if ev.get("usage"):
|
||||
usage = ev["usage"]
|
||||
for ch in ev.get("choices", []):
|
||||
if (ch.get("delta") or {}).get("content"):
|
||||
if ttft is None:
|
||||
ttft = time.time() - t0
|
||||
wall = time.time() - t0
|
||||
ct = int(usage.get("completion_tokens") or 0)
|
||||
if ct <= 0:
|
||||
# No usage on the stream: the endpoint is unhealthy or the model
|
||||
# returned nothing (e.g. HTTP 200 with an error payload).
|
||||
raise RuntimeError("no completion_tokens in usage: %r" % (usage,))
|
||||
dec_window = max(wall - (ttft or 0.0), 1e-9)
|
||||
return (ttft or wall), (ct / dec_window), ct
|
||||
|
||||
|
||||
def main():
|
||||
log("base=%s model=%s ceiling=%d" % (BASE, MODEL, PROMPT_CEILING))
|
||||
try:
|
||||
wpt = calibrate()
|
||||
except Exception as e:
|
||||
log("endpoint unhealthy during calibration: %s" % e)
|
||||
return 2
|
||||
log("words/token: %.3f" % wpt)
|
||||
|
||||
result = {"words_per_token": round(wpt, 5)}
|
||||
for tgt in DEPTHS:
|
||||
try:
|
||||
p, pt = prompt_for(tgt, wpt)
|
||||
measured(p, WARM_TOKENS) # warm the arena
|
||||
except Exception as e:
|
||||
log("warm pass failed at depth %d: %s" % (tgt, e))
|
||||
return 2
|
||||
rates, tts, cts = [], [], []
|
||||
for i in range(RUNS):
|
||||
try:
|
||||
tt, r, ct = measured(p, MAX_TOKENS)
|
||||
except Exception as e:
|
||||
log("run %d/%d failed at depth %d: %s" % (i + 1, RUNS, tgt, e))
|
||||
continue
|
||||
rates.append(r)
|
||||
tts.append(tt)
|
||||
cts.append(ct)
|
||||
if len(rates) < 2:
|
||||
log("only %d usable run(s) at depth %d — refusing to median" % (len(rates), tgt))
|
||||
return 2
|
||||
result["decode_tps_%d" % tgt] = round(statistics.median(rates), 2)
|
||||
result["ttft_ms_%d" % tgt] = round(1000.0 * statistics.median(tts), 1)
|
||||
result["completion_tokens_%d" % tgt] = int(statistics.median(cts))
|
||||
result["prompt_tokens_%d" % tgt] = pt
|
||||
log("depth %5d tok decode %.2f t/s ttft %.3f s (runs=%d)"
|
||||
% (pt, result["decode_tps_%d" % tgt], result["ttft_ms_%d" % tgt] / 1000.0, len(rates)))
|
||||
|
||||
meds = [result["decode_tps_%d" % t] for t in DEPTHS]
|
||||
result["decode_tps"] = round(statistics.median(meds), 2)
|
||||
print(json.dumps(result))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+200
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env bash
|
||||
# frank-primary-window.sh — one full bench window for frank-primary, run ON
|
||||
# DANKSERVER as the deploy user (invoked over ssh by ./autoresearch.sh).
|
||||
#
|
||||
# Discipline (AGENTS.md Benchmarking Policy): the production unit is
|
||||
# STOPPED + MASKED first, the candidate config boots AS THE SERVICE itself
|
||||
# (same serve script, same container name, same port 7770 — no scratch
|
||||
# ports, no co-tenant GPU), the in-container measure runs, and the window
|
||||
# ALWAYS ends by restoring the production unit and verifying its health.
|
||||
# The EXIT trap makes the restore unconditional: a boot failure, a measure
|
||||
# failure, or a dead ssh session all end with production back up.
|
||||
#
|
||||
# Staging layout (created by the orchestrator, ephemeral — never a config
|
||||
# source; the repo checkout is):
|
||||
# /home/dank/.autoresearch-frank-primary/
|
||||
# serve.sh staged candidate serve script
|
||||
# frank-primary.env staged candidate env (the candidate knobs)
|
||||
# measure.py the in-container measure client
|
||||
# hash candidate fingerprint
|
||||
# run.json result (metrics + boot + kv + fingerprint)
|
||||
# serve.out/measure.out/fail.log
|
||||
# last/ memo: run.json + hash of the last FULLY
|
||||
# successful window (bench AND verified restore)
|
||||
#
|
||||
# This script logs to stderr and exits 0 only when the measure produced a
|
||||
# usable result (the restore happens in the trap regardless of the bench
|
||||
# outcome; a failed restore forces exit 1).
|
||||
set -uo pipefail
|
||||
|
||||
STAGE="${FRANK_PRIMARY_AR_STAGE:-/home/dank/.autoresearch-frank-primary}"
|
||||
UNIT="${FRANK_PRIMARY_AR_UNIT:-frank-primary}"
|
||||
CTR="${FRANK_PRIMARY_AR_CTR:-frank-primary-qwen}"
|
||||
HEALTH_URL="http://127.0.0.1:7770/health"
|
||||
MODELS_URL="http://127.0.0.1:7770/v1/models"
|
||||
BOOT_TIMEOUT="${FRANK_PRIMARY_AR_BOOT_TIMEOUT:-1200}"
|
||||
RESTORE_TIMEOUT="${FRANK_PRIMARY_AR_RESTORE_TIMEOUT:-1200}"
|
||||
MEASURE_TIMEOUT="${FRANK_PRIMARY_AR_MEASURE_TIMEOUT:-1800}"
|
||||
SERVED="$(sed -n 's/^FRANK_PRIMARY_SERVED_NAME=//p' "$STAGE/frank-primary.env" 2>/dev/null | head -1)"
|
||||
SERVED="${SERVED:-frank-primary}"
|
||||
|
||||
RESTORED=0
|
||||
|
||||
log() { printf '[window %s] %s\n' "$(date '+%H:%M:%S')" "$*" >&2; }
|
||||
|
||||
container_state() { podman inspect -f '{{.State.Status}}' "$CTR" 2>/dev/null || echo absent; }
|
||||
|
||||
wait_port_free() {
|
||||
local i
|
||||
for i in $(seq 1 90); do
|
||||
[ "$(container_state)" != "running" ] && return 0
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_health() { # $1 = timeout seconds
|
||||
local t0 code
|
||||
t0=$(date +%s)
|
||||
while :; do
|
||||
code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$HEALTH_URL" 2>/dev/null || echo 000)"
|
||||
[ "$code" = "200" ] && return 0
|
||||
[ "$(( $(date +%s) - t0 ))" -ge "$1" ] && return 1
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
capture_bench_failure() {
|
||||
log "capturing bench-boot failure evidence"
|
||||
{
|
||||
echo "=== podman logs (tail 400) ==="
|
||||
podman logs --tail 400 "$CTR" 2>&1 || true
|
||||
echo "=== staged serve.out (tail 60) ==="
|
||||
tail -60 "$STAGE/serve.out" 2>/dev/null || true
|
||||
echo "=== engine log (tail 60) ==="
|
||||
tail -60 /home/dank/logs/frank_primary.log 2>/dev/null || true
|
||||
} >"$STAGE/fail.log" 2>&1 || true
|
||||
grep -E 'OutOfMemory|Tried to allocate|Engine core initialization failed|Error|error' \
|
||||
"$STAGE/fail.log" 2>/dev/null | tail -25 >&2 || true
|
||||
}
|
||||
|
||||
restore() {
|
||||
[ "$RESTORED" = 1 ] && return 0
|
||||
RESTORED=1
|
||||
log "restoring production unit $UNIT"
|
||||
if [ -f "$STAGE/serve.pid" ]; then
|
||||
kill "$(cat "$STAGE/serve.pid" 2>/dev/null)" 2>/dev/null || true
|
||||
fi
|
||||
podman stop -t 120 "$CTR" >/dev/null 2>&1 || true
|
||||
wait_port_free || log "WARNING: bench container still draining after 180s"
|
||||
sudo systemctl unmask "$UNIT" 2>/dev/null || true
|
||||
if ! sudo systemctl start "$UNIT"; then
|
||||
log "ERROR: systemctl start $UNIT failed — operator intervention required"
|
||||
return 1
|
||||
fi
|
||||
if wait_health "$RESTORE_TIMEOUT"; then
|
||||
if curl -s --max-time 10 "$MODELS_URL" 2>/dev/null | grep -q "$SERVED"; then
|
||||
log "production restored and verified ($SERVED answering on :7770)"
|
||||
# Memo promotion lives HERE: a cached result is only ever the
|
||||
# product of a fully successful window (bench AND verified
|
||||
# restore), never of a bare bench success.
|
||||
cp "$STAGE/run.json" "$STAGE/last/" 2>/dev/null || true
|
||||
cp "$STAGE/hash" "$STAGE/last/" 2>/dev/null || true
|
||||
else
|
||||
log "ERROR: health 200 but /v1/models does not list $SERVED"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log "ERROR: production not healthy within ${RESTORE_TIMEOUT}s — unit left to systemd (Restart=always)"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
trap 'restore; rc=$?; [ "$rc" -ne 0 ] && exit 1' EXIT
|
||||
|
||||
main() {
|
||||
local f
|
||||
for f in serve.sh frank-primary.env measure.py hash; do
|
||||
[ -f "$STAGE/$f" ] || { log "staged file missing: $STAGE/$f"; exit 1; }
|
||||
done
|
||||
mkdir -p "$STAGE/last"
|
||||
rm -f "$STAGE/run.json" "$STAGE/fail.log" "$STAGE/measure.out"
|
||||
|
||||
local t_start t0 i kvline kv_tokens kv_conc measure_json boot_seconds
|
||||
t_start="$(date +%s)"
|
||||
|
||||
log "stopping and masking production unit $UNIT"
|
||||
sudo systemctl stop "$UNIT" 2>/dev/null || true
|
||||
# Explicit stop is not auto-restarted even with Restart=always; the mask
|
||||
# is belt-and-braces against a stray start during the window.
|
||||
sudo systemctl mask "$UNIT" 2>/dev/null || \
|
||||
log "WARNING: mask failed — relying on explicit-stop semantics"
|
||||
wait_port_free || log "note: production container still draining"
|
||||
|
||||
t0="$(date +%s)"
|
||||
log "starting bench instance from staged config (fingerprint $(tr -d '[:space:]' <"$STAGE/hash"))"
|
||||
FRANK_PRIMARY_ENV_FILE="$STAGE/frank-primary.env" \
|
||||
setsid nohup bash "$STAGE/serve.sh" >"$STAGE/serve.out" 2>&1 </dev/null &
|
||||
echo "$!" >"$STAGE/serve.pid"
|
||||
|
||||
for i in $(seq 1 45); do
|
||||
[ "$(container_state)" = "running" ] && break
|
||||
sleep 2
|
||||
done
|
||||
if [ "$(container_state)" != "running" ]; then
|
||||
log "bench container never reached running state"
|
||||
tail -40 "$STAGE/serve.out" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if wait_health "$BOOT_TIMEOUT"; then
|
||||
boot_seconds=$(( $(date +%s) - t0 ))
|
||||
log "bench healthy after ${boot_seconds}s"
|
||||
else
|
||||
if [ "$(container_state)" = "running" ]; then
|
||||
log "bench boot timed out after ${BOOT_TIMEOUT}s (still loading)"
|
||||
else
|
||||
log "bench container died during boot"
|
||||
capture_bench_failure
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "running measure in container (timeout ${MEASURE_TIMEOUT}s)"
|
||||
if ! podman cp "$STAGE/measure.py" "$CTR":/tmp/ar-measure.py; then
|
||||
log "podman cp of measure client failed"
|
||||
exit 1
|
||||
fi
|
||||
if ! timeout "$MEASURE_TIMEOUT" podman exec "$CTR" python3 /tmp/ar-measure.py \
|
||||
>"$STAGE/measure.out" 2>&1; then
|
||||
log "measure failed or timed out"
|
||||
tail -40 "$STAGE/measure.out" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Capacity facts for the run (context-size experiments change these).
|
||||
kvline="$(podman logs "$CTR" 2>&1 | grep -E 'GPU KV cache size' | tail -1 || true)"
|
||||
kv_tokens="$(printf '%s' "$kvline" | sed -E 's/.*GPU KV cache size: ([0-9,]+) tokens.*/\1/' | tr -d ',' || true)"
|
||||
kv_conc="$(printf '%s' "$kvline" | sed -E 's/.*per request: ([0-9.]+x).*/\1/' || true)"
|
||||
|
||||
measure_json="$(grep -E '^\{' "$STAGE/measure.out" | tail -1 || true)"
|
||||
if [ -z "$measure_json" ]; then
|
||||
log "measure produced no JSON result line"
|
||||
tail -40 "$STAGE/measure.out" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The measure line is already JSON, so the run object can be assembled
|
||||
# without any host-side interpreter.
|
||||
printf '{"hash":"%s","boot_seconds":%s,"kv_tokens":%s,"kv_concurrency":"%s","total_seconds":%s,"measure":%s}\n' \
|
||||
"$(tr -d '[:space:]' <"$STAGE/hash")" \
|
||||
"$boot_seconds" "${kv_tokens:-0}" "${kv_conc:-unknown}" \
|
||||
"$(( $(date +%s) - t_start ))" "$measure_json" >"$STAGE/run.json"
|
||||
|
||||
log "bench window complete in $(( $(date +%s) - t_start ))s"
|
||||
# Memo only after a FULL success; the restore below still has to verify.
|
||||
exit 0
|
||||
}
|
||||
|
||||
main "$@"
|
||||
exit "$?"
|
||||
Reference in New Issue
Block a user