autoresearch: harness setup
Benchmark entrypoint: bash autoresearch.sh Goal: Make frank-primary faster: raise decode tok/s on the tcclaviger vLLM endpoint (dankserver :7770, Qwen3.8-Flash-Next-Uncensored-FP8, 2x R9700, automatic tiered-expert offload). Context size may be altered (MAXLEN/SEQS/KV_BYTES and friends); quantization may NOT be altered (the 131-shard FP8 payload and the digest-pinned image are frozen).
This commit is contained in:
Frank
+91
-67
@@ -1,80 +1,104 @@
|
||||
#!/usr/bin/env bash
|
||||
# autoresearch.sh — canonical speed benchmark for frank-large (Dankclaw iGPU).
|
||||
# autoresearch.sh — canonical benchmark entrypoint: speed of frank-primary.
|
||||
#
|
||||
# Window semantics per the repo Benchmarking Policy: the deployed unit is
|
||||
# STOPPED, the candidate (this worktree's scripts/frank-large.env +
|
||||
# scripts/serve-frank-large.sh, rsynced to ~/jobs/autoresearch so the host's
|
||||
# git checkout stays clean) boots standalone on :7770, the fixed greedy
|
||||
# battery runs against it ON the host, and a trap restores the deployed
|
||||
# baseline unit whatever happens. The candidate never touches the host
|
||||
# checkout; "keep" in the loop means a real deploy later.
|
||||
# Runs on the operator box and drives one bench window on DANKSERVER:
|
||||
# 1. fingerprint the candidate trio (env + serve script + unit)
|
||||
# 2. memo fast-path: an identical fingerprint that already produced a FULLY
|
||||
# successful window (bench + verified production restore) returns the
|
||||
# cached metrics without touching the fleet
|
||||
# 3. stage the candidate trio plus the measure client into
|
||||
# /home/dank/.autoresearch-frank-primary/ (ephemeral; the repo checkout
|
||||
# on the host stays the production source of truth)
|
||||
# 4. run the window (scripts/autoresearch/frank-primary-window.sh):
|
||||
# stop + mask the production unit, boot the staged config AS THE SERVICE
|
||||
# (same serve script, same container, same port 7770), wait for health,
|
||||
# run the in-container usage-counter measure, then restore the production
|
||||
# unit and verify its health — AGENTS.md Benchmarking Policy
|
||||
# 5. print METRIC lines (parse by run_experiment)
|
||||
#
|
||||
# Emits: METRIC decode_tokens_per_sec (primary, higher better),
|
||||
# METRIC prefill_tokens_per_sec, METRIC ttft_ms, ASI identity lines.
|
||||
# Primary metric: decode_tps (HIGHER is better) — median across the three
|
||||
# prompt depths (115/1056/4096) of per-depth medians of
|
||||
# completion_tokens / (wall - ttft), 3 runs each after a warm pass,
|
||||
# temperature 0, the campaign's deterministic word-seed workload.
|
||||
#
|
||||
# Exit 0 with METRIC lines on success; non-zero on any failure (the window
|
||||
# script's EXIT trap restores production on every path).
|
||||
set -euo pipefail
|
||||
|
||||
REPO="$(cd "$(dirname "$0")" && pwd)"
|
||||
HOST=dankclaw
|
||||
RDIR=/home/dank/jobs/autoresearch
|
||||
HEALTH_TRIES=200 # x5s = ~17 min ceiling (BF16 boots ~230s; INT8/dequant may differ)
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
HOST="${FRANK_PRIMARY_AR_HOST:-dankserver}"
|
||||
STAGE="/home/dank/.autoresearch-frank-primary"
|
||||
|
||||
restore() {
|
||||
local rc=$?
|
||||
trap - EXIT
|
||||
echo "[harness] tearing down candidate, restoring deployed baseline..." >&2
|
||||
ssh "$HOST" 'sudo -n systemctl stop frank-large.service 2>/dev/null;
|
||||
podman rm -f frank-large-mimo 2>/dev/null;
|
||||
pkill -x llama-server 2>/dev/null; true' >/dev/null 2>&1 || true
|
||||
ssh "$HOST" 'sudo -n systemctl start frank-large.service' >/dev/null 2>&1 || true
|
||||
exit $rc
|
||||
ENV_FILE="$SCRIPT_DIR/scripts/frank-primary.env"
|
||||
SERVE="$SCRIPT_DIR/scripts/serve-frank-primary.sh"
|
||||
UNIT="$SCRIPT_DIR/scripts/frank-primary.service"
|
||||
MEASURE="$SCRIPT_DIR/scripts/autoresearch/frank-primary-measure.py"
|
||||
WINDOW="$SCRIPT_DIR/scripts/autoresearch/frank-primary-window.sh"
|
||||
|
||||
for f in "$ENV_FILE" "$SERVE" "$UNIT" "$MEASURE" "$WINDOW"; do
|
||||
[ -f "$f" ] || { echo "[autoresearch] missing harness file: $f" >&2; exit 1; }
|
||||
done
|
||||
command -v python3 >/dev/null 2>&1 || { echo "[autoresearch] python3 required on operator box" >&2; exit 1; }
|
||||
|
||||
# Fingerprint of everything that can change the engine's argv or environment.
|
||||
HASH="$(cat "$ENV_FILE" "$SERVE" "$UNIT" | sha256sum | cut -d' ' -f1)"
|
||||
echo "[autoresearch] candidate fingerprint $HASH" >&2
|
||||
|
||||
ssh_run() { ssh -o BatchMode=yes -o ConnectTimeout=15 "$HOST" "$@"; }
|
||||
|
||||
emit_metrics() { # $1 = run.json
|
||||
python3 -c '
|
||||
import json, sys
|
||||
run = json.loads(sys.argv[1])
|
||||
m = run["measure"]
|
||||
print("METRIC decode_tps=%s" % m["decode_tps"])
|
||||
for k in ("decode_tps_115", "decode_tps_1056", "decode_tps_4096"):
|
||||
print("METRIC %s=%s" % (k, m[k]))
|
||||
print("METRIC ttft_ms_4096=%s" % m["ttft_ms_4096"])
|
||||
print("METRIC kv_tokens=%s" % (run.get("kv_tokens") or 0))
|
||||
print("METRIC boot_seconds=%s" % (run.get("boot_seconds") or -1))
|
||||
print("METRIC completion_tokens_4096=%s" % m.get("completion_tokens_4096", 0))
|
||||
' "$1"
|
||||
}
|
||||
trap restore EXIT
|
||||
|
||||
# 1. Ship the candidate (env + serve + battery) outside the git checkout.
|
||||
ssh "$HOST" "mkdir -p $RDIR"
|
||||
rsync -a "$REPO/scripts/frank-large.env" "$REPO/scripts/serve-frank-large.sh" \
|
||||
"$REPO/scripts/autoresearch-bench.py" "$HOST:$RDIR/"
|
||||
# --- memo fast-path -------------------------------------------------------
|
||||
cached_hash="$(ssh_run "cat '$STAGE/last/hash' 2>/dev/null" 2>/dev/null || true)"
|
||||
if [ -n "$cached_hash" ] && [ "$cached_hash" = "$HASH" ]; then
|
||||
cached_run="$(ssh_run "cat '$STAGE/last/run.json' 2>/dev/null" || true)"
|
||||
if [ -n "$cached_run" ]; then
|
||||
echo "[autoresearch] memo hit — cached metrics for $HASH, fleet untouched" >&2
|
||||
emit_metrics "$cached_run"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# 2. Free the GPU and the port: stop the deployed unit (mask is unnecessary—
|
||||
# the unit only starts via systemd, and we never enable more of it).
|
||||
ssh "$HOST" 'sudo -n systemctl stop frank-large.service;
|
||||
podman rm -f frank-large-mimo 2>/dev/null;
|
||||
pkill -x llama-server 2>/dev/null; true' >/dev/null
|
||||
# --- stage the candidate ----------------------------------------------------
|
||||
echo "[autoresearch] staging candidate on $HOST:$STAGE" >&2
|
||||
ssh_run "mkdir -p '$STAGE'"
|
||||
scp -q "$ENV_FILE" "$HOST:$STAGE/frank-primary.env"
|
||||
scp -q "$SERVE" "$HOST:$STAGE/serve.sh"
|
||||
scp -q "$MEASURE" "$HOST:$STAGE/measure.py"
|
||||
scp -q "$WINDOW" "$HOST:$STAGE/window.sh"
|
||||
ssh_run "printf '%s\n' '$HASH' > '$STAGE/hash'"
|
||||
|
||||
# 3. Boot the candidate detached; the launcher's own output IS the serve
|
||||
# stage output (gate failures print there before any container exists).
|
||||
echo "[harness] starting candidate..." >&2
|
||||
ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \
|
||||
bash $RDIR/serve-frank-large.sh >$RDIR/serve.log 2>&1 </dev/null & echo ok" \
|
||||
| tee /tmp/autoresearch-launch.log >&2
|
||||
grep -q ok /tmp/autoresearch-launch.log || { echo "ERROR: launch failed" >&2; exit 1; }
|
||||
|
||||
# 4. Wait for health (503 while loading; anything else = boot failure).
|
||||
echo "[harness] waiting for health..." >&2
|
||||
if ! ssh "$HOST" "for i in \$(seq 1 $HEALTH_TRIES); do
|
||||
code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 \
|
||||
http://127.0.0.1:7770/health 2>/dev/null || echo 000)
|
||||
[ \"\$code\" = 200 ] && echo UP && exit 0
|
||||
st=\$(podman inspect -f '{{.State.Status}}' frank-large-mimo 2>/dev/null || echo gone)
|
||||
if [ "\$st" = exited ] || { [ "\$st" = gone ] && ! pgrep -f serve-frank-large >/dev/null && ! pgrep -x llama-server >/dev/null; }; then echo DOWN; exit 1; fi
|
||||
sleep 5
|
||||
done; exit 1" | grep -q UP; then
|
||||
echo "ERROR: candidate never became healthy; serve.log tail:" >&2
|
||||
ssh "$HOST" "tail -30 $RDIR/serve.log; echo '--- engine log ---';
|
||||
podman logs --tail 120 frank-large-mimo 2>&1 |
|
||||
grep -aiE 'error|raise|failed|traceback|assert|kernel' |
|
||||
tail -30" >&2 || true
|
||||
# --- the window -------------------------------------------------------------
|
||||
# Budgets: bench boot up to 20 min (warm boots measured ~7 min), measure up
|
||||
# to 30 min, restore boot up to 20 min — call run_experiment with
|
||||
# timeout_seconds >= 3600.
|
||||
echo "[autoresearch] opening bench window (stop+mask -> boot -> measure -> restore)" >&2
|
||||
set +e
|
||||
ssh_run "bash '$STAGE/window.sh'"
|
||||
rc=$?
|
||||
set -e
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "[autoresearch] window failed (rc=$rc); host-side evidence:" >&2
|
||||
ssh_run "tail -60 '$STAGE/fail.log' 2>/dev/null || tail -40 '$STAGE/serve.out' 2>/dev/null || tail -40 '$STAGE/measure.out' 2>/dev/null || true" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 5. Identity evidence for the loop's records (exact engine argv).
|
||||
ASI_ARGS="$(ssh "$HOST" 'podman inspect frank-large-mimo \
|
||||
--format "{{join .Config.Cmd \" \"}}" 2>/dev/null' | tr -d "\n" | head -c 400 || true)"
|
||||
if [ -z "$ASI_ARGS" ]; then
|
||||
ASI_ARGS="$(ssh "$HOST" "sh -c 'tr \"\\0\" \" \" < /proc/\\$(pgrep -x llama-server | head -1)/cmdline 2>/dev/null || true'" | head -c 400 || true)"
|
||||
fi
|
||||
echo "ASI engine_args=${ASI_ARGS:0:400}"
|
||||
|
||||
# 6. The battery, on the host, medians over 3 reps x 2 greedy prompts.
|
||||
ssh "$HOST" "python3 $RDIR/autoresearch-bench.py \
|
||||
--url http://127.0.0.1:7770 --model frank-large"
|
||||
run_json="$(ssh_run "cat '$STAGE/run.json'")" || {
|
||||
echo "[autoresearch] could not read $STAGE/run.json back" >&2; exit 1; }
|
||||
echo "[autoresearch] window result:" >&2
|
||||
printf '%s\n' "$run_json" >&2
|
||||
emit_metrics "$run_json"
|
||||
exit 0
|
||||
|
||||
Reference in New Issue
Block a user