autoresearch: harness setup

Benchmark entrypoint: bash autoresearch.sh Goal: Maximize frank-large decode throughput on the Strix Halo iGPU (gfx1151) via quantization >=q6/fp6, engine flags, spec-decode shape, or runtime swap — context limits unchanged, KV stays bf16.
2026-10-01 20:41:51 -07:00
parent 3c339eeba2
commit 5df6ef3166
2 changed files with 212 additions and 0 deletions
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# autoresearch.sh — canonical speed benchmark for frank-large (Dankclaw iGPU).
+#
+# Window semantics per the repo Benchmarking Policy: the deployed unit is
+# STOPPED, the candidate (this worktree's scripts/frank-large.env +
+# scripts/serve-frank-large.sh, rsynced to ~/jobs/autoresearch so the host's
+# git checkout stays clean) boots standalone on :7770, the fixed greedy
+# battery runs against it ON the host, and a trap restores the deployed
+# baseline unit whatever happens. The candidate never touches the host
+# checkout; "keep" in the loop means a real deploy later.
+#
+# Emits: METRIC decode_tokens_per_sec (primary, higher better),
+#        METRIC prefill_tokens_per_sec, METRIC ttft_ms, ASI identity lines.
+set -euo pipefail
+
+REPO="$(cd "$(dirname "$0")" && pwd)"
+HOST=dankclaw
+RDIR=/home/dank/jobs/autoresearch
+HEALTH_TRIES=200   # x5s = ~17 min ceiling (BF16 boots ~230s; INT8/dequant may differ)
+
+restore() {
+    local rc=$?
+    trap - EXIT
+    echo "[harness] tearing down candidate, restoring deployed baseline..." >&2
+    ssh "$HOST" 'sudo -n systemctl stop frank-large.service 2>/dev/null;
+                 podman rm -f frank-large-mimo 2>/dev/null;
+                 true' >/dev/null 2>&1 || true
+    ssh "$HOST" 'sudo -n systemctl start frank-large.service' >/dev/null 2>&1 || true
+    exit $rc
+}
+trap restore EXIT
+
+# 1. Ship the candidate (env + serve + battery) outside the git checkout.
+ssh "$HOST" "mkdir -p $RDIR"
+rsync -a "$REPO/scripts/frank-large.env" "$REPO/scripts/serve-frank-large.sh" \
+    "$REPO/scripts/autoresearch-bench.py" "$HOST:$RDIR/"
+
+# 2. Free the GPU and the port: stop the deployed unit (mask is unnecessary—
+#    the unit only starts via systemd, and we never enable more of it).
+ssh "$HOST" 'sudo -n systemctl stop frank-large.service;
+             podman rm -f frank-large-mimo 2>/dev/null; true' >/dev/null
+
+# 3. Boot the candidate detached. The serve script's own gates run first;
+#    a gate failure exits here with the reason.
+echo "[harness] starting candidate..." >&2
+if ! ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \
+        bash $RDIR/serve-frank-large.sh >$RDIR/serve.log 2>&1 </dev/null & echo ok" \
+        | grep -q ok; then
+    echo "ERROR: could not launch candidate" >&2; exit 1
+fi
+
+# 4. Wait for health (503 while loading; anything else = boot failure).
+echo "[harness] waiting for health..." >&2
+if ! ssh "$HOST" "for i in \$(seq 1 $HEALTH_TRIES); do
+        code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 \
+              http://127.0.0.1:7770/health 2>/dev/null || echo 000)
+        [ \"\$code\" = 200 ] && echo UP && exit 0
+        sleep 5
+    done; exit 1" | grep -q UP; then
+    echo "ERROR: candidate never became healthy; serve.log tail:" >&2
+    ssh "$HOST" "tail -30 $RDIR/serve.log" >&2 || true
+    exit 1
+fi
+
+# 5. Identity evidence for the loop's records (exact engine argv).
+ASI_ARGS="$(ssh "$HOST" 'podman inspect frank-large-mimo \
+    --format "{{join .Config.Cmd \" \"}}" 2>/dev/null' | tr -d "\n" | head -c 400)"
+echo "ASI engine_args=${ASI_ARGS:0:400}"
+
+# 6. The battery, on the host, medians over 3 reps x 2 greedy prompts.
+ssh "$HOST" "python3 $RDIR/autoresearch-bench.py \
+    --url http://127.0.0.1:7770 --model frank-large"