78 lines
3.8 KiB
Bash
Executable File
78 lines
3.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# autoresearch.sh — canonical speed benchmark for frank-large (Dankclaw iGPU).
|
|
#
|
|
# Window semantics per the repo Benchmarking Policy: the deployed unit is
|
|
# STOPPED, the candidate (this worktree's scripts/frank-large.env +
|
|
# scripts/serve-frank-large.sh, rsynced to ~/jobs/autoresearch so the host's
|
|
# git checkout stays clean) boots standalone on :7770, the fixed greedy
|
|
# battery runs against it ON the host, and a trap restores the deployed
|
|
# baseline unit whatever happens. The candidate never touches the host
|
|
# checkout; "keep" in the loop means a real deploy later.
|
|
#
|
|
# Emits: METRIC decode_tokens_per_sec (primary, higher better),
|
|
# METRIC prefill_tokens_per_sec, METRIC ttft_ms, ASI identity lines.
|
|
set -euo pipefail
|
|
|
|
REPO="$(cd "$(dirname "$0")" && pwd)"
|
|
HOST=dankclaw
|
|
RDIR=/home/dank/jobs/autoresearch
|
|
HEALTH_TRIES=200 # x5s = ~17 min ceiling (BF16 boots ~230s; INT8/dequant may differ)
|
|
|
|
restore() {
|
|
local rc=$?
|
|
trap - EXIT
|
|
echo "[harness] tearing down candidate, restoring deployed baseline..." >&2
|
|
ssh "$HOST" 'sudo -n systemctl stop frank-large.service 2>/dev/null;
|
|
podman rm -f frank-large-mimo 2>/dev/null;
|
|
pkill -x llama-server 2>/dev/null; true' >/dev/null 2>&1 || true
|
|
ssh "$HOST" 'sudo -n systemctl start frank-large.service' >/dev/null 2>&1 || true
|
|
exit $rc
|
|
}
|
|
trap restore EXIT
|
|
|
|
# 1. Ship the candidate (env + serve + battery) outside the git checkout.
|
|
ssh "$HOST" "mkdir -p $RDIR"
|
|
rsync -a "$REPO/scripts/frank-large.env" "$REPO/scripts/serve-frank-large.sh" \
|
|
"$REPO/scripts/autoresearch-bench.py" "$HOST:$RDIR/"
|
|
|
|
# 2. Free the GPU and the port: stop the deployed unit (mask is unnecessary—
|
|
# the unit only starts via systemd, and we never enable more of it).
|
|
ssh "$HOST" 'sudo -n systemctl stop frank-large.service;
|
|
podman rm -f frank-large-mimo 2>/dev/null;
|
|
pkill -x llama-server 2>/dev/null; true' >/dev/null
|
|
|
|
# 3. Boot the candidate detached; the launcher's own output IS the serve
|
|
# stage output (gate failures print there before any container exists).
|
|
echo "[harness] starting candidate..." >&2
|
|
ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \
|
|
bash $RDIR/serve-frank-large.sh >$RDIR/serve.log 2>&1 </dev/null & echo ok" \
|
|
| tee /tmp/autoresearch-launch.log >&2
|
|
grep -q ok /tmp/autoresearch-launch.log || { echo "ERROR: launch failed" >&2; exit 1; }
|
|
|
|
# 4. Wait for health (503 while loading; anything else = boot failure).
|
|
echo "[harness] waiting for health..." >&2
|
|
if ! ssh "$HOST" "for i in \$(seq 1 $HEALTH_TRIES); do
|
|
code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 \
|
|
http://127.0.0.1:7770/health 2>/dev/null || echo 000)
|
|
[ \"\$code\" = 200 ] && echo UP && exit 0
|
|
st=\$(podman inspect -f '{{.State.Status}}' frank-large-mimo 2>/dev/null || echo gone)
|
|
if [ "\$st" = exited ] || { [ "\$st" = gone ] && ! pgrep -f serve-frank-large >/dev/null && ! pgrep -x llama-server >/dev/null; }; then echo DOWN; exit 1; fi
|
|
sleep 5
|
|
done; exit 1" | grep -q UP; then
|
|
echo "ERROR: candidate never became healthy; serve.log tail:" >&2
|
|
ssh "$HOST" "tail -30 $RDIR/serve.log; echo '--- engine log ---';
|
|
podman logs --tail 120 frank-large-mimo 2>&1 |
|
|
grep -aiE 'error|raise|failed|traceback|assert|kernel' |
|
|
tail -30" >&2 || true
|
|
exit 1
|
|
fi
|
|
|
|
# 5. Identity evidence for the loop's records (exact engine argv).
|
|
ASI_ARGS="$(ssh "$HOST" 'podman inspect frank-large-mimo \
|
|
--format "{{join .Config.Cmd \" \"}}" 2>/dev/null' | tr -d "\n" | head -c 400)"
|
|
echo "ASI engine_args=${ASI_ARGS:0:400}"
|
|
|
|
# 6. The battery, on the host, medians over 3 reps x 2 greedy prompts.
|
|
ssh "$HOST" "python3 $RDIR/autoresearch-bench.py \
|
|
--url http://127.0.0.1:7770 --model frank-large"
|