Frank
f8f845d86b
Benchmark entrypoint: bash autoresearch.sh Goal: Make frank-primary faster: raise decode tok/s on the tcclaviger vLLM endpoint (dankserver :7770, Qwen3.8-Flash-Next-Uncensored-FP8, 2x R9700, automatic tiered-expert offload). Context size may be altered (MAXLEN/SEQS/KV_BYTES and friends); quantization may NOT be altered (the 131-shard FP8 payload and the digest-pinned image are frozen).
105 lines
4.7 KiB
Bash
Executable File
105 lines
4.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# autoresearch.sh — canonical benchmark entrypoint: speed of frank-primary.
|
|
#
|
|
# Runs on the operator box and drives one bench window on DANKSERVER:
|
|
# 1. fingerprint the candidate trio (env + serve script + unit)
|
|
# 2. memo fast-path: an identical fingerprint that already produced a FULLY
|
|
# successful window (bench + verified production restore) returns the
|
|
# cached metrics without touching the fleet
|
|
# 3. stage the candidate trio plus the measure client into
|
|
# /home/dank/.autoresearch-frank-primary/ (ephemeral; the repo checkout
|
|
# on the host stays the production source of truth)
|
|
# 4. run the window (scripts/autoresearch/frank-primary-window.sh):
|
|
# stop + mask the production unit, boot the staged config AS THE SERVICE
|
|
# (same serve script, same container, same port 7770), wait for health,
|
|
# run the in-container usage-counter measure, then restore the production
|
|
# unit and verify its health — AGENTS.md Benchmarking Policy
|
|
# 5. print METRIC lines (parse by run_experiment)
|
|
#
|
|
# Primary metric: decode_tps (HIGHER is better) — median across the three
|
|
# prompt depths (115/1056/4096) of per-depth medians of
|
|
# completion_tokens / (wall - ttft), 3 runs each after a warm pass,
|
|
# temperature 0, the campaign's deterministic word-seed workload.
|
|
#
|
|
# Exit 0 with METRIC lines on success; non-zero on any failure (the window
|
|
# script's EXIT trap restores production on every path).
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
HOST="${FRANK_PRIMARY_AR_HOST:-dankserver}"
|
|
STAGE="/home/dank/.autoresearch-frank-primary"
|
|
|
|
ENV_FILE="$SCRIPT_DIR/scripts/frank-primary.env"
|
|
SERVE="$SCRIPT_DIR/scripts/serve-frank-primary.sh"
|
|
UNIT="$SCRIPT_DIR/scripts/frank-primary.service"
|
|
MEASURE="$SCRIPT_DIR/scripts/autoresearch/frank-primary-measure.py"
|
|
WINDOW="$SCRIPT_DIR/scripts/autoresearch/frank-primary-window.sh"
|
|
|
|
for f in "$ENV_FILE" "$SERVE" "$UNIT" "$MEASURE" "$WINDOW"; do
|
|
[ -f "$f" ] || { echo "[autoresearch] missing harness file: $f" >&2; exit 1; }
|
|
done
|
|
command -v python3 >/dev/null 2>&1 || { echo "[autoresearch] python3 required on operator box" >&2; exit 1; }
|
|
|
|
# Fingerprint of everything that can change the engine's argv or environment.
|
|
HASH="$(cat "$ENV_FILE" "$SERVE" "$UNIT" | sha256sum | cut -d' ' -f1)"
|
|
echo "[autoresearch] candidate fingerprint $HASH" >&2
|
|
|
|
ssh_run() { ssh -o BatchMode=yes -o ConnectTimeout=15 "$HOST" "$@"; }
|
|
|
|
emit_metrics() { # $1 = run.json
|
|
python3 -c '
|
|
import json, sys
|
|
run = json.loads(sys.argv[1])
|
|
m = run["measure"]
|
|
print("METRIC decode_tps=%s" % m["decode_tps"])
|
|
for k in ("decode_tps_115", "decode_tps_1056", "decode_tps_4096"):
|
|
print("METRIC %s=%s" % (k, m[k]))
|
|
print("METRIC ttft_ms_4096=%s" % m["ttft_ms_4096"])
|
|
print("METRIC kv_tokens=%s" % (run.get("kv_tokens") or 0))
|
|
print("METRIC boot_seconds=%s" % (run.get("boot_seconds") or -1))
|
|
print("METRIC completion_tokens_4096=%s" % m.get("completion_tokens_4096", 0))
|
|
' "$1"
|
|
}
|
|
|
|
# --- memo fast-path -------------------------------------------------------
|
|
cached_hash="$(ssh_run "cat '$STAGE/last/hash' 2>/dev/null" 2>/dev/null || true)"
|
|
if [ -n "$cached_hash" ] && [ "$cached_hash" = "$HASH" ]; then
|
|
cached_run="$(ssh_run "cat '$STAGE/last/run.json' 2>/dev/null" || true)"
|
|
if [ -n "$cached_run" ]; then
|
|
echo "[autoresearch] memo hit — cached metrics for $HASH, fleet untouched" >&2
|
|
emit_metrics "$cached_run"
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
# --- stage the candidate ----------------------------------------------------
|
|
echo "[autoresearch] staging candidate on $HOST:$STAGE" >&2
|
|
ssh_run "mkdir -p '$STAGE'"
|
|
scp -q "$ENV_FILE" "$HOST:$STAGE/frank-primary.env"
|
|
scp -q "$SERVE" "$HOST:$STAGE/serve.sh"
|
|
scp -q "$MEASURE" "$HOST:$STAGE/measure.py"
|
|
scp -q "$WINDOW" "$HOST:$STAGE/window.sh"
|
|
ssh_run "printf '%s\n' '$HASH' > '$STAGE/hash'"
|
|
|
|
# --- the window -------------------------------------------------------------
|
|
# Budgets: bench boot up to 20 min (warm boots measured ~7 min), measure up
|
|
# to 30 min, restore boot up to 20 min — call run_experiment with
|
|
# timeout_seconds >= 3600.
|
|
echo "[autoresearch] opening bench window (stop+mask -> boot -> measure -> restore)" >&2
|
|
set +e
|
|
ssh_run "bash '$STAGE/window.sh'"
|
|
rc=$?
|
|
set -e
|
|
if [ "$rc" -ne 0 ]; then
|
|
echo "[autoresearch] window failed (rc=$rc); host-side evidence:" >&2
|
|
ssh_run "tail -60 '$STAGE/fail.log' 2>/dev/null || tail -40 '$STAGE/serve.out' 2>/dev/null || tail -40 '$STAGE/measure.out' 2>/dev/null || true" >&2 || true
|
|
exit 1
|
|
fi
|
|
|
|
run_json="$(ssh_run "cat '$STAGE/run.json'")" || {
|
|
echo "[autoresearch] could not read $STAGE/run.json back" >&2; exit 1; }
|
|
echo "[autoresearch] window result:" >&2
|
|
printf '%s\n' "$run_json" >&2
|
|
emit_metrics "$run_json"
|
|
exit 0
|