Frank
b7c3bdb2be
Four harness gaps that block an honest MTP re-measurement:
- measure: AR_TEMPS sweeps sampling temperatures (default "0,0.7"). Draft
ACCEPTANCE is sampling-dependent — a greedy draft matches a greedy target
exactly and is rejected more under a non-greedy target — so judging a spec
arm only at temperature 0 judges half the contract. The unsuffixed keys stay
temperature 0, so every historical number remains comparable.
- window: capture an evidence.log sidecar (expert-plan lines, compile path,
graph sizes, /metrics spec-decode and cache counters, OOM signatures). The
arena/overhead/reserve numbers exist ONLY in the boot log; an arm that loses
decode speed is otherwise unexplainable afterwards.
- orchestrator: FRANK_PRIMARY_AR_ENV_FILE stages an env other than the repo's,
which is how arms A/B without mutating the checkout (the fingerprint covers
whatever is staged, so the memo can never answer one arm with another's
numbers), plus METRIC lines for the swept temperatures and the evidence dump.
- scripts/autoresearch/frank-primary-arms.sh: runs a sequence of full windows
(baseline, MTP3+probabilistic with a step-4 capture ladder at precise memory
ON, the same at the default overhead, and the vendor SEQS-16 shape), gating
each staged env on its JSON knobs still parsing AFTER sourcing — an unquoted
{"temperature":0.7} loses its quotes at source time and reaches the engine as
{temperature:0.7}, which is a 20-minute boot wasted.
118 lines
5.5 KiB
Bash
Executable File
118 lines
5.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# autoresearch.sh — canonical benchmark entrypoint: speed of frank-primary.
|
|
#
|
|
# Runs on the operator box and drives one bench window on DANKSERVER:
|
|
# 1. fingerprint the candidate trio (env + serve script + unit)
|
|
# 2. memo fast-path: an identical fingerprint that already produced a FULLY
|
|
# successful window (bench + verified production restore) returns the
|
|
# cached metrics without touching the fleet
|
|
# 3. stage the candidate trio plus the measure client into
|
|
# /home/dank/.autoresearch-frank-primary/ (ephemeral; the repo checkout
|
|
# on the host stays the production source of truth)
|
|
# 4. run the window (scripts/autoresearch/frank-primary-window.sh):
|
|
# stop + mask the production unit, boot the staged config AS THE SERVICE
|
|
# (same serve script, same container, same port 7770), wait for health,
|
|
# run the in-container usage-counter measure, then restore the production
|
|
# unit and verify its health — AGENTS.md Benchmarking Policy
|
|
# 5. print METRIC lines (parse by run_experiment)
|
|
#
|
|
# Primary metric: decode_tps (HIGHER is better) — median across the three
|
|
# prompt depths (115/1056/4096) of per-depth medians of
|
|
# completion_tokens / (wall - ttft), 3 runs each after a warm pass,
|
|
# temperature 0, the campaign's deterministic word-seed workload.
|
|
#
|
|
# Exit 0 with METRIC lines on success; non-zero on any failure (the window
|
|
# script's EXIT trap restores production on every path).
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
HOST="${FRANK_PRIMARY_AR_HOST:-dankserver}"
|
|
STAGE="/home/dank/.autoresearch-frank-primary"
|
|
|
|
# FRANK_PRIMARY_AR_ENV_FILE stages a DIFFERENT env than the repo's — that is how
|
|
# the arms driver A/Bs engine knobs without mutating the checkout. The
|
|
# fingerprint covers whatever file is staged, so the memo cannot cross arms.
|
|
ENV_FILE="${FRANK_PRIMARY_AR_ENV_FILE:-$SCRIPT_DIR/scripts/frank-primary.env}"
|
|
SERVE="$SCRIPT_DIR/scripts/serve-frank-primary.sh"
|
|
UNIT="$SCRIPT_DIR/scripts/frank-primary.service"
|
|
MEASURE="$SCRIPT_DIR/scripts/autoresearch/frank-primary-measure.py"
|
|
WINDOW="$SCRIPT_DIR/scripts/autoresearch/frank-primary-window.sh"
|
|
|
|
for f in "$ENV_FILE" "$SERVE" "$UNIT" "$MEASURE" "$WINDOW"; do
|
|
[ -f "$f" ] || { echo "[autoresearch] missing harness file: $f" >&2; exit 1; }
|
|
done
|
|
command -v python3 >/dev/null 2>&1 || { echo "[autoresearch] python3 required on operator box" >&2; exit 1; }
|
|
|
|
# Fingerprint of everything that can change the engine's argv or environment.
|
|
HASH="$(cat "$ENV_FILE" "$SERVE" "$UNIT" | sha256sum | cut -d' ' -f1)"
|
|
echo "[autoresearch] candidate fingerprint $HASH" >&2
|
|
|
|
ssh_run() { ssh -o BatchMode=yes -o ConnectTimeout=15 "$HOST" "$@"; }
|
|
|
|
emit_metrics() { # $1 = run.json
|
|
python3 -c '
|
|
import json, sys
|
|
run = json.loads(sys.argv[1])
|
|
m = run["measure"]
|
|
print("METRIC decode_tps=%s" % m["decode_tps"])
|
|
for k in ("decode_tps_115", "decode_tps_1056", "decode_tps_4096"):
|
|
print("METRIC %s=%s" % (k, m[k]))
|
|
print("METRIC ttft_ms_4096=%s" % m["ttft_ms_4096"])
|
|
print("METRIC kv_tokens=%s" % (run.get("kv_tokens") or 0))
|
|
print("METRIC boot_seconds=%s" % (run.get("boot_seconds") or -1))
|
|
print("METRIC completion_tokens_4096=%s" % m.get("completion_tokens_4096", 0))
|
|
# Any extra sampling temperature the measure swept (AR_TEMPS), e.g. the 0.7
|
|
# arm — that is where speculative-decode acceptance shows up, since a greedy
|
|
# draft matches a greedy target exactly and is rejected more under real
|
|
# sampling. Suffixed keys only (_t0.7), so the historical lines are untouched.
|
|
for k in sorted(m):
|
|
if "_t" in k and (k.startswith("decode_tps") or k.startswith("ttft_ms")):
|
|
print("METRIC %s=%s" % (k, m[k]))
|
|
' "$1"
|
|
}
|
|
|
|
# --- memo fast-path -------------------------------------------------------
|
|
cached_hash="$(ssh_run "cat '$STAGE/last/hash' 2>/dev/null" 2>/dev/null || true)"
|
|
if [ -n "$cached_hash" ] && [ "$cached_hash" = "$HASH" ]; then
|
|
cached_run="$(ssh_run "cat '$STAGE/last/run.json' 2>/dev/null" || true)"
|
|
if [ -n "$cached_run" ]; then
|
|
echo "[autoresearch] memo hit — cached metrics for $HASH, fleet untouched" >&2
|
|
emit_metrics "$cached_run"
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
# --- stage the candidate ----------------------------------------------------
|
|
echo "[autoresearch] staging candidate on $HOST:$STAGE" >&2
|
|
ssh_run "mkdir -p '$STAGE'"
|
|
scp -q "$ENV_FILE" "$HOST:$STAGE/frank-primary.env"
|
|
scp -q "$SERVE" "$HOST:$STAGE/serve.sh"
|
|
scp -q "$MEASURE" "$HOST:$STAGE/measure.py"
|
|
scp -q "$WINDOW" "$HOST:$STAGE/window.sh"
|
|
ssh_run "printf '%s\n' '$HASH' > '$STAGE/hash'"
|
|
|
|
# --- the window -------------------------------------------------------------
|
|
# Budgets: bench boot up to 20 min (warm boots measured ~7 min), measure up
|
|
# to 30 min, restore boot up to 20 min — call run_experiment with
|
|
# timeout_seconds >= 3600.
|
|
echo "[autoresearch] opening bench window (stop+mask -> boot -> measure -> restore)" >&2
|
|
set +e
|
|
ssh_run "bash '$STAGE/window.sh'"
|
|
rc=$?
|
|
set -e
|
|
if [ "$rc" -ne 0 ]; then
|
|
echo "[autoresearch] window failed (rc=$rc); host-side evidence:" >&2
|
|
ssh_run "tail -60 '$STAGE/fail.log' 2>/dev/null || tail -40 '$STAGE/serve.out' 2>/dev/null || tail -40 '$STAGE/measure.out' 2>/dev/null || true" >&2 || true
|
|
exit 1
|
|
fi
|
|
|
|
run_json="$(ssh_run "cat '$STAGE/run.json'")" || {
|
|
echo "[autoresearch] could not read $STAGE/run.json back" >&2; exit 1; }
|
|
echo "[autoresearch] window result:" >&2
|
|
printf '%s\n' "$run_json" >&2
|
|
emit_metrics "$run_json"
|
|
|
|
echo "[autoresearch] engine evidence (plan / compile path / spec counters):" >&2
|
|
ssh_run "cat '$STAGE/evidence.log' 2>/dev/null" >&2 || true
|
|
exit 0
|