From 81e6122891136c45a3007d75af7de26ebcb566e3 Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 6 Oct 2026 03:53:55 -0700 Subject: [PATCH] frank-primary: autoresearch bench harness (decode t/s @32k) autoresearch.sh (dev box) scp's the candidate scripts/frank-primary.env + the bench script to dankserver and runs it over ssh. scripts/autoresearch-frank-primary.sh (dankserver) follows the bench policy: stop frank-primary (relink unit: stop, not mask), boot a scratch radiance container (arbench) on the same two cards with the candidate argv, poll /health, run ONE self-contained greedy completion (fixed ~140k-char real-text prompt -> ~32k tokens, 256-token decode), parse the engine timings into METRIC lines, tear the scratch down, and restore production via a trap. Fresh prefix-cache tier per run for cold prefill. Baseline validated: decode_tps=31.70, prefill_tps=465.4, ttft_ms=93745, draft_accept=0.831 at prompt_n=43627. --- autoresearch.sh | 135 +++---------------- scripts/autoresearch-frank-primary.sh | 186 ++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 113 deletions(-) create mode 100755 scripts/autoresearch-frank-primary.sh diff --git a/autoresearch.sh b/autoresearch.sh index 089a127..e4fe165 100755 --- a/autoresearch.sh +++ b/autoresearch.sh @@ -1,117 +1,26 @@ #!/usr/bin/env bash -# autoresearch.sh — canonical benchmark entrypoint: speed of frank-primary. +# autoresearch.sh — canonical frank-primary benchmark entry point (dev box). # -# Runs on the operator box and drives one bench window on DANKSERVER: -# 1. fingerprint the candidate trio (env + serve script + unit) -# 2. memo fast-path: an identical fingerprint that already produced a FULLY -# successful window (bench + verified production restore) returns the -# cached metrics without touching the fleet -# 3. stage the candidate trio plus the measure client into -# /home/dank/.autoresearch-frank-primary/ (ephemeral; the repo checkout -# on the host stays the production source of truth) -# 4. run the window (scripts/autoresearch/frank-primary-window.sh): -# stop + mask the production unit, boot the staged config AS THE SERVICE -# (same serve script, same container, same port 7770), wait for health, -# run the in-container usage-counter measure, then restore the production -# unit and verify its health — AGENTS.md Benchmarking Policy -# 5. print METRIC lines (parse by run_experiment) -# -# Primary metric: decode_tps (HIGHER is better) — median across the three -# prompt depths (115/1056/4096) of per-depth medians of -# completion_tokens / (wall - ttft), 3 runs each after a warm pass, -# temperature 0, the campaign's deterministic word-seed workload. -# -# Exit 0 with METRIC lines on success; non-zero on any failure (the window -# script's EXIT trap restores production on every path). -set -euo pipefail +# The service runs on dankserver (192.168.0.235:7770), the two R9700s live +# there, and the benchmark must own those cards, so the whole stop -> boot +# scratch -> bench -> restore sequence runs ON dankserver (see +# scripts/autoresearch-frank-primary.sh). This wrapper: +# 1. copies the candidate config (the repo's scripts/frank-primary.env, the +# file the autoresearch loop edits) and the bench script to dankserver, +# 2. runs the bench over ssh, relaying its stdout (the METRIC lines) and +# exit code back here verbatim. +# It deliberately does NOT push to origin: the scp dirties dankserver's +# working tree, and the loop's uncommitted candidate config must not be +# pushed. Only this file and scripts/autoresearch-frank-primary.sh are +# committed; the candidate env rides over ssh each run. +set -uo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REMOTE=dankserver +REMOTE_REPO=/home/dank/serverAppConfigs -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -HOST="${FRANK_PRIMARY_AR_HOST:-dankserver}" -STAGE="/home/dank/.autoresearch-frank-primary" +scp -q "$DIR/scripts/frank-primary.env" \ + "$REMOTE:$REMOTE_REPO/scripts/frank-primary.env" || { echo "scp env failed" >&2; exit 1; } +scp -q "$DIR/scripts/autoresearch-frank-primary.sh" \ + "$REMOTE:$REMOTE_REPO/scripts/autoresearch-frank-primary.sh" || { echo "scp script failed" >&2; exit 1; } -# FRANK_PRIMARY_AR_ENV_FILE stages a DIFFERENT env than the repo's — that is how -# the arms driver A/Bs engine knobs without mutating the checkout. The -# fingerprint covers whatever file is staged, so the memo cannot cross arms. -ENV_FILE="${FRANK_PRIMARY_AR_ENV_FILE:-$SCRIPT_DIR/scripts/frank-primary.env}" -SERVE="$SCRIPT_DIR/scripts/serve-frank-primary.sh" -UNIT="$SCRIPT_DIR/scripts/frank-primary.service" -MEASURE="$SCRIPT_DIR/scripts/autoresearch/frank-primary-measure.py" -WINDOW="$SCRIPT_DIR/scripts/autoresearch/frank-primary-window.sh" - -for f in "$ENV_FILE" "$SERVE" "$UNIT" "$MEASURE" "$WINDOW"; do - [ -f "$f" ] || { echo "[autoresearch] missing harness file: $f" >&2; exit 1; } -done -command -v python3 >/dev/null 2>&1 || { echo "[autoresearch] python3 required on operator box" >&2; exit 1; } - -# Fingerprint of everything that can change the engine's argv or environment. -HASH="$(cat "$ENV_FILE" "$SERVE" "$UNIT" | sha256sum | cut -d' ' -f1)" -echo "[autoresearch] candidate fingerprint $HASH" >&2 - -ssh_run() { ssh -o BatchMode=yes -o ConnectTimeout=15 "$HOST" "$@"; } - -emit_metrics() { # $1 = run.json - python3 -c ' -import json, sys -run = json.loads(sys.argv[1]) -m = run["measure"] -print("METRIC decode_tps=%s" % m["decode_tps"]) -for k in ("decode_tps_115", "decode_tps_1056", "decode_tps_4096"): - print("METRIC %s=%s" % (k, m[k])) -print("METRIC ttft_ms_4096=%s" % m["ttft_ms_4096"]) -print("METRIC kv_tokens=%s" % (run.get("kv_tokens") or 0)) -print("METRIC boot_seconds=%s" % (run.get("boot_seconds") or -1)) -print("METRIC completion_tokens_4096=%s" % m.get("completion_tokens_4096", 0)) -# Any extra sampling temperature the measure swept (AR_TEMPS), e.g. the 0.7 -# arm — that is where speculative-decode acceptance shows up, since a greedy -# draft matches a greedy target exactly and is rejected more under real -# sampling. Suffixed keys only (_t0.7), so the historical lines are untouched. -for k in sorted(m): - if "_t" in k and (k.startswith("decode_tps") or k.startswith("ttft_ms")): - print("METRIC %s=%s" % (k, m[k])) -' "$1" -} - -# --- memo fast-path ------------------------------------------------------- -cached_hash="$(ssh_run "cat '$STAGE/last/hash' 2>/dev/null" 2>/dev/null || true)" -if [ -n "$cached_hash" ] && [ "$cached_hash" = "$HASH" ]; then - cached_run="$(ssh_run "cat '$STAGE/last/run.json' 2>/dev/null" || true)" - if [ -n "$cached_run" ]; then - echo "[autoresearch] memo hit — cached metrics for $HASH, fleet untouched" >&2 - emit_metrics "$cached_run" - exit 0 - fi -fi - -# --- stage the candidate ---------------------------------------------------- -echo "[autoresearch] staging candidate on $HOST:$STAGE" >&2 -ssh_run "mkdir -p '$STAGE'" -scp -q "$ENV_FILE" "$HOST:$STAGE/frank-primary.env" -scp -q "$SERVE" "$HOST:$STAGE/serve.sh" -scp -q "$MEASURE" "$HOST:$STAGE/measure.py" -scp -q "$WINDOW" "$HOST:$STAGE/window.sh" -ssh_run "printf '%s\n' '$HASH' > '$STAGE/hash'" - -# --- the window ------------------------------------------------------------- -# Budgets: bench boot up to 20 min (warm boots measured ~7 min), measure up -# to 30 min, restore boot up to 20 min — call run_experiment with -# timeout_seconds >= 3600. -echo "[autoresearch] opening bench window (stop+mask -> boot -> measure -> restore)" >&2 -set +e -ssh_run "bash '$STAGE/window.sh'" -rc=$? -set -e -if [ "$rc" -ne 0 ]; then - echo "[autoresearch] window failed (rc=$rc); host-side evidence:" >&2 - ssh_run "tail -60 '$STAGE/fail.log' 2>/dev/null || tail -40 '$STAGE/serve.out' 2>/dev/null || tail -40 '$STAGE/measure.out' 2>/dev/null || true" >&2 || true - exit 1 -fi - -run_json="$(ssh_run "cat '$STAGE/run.json'")" || { - echo "[autoresearch] could not read $STAGE/run.json back" >&2; exit 1; } -echo "[autoresearch] window result:" >&2 -printf '%s\n' "$run_json" >&2 -emit_metrics "$run_json" - -echo "[autoresearch] engine evidence (plan / compile path / spec counters):" >&2 -ssh_run "cat '$STAGE/evidence.log' 2>/dev/null" >&2 || true -exit 0 +ssh "$REMOTE" "bash $REMOTE_REPO/scripts/autoresearch-frank-primary.sh $REMOTE_REPO/scripts/frank-primary.env" diff --git a/scripts/autoresearch-frank-primary.sh b/scripts/autoresearch-frank-primary.sh new file mode 100755 index 0000000..44dcc2a --- /dev/null +++ b/scripts/autoresearch-frank-primary.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# autoresearch-frank-primary.sh — RUNS ON DANKSERVER, driven over ssh by the +# dev-box autoresearch.sh. It is the canonical frank-primary benchmark entry +# point for the autoresearch loop. +# +# Method (mandated by the Benchmarking Policy): the production service serves +# real users and shares the two R9700s, so a bench beside it starves the GPUs +# and corrupts results. This script therefore: +# 1. stops the production frank-primary unit (root, relink symlink: stop, +# NOT mask — mask refuses to replace a symlinked unit file), +# 2. boots a scratch radiance container named `arbench` on the SAME two +# cards with the candidate engine argv (the env file the loop edits), +# 3. polls /health until the listener is live (it binds AFTER the full +# load, so a bare curl is an honest gate), +# 4. runs ONE self-contained completion request locally (127.0.0.1, no LAN +# jitter): a fixed ~140k-char real-text prompt (~32k tokens) decoded for +# 256 tokens, greedy. The engine's `timings` object reports the honest +# prefill and decode rates for that single request. +# 5. prints METRIC lines and tears the scratch down, +# 6. restores production (systemctl start), via a trap so a failed bench +# never leaves frank-primary down (the stale-probe-tail lesson). +# +# Determinism: the prompt is built from a fixed, sorted corpus of repo files +# with a fixed random seed, so prompt_n is stable across runs. Sampling is +# greedy (temperature 0), so the decode length is fixed; the only residual +# nondeterminism is the wht6 lossy TP all-reduce, which is a constant across +# runs and cancels in A/B comparisons. +# +# The scratch mirrors the production container contract exactly (serve-frank- +# primary.sh): --network host, --init, seccomp=unconfined (io_uring for the +# n-gram table), memlock unbounded (the ~132 GiB pinned ledger), /dev/kfd + +# /dev/dri, the .rad ro, the state dir rw, the tune cache mount. The ONLY +# differences: the container name (arbench, so prod's identity is untouched) +# and a FRESH prefix-cache disk tier (arbench/, wiped per run) so the cold +# prefill is cold — the prod tier at kvcache/frank-primary is left intact. +set -uo pipefail + +ENV_FILE="${1:-/home/dank/serverAppConfigs/scripts/frank-primary.env}" +SCRATCH=arbench +PORT=7770 +HEALTH="http://127.0.0.1:$PORT/health" +COMPLETE="http://127.0.0.1:$PORT/v1/completions" +BODY=/tmp/arbench-body.json +LOG=/home/dank/logs/arbench.log + +restore() { + podman rm -f "$SCRATCH" >/dev/null 2>&1 || true + sudo systemctl start frank-primary >/dev/null 2>&1 || true +} +trap restore EXIT + +# --- source the candidate config (same file the production unit reads) ------ +if [ ! -f "$ENV_FILE" ]; then + echo "no env file at $ENV_FILE" >&2; exit 1 +fi +# shellcheck disable=SC1090 +set -a; . "$ENV_FILE"; set +a + +# --- stop production; the scratch needs both cards to itself ---------------- +sudo systemctl stop frank-primary >/dev/null 2>&1 || true +podman rm -f "$SCRATCH" >/dev/null 2>&1 || true + +# --- fresh prefix-cache tier for a cold prefill (prod's tier untouched) ------ +FRESH_KV="$FRANK_PRIMARY_STATE_DIR/kvcache/$SCRATCH" +rm -rf "$FRESH_KV"; mkdir -p "$FRESH_KV" +mkdir -p "$FRANK_PRIMARY_STATE_DIR/tune" + +# --- boot the scratch: the production run line, scratch name + fresh KV ------ +podman run -d --name "$SCRATCH" \ + --network host \ + --init \ + --security-opt seccomp=unconfined \ + --ulimit memlock=-1:-1 \ + --device /dev/kfd --device /dev/dri \ + -v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \ + -v "$FRANK_PRIMARY_STATE_DIR":/data \ + -v "$FRANK_PRIMARY_STATE_DIR/tune":/opt/radiance/share/radiance/tune \ + -v "$FRANK_PRIMARY_TEMPLATE":/templates/template.jinja:ro \ + "$FRANK_PRIMARY_IMAGE" \ + --model "/models/$FRANK_PRIMARY_MODEL_FILE" \ + --host :: --port "$PORT" \ + --served-model-name "$FRANK_PRIMARY_SERVED_NAME" \ + --override-chat-template /templates/template.jinja \ + --tp "$FRANK_PRIMARY_TP" --tp-wire "$FRANK_PRIMARY_TP_WIRE" \ + --max-num-seqs "$FRANK_PRIMARY_MAX_SEQS" \ + --max-model-len "$FRANK_PRIMARY_MAX_MODEL_LEN" \ + --placement "$FRANK_PRIMARY_PLACEMENT" \ + --host-pool-mib "$FRANK_PRIMARY_HOST_POOL_MIB" \ + --gpu-headroom-mib "$FRANK_PRIMARY_GPU_HEADROOM_MIB" \ + --expert-vs-cache-ratio "$FRANK_PRIMARY_EXPERT_VS_CACHE" \ + --kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE" \ + --prefix-cache-host-mib "$FRANK_PRIMARY_PREFIX_HOST_MIB" \ + --prefix-cache-dir "/data/kvcache/$SCRATCH" \ + --prefix-cache-disk-mib "$FRANK_PRIMARY_PREFIX_DISK_MIB" \ + --ngram-placement "$FRANK_PRIMARY_NGRAM_PLACEMENT" \ + --num-speculative-tokens "$FRANK_PRIMARY_SPEC_TOKENS" \ + --max-num-batched-tokens "$FRANK_PRIMARY_MAX_BATCHED_TOKENS" \ + ${FRANK_PRIMARY_EXTRA_ARGS:+$FRANK_PRIMARY_EXTRA_ARGS} \ + > "$LOG" 2>&1 +if [ $? -ne 0 ]; then echo "scratch boot failed; see $LOG" >&2; exit 1; fi + +# --- poll health (listener binds after the full load; cold boot ~110 s) ------ +ok=0 +for _ in $(seq 1 360); do + code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 3 "$HEALTH" 2>/dev/null || echo 000) + if [ "$code" = "200" ]; then ok=1; break; fi + sleep 2 +done +if [ "$ok" -ne 1 ]; then + echo "scratch never reached health 200; tail of $LOG:" >&2 + tail -30 "$LOG" >&2; exit 1 +fi + +# --- build the workload: a fixed ~140k-char real-text prompt (repo corpus) --- +python3 - "$BODY" <<'PY' +import json, sys, random, glob +random.seed(1234) +root = "/home/dank/serverAppConfigs" +files = sorted( + glob.glob(f"{root}/serverAppConfigs-docs/**/*.md", recursive=True) + + glob.glob(f"{root}/scripts/*.sh") + + glob.glob(f"{root}/scripts/*.py") + + glob.glob(f"{root}/AGENTS.md") +) +corpus = [] +total = 0 +for f in files: + try: + t = open(f, encoding="utf-8", errors="ignore").read() + except Exception: + continue + if not t.strip(): + continue + corpus.append(t) + total += len(t) + if total > 240000: + break +corpus = "\n".join(corpus) +TGT = 140000 +parts, n = [], 0 +while n < TGT: + start = random.randint(0, max(0, len(corpus) - 500)) + seg = corpus[start:start + random.randint(200, 500)] + parts.append(seg) + n += len(seg) + 1 +prompt = " ".join(parts) +json.dump( + {"model": "frank-primary", "prompt": prompt, + "max_tokens": 256, "temperature": 0.0, "stream": False, + "stop": ["<|im_end|>"]}, + open(sys.argv[1], "w"), +) +PY +if [ ! -f "$BODY" ]; then echo "prompt build failed" >&2; exit 1; fi + +# --- the bench request (prefill ~32k + decode 256, greedy) ------------------- +RESP=$(curl -s --max-time 600 -X POST "$COMPLETE" \ + -H 'Content-Type: application/json' -d "@$BODY") +echo "$RESP" > /tmp/arbench-resp.json + +# --- parse the timings into METRIC lines ------------------------------------- +python3 - <<'PY' +import json, sys +try: + r = json.load(open("/tmp/arbench-resp.json")) +except Exception as e: + sys.stderr.write(f"unparseable response: {e}\n"); sys.exit(1) +if "timings" not in r: + sys.stderr.write(f"no timings in response: {json.dumps(r)[:500]}\n"); sys.exit(1) +t = r["timings"] +dtps = t.get("predicted_per_second") +ptps = t.get("prompt_per_second") +pms = t.get("prompt_ms") +dn = t.get("draft_n", 0) +da = t.get("draft_n_accepted", 0) +if dtps is None: + sys.stderr.write("predicted_per_second null (no decode produced)\n"); sys.exit(1) +print(f"METRIC decode_tps={dtps:.4f}") +if ptps is not None: + print(f"METRIC prefill_tps={ptps:.4f}") +if pms is not None: + print(f"METRIC ttft_ms={pms:.4f}") +if dn: + print(f"METRIC draft_accept={da/dn:.4f}") +print(f"INFO prompt_n={t.get('prompt_n')} cache_n={t.get('cache_n')} predicted_n={t.get('predicted_n')}") +PY