frank-primary: autoresearch bench harness (decode t/s @32k)

autoresearch.sh (dev box) scp's the candidate scripts/frank-primary.env + the bench script to dankserver and runs it over ssh. scripts/autoresearch-frank-primary.sh (dankserver) follows the bench policy: stop frank-primary (relink unit: stop, not mask), boot a scratch radiance container (arbench) on the same two cards with the candidate argv, poll /health, run ONE self-contained greedy completion (fixed ~140k-char real-text prompt -> ~32k tokens, 256-token decode), parse the engine timings into METRIC lines, tear the scratch down, and restore production via a trap. Fresh prefix-cache tier per run for cold prefill. Baseline validated: decode_tps=31.70, prefill_tps=465.4, ttft_ms=93745, draft_accept=0.831 at prompt_n=43627.
2026-10-06 03:53:55 -07:00
1 parent 2d89bf97cc
commit 81e6122891
@@ -1,117 +1,26 @@
 #!/usr/bin/env bash
-# autoresearch.sh — canonical benchmark entrypoint: speed of frank-primary.
+# autoresearch.sh — canonical frank-primary benchmark entry point (dev box).
 #
-# Runs on the operator box and drives one bench window on DANKSERVER:
-#   1. fingerprint the candidate trio (env + serve script + unit)
-#   2. memo fast-path: an identical fingerprint that already produced a FULLY
-#      successful window (bench + verified production restore) returns the
-#      cached metrics without touching the fleet
-#   3. stage the candidate trio plus the measure client into
-#      /home/dank/.autoresearch-frank-primary/ (ephemeral; the repo checkout
-#      on the host stays the production source of truth)
-#   4. run the window (scripts/autoresearch/frank-primary-window.sh):
-#      stop + mask the production unit, boot the staged config AS THE SERVICE
-#      (same serve script, same container, same port 7770), wait for health,
-#      run the in-container usage-counter measure, then restore the production
-#      unit and verify its health — AGENTS.md Benchmarking Policy
-#   5. print METRIC lines (parse by run_experiment)
-#
-# Primary metric: decode_tps (HIGHER is better) — median across the three
-# prompt depths (115/1056/4096) of per-depth medians of
-# completion_tokens / (wall - ttft), 3 runs each after a warm pass,
-# temperature 0, the campaign's deterministic word-seed workload.
-#
-# Exit 0 with METRIC lines on success; non-zero on any failure (the window
-# script's EXIT trap restores production on every path).
-set -euo pipefail
+# The service runs on dankserver (192.168.0.235:7770), the two R9700s live
+# there, and the benchmark must own those cards, so the whole stop -> boot
+# scratch -> bench -> restore sequence runs ON dankserver (see
+# scripts/autoresearch-frank-primary.sh). This wrapper:
+#   1. copies the candidate config (the repo's scripts/frank-primary.env, the
+#      file the autoresearch loop edits) and the bench script to dankserver,
+#   2. runs the bench over ssh, relaying its stdout (the METRIC lines) and
+#      exit code back here verbatim.
+# It deliberately does NOT push to origin: the scp dirties dankserver's
+# working tree, and the loop's uncommitted candidate config must not be
+# pushed. Only this file and scripts/autoresearch-frank-primary.sh are
+# committed; the candidate env rides over ssh each run.
+set -uo pipefail
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REMOTE=dankserver
+REMOTE_REPO=/home/dank/serverAppConfigs

-SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-HOST="${FRANK_PRIMARY_AR_HOST:-dankserver}"
-STAGE="/home/dank/.autoresearch-frank-primary"
+scp -q "$DIR/scripts/frank-primary.env" \
+    "$REMOTE:$REMOTE_REPO/scripts/frank-primary.env" || { echo "scp env failed" >&2; exit 1; }
+scp -q "$DIR/scripts/autoresearch-frank-primary.sh" \
+    "$REMOTE:$REMOTE_REPO/scripts/autoresearch-frank-primary.sh" || { echo "scp script failed" >&2; exit 1; }

-# FRANK_PRIMARY_AR_ENV_FILE stages a DIFFERENT env than the repo's — that is how
-# the arms driver A/Bs engine knobs without mutating the checkout. The
-# fingerprint covers whatever file is staged, so the memo cannot cross arms.
-ENV_FILE="${FRANK_PRIMARY_AR_ENV_FILE:-$SCRIPT_DIR/scripts/frank-primary.env}"
-SERVE="$SCRIPT_DIR/scripts/serve-frank-primary.sh"
-UNIT="$SCRIPT_DIR/scripts/frank-primary.service"
-MEASURE="$SCRIPT_DIR/scripts/autoresearch/frank-primary-measure.py"
-WINDOW="$SCRIPT_DIR/scripts/autoresearch/frank-primary-window.sh"
-
-for f in "$ENV_FILE" "$SERVE" "$UNIT" "$MEASURE" "$WINDOW"; do
-    [ -f "$f" ] || { echo "[autoresearch] missing harness file: $f" >&2; exit 1; }
-done
-command -v python3 >/dev/null 2>&1 || { echo "[autoresearch] python3 required on operator box" >&2; exit 1; }
-
-# Fingerprint of everything that can change the engine's argv or environment.
-HASH="$(cat "$ENV_FILE" "$SERVE" "$UNIT" | sha256sum | cut -d' ' -f1)"
-echo "[autoresearch] candidate fingerprint $HASH" >&2
-
-ssh_run() { ssh -o BatchMode=yes -o ConnectTimeout=15 "$HOST" "$@"; }
-
-emit_metrics() { # $1 = run.json
-    python3 -c '
-import json, sys
-run = json.loads(sys.argv[1])
-m = run["measure"]
-print("METRIC decode_tps=%s" % m["decode_tps"])
-for k in ("decode_tps_115", "decode_tps_1056", "decode_tps_4096"):
-    print("METRIC %s=%s" % (k, m[k]))
-print("METRIC ttft_ms_4096=%s" % m["ttft_ms_4096"])
-print("METRIC kv_tokens=%s" % (run.get("kv_tokens") or 0))
-print("METRIC boot_seconds=%s" % (run.get("boot_seconds") or -1))
-print("METRIC completion_tokens_4096=%s" % m.get("completion_tokens_4096", 0))
-# Any extra sampling temperature the measure swept (AR_TEMPS), e.g. the 0.7
-# arm — that is where speculative-decode acceptance shows up, since a greedy
-# draft matches a greedy target exactly and is rejected more under real
-# sampling. Suffixed keys only (_t0.7), so the historical lines are untouched.
-for k in sorted(m):
-    if "_t" in k and (k.startswith("decode_tps") or k.startswith("ttft_ms")):
-        print("METRIC %s=%s" % (k, m[k]))
-' "$1"
-}
-
-# --- memo fast-path -------------------------------------------------------
-cached_hash="$(ssh_run "cat '$STAGE/last/hash' 2>/dev/null" 2>/dev/null || true)"
-if [ -n "$cached_hash" ] && [ "$cached_hash" = "$HASH" ]; then
-    cached_run="$(ssh_run "cat '$STAGE/last/run.json' 2>/dev/null" || true)"
-    if [ -n "$cached_run" ]; then
-        echo "[autoresearch] memo hit — cached metrics for $HASH, fleet untouched" >&2
-        emit_metrics "$cached_run"
-        exit 0
-    fi
-fi
-
-# --- stage the candidate ----------------------------------------------------
-echo "[autoresearch] staging candidate on $HOST:$STAGE" >&2
-ssh_run "mkdir -p '$STAGE'"
-scp -q "$ENV_FILE" "$HOST:$STAGE/frank-primary.env"
-scp -q "$SERVE" "$HOST:$STAGE/serve.sh"
-scp -q "$MEASURE" "$HOST:$STAGE/measure.py"
-scp -q "$WINDOW" "$HOST:$STAGE/window.sh"
-ssh_run "printf '%s\n' '$HASH' > '$STAGE/hash'"
-
-# --- the window -------------------------------------------------------------
-# Budgets: bench boot up to 20 min (warm boots measured ~7 min), measure up
-# to 30 min, restore boot up to 20 min — call run_experiment with
-# timeout_seconds >= 3600.
-echo "[autoresearch] opening bench window (stop+mask -> boot -> measure -> restore)" >&2
-set +e
-ssh_run "bash '$STAGE/window.sh'"
-rc=$?
-set -e
-if [ "$rc" -ne 0 ]; then
-    echo "[autoresearch] window failed (rc=$rc); host-side evidence:" >&2
-    ssh_run "tail -60 '$STAGE/fail.log' 2>/dev/null || tail -40 '$STAGE/serve.out' 2>/dev/null || tail -40 '$STAGE/measure.out' 2>/dev/null || true" >&2 || true
-    exit 1
-fi
-
-run_json="$(ssh_run "cat '$STAGE/run.json'")" || {
-    echo "[autoresearch] could not read $STAGE/run.json back" >&2; exit 1; }
-echo "[autoresearch] window result:" >&2
-printf '%s\n' "$run_json" >&2
-emit_metrics "$run_json"
-
-echo "[autoresearch] engine evidence (plan / compile path / spec counters):" >&2
-ssh_run "cat '$STAGE/evidence.log' 2>/dev/null" >&2 || true
-exit 0
+ssh "$REMOTE" "bash $REMOTE_REPO/scripts/autoresearch-frank-primary.sh $REMOTE_REPO/scripts/frank-primary.env"
@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+# autoresearch-frank-primary.sh — RUNS ON DANKSERVER, driven over ssh by the
+# dev-box autoresearch.sh. It is the canonical frank-primary benchmark entry
+# point for the autoresearch loop.
+#
+# Method (mandated by the Benchmarking Policy): the production service serves
+# real users and shares the two R9700s, so a bench beside it starves the GPUs
+# and corrupts results. This script therefore:
+#   1. stops the production frank-primary unit (root, relink symlink: stop,
+#      NOT mask — mask refuses to replace a symlinked unit file),
+#   2. boots a scratch radiance container named `arbench` on the SAME two
+#      cards with the candidate engine argv (the env file the loop edits),
+#   3. polls /health until the listener is live (it binds AFTER the full
+#      load, so a bare curl is an honest gate),
+#   4. runs ONE self-contained completion request locally (127.0.0.1, no LAN
+#      jitter): a fixed ~140k-char real-text prompt (~32k tokens) decoded for
+#      256 tokens, greedy. The engine's `timings` object reports the honest
+#      prefill and decode rates for that single request.
+#   5. prints METRIC lines and tears the scratch down,
+#   6. restores production (systemctl start), via a trap so a failed bench
+#      never leaves frank-primary down (the stale-probe-tail lesson).
+#
+# Determinism: the prompt is built from a fixed, sorted corpus of repo files
+# with a fixed random seed, so prompt_n is stable across runs. Sampling is
+# greedy (temperature 0), so the decode length is fixed; the only residual
+# nondeterminism is the wht6 lossy TP all-reduce, which is a constant across
+# runs and cancels in A/B comparisons.
+#
+# The scratch mirrors the production container contract exactly (serve-frank-
+# primary.sh): --network host, --init, seccomp=unconfined (io_uring for the
+# n-gram table), memlock unbounded (the ~132 GiB pinned ledger), /dev/kfd +
+# /dev/dri, the .rad ro, the state dir rw, the tune cache mount. The ONLY
+# differences: the container name (arbench, so prod's identity is untouched)
+# and a FRESH prefix-cache disk tier (arbench/, wiped per run) so the cold
+# prefill is cold — the prod tier at kvcache/frank-primary is left intact.
+set -uo pipefail
+
+ENV_FILE="${1:-/home/dank/serverAppConfigs/scripts/frank-primary.env}"
+SCRATCH=arbench
+PORT=7770
+HEALTH="http://127.0.0.1:$PORT/health"
+COMPLETE="http://127.0.0.1:$PORT/v1/completions"
+BODY=/tmp/arbench-body.json
+LOG=/home/dank/logs/arbench.log
+
+restore() {
+  podman rm -f "$SCRATCH" >/dev/null 2>&1 || true
+  sudo systemctl start frank-primary >/dev/null 2>&1 || true
+}
+trap restore EXIT
+
+# --- source the candidate config (same file the production unit reads) ------
+if [ ! -f "$ENV_FILE" ]; then
+  echo "no env file at $ENV_FILE" >&2; exit 1
+fi
+# shellcheck disable=SC1090
+set -a; . "$ENV_FILE"; set +a
+
+# --- stop production; the scratch needs both cards to itself ----------------
+sudo systemctl stop frank-primary >/dev/null 2>&1 || true
+podman rm -f "$SCRATCH" >/dev/null 2>&1 || true
+
+# --- fresh prefix-cache tier for a cold prefill (prod's tier untouched) ------
+FRESH_KV="$FRANK_PRIMARY_STATE_DIR/kvcache/$SCRATCH"
+rm -rf "$FRESH_KV"; mkdir -p "$FRESH_KV"
+mkdir -p "$FRANK_PRIMARY_STATE_DIR/tune"
+
+# --- boot the scratch: the production run line, scratch name + fresh KV ------
+podman run -d --name "$SCRATCH" \
+  --network host \
+  --init \
+  --security-opt seccomp=unconfined \
+  --ulimit memlock=-1:-1 \
+  --device /dev/kfd --device /dev/dri \
+  -v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \
+  -v "$FRANK_PRIMARY_STATE_DIR":/data \
+  -v "$FRANK_PRIMARY_STATE_DIR/tune":/opt/radiance/share/radiance/tune \
+  -v "$FRANK_PRIMARY_TEMPLATE":/templates/template.jinja:ro \
+  "$FRANK_PRIMARY_IMAGE" \
+  --model "/models/$FRANK_PRIMARY_MODEL_FILE" \
+  --host :: --port "$PORT" \
+  --served-model-name "$FRANK_PRIMARY_SERVED_NAME" \
+  --override-chat-template /templates/template.jinja \
+  --tp "$FRANK_PRIMARY_TP" --tp-wire "$FRANK_PRIMARY_TP_WIRE" \
+  --max-num-seqs "$FRANK_PRIMARY_MAX_SEQS" \
+  --max-model-len "$FRANK_PRIMARY_MAX_MODEL_LEN" \
+  --placement "$FRANK_PRIMARY_PLACEMENT" \
+  --host-pool-mib "$FRANK_PRIMARY_HOST_POOL_MIB" \
+  --gpu-headroom-mib "$FRANK_PRIMARY_GPU_HEADROOM_MIB" \
+  --expert-vs-cache-ratio "$FRANK_PRIMARY_EXPERT_VS_CACHE" \
+  --kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE" \
+  --prefix-cache-host-mib "$FRANK_PRIMARY_PREFIX_HOST_MIB" \
+  --prefix-cache-dir "/data/kvcache/$SCRATCH" \
+  --prefix-cache-disk-mib "$FRANK_PRIMARY_PREFIX_DISK_MIB" \
+  --ngram-placement "$FRANK_PRIMARY_NGRAM_PLACEMENT" \
+  --num-speculative-tokens "$FRANK_PRIMARY_SPEC_TOKENS" \
+  --max-num-batched-tokens "$FRANK_PRIMARY_MAX_BATCHED_TOKENS" \
+  ${FRANK_PRIMARY_EXTRA_ARGS:+$FRANK_PRIMARY_EXTRA_ARGS} \
+  > "$LOG" 2>&1
+if [ $? -ne 0 ]; then echo "scratch boot failed; see $LOG" >&2; exit 1; fi
+
+# --- poll health (listener binds after the full load; cold boot ~110 s) ------
+ok=0
+for _ in $(seq 1 360); do
+  code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 3 "$HEALTH" 2>/dev/null || echo 000)
+  if [ "$code" = "200" ]; then ok=1; break; fi
+  sleep 2
+done
+if [ "$ok" -ne 1 ]; then
+  echo "scratch never reached health 200; tail of $LOG:" >&2
+  tail -30 "$LOG" >&2; exit 1
+fi
+
+# --- build the workload: a fixed ~140k-char real-text prompt (repo corpus) ---
+python3 - "$BODY" <<'PY'
+import json, sys, random, glob
+random.seed(1234)
+root = "/home/dank/serverAppConfigs"
+files = sorted(
+    glob.glob(f"{root}/serverAppConfigs-docs/**/*.md", recursive=True)
+    + glob.glob(f"{root}/scripts/*.sh")
+    + glob.glob(f"{root}/scripts/*.py")
+    + glob.glob(f"{root}/AGENTS.md")
+)
+corpus = []
+total = 0
+for f in files:
+    try:
+        t = open(f, encoding="utf-8", errors="ignore").read()
+    except Exception:
+        continue
+    if not t.strip():
+        continue
+    corpus.append(t)
+    total += len(t)
+    if total > 240000:
+        break
+corpus = "\n".join(corpus)
+TGT = 140000
+parts, n = [], 0
+while n < TGT:
+    start = random.randint(0, max(0, len(corpus) - 500))
+    seg = corpus[start:start + random.randint(200, 500)]
+    parts.append(seg)
+    n += len(seg) + 1
+prompt = " ".join(parts)
+json.dump(
+    {"model": "frank-primary", "prompt": prompt,
+     "max_tokens": 256, "temperature": 0.0, "stream": False,
+     "stop": ["<|im_end|>"]},
+    open(sys.argv[1], "w"),
+)
+PY
+if [ ! -f "$BODY" ]; then echo "prompt build failed" >&2; exit 1; fi
+
+# --- the bench request (prefill ~32k + decode 256, greedy) -------------------
+RESP=$(curl -s --max-time 600 -X POST "$COMPLETE" \
+  -H 'Content-Type: application/json' -d "@$BODY")
+echo "$RESP" > /tmp/arbench-resp.json
+
+# --- parse the timings into METRIC lines -------------------------------------
+python3 - <<'PY'
+import json, sys
+try:
+    r = json.load(open("/tmp/arbench-resp.json"))
+except Exception as e:
+    sys.stderr.write(f"unparseable response: {e}\n"); sys.exit(1)
+if "timings" not in r:
+    sys.stderr.write(f"no timings in response: {json.dumps(r)[:500]}\n"); sys.exit(1)
+t = r["timings"]
+dtps = t.get("predicted_per_second")
+ptps = t.get("prompt_per_second")
+pms  = t.get("prompt_ms")
+dn   = t.get("draft_n", 0)
+da   = t.get("draft_n_accepted", 0)
+if dtps is None:
+    sys.stderr.write("predicted_per_second null (no decode produced)\n"); sys.exit(1)
+print(f"METRIC decode_tps={dtps:.4f}")
+if ptps is not None:
+    print(f"METRIC prefill_tps={ptps:.4f}")
+if pms is not None:
+    print(f"METRIC ttft_ms={pms:.4f}")
+if dn:
+    print(f"METRIC draft_accept={da/dn:.4f}")
+print(f"INFO prompt_n={t.get('prompt_n')} cache_n={t.get('cache_n')} predicted_n={t.get('predicted_n')}")
+PY