autoresearch: temp sweep, engine evidence, env overlays, arms driver

Four harness gaps that block an honest MTP re-measurement:

- measure: AR_TEMPS sweeps sampling temperatures (default "0,0.7"). Draft
  ACCEPTANCE is sampling-dependent — a greedy draft matches a greedy target
  exactly and is rejected more under a non-greedy target — so judging a spec
  arm only at temperature 0 judges half the contract. The unsuffixed keys stay
  temperature 0, so every historical number remains comparable.
- window: capture an evidence.log sidecar (expert-plan lines, compile path,
  graph sizes, /metrics spec-decode and cache counters, OOM signatures). The
  arena/overhead/reserve numbers exist ONLY in the boot log; an arm that loses
  decode speed is otherwise unexplainable afterwards.
- orchestrator: FRANK_PRIMARY_AR_ENV_FILE stages an env other than the repo's,
  which is how arms A/B without mutating the checkout (the fingerprint covers
  whatever is staged, so the memo can never answer one arm with another's
  numbers), plus METRIC lines for the swept temperatures and the evidence dump.
- scripts/autoresearch/frank-primary-arms.sh: runs a sequence of full windows
  (baseline, MTP3+probabilistic with a step-4 capture ladder at precise memory
  ON, the same at the default overhead, and the vendor SEQS-16 shape), gating
  each staged env on its JSON knobs still parsing AFTER sourcing — an unquoted
  {"temperature":0.7} loses its quotes at source time and reaches the engine as
  {temperature:0.7}, which is a 20-minute boot wasted.

autoresearch.sh

@@ -29,7 +29,10 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 HOST="${FRANK_PRIMARY_AR_HOST:-dankserver}"
 STAGE="/home/dank/.autoresearch-frank-primary"
 
-ENV_FILE="$SCRIPT_DIR/scripts/frank-primary.env"
+# FRANK_PRIMARY_AR_ENV_FILE stages a DIFFERENT env than the repo's — that is how
+# the arms driver A/Bs engine knobs without mutating the checkout. The
+# fingerprint covers whatever file is staged, so the memo cannot cross arms.
+ENV_FILE="${FRANK_PRIMARY_AR_ENV_FILE:-$SCRIPT_DIR/scripts/frank-primary.env}"
 SERVE="$SCRIPT_DIR/scripts/serve-frank-primary.sh"
 UNIT="$SCRIPT_DIR/scripts/frank-primary.service"
 MEASURE="$SCRIPT_DIR/scripts/autoresearch/frank-primary-measure.py"
@@ -58,6 +61,13 @@ print("METRIC ttft_ms_4096=%s" % m["ttft_ms_4096"])
 print("METRIC kv_tokens=%s" % (run.get("kv_tokens") or 0))
 print("METRIC boot_seconds=%s" % (run.get("boot_seconds") or -1))
 print("METRIC completion_tokens_4096=%s" % m.get("completion_tokens_4096", 0))
+# Any extra sampling temperature the measure swept (AR_TEMPS), e.g. the 0.7
+# arm — that is where speculative-decode acceptance shows up, since a greedy
+# draft matches a greedy target exactly and is rejected more under real
+# sampling. Suffixed keys only (_t0.7), so the historical lines are untouched.
+for k in sorted(m):
+    if "_t" in k and (k.startswith("decode_tps") or k.startswith("ttft_ms")):
+        print("METRIC %s=%s" % (k, m[k]))
 ' "$1"
 }
 
@@ -101,4 +111,7 @@ run_json="$(ssh_run "cat '$STAGE/run.json'")" || {
 echo "[autoresearch] window result:" >&2
 printf '%s\n' "$run_json" >&2
 emit_metrics "$run_json"
+
+echo "[autoresearch] engine evidence (plan / compile path / spec counters):" >&2
+ssh_run "cat '$STAGE/evidence.log' 2>/dev/null" >&2 || true
 exit 0

scripts/autoresearch/frank-primary-arms.sh

@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+# frank-primary-arms.sh — run a SEQUENCE of frank-primary bench arms and keep
+# every arm's evidence. Runs on the operator box; each arm is one full window
+# of ./autoresearch.sh (stop+mask -> boot the arm as the service -> measure ->
+# restore production and verify), so the fleet is only ever down for one arm at
+# a time and the production unit is back up between arms.
+#
+# An arm is the repo's own frank-primary.env plus an OVERLAY of engine knobs
+# appended after it (later assignment wins when the serve script sources it),
+# staged through FRANK_PRIMARY_AR_ENV_FILE so the checkout is never mutated and
+# each arm gets its own fingerprint (the memo can never answer one arm with
+# another arm's numbers).
+#
+# Usage:  scripts/autoresearch/frank-primary-arms.sh [arm ...]
+#         scripts/autoresearch/frank-primary-arms.sh --list
+# Output, under /tmp/frank-primary-arms-<UTC stamp>/ (override with ARMS_DIR):
+#
+#   <arm>.out        full autoresearch output (metrics + engine evidence)
+#   <arm>.env        the exact staged env
+#   <arm>.rc         window exit code
+#   summary.tsv      arm, rc, decode_tps, decode_tps_t0.7, kv_tokens, boot_s
+#
+# Exit 0 when every arm produced metrics; 1 if any arm failed (a failed boot is
+# a RESULT — the window still restored production, and the rc says which arm).
+set -uo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+REPO="$(cd -- "$SCRIPT_DIR/../.." && pwd)"
+BASE_ENV="$REPO/scripts/frank-primary.env"
+ARMS_DIR="${ARMS_DIR:-/tmp/frank-primary-arms-$(date -u +%Y%m%dT%H%M%SZ)}"
+
+# --- arms ------------------------------------------------------------------
+# baseline: the shipped shape E, measured with THIS harness so the comparison is
+# matched (the ~29.6 t/s figure in the docs is an older shape, and no shape-E
+# decode number exists yet).
+OVER_baseline=()
+# B: MTP3 with the vendor's probabilistic draft sampling, a capture ladder whose
+# step is num_speculative_tokens + 1 = 4, and the sampling override. Precise
+# memory stays ON — this arm is simultaneously the MTP test and the test of
+# whether MTP's drafter workspace and the fitted 5.25 GiB overhead can coexist.
+SPEC3='{"method":"mtp","num_speculative_tokens":3,"draft_sample_method":"probabilistic"}'
+LADDER3='{"cudagraph_capture_sizes":[4,8,12],"max_cudagraph_capture_size":12,"inductor_compile_config":{"combo_kernels":false,"benchmark_combo_kernel":false}}'
+# The value must arrive in the staged env file SINGLE-QUOTED: sourcing an env
+# file performs quote removal, so an unquoted {"temperature":0.7} would reach
+# the engine as {temperature:0.7}.
+GEN07='{"temperature":0.7}'
+OVER_B=(
+    "FRANK_PRIMARY_SPEC='$SPEC3'"
+    "FRANK_PRIMARY_COMPCONFIG='$LADDER3'"
+    'FRANK_PRIMARY_GEN_CONFIG='"'$GEN07'"
+)
+# B2: identical to B with the default (7.60 GiB) overhead, i.e. precise memory
+# OFF. It is the fallback if B dies at graph capture, and it prices the arena
+# that precise memory buys (7,522 slots -> ~6,496).
+OVER_B2=(
+    "FRANK_PRIMARY_SPEC='$SPEC3'"
+    "FRANK_PRIMARY_COMPCONFIG='$LADDER3'"
+    'FRANK_PRIMARY_GEN_CONFIG='"'$GEN07'"
+    'FRANK_PRIMARY_PRECISE_MEMORY='
+)
+# C: the vendor's concurrency shape — SEQS 16 with the [4..32] ladder and util
+# 0.95. This is also the direct test of the theory that pinning the capture
+# ladder unlocks SEQS > 3 (the SEQS=2 failure was query_start_loc specialised to
+# the constant 3, which is NOT in the default ladder; 4 is).
+LADDER16='{"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"max_cudagraph_capture_size":32,"inductor_compile_config":{"combo_kernels":false,"benchmark_combo_kernel":false}}'
+OVER_C=(
+    "FRANK_PRIMARY_SPEC='$SPEC3'"
+    "FRANK_PRIMARY_COMPCONFIG='$LADDER16'"
+    'FRANK_PRIMARY_GEN_CONFIG='"'$GEN07'"
+    'FRANK_PRIMARY_SEQS=16'
+    'FRANK_PRIMARY_MNBT=4096'
+    'FRANK_PRIMARY_UTIL=0.95'
+)
+
+arm_overlay() {
+    case "$1" in
+        baseline) ;;                       # the shipped env, no overlay
+        B)  printf '%s\n' "${OVER_B[@]}" ;;
+        B2) printf '%s\n' "${OVER_B2[@]}" ;;
+        C)  printf '%s\n' "${OVER_C[@]}" ;;
+        *) return 1 ;;
+    esac
+}
+
+if [ "${1:-}" = "--list" ]; then
+    printf 'baseline B B2 C\n'
+    exit 0
+fi
+
+[ -f "$BASE_ENV" ] || { echo "[arms] missing $BASE_ENV" >&2; exit 1; }
+if [ "$#" -gt 0 ]; then ARMS=("$@"); else ARMS=(baseline B B2 C); fi
+mkdir -p "$ARMS_DIR"
+echo "[arms] dir $ARMS_DIR  arms: ${ARMS[*]}" >&2
+
+: >"$ARMS_DIR/summary.tsv"
+printf 'arm\trc\tdecode_tps\tdecode_tps_t0.7\tkv_tokens\tboot_s\n' >>"$ARMS_DIR/summary.tsv"
+overall=0
+
+for arm in "${ARMS[@]}"; do
+    if ! arm_overlay "$arm" >"$ARMS_DIR/$arm.overlay"; then
+        echo "[arms] unknown arm: $arm" >&2
+        overall=1
+        continue
+    fi
+    { cat "$BASE_ENV"; echo; echo "# --- arm $arm overlay ($(date -u +%FT%TZ)) ---"
+      arm_overlay "$arm"; } >"$ARMS_DIR/$arm.env"
+    # Gate the staged env BEFORE burning a ~20-minute window: every knob that
+    # holds JSON must still be one valid JSON object after the file is SOURCED
+    # (that is the step that silently eats unquoted braces).
+    if ! bash -c 'set -a; . "$1" >/dev/null; set +a
+        for k in FRANK_PRIMARY_SPEC FRANK_PRIMARY_COMPCONFIG FRANK_PRIMARY_GEN_CONFIG; do
+            v="${!k:-}"
+            [ -n "$v" ] || continue
+            case "$v" in
+                "{"*) python3 -c "import json,sys; json.loads(sys.argv[1])" "$v" \
+                        || { echo "[arms] $k is not valid JSON: $v" >&2; exit 1; } ;;
+                *) echo "[arms] $k is not a JSON object: $v" >&2; exit 1 ;;
+            esac
+        done' _ "$ARMS_DIR/$arm.env"; then
+        echo "[arms] $arm: staged env rejected, window not opened" >&2
+        echo 99 >"$ARMS_DIR/$arm.rc"
+        overall=1
+        continue
+    fi
+
+    echo "[arms] === $arm: opening window $(date -u +%H:%M:%S)Z ===" >&2
+    FRANK_PRIMARY_AR_ENV_FILE="$ARMS_DIR/$arm.env" \
+        "$REPO/autoresearch.sh" >"$ARMS_DIR/$arm.out" 2>&1
+    rc=$?
+    echo "$rc" >"$ARMS_DIR/$arm.rc"
+    echo "[arms] === $arm: rc=$rc $(date -u +%H:%M:%S)Z ===" >&2
+    grep -E '^METRIC ' "$ARMS_DIR/$arm.out" | sed 's/^METRIC //' >"$ARMS_DIR/$arm.metrics" || true
+    metric() { local v; v="$(sed -n "s/^$1=//p" "$ARMS_DIR/$arm.metrics" | tail -1)"; printf '%s' "${v:-$2}"; }
+    printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$arm" "$rc" \
+        "$(metric decode_tps FAIL)" "$(metric decode_tps_t0.7 FAIL)" \
+        "$(metric kv_tokens ?)" "$(metric boot_seconds ?)" >>"$ARMS_DIR/summary.tsv"
+    [ "$rc" -ne 0 ] && overall=1
+done
+
+echo "[arms] summary:" >&2
+cat "$ARMS_DIR/summary.tsv" >&2
+exit "$overall"

scripts/autoresearch/frank-primary-measure.py

@@ -19,14 +19,18 @@ Workload (fixed, deterministic, offline):
     the server's own /v1/messages/count_tokens counter
   * depths 115 / 1056 / 4096 target prompt tokens
   * one 64-token warm pass per depth (warms the expert arena), then
-    max_tokens=256 at temperature 0, 3 runs, median
+    max_tokens=256, 3 runs, median — repeated for every temperature in AR_TEMPS
-Everything is greedy with a fixed seed — the same workload every run, no
+    (default 0 and 0.7). The prompt workload is deterministic (fixed seed, no
-network, no time-of-day dependence.
+    network, no time-of-day dependence); at temperature > 0 the generated text
+    is not, but the measured window is bounded by max_tokens, so the rate stays
+    comparable and the non-determinism is exactly the sampling that a
+    speculative-decode arm must survive.
 
 Env:
   AR_BASE             endpoint (default http://127.0.0.1:7770)
   AR_MODEL            served name (default frank-primary)
   AR_PROMPT_CEILING   hard prompt ceiling in tokens (default 100000)
+  AR_TEMPS            comma-separated sampling temperatures (default "0,0.7")
 
 Output: progress on stderr, ONE json object on stdout (the last line).
 Exit codes: 0 ok, 2 endpoint unhealthy / no usable completion, 3 bad usage.
@@ -47,6 +51,14 @@ DEPTHS = (115, 1056, 4096)
 WARM_TOKENS = 64
 MAX_TOKENS = 256
 RUNS = 3
+# Temperature sweep. The unsuffixed keys are temperature 0 — byte-comparable to
+# every historical measurement — and AR_TEMPS adds more. The second one matters
+# for speculative decode: draft ACCEPTANCE is sampling-dependent (a greedy draft
+# matches a greedy target exactly and is rejected more under a non-greedy
+# target), so an MTP arm judged only at temp 0 would be judged on half the
+# contract. Production traffic is non-zero temperature (the payload's
+# generation_config says 1.0; an override to 0.7 is under test).
+TEMPS = tuple(float(x) for x in os.environ.get("AR_TEMPS", "0,0.7").split(","))
 
 # 43-word x 12 = 516-word vocabulary, seed 1234 — the campaign's exact list
 # and seed, so prompts are byte-identical to the historical measurements.
@@ -105,11 +117,11 @@ def prompt_for(target_tokens, wpt):
     return text, pt
 
 
-def measured(prompt, max_tokens):
+def measured(prompt, max_tokens, temperature=0):
     """Return (ttft_s, decode_tok_s, completion_tokens) via usage counters."""
     body = {"model": MODEL,
             "messages": [{"role": "user", "content": prompt}],
-            "max_tokens": max_tokens, "temperature": 0,
+            "max_tokens": max_tokens, "temperature": temperature,
             "stream": True, "stream_options": {"include_usage": True}}
     req = urllib.request.Request(
         BASE + "/v1/chat/completions", data=json.dumps(body).encode(),
@@ -145,8 +157,13 @@ def measured(prompt, max_tokens):
     return (ttft or wall), (ct / dec_window), ct
 
 
+def fmt_temp(t):
+    return ("%.1f" % t).rstrip("0").rstrip(".") if t else "0"
+
+
 def main():
-    log("base=%s model=%s ceiling=%d" % (BASE, MODEL, PROMPT_CEILING))
+    log("base=%s model=%s ceiling=%d temps=%s"
+        % (BASE, MODEL, PROMPT_CEILING, ",".join(fmt_temp(t) for t in TEMPS)))
     try:
         wpt = calibrate()
     except Exception as e:
@@ -158,32 +175,39 @@ def main():
     for tgt in DEPTHS:
         try:
             p, pt = prompt_for(tgt, wpt)
-            measured(p, WARM_TOKENS)  # warm the arena
+            measured(p, WARM_TOKENS)  # warm the arena and the prefix cache
         except Exception as e:
             log("warm pass failed at depth %d: %s" % (tgt, e))
             return 2
-        rates, tts, cts = [], [], []
+        for temp in TEMPS:
-        for i in range(RUNS):
+            suffix = "" if temp == 0 else "_t%s" % fmt_temp(temp)
-            try:
+            rates, tts, cts = [], [], []
-                tt, r, ct = measured(p, MAX_TOKENS)
+            for i in range(RUNS):
-            except Exception as e:
+                try:
-                log("run %d/%d failed at depth %d: %s" % (i + 1, RUNS, tgt, e))
+                    tt, r, ct = measured(p, MAX_TOKENS, temp)
-                continue
+                except Exception as e:
-            rates.append(r)
+                    log("run %d/%d failed at depth %d temp %s: %s"
-            tts.append(tt)
+                        % (i + 1, RUNS, tgt, fmt_temp(temp), e))
-            cts.append(ct)
+                    continue
-        if len(rates) < 2:
+                rates.append(r)
-            log("only %d usable run(s) at depth %d — refusing to median" % (len(rates), tgt))
+                tts.append(tt)
-            return 2
+                cts.append(ct)
-        result["decode_tps_%d" % tgt] = round(statistics.median(rates), 2)
+            if len(rates) < 2:
-        result["ttft_ms_%d" % tgt] = round(1000.0 * statistics.median(tts), 1)
+                log("only %d usable run(s) at depth %d temp %s — refusing to median"
-        result["completion_tokens_%d" % tgt] = int(statistics.median(cts))
+                    % (len(rates), tgt, fmt_temp(temp)))
-        result["prompt_tokens_%d" % tgt] = pt
+                return 2
-        log("depth %5d tok  decode %.2f t/s  ttft %.3f s  (runs=%d)"
+            result["decode_tps_%d%s" % (tgt, suffix)] = round(statistics.median(rates), 2)
-            % (pt, result["decode_tps_%d" % tgt], result["ttft_ms_%d" % tgt] / 1000.0, len(rates)))
+            result["ttft_ms_%d%s" % (tgt, suffix)] = round(1000.0 * statistics.median(tts), 1)
+            result["completion_tokens_%d%s" % (tgt, suffix)] = int(statistics.median(cts))
+            result["prompt_tokens_%d%s" % (tgt, suffix)] = pt
+            log("depth %5d tok temp %-4s decode %7.2f t/s  ttft %.3f s  (runs=%d)"
+                % (pt, fmt_temp(temp), result["decode_tps_%d%s" % (tgt, suffix)],
+                   result["ttft_ms_%d%s" % (tgt, suffix)] / 1000.0, len(rates)))
 
-    meds = [result["decode_tps_%d" % t] for t in DEPTHS]
+    for temp in TEMPS:
-    result["decode_tps"] = round(statistics.median(meds), 2)
+        suffix = "" if temp == 0 else "_t%s" % fmt_temp(temp)
+        meds = [result["decode_tps_%d%s" % (t, suffix)] for t in DEPTHS]
+        result["decode_tps%s" % suffix] = round(statistics.median(meds), 2)
     print(json.dumps(result))
     return 0
 

scripts/autoresearch/frank-primary-window.sh

@@ -118,7 +118,7 @@ main() {
         [ -f "$STAGE/$f" ] || { log "staged file missing: $STAGE/$f"; exit 1; }
     done
     mkdir -p "$STAGE/last"
-    rm -f "$STAGE/run.json" "$STAGE/fail.log" "$STAGE/measure.out"
+    rm -f "$STAGE/run.json" "$STAGE/fail.log" "$STAGE/measure.out" "$STAGE/evidence.log"
 
     local t_start t0 i kvline kv_tokens kv_conc measure_json boot_seconds
     t_start="$(date +%s)"
@@ -177,6 +177,24 @@ main() {
     kv_tokens="$(printf '%s' "$kvline" | sed -E 's/.*GPU KV cache size: ([0-9,]+) tokens.*/\1/' | tr -d ',' || true)"
     kv_conc="$(printf '%s' "$kvline" | sed -E 's/.*per request: ([0-9.]+x).*/\1/' || true)"
 
+    # Human-facing evidence, in a sidecar file because it is multi-line prose,
+    # not metrics: the expert-plan lines (the ONLY place the arena / runtime
+    # overhead / KV-reserve numbers exist — they are never in /metrics), the
+    # compile path (cold build vs cache hit), the graph-capture sizes, the
+    # speculative-decode counters, and any OOM signature. Without this an arm
+    # that loses decode speed is unexplainable afterwards.
+    {
+        echo "=== engine plan / capacity / spec evidence ==="
+        podman logs "$CTR" 2>&1 | grep -E \
+'KV cache size|Maximum concurrency|layers split|expert cache|runtime overhead|non-default args|r4d all-reduce|Using \[|cudagraph|CUDA graph|Capturing|graph for compile range|torch.compile took|Directly load the compiled|speculative|Speculative|acceptance|Acceptance|offload|precise|OutOfMemory|Tried to allocate|Engine core initialization failed' \
+            | tail -150 || true
+        echo "=== /metrics counters (spec decode, caches, success) ==="
+        curl -s --max-time 15 http://127.0.0.1:7770/metrics 2>/dev/null \
+            | grep -E '^vllm:(spec_decode|gpu_cache|prefix_cache|request_success|generation_tokens|iteration)' \
+            | head -80 || true
+    } >"$STAGE/evidence.log" 2>&1 || true
+    log "engine evidence -> $STAGE/evidence.log ($(wc -l <"$STAGE/evidence.log") lines)"
+
     measure_json="$(grep -E '^\{' "$STAGE/measure.out" | tail -1 || true)"
     if [ -z "$measure_json" ]; then
         log "measure produced no JSON result line"