autoresearch: temp sweep, engine evidence, env overlays, arms driver

Four harness gaps that block an honest MTP re-measurement:

- measure: AR_TEMPS sweeps sampling temperatures (default "0,0.7"). Draft
  ACCEPTANCE is sampling-dependent — a greedy draft matches a greedy target
  exactly and is rejected more under a non-greedy target — so judging a spec
  arm only at temperature 0 judges half the contract. The unsuffixed keys stay
  temperature 0, so every historical number remains comparable.
- window: capture an evidence.log sidecar (expert-plan lines, compile path,
  graph sizes, /metrics spec-decode and cache counters, OOM signatures). The
  arena/overhead/reserve numbers exist ONLY in the boot log; an arm that loses
  decode speed is otherwise unexplainable afterwards.
- orchestrator: FRANK_PRIMARY_AR_ENV_FILE stages an env other than the repo's,
  which is how arms A/B without mutating the checkout (the fingerprint covers
  whatever is staged, so the memo can never answer one arm with another's
  numbers), plus METRIC lines for the swept temperatures and the evidence dump.
- scripts/autoresearch/frank-primary-arms.sh: runs a sequence of full windows
  (baseline, MTP3+probabilistic with a step-4 capture ladder at precise memory
  ON, the same at the default overhead, and the vendor SEQS-16 shape), gating
  each staged env on its JSON knobs still parsing AFTER sourcing — an unquoted
  {"temperature":0.7} loses its quotes at source time and reaches the engine as
  {temperature:0.7}, which is a 20-minute boot wasted.

scripts/autoresearch/frank-primary-measure.py

@@ -19,14 +19,18 @@ Workload (fixed, deterministic, offline):
     the server's own /v1/messages/count_tokens counter
   * depths 115 / 1056 / 4096 target prompt tokens
   * one 64-token warm pass per depth (warms the expert arena), then
-    max_tokens=256 at temperature 0, 3 runs, median
-Everything is greedy with a fixed seed — the same workload every run, no
-network, no time-of-day dependence.
+    max_tokens=256, 3 runs, median — repeated for every temperature in AR_TEMPS
+    (default 0 and 0.7). The prompt workload is deterministic (fixed seed, no
+    network, no time-of-day dependence); at temperature > 0 the generated text
+    is not, but the measured window is bounded by max_tokens, so the rate stays
+    comparable and the non-determinism is exactly the sampling that a
+    speculative-decode arm must survive.
 
 Env:
   AR_BASE             endpoint (default http://127.0.0.1:7770)
   AR_MODEL            served name (default frank-primary)
   AR_PROMPT_CEILING   hard prompt ceiling in tokens (default 100000)
+  AR_TEMPS            comma-separated sampling temperatures (default "0,0.7")
 
 Output: progress on stderr, ONE json object on stdout (the last line).
 Exit codes: 0 ok, 2 endpoint unhealthy / no usable completion, 3 bad usage.
@@ -47,6 +51,14 @@ DEPTHS = (115, 1056, 4096)
 WARM_TOKENS = 64
 MAX_TOKENS = 256
 RUNS = 3
+# Temperature sweep. The unsuffixed keys are temperature 0 — byte-comparable to
+# every historical measurement — and AR_TEMPS adds more. The second one matters
+# for speculative decode: draft ACCEPTANCE is sampling-dependent (a greedy draft
+# matches a greedy target exactly and is rejected more under a non-greedy
+# target), so an MTP arm judged only at temp 0 would be judged on half the
+# contract. Production traffic is non-zero temperature (the payload's
+# generation_config says 1.0; an override to 0.7 is under test).
+TEMPS = tuple(float(x) for x in os.environ.get("AR_TEMPS", "0,0.7").split(","))
 
 # 43-word x 12 = 516-word vocabulary, seed 1234 — the campaign's exact list
 # and seed, so prompts are byte-identical to the historical measurements.
@@ -105,11 +117,11 @@ def prompt_for(target_tokens, wpt):
     return text, pt
 
 
-def measured(prompt, max_tokens):
+def measured(prompt, max_tokens, temperature=0):
     """Return (ttft_s, decode_tok_s, completion_tokens) via usage counters."""
     body = {"model": MODEL,
             "messages": [{"role": "user", "content": prompt}],
-            "max_tokens": max_tokens, "temperature": 0,
+            "max_tokens": max_tokens, "temperature": temperature,
             "stream": True, "stream_options": {"include_usage": True}}
     req = urllib.request.Request(
         BASE + "/v1/chat/completions", data=json.dumps(body).encode(),
@@ -145,8 +157,13 @@ def measured(prompt, max_tokens):
     return (ttft or wall), (ct / dec_window), ct
 
 
+def fmt_temp(t):
+    return ("%.1f" % t).rstrip("0").rstrip(".") if t else "0"
+
+
 def main():
-    log("base=%s model=%s ceiling=%d" % (BASE, MODEL, PROMPT_CEILING))
+    log("base=%s model=%s ceiling=%d temps=%s"
+        % (BASE, MODEL, PROMPT_CEILING, ",".join(fmt_temp(t) for t in TEMPS)))
     try:
         wpt = calibrate()
     except Exception as e:
@@ -158,32 +175,39 @@ def main():
     for tgt in DEPTHS:
         try:
             p, pt = prompt_for(tgt, wpt)
-            measured(p, WARM_TOKENS)  # warm the arena
+            measured(p, WARM_TOKENS)  # warm the arena and the prefix cache
         except Exception as e:
             log("warm pass failed at depth %d: %s" % (tgt, e))
             return 2
-        rates, tts, cts = [], [], []
-        for i in range(RUNS):
-            try:
-                tt, r, ct = measured(p, MAX_TOKENS)
-            except Exception as e:
-                log("run %d/%d failed at depth %d: %s" % (i + 1, RUNS, tgt, e))
-                continue
-            rates.append(r)
-            tts.append(tt)
-            cts.append(ct)
-        if len(rates) < 2:
-            log("only %d usable run(s) at depth %d — refusing to median" % (len(rates), tgt))
-            return 2
-        result["decode_tps_%d" % tgt] = round(statistics.median(rates), 2)
-        result["ttft_ms_%d" % tgt] = round(1000.0 * statistics.median(tts), 1)
-        result["completion_tokens_%d" % tgt] = int(statistics.median(cts))
-        result["prompt_tokens_%d" % tgt] = pt
-        log("depth %5d tok  decode %.2f t/s  ttft %.3f s  (runs=%d)"
-            % (pt, result["decode_tps_%d" % tgt], result["ttft_ms_%d" % tgt] / 1000.0, len(rates)))
+        for temp in TEMPS:
+            suffix = "" if temp == 0 else "_t%s" % fmt_temp(temp)
+            rates, tts, cts = [], [], []
+            for i in range(RUNS):
+                try:
+                    tt, r, ct = measured(p, MAX_TOKENS, temp)
+                except Exception as e:
+                    log("run %d/%d failed at depth %d temp %s: %s"
+                        % (i + 1, RUNS, tgt, fmt_temp(temp), e))
+                    continue
+                rates.append(r)
+                tts.append(tt)
+                cts.append(ct)
+            if len(rates) < 2:
+                log("only %d usable run(s) at depth %d temp %s — refusing to median"
+                    % (len(rates), tgt, fmt_temp(temp)))
+                return 2
+            result["decode_tps_%d%s" % (tgt, suffix)] = round(statistics.median(rates), 2)
+            result["ttft_ms_%d%s" % (tgt, suffix)] = round(1000.0 * statistics.median(tts), 1)
+            result["completion_tokens_%d%s" % (tgt, suffix)] = int(statistics.median(cts))
+            result["prompt_tokens_%d%s" % (tgt, suffix)] = pt
+            log("depth %5d tok temp %-4s decode %7.2f t/s  ttft %.3f s  (runs=%d)"
+                % (pt, fmt_temp(temp), result["decode_tps_%d%s" % (tgt, suffix)],
+                   result["ttft_ms_%d%s" % (tgt, suffix)] / 1000.0, len(rates)))
 
-    meds = [result["decode_tps_%d" % t] for t in DEPTHS]
-    result["decode_tps"] = round(statistics.median(meds), 2)
+    for temp in TEMPS:
+        suffix = "" if temp == 0 else "_t%s" % fmt_temp(temp)
+        meds = [result["decode_tps_%d%s" % (t, suffix)] for t in DEPTHS]
+        result["decode_tps%s" % suffix] = round(statistics.median(meds), 2)
     print(json.dumps(result))
     return 0