diff --git a/autoresearch.sh b/autoresearch.sh index a5aea10..089a127 100755 --- a/autoresearch.sh +++ b/autoresearch.sh @@ -29,7 +29,10 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" HOST="${FRANK_PRIMARY_AR_HOST:-dankserver}" STAGE="/home/dank/.autoresearch-frank-primary" -ENV_FILE="$SCRIPT_DIR/scripts/frank-primary.env" +# FRANK_PRIMARY_AR_ENV_FILE stages a DIFFERENT env than the repo's — that is how +# the arms driver A/Bs engine knobs without mutating the checkout. The +# fingerprint covers whatever file is staged, so the memo cannot cross arms. +ENV_FILE="${FRANK_PRIMARY_AR_ENV_FILE:-$SCRIPT_DIR/scripts/frank-primary.env}" SERVE="$SCRIPT_DIR/scripts/serve-frank-primary.sh" UNIT="$SCRIPT_DIR/scripts/frank-primary.service" MEASURE="$SCRIPT_DIR/scripts/autoresearch/frank-primary-measure.py" @@ -58,6 +61,13 @@ print("METRIC ttft_ms_4096=%s" % m["ttft_ms_4096"]) print("METRIC kv_tokens=%s" % (run.get("kv_tokens") or 0)) print("METRIC boot_seconds=%s" % (run.get("boot_seconds") or -1)) print("METRIC completion_tokens_4096=%s" % m.get("completion_tokens_4096", 0)) +# Any extra sampling temperature the measure swept (AR_TEMPS), e.g. the 0.7 +# arm — that is where speculative-decode acceptance shows up, since a greedy +# draft matches a greedy target exactly and is rejected more under real +# sampling. Suffixed keys only (_t0.7), so the historical lines are untouched. +for k in sorted(m): + if "_t" in k and (k.startswith("decode_tps") or k.startswith("ttft_ms")): + print("METRIC %s=%s" % (k, m[k])) ' "$1" } @@ -101,4 +111,7 @@ run_json="$(ssh_run "cat '$STAGE/run.json'")" || { echo "[autoresearch] window result:" >&2 printf '%s\n' "$run_json" >&2 emit_metrics "$run_json" + +echo "[autoresearch] engine evidence (plan / compile path / spec counters):" >&2 +ssh_run "cat '$STAGE/evidence.log' 2>/dev/null" >&2 || true exit 0 diff --git a/scripts/autoresearch/frank-primary-arms.sh b/scripts/autoresearch/frank-primary-arms.sh new file mode 100755 index 0000000..90caa3d --- /dev/null +++ b/scripts/autoresearch/frank-primary-arms.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# frank-primary-arms.sh — run a SEQUENCE of frank-primary bench arms and keep +# every arm's evidence. Runs on the operator box; each arm is one full window +# of ./autoresearch.sh (stop+mask -> boot the arm as the service -> measure -> +# restore production and verify), so the fleet is only ever down for one arm at +# a time and the production unit is back up between arms. +# +# An arm is the repo's own frank-primary.env plus an OVERLAY of engine knobs +# appended after it (later assignment wins when the serve script sources it), +# staged through FRANK_PRIMARY_AR_ENV_FILE so the checkout is never mutated and +# each arm gets its own fingerprint (the memo can never answer one arm with +# another arm's numbers). +# +# Usage: scripts/autoresearch/frank-primary-arms.sh [arm ...] +# scripts/autoresearch/frank-primary-arms.sh --list +# Output, under /tmp/frank-primary-arms-/ (override with ARMS_DIR): +# +# .out full autoresearch output (metrics + engine evidence) +# .env the exact staged env +# .rc window exit code +# summary.tsv arm, rc, decode_tps, decode_tps_t0.7, kv_tokens, boot_s +# +# Exit 0 when every arm produced metrics; 1 if any arm failed (a failed boot is +# a RESULT — the window still restored production, and the rc says which arm). +set -uo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd -- "$SCRIPT_DIR/../.." && pwd)" +BASE_ENV="$REPO/scripts/frank-primary.env" +ARMS_DIR="${ARMS_DIR:-/tmp/frank-primary-arms-$(date -u +%Y%m%dT%H%M%SZ)}" + +# --- arms ------------------------------------------------------------------ +# baseline: the shipped shape E, measured with THIS harness so the comparison is +# matched (the ~29.6 t/s figure in the docs is an older shape, and no shape-E +# decode number exists yet). +OVER_baseline=() +# B: MTP3 with the vendor's probabilistic draft sampling, a capture ladder whose +# step is num_speculative_tokens + 1 = 4, and the sampling override. Precise +# memory stays ON — this arm is simultaneously the MTP test and the test of +# whether MTP's drafter workspace and the fitted 5.25 GiB overhead can coexist. +SPEC3='{"method":"mtp","num_speculative_tokens":3,"draft_sample_method":"probabilistic"}' +LADDER3='{"cudagraph_capture_sizes":[4,8,12],"max_cudagraph_capture_size":12,"inductor_compile_config":{"combo_kernels":false,"benchmark_combo_kernel":false}}' +# The value must arrive in the staged env file SINGLE-QUOTED: sourcing an env +# file performs quote removal, so an unquoted {"temperature":0.7} would reach +# the engine as {temperature:0.7}. +GEN07='{"temperature":0.7}' +OVER_B=( + "FRANK_PRIMARY_SPEC='$SPEC3'" + "FRANK_PRIMARY_COMPCONFIG='$LADDER3'" + 'FRANK_PRIMARY_GEN_CONFIG='"'$GEN07'" +) +# B2: identical to B with the default (7.60 GiB) overhead, i.e. precise memory +# OFF. It is the fallback if B dies at graph capture, and it prices the arena +# that precise memory buys (7,522 slots -> ~6,496). +OVER_B2=( + "FRANK_PRIMARY_SPEC='$SPEC3'" + "FRANK_PRIMARY_COMPCONFIG='$LADDER3'" + 'FRANK_PRIMARY_GEN_CONFIG='"'$GEN07'" + 'FRANK_PRIMARY_PRECISE_MEMORY=' +) +# C: the vendor's concurrency shape — SEQS 16 with the [4..32] ladder and util +# 0.95. This is also the direct test of the theory that pinning the capture +# ladder unlocks SEQS > 3 (the SEQS=2 failure was query_start_loc specialised to +# the constant 3, which is NOT in the default ladder; 4 is). +LADDER16='{"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"max_cudagraph_capture_size":32,"inductor_compile_config":{"combo_kernels":false,"benchmark_combo_kernel":false}}' +OVER_C=( + "FRANK_PRIMARY_SPEC='$SPEC3'" + "FRANK_PRIMARY_COMPCONFIG='$LADDER16'" + 'FRANK_PRIMARY_GEN_CONFIG='"'$GEN07'" + 'FRANK_PRIMARY_SEQS=16' + 'FRANK_PRIMARY_MNBT=4096' + 'FRANK_PRIMARY_UTIL=0.95' +) + +arm_overlay() { + case "$1" in + baseline) ;; # the shipped env, no overlay + B) printf '%s\n' "${OVER_B[@]}" ;; + B2) printf '%s\n' "${OVER_B2[@]}" ;; + C) printf '%s\n' "${OVER_C[@]}" ;; + *) return 1 ;; + esac +} + +if [ "${1:-}" = "--list" ]; then + printf 'baseline B B2 C\n' + exit 0 +fi + +[ -f "$BASE_ENV" ] || { echo "[arms] missing $BASE_ENV" >&2; exit 1; } +if [ "$#" -gt 0 ]; then ARMS=("$@"); else ARMS=(baseline B B2 C); fi +mkdir -p "$ARMS_DIR" +echo "[arms] dir $ARMS_DIR arms: ${ARMS[*]}" >&2 + +: >"$ARMS_DIR/summary.tsv" +printf 'arm\trc\tdecode_tps\tdecode_tps_t0.7\tkv_tokens\tboot_s\n' >>"$ARMS_DIR/summary.tsv" +overall=0 + +for arm in "${ARMS[@]}"; do + if ! arm_overlay "$arm" >"$ARMS_DIR/$arm.overlay"; then + echo "[arms] unknown arm: $arm" >&2 + overall=1 + continue + fi + { cat "$BASE_ENV"; echo; echo "# --- arm $arm overlay ($(date -u +%FT%TZ)) ---" + arm_overlay "$arm"; } >"$ARMS_DIR/$arm.env" + # Gate the staged env BEFORE burning a ~20-minute window: every knob that + # holds JSON must still be one valid JSON object after the file is SOURCED + # (that is the step that silently eats unquoted braces). + if ! bash -c 'set -a; . "$1" >/dev/null; set +a + for k in FRANK_PRIMARY_SPEC FRANK_PRIMARY_COMPCONFIG FRANK_PRIMARY_GEN_CONFIG; do + v="${!k:-}" + [ -n "$v" ] || continue + case "$v" in + "{"*) python3 -c "import json,sys; json.loads(sys.argv[1])" "$v" \ + || { echo "[arms] $k is not valid JSON: $v" >&2; exit 1; } ;; + *) echo "[arms] $k is not a JSON object: $v" >&2; exit 1 ;; + esac + done' _ "$ARMS_DIR/$arm.env"; then + echo "[arms] $arm: staged env rejected, window not opened" >&2 + echo 99 >"$ARMS_DIR/$arm.rc" + overall=1 + continue + fi + + echo "[arms] === $arm: opening window $(date -u +%H:%M:%S)Z ===" >&2 + FRANK_PRIMARY_AR_ENV_FILE="$ARMS_DIR/$arm.env" \ + "$REPO/autoresearch.sh" >"$ARMS_DIR/$arm.out" 2>&1 + rc=$? + echo "$rc" >"$ARMS_DIR/$arm.rc" + echo "[arms] === $arm: rc=$rc $(date -u +%H:%M:%S)Z ===" >&2 + grep -E '^METRIC ' "$ARMS_DIR/$arm.out" | sed 's/^METRIC //' >"$ARMS_DIR/$arm.metrics" || true + metric() { local v; v="$(sed -n "s/^$1=//p" "$ARMS_DIR/$arm.metrics" | tail -1)"; printf '%s' "${v:-$2}"; } + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$arm" "$rc" \ + "$(metric decode_tps FAIL)" "$(metric decode_tps_t0.7 FAIL)" \ + "$(metric kv_tokens ?)" "$(metric boot_seconds ?)" >>"$ARMS_DIR/summary.tsv" + [ "$rc" -ne 0 ] && overall=1 +done + +echo "[arms] summary:" >&2 +cat "$ARMS_DIR/summary.tsv" >&2 +exit "$overall" diff --git a/scripts/autoresearch/frank-primary-measure.py b/scripts/autoresearch/frank-primary-measure.py index d476d02..f3d3b09 100755 --- a/scripts/autoresearch/frank-primary-measure.py +++ b/scripts/autoresearch/frank-primary-measure.py @@ -19,14 +19,18 @@ Workload (fixed, deterministic, offline): the server's own /v1/messages/count_tokens counter * depths 115 / 1056 / 4096 target prompt tokens * one 64-token warm pass per depth (warms the expert arena), then - max_tokens=256 at temperature 0, 3 runs, median -Everything is greedy with a fixed seed — the same workload every run, no -network, no time-of-day dependence. + max_tokens=256, 3 runs, median — repeated for every temperature in AR_TEMPS + (default 0 and 0.7). The prompt workload is deterministic (fixed seed, no + network, no time-of-day dependence); at temperature > 0 the generated text + is not, but the measured window is bounded by max_tokens, so the rate stays + comparable and the non-determinism is exactly the sampling that a + speculative-decode arm must survive. Env: AR_BASE endpoint (default http://127.0.0.1:7770) AR_MODEL served name (default frank-primary) AR_PROMPT_CEILING hard prompt ceiling in tokens (default 100000) + AR_TEMPS comma-separated sampling temperatures (default "0,0.7") Output: progress on stderr, ONE json object on stdout (the last line). Exit codes: 0 ok, 2 endpoint unhealthy / no usable completion, 3 bad usage. @@ -47,6 +51,14 @@ DEPTHS = (115, 1056, 4096) WARM_TOKENS = 64 MAX_TOKENS = 256 RUNS = 3 +# Temperature sweep. The unsuffixed keys are temperature 0 — byte-comparable to +# every historical measurement — and AR_TEMPS adds more. The second one matters +# for speculative decode: draft ACCEPTANCE is sampling-dependent (a greedy draft +# matches a greedy target exactly and is rejected more under a non-greedy +# target), so an MTP arm judged only at temp 0 would be judged on half the +# contract. Production traffic is non-zero temperature (the payload's +# generation_config says 1.0; an override to 0.7 is under test). +TEMPS = tuple(float(x) for x in os.environ.get("AR_TEMPS", "0,0.7").split(",")) # 43-word x 12 = 516-word vocabulary, seed 1234 — the campaign's exact list # and seed, so prompts are byte-identical to the historical measurements. @@ -105,11 +117,11 @@ def prompt_for(target_tokens, wpt): return text, pt -def measured(prompt, max_tokens): +def measured(prompt, max_tokens, temperature=0): """Return (ttft_s, decode_tok_s, completion_tokens) via usage counters.""" body = {"model": MODEL, "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, "temperature": 0, + "max_tokens": max_tokens, "temperature": temperature, "stream": True, "stream_options": {"include_usage": True}} req = urllib.request.Request( BASE + "/v1/chat/completions", data=json.dumps(body).encode(), @@ -145,8 +157,13 @@ def measured(prompt, max_tokens): return (ttft or wall), (ct / dec_window), ct +def fmt_temp(t): + return ("%.1f" % t).rstrip("0").rstrip(".") if t else "0" + + def main(): - log("base=%s model=%s ceiling=%d" % (BASE, MODEL, PROMPT_CEILING)) + log("base=%s model=%s ceiling=%d temps=%s" + % (BASE, MODEL, PROMPT_CEILING, ",".join(fmt_temp(t) for t in TEMPS))) try: wpt = calibrate() except Exception as e: @@ -158,32 +175,39 @@ def main(): for tgt in DEPTHS: try: p, pt = prompt_for(tgt, wpt) - measured(p, WARM_TOKENS) # warm the arena + measured(p, WARM_TOKENS) # warm the arena and the prefix cache except Exception as e: log("warm pass failed at depth %d: %s" % (tgt, e)) return 2 - rates, tts, cts = [], [], [] - for i in range(RUNS): - try: - tt, r, ct = measured(p, MAX_TOKENS) - except Exception as e: - log("run %d/%d failed at depth %d: %s" % (i + 1, RUNS, tgt, e)) - continue - rates.append(r) - tts.append(tt) - cts.append(ct) - if len(rates) < 2: - log("only %d usable run(s) at depth %d — refusing to median" % (len(rates), tgt)) - return 2 - result["decode_tps_%d" % tgt] = round(statistics.median(rates), 2) - result["ttft_ms_%d" % tgt] = round(1000.0 * statistics.median(tts), 1) - result["completion_tokens_%d" % tgt] = int(statistics.median(cts)) - result["prompt_tokens_%d" % tgt] = pt - log("depth %5d tok decode %.2f t/s ttft %.3f s (runs=%d)" - % (pt, result["decode_tps_%d" % tgt], result["ttft_ms_%d" % tgt] / 1000.0, len(rates))) + for temp in TEMPS: + suffix = "" if temp == 0 else "_t%s" % fmt_temp(temp) + rates, tts, cts = [], [], [] + for i in range(RUNS): + try: + tt, r, ct = measured(p, MAX_TOKENS, temp) + except Exception as e: + log("run %d/%d failed at depth %d temp %s: %s" + % (i + 1, RUNS, tgt, fmt_temp(temp), e)) + continue + rates.append(r) + tts.append(tt) + cts.append(ct) + if len(rates) < 2: + log("only %d usable run(s) at depth %d temp %s — refusing to median" + % (len(rates), tgt, fmt_temp(temp))) + return 2 + result["decode_tps_%d%s" % (tgt, suffix)] = round(statistics.median(rates), 2) + result["ttft_ms_%d%s" % (tgt, suffix)] = round(1000.0 * statistics.median(tts), 1) + result["completion_tokens_%d%s" % (tgt, suffix)] = int(statistics.median(cts)) + result["prompt_tokens_%d%s" % (tgt, suffix)] = pt + log("depth %5d tok temp %-4s decode %7.2f t/s ttft %.3f s (runs=%d)" + % (pt, fmt_temp(temp), result["decode_tps_%d%s" % (tgt, suffix)], + result["ttft_ms_%d%s" % (tgt, suffix)] / 1000.0, len(rates))) - meds = [result["decode_tps_%d" % t] for t in DEPTHS] - result["decode_tps"] = round(statistics.median(meds), 2) + for temp in TEMPS: + suffix = "" if temp == 0 else "_t%s" % fmt_temp(temp) + meds = [result["decode_tps_%d%s" % (t, suffix)] for t in DEPTHS] + result["decode_tps%s" % suffix] = round(statistics.median(meds), 2) print(json.dumps(result)) return 0 diff --git a/scripts/autoresearch/frank-primary-window.sh b/scripts/autoresearch/frank-primary-window.sh index a2936b9..4d9137e 100755 --- a/scripts/autoresearch/frank-primary-window.sh +++ b/scripts/autoresearch/frank-primary-window.sh @@ -118,7 +118,7 @@ main() { [ -f "$STAGE/$f" ] || { log "staged file missing: $STAGE/$f"; exit 1; } done mkdir -p "$STAGE/last" - rm -f "$STAGE/run.json" "$STAGE/fail.log" "$STAGE/measure.out" + rm -f "$STAGE/run.json" "$STAGE/fail.log" "$STAGE/measure.out" "$STAGE/evidence.log" local t_start t0 i kvline kv_tokens kv_conc measure_json boot_seconds t_start="$(date +%s)" @@ -177,6 +177,24 @@ main() { kv_tokens="$(printf '%s' "$kvline" | sed -E 's/.*GPU KV cache size: ([0-9,]+) tokens.*/\1/' | tr -d ',' || true)" kv_conc="$(printf '%s' "$kvline" | sed -E 's/.*per request: ([0-9.]+x).*/\1/' || true)" + # Human-facing evidence, in a sidecar file because it is multi-line prose, + # not metrics: the expert-plan lines (the ONLY place the arena / runtime + # overhead / KV-reserve numbers exist — they are never in /metrics), the + # compile path (cold build vs cache hit), the graph-capture sizes, the + # speculative-decode counters, and any OOM signature. Without this an arm + # that loses decode speed is unexplainable afterwards. + { + echo "=== engine plan / capacity / spec evidence ===" + podman logs "$CTR" 2>&1 | grep -E \ +'KV cache size|Maximum concurrency|layers split|expert cache|runtime overhead|non-default args|r4d all-reduce|Using \[|cudagraph|CUDA graph|Capturing|graph for compile range|torch.compile took|Directly load the compiled|speculative|Speculative|acceptance|Acceptance|offload|precise|OutOfMemory|Tried to allocate|Engine core initialization failed' \ + | tail -150 || true + echo "=== /metrics counters (spec decode, caches, success) ===" + curl -s --max-time 15 http://127.0.0.1:7770/metrics 2>/dev/null \ + | grep -E '^vllm:(spec_decode|gpu_cache|prefix_cache|request_success|generation_tokens|iteration)' \ + | head -80 || true + } >"$STAGE/evidence.log" 2>&1 || true + log "engine evidence -> $STAGE/evidence.log ($(wc -l <"$STAGE/evidence.log") lines)" + measure_json="$(grep -E '^\{' "$STAGE/measure.out" | tail -1 || true)" if [ -z "$measure_json" ]; then log "measure produced no JSON result line"