diff --git a/autoresearch.sh b/autoresearch.sh index 0e98e42..a5aea10 100755 --- a/autoresearch.sh +++ b/autoresearch.sh @@ -1,80 +1,104 @@ #!/usr/bin/env bash -# autoresearch.sh — canonical speed benchmark for frank-large (Dankclaw iGPU). +# autoresearch.sh — canonical benchmark entrypoint: speed of frank-primary. # -# Window semantics per the repo Benchmarking Policy: the deployed unit is -# STOPPED, the candidate (this worktree's scripts/frank-large.env + -# scripts/serve-frank-large.sh, rsynced to ~/jobs/autoresearch so the host's -# git checkout stays clean) boots standalone on :7770, the fixed greedy -# battery runs against it ON the host, and a trap restores the deployed -# baseline unit whatever happens. The candidate never touches the host -# checkout; "keep" in the loop means a real deploy later. +# Runs on the operator box and drives one bench window on DANKSERVER: +# 1. fingerprint the candidate trio (env + serve script + unit) +# 2. memo fast-path: an identical fingerprint that already produced a FULLY +# successful window (bench + verified production restore) returns the +# cached metrics without touching the fleet +# 3. stage the candidate trio plus the measure client into +# /home/dank/.autoresearch-frank-primary/ (ephemeral; the repo checkout +# on the host stays the production source of truth) +# 4. run the window (scripts/autoresearch/frank-primary-window.sh): +# stop + mask the production unit, boot the staged config AS THE SERVICE +# (same serve script, same container, same port 7770), wait for health, +# run the in-container usage-counter measure, then restore the production +# unit and verify its health — AGENTS.md Benchmarking Policy +# 5. print METRIC lines (parse by run_experiment) # -# Emits: METRIC decode_tokens_per_sec (primary, higher better), -# METRIC prefill_tokens_per_sec, METRIC ttft_ms, ASI identity lines. +# Primary metric: decode_tps (HIGHER is better) — median across the three +# prompt depths (115/1056/4096) of per-depth medians of +# completion_tokens / (wall - ttft), 3 runs each after a warm pass, +# temperature 0, the campaign's deterministic word-seed workload. +# +# Exit 0 with METRIC lines on success; non-zero on any failure (the window +# script's EXIT trap restores production on every path). set -euo pipefail -REPO="$(cd "$(dirname "$0")" && pwd)" -HOST=dankclaw -RDIR=/home/dank/jobs/autoresearch -HEALTH_TRIES=200 # x5s = ~17 min ceiling (BF16 boots ~230s; INT8/dequant may differ) +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +HOST="${FRANK_PRIMARY_AR_HOST:-dankserver}" +STAGE="/home/dank/.autoresearch-frank-primary" -restore() { - local rc=$? - trap - EXIT - echo "[harness] tearing down candidate, restoring deployed baseline..." >&2 - ssh "$HOST" 'sudo -n systemctl stop frank-large.service 2>/dev/null; - podman rm -f frank-large-mimo 2>/dev/null; - pkill -x llama-server 2>/dev/null; true' >/dev/null 2>&1 || true - ssh "$HOST" 'sudo -n systemctl start frank-large.service' >/dev/null 2>&1 || true - exit $rc +ENV_FILE="$SCRIPT_DIR/scripts/frank-primary.env" +SERVE="$SCRIPT_DIR/scripts/serve-frank-primary.sh" +UNIT="$SCRIPT_DIR/scripts/frank-primary.service" +MEASURE="$SCRIPT_DIR/scripts/autoresearch/frank-primary-measure.py" +WINDOW="$SCRIPT_DIR/scripts/autoresearch/frank-primary-window.sh" + +for f in "$ENV_FILE" "$SERVE" "$UNIT" "$MEASURE" "$WINDOW"; do + [ -f "$f" ] || { echo "[autoresearch] missing harness file: $f" >&2; exit 1; } +done +command -v python3 >/dev/null 2>&1 || { echo "[autoresearch] python3 required on operator box" >&2; exit 1; } + +# Fingerprint of everything that can change the engine's argv or environment. +HASH="$(cat "$ENV_FILE" "$SERVE" "$UNIT" | sha256sum | cut -d' ' -f1)" +echo "[autoresearch] candidate fingerprint $HASH" >&2 + +ssh_run() { ssh -o BatchMode=yes -o ConnectTimeout=15 "$HOST" "$@"; } + +emit_metrics() { # $1 = run.json + python3 -c ' +import json, sys +run = json.loads(sys.argv[1]) +m = run["measure"] +print("METRIC decode_tps=%s" % m["decode_tps"]) +for k in ("decode_tps_115", "decode_tps_1056", "decode_tps_4096"): + print("METRIC %s=%s" % (k, m[k])) +print("METRIC ttft_ms_4096=%s" % m["ttft_ms_4096"]) +print("METRIC kv_tokens=%s" % (run.get("kv_tokens") or 0)) +print("METRIC boot_seconds=%s" % (run.get("boot_seconds") or -1)) +print("METRIC completion_tokens_4096=%s" % m.get("completion_tokens_4096", 0)) +' "$1" } -trap restore EXIT -# 1. Ship the candidate (env + serve + battery) outside the git checkout. -ssh "$HOST" "mkdir -p $RDIR" -rsync -a "$REPO/scripts/frank-large.env" "$REPO/scripts/serve-frank-large.sh" \ - "$REPO/scripts/autoresearch-bench.py" "$HOST:$RDIR/" +# --- memo fast-path ------------------------------------------------------- +cached_hash="$(ssh_run "cat '$STAGE/last/hash' 2>/dev/null" 2>/dev/null || true)" +if [ -n "$cached_hash" ] && [ "$cached_hash" = "$HASH" ]; then + cached_run="$(ssh_run "cat '$STAGE/last/run.json' 2>/dev/null" || true)" + if [ -n "$cached_run" ]; then + echo "[autoresearch] memo hit — cached metrics for $HASH, fleet untouched" >&2 + emit_metrics "$cached_run" + exit 0 + fi +fi -# 2. Free the GPU and the port: stop the deployed unit (mask is unnecessary— -# the unit only starts via systemd, and we never enable more of it). -ssh "$HOST" 'sudo -n systemctl stop frank-large.service; - podman rm -f frank-large-mimo 2>/dev/null; - pkill -x llama-server 2>/dev/null; true' >/dev/null +# --- stage the candidate ---------------------------------------------------- +echo "[autoresearch] staging candidate on $HOST:$STAGE" >&2 +ssh_run "mkdir -p '$STAGE'" +scp -q "$ENV_FILE" "$HOST:$STAGE/frank-primary.env" +scp -q "$SERVE" "$HOST:$STAGE/serve.sh" +scp -q "$MEASURE" "$HOST:$STAGE/measure.py" +scp -q "$WINDOW" "$HOST:$STAGE/window.sh" +ssh_run "printf '%s\n' '$HASH' > '$STAGE/hash'" -# 3. Boot the candidate detached; the launcher's own output IS the serve -# stage output (gate failures print there before any container exists). -echo "[harness] starting candidate..." >&2 -ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \ - bash $RDIR/serve-frank-large.sh >$RDIR/serve.log 2>&1 &2 -grep -q ok /tmp/autoresearch-launch.log || { echo "ERROR: launch failed" >&2; exit 1; } - -# 4. Wait for health (503 while loading; anything else = boot failure). -echo "[harness] waiting for health..." >&2 -if ! ssh "$HOST" "for i in \$(seq 1 $HEALTH_TRIES); do - code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 \ - http://127.0.0.1:7770/health 2>/dev/null || echo 000) - [ \"\$code\" = 200 ] && echo UP && exit 0 - st=\$(podman inspect -f '{{.State.Status}}' frank-large-mimo 2>/dev/null || echo gone) - if [ "\$st" = exited ] || { [ "\$st" = gone ] && ! pgrep -f serve-frank-large >/dev/null && ! pgrep -x llama-server >/dev/null; }; then echo DOWN; exit 1; fi - sleep 5 - done; exit 1" | grep -q UP; then - echo "ERROR: candidate never became healthy; serve.log tail:" >&2 - ssh "$HOST" "tail -30 $RDIR/serve.log; echo '--- engine log ---'; - podman logs --tail 120 frank-large-mimo 2>&1 | - grep -aiE 'error|raise|failed|traceback|assert|kernel' | - tail -30" >&2 || true +# --- the window ------------------------------------------------------------- +# Budgets: bench boot up to 20 min (warm boots measured ~7 min), measure up +# to 30 min, restore boot up to 20 min — call run_experiment with +# timeout_seconds >= 3600. +echo "[autoresearch] opening bench window (stop+mask -> boot -> measure -> restore)" >&2 +set +e +ssh_run "bash '$STAGE/window.sh'" +rc=$? +set -e +if [ "$rc" -ne 0 ]; then + echo "[autoresearch] window failed (rc=$rc); host-side evidence:" >&2 + ssh_run "tail -60 '$STAGE/fail.log' 2>/dev/null || tail -40 '$STAGE/serve.out' 2>/dev/null || tail -40 '$STAGE/measure.out' 2>/dev/null || true" >&2 || true exit 1 fi -# 5. Identity evidence for the loop's records (exact engine argv). -ASI_ARGS="$(ssh "$HOST" 'podman inspect frank-large-mimo \ - --format "{{join .Config.Cmd \" \"}}" 2>/dev/null' | tr -d "\n" | head -c 400 || true)" -if [ -z "$ASI_ARGS" ]; then - ASI_ARGS="$(ssh "$HOST" "sh -c 'tr \"\\0\" \" \" < /proc/\\$(pgrep -x llama-server | head -1)/cmdline 2>/dev/null || true'" | head -c 400 || true)" -fi -echo "ASI engine_args=${ASI_ARGS:0:400}" - -# 6. The battery, on the host, medians over 3 reps x 2 greedy prompts. -ssh "$HOST" "python3 $RDIR/autoresearch-bench.py \ - --url http://127.0.0.1:7770 --model frank-large" +run_json="$(ssh_run "cat '$STAGE/run.json'")" || { + echo "[autoresearch] could not read $STAGE/run.json back" >&2; exit 1; } +echo "[autoresearch] window result:" >&2 +printf '%s\n' "$run_json" >&2 +emit_metrics "$run_json" +exit 0 diff --git a/scripts/autoresearch/frank-primary-measure.py b/scripts/autoresearch/frank-primary-measure.py new file mode 100755 index 0000000..d476d02 --- /dev/null +++ b/scripts/autoresearch/frank-primary-measure.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""frank-primary decode measurement — runs INSIDE the engine container. + +The method is the fleet's established one (the 2026-09-29 / 2026-10-01 +tuning campaigns, /home/dank/mtp_measure.py): SSE chunk counting is invalid +under speculative decoding because rejected drafts change the accounting, so +decode is measured from the server's own usage counters: + + decode_tok_s = completion_tokens / (wall - ttft) + +with ttft = wall time to the FIRST content-bearing chunk (thinking arrives in +`reasoning`/`delta.reasoning` on this stack and is excluded from the decode +window — and is deterministic per prompt at temperature 0, so the exclusion +cancels in any A/B on the same workload). + +Workload (fixed, deterministic, offline): + * prompts built from a large fixed-seed shuffled word list so tokens rarely + repeat (no n-gram/prefix predictability of the text itself), sized with + the server's own /v1/messages/count_tokens counter + * depths 115 / 1056 / 4096 target prompt tokens + * one 64-token warm pass per depth (warms the expert arena), then + max_tokens=256 at temperature 0, 3 runs, median +Everything is greedy with a fixed seed — the same workload every run, no +network, no time-of-day dependence. + +Env: + AR_BASE endpoint (default http://127.0.0.1:7770) + AR_MODEL served name (default frank-primary) + AR_PROMPT_CEILING hard prompt ceiling in tokens (default 100000) + +Output: progress on stderr, ONE json object on stdout (the last line). +Exit codes: 0 ok, 2 endpoint unhealthy / no usable completion, 3 bad usage. +""" +import json +import os +import random +import statistics +import sys +import time +import urllib.error +import urllib.request + +BASE = os.environ.get("AR_BASE", "http://127.0.0.1:7770") +MODEL = os.environ.get("AR_MODEL", "frank-primary") +PROMPT_CEILING = int(os.environ.get("AR_PROMPT_CEILING", "100000")) +DEPTHS = (115, 1056, 4096) +WARM_TOKENS = 64 +MAX_TOKENS = 256 +RUNS = 3 + +# 43-word x 12 = 516-word vocabulary, seed 1234 — the campaign's exact list +# and seed, so prompts are byte-identical to the historical measurements. +_VOCAB = ("alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima " + "mike november oscar papa quebec romeo sierra tango uniform victor whiskey " + "xray yankee zulu cardinal ordinal topology kernel scheduler resident " + "unified aperture expert arena slab offload prefetch gemm triton quant " + "tensor shard checkpoint manifest embed router cache page table window " + "gradient sparse dense moe attention head dim block stream decode prefill " + "bandwidth latency throughput residency eviction tier swap pin memlock").split() +_R = random.Random(1234) +_ORDER = list(range(len(_VOCAB))) +_R.shuffle(_ORDER) + + +def log(msg): + print("[measure] %s" % msg, file=sys.stderr, flush=True) + + +def post_json(path, body, timeout): + req = urllib.request.Request( + BASE + path, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.load(r) + + +def count_tokens(text): + """Token count without a prefill, from the server's own counter.""" + body = {"model": MODEL, + "messages": [{"role": "user", "content": text}]} + r = post_json("/v1/messages/count_tokens", body, timeout=300) + n = r.get("input_tokens") or r.get("count") or r.get("token_count") + if not n: + raise RuntimeError("count_tokens returned no count: %r" % (r,)) + return int(n) + + +def words(n): + return " ".join(_VOCAB[_ORDER[i % len(_VOCAB)]] for i in range(n)) + + +def calibrate(): + """Measure words->tokens from the server's own counter.""" + return 2000.0 / count_tokens(words(2000)) + + +def prompt_for(target_tokens, wpt): + n = max(1, int(target_tokens * wpt)) + text = words(n) + pt = count_tokens(text) + while pt > PROMPT_CEILING: + n = max(1, int(n * (PROMPT_CEILING / pt) * 0.98)) + text = words(n) + pt = count_tokens(text) + return text, pt + + +def measured(prompt, max_tokens): + """Return (ttft_s, decode_tok_s, completion_tokens) via usage counters.""" + body = {"model": MODEL, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, "temperature": 0, + "stream": True, "stream_options": {"include_usage": True}} + req = urllib.request.Request( + BASE + "/v1/chat/completions", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + t0 = time.time() + ttft = None + usage = {} + with urllib.request.urlopen(req, timeout=1800) as r: + for raw in r: + line = raw.decode("utf-8", "replace").strip() + if not line.startswith("data:"): + continue + s = line[5:].strip() + if s == "[DONE]": + break + try: + ev = json.loads(s) + except json.JSONDecodeError: + continue + if ev.get("usage"): + usage = ev["usage"] + for ch in ev.get("choices", []): + if (ch.get("delta") or {}).get("content"): + if ttft is None: + ttft = time.time() - t0 + wall = time.time() - t0 + ct = int(usage.get("completion_tokens") or 0) + if ct <= 0: + # No usage on the stream: the endpoint is unhealthy or the model + # returned nothing (e.g. HTTP 200 with an error payload). + raise RuntimeError("no completion_tokens in usage: %r" % (usage,)) + dec_window = max(wall - (ttft or 0.0), 1e-9) + return (ttft or wall), (ct / dec_window), ct + + +def main(): + log("base=%s model=%s ceiling=%d" % (BASE, MODEL, PROMPT_CEILING)) + try: + wpt = calibrate() + except Exception as e: + log("endpoint unhealthy during calibration: %s" % e) + return 2 + log("words/token: %.3f" % wpt) + + result = {"words_per_token": round(wpt, 5)} + for tgt in DEPTHS: + try: + p, pt = prompt_for(tgt, wpt) + measured(p, WARM_TOKENS) # warm the arena + except Exception as e: + log("warm pass failed at depth %d: %s" % (tgt, e)) + return 2 + rates, tts, cts = [], [], [] + for i in range(RUNS): + try: + tt, r, ct = measured(p, MAX_TOKENS) + except Exception as e: + log("run %d/%d failed at depth %d: %s" % (i + 1, RUNS, tgt, e)) + continue + rates.append(r) + tts.append(tt) + cts.append(ct) + if len(rates) < 2: + log("only %d usable run(s) at depth %d — refusing to median" % (len(rates), tgt)) + return 2 + result["decode_tps_%d" % tgt] = round(statistics.median(rates), 2) + result["ttft_ms_%d" % tgt] = round(1000.0 * statistics.median(tts), 1) + result["completion_tokens_%d" % tgt] = int(statistics.median(cts)) + result["prompt_tokens_%d" % tgt] = pt + log("depth %5d tok decode %.2f t/s ttft %.3f s (runs=%d)" + % (pt, result["decode_tps_%d" % tgt], result["ttft_ms_%d" % tgt] / 1000.0, len(rates))) + + meds = [result["decode_tps_%d" % t] for t in DEPTHS] + result["decode_tps"] = round(statistics.median(meds), 2) + print(json.dumps(result)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/autoresearch/frank-primary-window.sh b/scripts/autoresearch/frank-primary-window.sh new file mode 100755 index 0000000..a2936b9 --- /dev/null +++ b/scripts/autoresearch/frank-primary-window.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# frank-primary-window.sh — one full bench window for frank-primary, run ON +# DANKSERVER as the deploy user (invoked over ssh by ./autoresearch.sh). +# +# Discipline (AGENTS.md Benchmarking Policy): the production unit is +# STOPPED + MASKED first, the candidate config boots AS THE SERVICE itself +# (same serve script, same container name, same port 7770 — no scratch +# ports, no co-tenant GPU), the in-container measure runs, and the window +# ALWAYS ends by restoring the production unit and verifying its health. +# The EXIT trap makes the restore unconditional: a boot failure, a measure +# failure, or a dead ssh session all end with production back up. +# +# Staging layout (created by the orchestrator, ephemeral — never a config +# source; the repo checkout is): +# /home/dank/.autoresearch-frank-primary/ +# serve.sh staged candidate serve script +# frank-primary.env staged candidate env (the candidate knobs) +# measure.py the in-container measure client +# hash candidate fingerprint +# run.json result (metrics + boot + kv + fingerprint) +# serve.out/measure.out/fail.log +# last/ memo: run.json + hash of the last FULLY +# successful window (bench AND verified restore) +# +# This script logs to stderr and exits 0 only when the measure produced a +# usable result (the restore happens in the trap regardless of the bench +# outcome; a failed restore forces exit 1). +set -uo pipefail + +STAGE="${FRANK_PRIMARY_AR_STAGE:-/home/dank/.autoresearch-frank-primary}" +UNIT="${FRANK_PRIMARY_AR_UNIT:-frank-primary}" +CTR="${FRANK_PRIMARY_AR_CTR:-frank-primary-qwen}" +HEALTH_URL="http://127.0.0.1:7770/health" +MODELS_URL="http://127.0.0.1:7770/v1/models" +BOOT_TIMEOUT="${FRANK_PRIMARY_AR_BOOT_TIMEOUT:-1200}" +RESTORE_TIMEOUT="${FRANK_PRIMARY_AR_RESTORE_TIMEOUT:-1200}" +MEASURE_TIMEOUT="${FRANK_PRIMARY_AR_MEASURE_TIMEOUT:-1800}" +SERVED="$(sed -n 's/^FRANK_PRIMARY_SERVED_NAME=//p' "$STAGE/frank-primary.env" 2>/dev/null | head -1)" +SERVED="${SERVED:-frank-primary}" + +RESTORED=0 + +log() { printf '[window %s] %s\n' "$(date '+%H:%M:%S')" "$*" >&2; } + +container_state() { podman inspect -f '{{.State.Status}}' "$CTR" 2>/dev/null || echo absent; } + +wait_port_free() { + local i + for i in $(seq 1 90); do + [ "$(container_state)" != "running" ] && return 0 + sleep 2 + done + return 1 +} + +wait_health() { # $1 = timeout seconds + local t0 code + t0=$(date +%s) + while :; do + code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$HEALTH_URL" 2>/dev/null || echo 000)" + [ "$code" = "200" ] && return 0 + [ "$(( $(date +%s) - t0 ))" -ge "$1" ] && return 1 + sleep 2 + done +} + +capture_bench_failure() { + log "capturing bench-boot failure evidence" + { + echo "=== podman logs (tail 400) ===" + podman logs --tail 400 "$CTR" 2>&1 || true + echo "=== staged serve.out (tail 60) ===" + tail -60 "$STAGE/serve.out" 2>/dev/null || true + echo "=== engine log (tail 60) ===" + tail -60 /home/dank/logs/frank_primary.log 2>/dev/null || true + } >"$STAGE/fail.log" 2>&1 || true + grep -E 'OutOfMemory|Tried to allocate|Engine core initialization failed|Error|error' \ + "$STAGE/fail.log" 2>/dev/null | tail -25 >&2 || true +} + +restore() { + [ "$RESTORED" = 1 ] && return 0 + RESTORED=1 + log "restoring production unit $UNIT" + if [ -f "$STAGE/serve.pid" ]; then + kill "$(cat "$STAGE/serve.pid" 2>/dev/null)" 2>/dev/null || true + fi + podman stop -t 120 "$CTR" >/dev/null 2>&1 || true + wait_port_free || log "WARNING: bench container still draining after 180s" + sudo systemctl unmask "$UNIT" 2>/dev/null || true + if ! sudo systemctl start "$UNIT"; then + log "ERROR: systemctl start $UNIT failed — operator intervention required" + return 1 + fi + if wait_health "$RESTORE_TIMEOUT"; then + if curl -s --max-time 10 "$MODELS_URL" 2>/dev/null | grep -q "$SERVED"; then + log "production restored and verified ($SERVED answering on :7770)" + # Memo promotion lives HERE: a cached result is only ever the + # product of a fully successful window (bench AND verified + # restore), never of a bare bench success. + cp "$STAGE/run.json" "$STAGE/last/" 2>/dev/null || true + cp "$STAGE/hash" "$STAGE/last/" 2>/dev/null || true + else + log "ERROR: health 200 but /v1/models does not list $SERVED" + return 1 + fi + else + log "ERROR: production not healthy within ${RESTORE_TIMEOUT}s — unit left to systemd (Restart=always)" + return 1 + fi + return 0 +} +trap 'restore; rc=$?; [ "$rc" -ne 0 ] && exit 1' EXIT + +main() { + local f + for f in serve.sh frank-primary.env measure.py hash; do + [ -f "$STAGE/$f" ] || { log "staged file missing: $STAGE/$f"; exit 1; } + done + mkdir -p "$STAGE/last" + rm -f "$STAGE/run.json" "$STAGE/fail.log" "$STAGE/measure.out" + + local t_start t0 i kvline kv_tokens kv_conc measure_json boot_seconds + t_start="$(date +%s)" + + log "stopping and masking production unit $UNIT" + sudo systemctl stop "$UNIT" 2>/dev/null || true + # Explicit stop is not auto-restarted even with Restart=always; the mask + # is belt-and-braces against a stray start during the window. + sudo systemctl mask "$UNIT" 2>/dev/null || \ + log "WARNING: mask failed — relying on explicit-stop semantics" + wait_port_free || log "note: production container still draining" + + t0="$(date +%s)" + log "starting bench instance from staged config (fingerprint $(tr -d '[:space:]' <"$STAGE/hash"))" + FRANK_PRIMARY_ENV_FILE="$STAGE/frank-primary.env" \ + setsid nohup bash "$STAGE/serve.sh" >"$STAGE/serve.out" 2>&1 "$STAGE/serve.pid" + + for i in $(seq 1 45); do + [ "$(container_state)" = "running" ] && break + sleep 2 + done + if [ "$(container_state)" != "running" ]; then + log "bench container never reached running state" + tail -40 "$STAGE/serve.out" >&2 || true + exit 1 + fi + + if wait_health "$BOOT_TIMEOUT"; then + boot_seconds=$(( $(date +%s) - t0 )) + log "bench healthy after ${boot_seconds}s" + else + if [ "$(container_state)" = "running" ]; then + log "bench boot timed out after ${BOOT_TIMEOUT}s (still loading)" + else + log "bench container died during boot" + capture_bench_failure + fi + exit 1 + fi + + log "running measure in container (timeout ${MEASURE_TIMEOUT}s)" + if ! podman cp "$STAGE/measure.py" "$CTR":/tmp/ar-measure.py; then + log "podman cp of measure client failed" + exit 1 + fi + if ! timeout "$MEASURE_TIMEOUT" podman exec "$CTR" python3 /tmp/ar-measure.py \ + >"$STAGE/measure.out" 2>&1; then + log "measure failed or timed out" + tail -40 "$STAGE/measure.out" >&2 || true + exit 1 + fi + + # Capacity facts for the run (context-size experiments change these). + kvline="$(podman logs "$CTR" 2>&1 | grep -E 'GPU KV cache size' | tail -1 || true)" + kv_tokens="$(printf '%s' "$kvline" | sed -E 's/.*GPU KV cache size: ([0-9,]+) tokens.*/\1/' | tr -d ',' || true)" + kv_conc="$(printf '%s' "$kvline" | sed -E 's/.*per request: ([0-9.]+x).*/\1/' || true)" + + measure_json="$(grep -E '^\{' "$STAGE/measure.out" | tail -1 || true)" + if [ -z "$measure_json" ]; then + log "measure produced no JSON result line" + tail -40 "$STAGE/measure.out" >&2 || true + exit 1 + fi + + # The measure line is already JSON, so the run object can be assembled + # without any host-side interpreter. + printf '{"hash":"%s","boot_seconds":%s,"kv_tokens":%s,"kv_concurrency":"%s","total_seconds":%s,"measure":%s}\n' \ + "$(tr -d '[:space:]' <"$STAGE/hash")" \ + "$boot_seconds" "${kv_tokens:-0}" "${kv_conc:-unknown}" \ + "$(( $(date +%s) - t_start ))" "$measure_json" >"$STAGE/run.json" + + log "bench window complete in $(( $(date +%s) - t_start ))s" + # Memo only after a FULL success; the restore below still has to verify. + exit 0 +} + +main "$@" +exit "$?" diff --git a/tests/autoresearch_measure.bats b/tests/autoresearch_measure.bats new file mode 100755 index 0000000..1a7afa9 --- /dev/null +++ b/tests/autoresearch_measure.bats @@ -0,0 +1,150 @@ +#!/usr/bin/env bats +# Seam: autoresearch measure client behavior. +# +# The measure client is the whole yardstick of the frank-primary speed +# session, so its METRIC arithmetic gets a behavior test: against a +# deterministic mock endpoint that emits a KNOWN number of completion +# tokens at a KNOWN per-token delay, the reported decode rate must land +# near the nominal rate, the usage accounting must come back exact, and an +# unhealthy endpoint must fail loudly (no half-measured numbers). +# +# The mock implements only what the client calls: POST +# /v1/messages/count_tokens (word count as the token count, which makes the +# client's words/token calibration come out to exactly 1.0) and a streaming +# POST /v1/chat/completions (one content chunk per completion token, then a +# usage chunk, then [DONE]). + +load helpers + +MEASURE="$REPO_ROOT/scripts/autoresearch/frank-primary-measure.py" +MOCK="$BATS_TEST_TMPDIR/mock.py" +PORT="" +MOCK_PID="" + +setup() { + # Deterministic mock; argv: ; env MOCK_FAIL=1 + # makes the chat endpoint answer 500. + cat >"$MOCK" <<'PYEOF' +import json, os, sys, time +from http.server import BaseHTTPRequestHandler, HTTPServer + +PORT = int(sys.argv[1]) +DELAY = float(sys.argv[2]) +FAIL = os.environ.get("MOCK_FAIL", "0") == "1" + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def _send(self, obj, code=200): + b = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(b))) + self.end_headers() + self.wfile.write(b) + + def do_POST(self): + n = int(self.headers.get("Content-Length", "0")) + body = json.loads(self.rfile.read(n) or b"{}") + if self.path == "/v1/messages/count_tokens": + text = body["messages"][0]["content"] + self._send({"input_tokens": max(1, len(text.split()))}) + elif self.path == "/v1/chat/completions": + if FAIL: + self._send({"error": "engine down"}, 500) + return + max_tokens = int(body.get("max_tokens", 64)) + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + for i in range(max_tokens): + ev = {"choices": [{"delta": {"content": "tok%d " % i}}]} + self.wfile.write(("data: %s\n\n" % json.dumps(ev)).encode()) + self.wfile.flush() + time.sleep(DELAY) + prompt_words = max(1, len(body["messages"][0]["content"].split())) + ev = {"choices": [{"delta": {}}], + "usage": {"prompt_tokens": prompt_words, + "completion_tokens": max_tokens}} + self.wfile.write(("data: %s\n\n" % json.dumps(ev)).encode()) + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + else: + self._send({"error": "unknown path"}, 404) + + +HTTPServer(("127.0.0.1", PORT), Handler).serve_forever() +PYEOF +} +free_port() { + python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()' +} + +start_mock() { # $1 = ok|fail + PORT="$(free_port)" + if [ "$1" = "fail" ]; then + MOCK_FAIL=1 python3 "$MOCK" "$PORT" 0.005 & + else + python3 "$MOCK" "$PORT" 0.005 & + fi + MOCK_PID=$! + local i + for i in $(seq 1 100); do + if python3 -c "import socket; socket.create_connection(('127.0.0.1', $PORT), 0.2).close()" 2>/dev/null; then + return 0 + fi + sleep 0.1 + done + echo "mock did not start" >&2 + return 1 +} + +stop_mock() { + if [ -n "$MOCK_PID" ]; then + kill "$MOCK_PID" 2>/dev/null || true + wait "$MOCK_PID" 2>/dev/null || true + MOCK_PID="" + fi +} + +@test "measure exists and compiles" { + [ -f "$MEASURE" ] + python3 -m py_compile "$MEASURE" +} + +@test "measure computes decode rate and exact usage from a controlled endpoint" { + start_mock ok || return 1 + run env AR_BASE="http://127.0.0.1:$PORT" AR_MODEL=mock python3 "$MEASURE" + stop_mock + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + # run() merges stderr in; the contract is: exactly ONE json result line + # (the window script parses the same way), progress on stderr. + printf '%s\n' "$output" | python3 -c ' +import json, sys +lines = [l for l in sys.stdin.read().splitlines() if l.startswith("{")] +assert len(lines) == 1, "expected exactly one JSON line, got %d" % len(lines) +d = json.loads(lines[0]) +# nominal 200 t/s (0.005 s/chunk) minus small per-chunk overhead +for k in ("decode_tps", "decode_tps_115", "decode_tps_1056", "decode_tps_4096"): + v = d[k] + assert 60.0 <= v <= 320.0, "%s=%s outside [60, 320]" % (k, v) +# the mock emits exactly max_tokens completion tokens; the accounting must be exact +assert d["completion_tokens_4096"] == 256, d["completion_tokens_4096"] +assert d["completion_tokens_115"] == 256 +# prompt sizing: calibration is exactly 1.0 words/token in the mock, so the +# measured prompt lengths equal the targets +assert d["prompt_tokens_115"] == 115 +assert d["prompt_tokens_4096"] == 4096 +print("measure claims verified") +' +} + +@test "measure fails loudly on an unhealthy endpoint instead of half-measuring" { + start_mock fail || return 1 + run env AR_BASE="http://127.0.0.1:$PORT" AR_MODEL=mock python3 "$MEASURE" + stop_mock + [ "$status" -ne 0 ] + ! printf '%s\n' "$output" | grep -qE '^\{' +}