autoresearch: harness setup
Benchmark entrypoint: bash autoresearch.sh Goal: Make frank-primary faster: raise decode tok/s on the tcclaviger vLLM endpoint (dankserver :7770, Qwen3.8-Flash-Next-Uncensored-FP8, 2x R9700, automatic tiered-expert offload). Context size may be altered (MAXLEN/SEQS/KV_BYTES and friends); quantization may NOT be altered (the 131-shard FP8 payload and the digest-pinned image are frozen).
This commit is contained in:
Frank
+91
-67
@@ -1,80 +1,104 @@
|
||||
#!/usr/bin/env bash
|
||||
# autoresearch.sh — canonical speed benchmark for frank-large (Dankclaw iGPU).
|
||||
# autoresearch.sh — canonical benchmark entrypoint: speed of frank-primary.
|
||||
#
|
||||
# Window semantics per the repo Benchmarking Policy: the deployed unit is
|
||||
# STOPPED, the candidate (this worktree's scripts/frank-large.env +
|
||||
# scripts/serve-frank-large.sh, rsynced to ~/jobs/autoresearch so the host's
|
||||
# git checkout stays clean) boots standalone on :7770, the fixed greedy
|
||||
# battery runs against it ON the host, and a trap restores the deployed
|
||||
# baseline unit whatever happens. The candidate never touches the host
|
||||
# checkout; "keep" in the loop means a real deploy later.
|
||||
# Runs on the operator box and drives one bench window on DANKSERVER:
|
||||
# 1. fingerprint the candidate trio (env + serve script + unit)
|
||||
# 2. memo fast-path: an identical fingerprint that already produced a FULLY
|
||||
# successful window (bench + verified production restore) returns the
|
||||
# cached metrics without touching the fleet
|
||||
# 3. stage the candidate trio plus the measure client into
|
||||
# /home/dank/.autoresearch-frank-primary/ (ephemeral; the repo checkout
|
||||
# on the host stays the production source of truth)
|
||||
# 4. run the window (scripts/autoresearch/frank-primary-window.sh):
|
||||
# stop + mask the production unit, boot the staged config AS THE SERVICE
|
||||
# (same serve script, same container, same port 7770), wait for health,
|
||||
# run the in-container usage-counter measure, then restore the production
|
||||
# unit and verify its health — AGENTS.md Benchmarking Policy
|
||||
# 5. print METRIC lines (parse by run_experiment)
|
||||
#
|
||||
# Emits: METRIC decode_tokens_per_sec (primary, higher better),
|
||||
# METRIC prefill_tokens_per_sec, METRIC ttft_ms, ASI identity lines.
|
||||
# Primary metric: decode_tps (HIGHER is better) — median across the three
|
||||
# prompt depths (115/1056/4096) of per-depth medians of
|
||||
# completion_tokens / (wall - ttft), 3 runs each after a warm pass,
|
||||
# temperature 0, the campaign's deterministic word-seed workload.
|
||||
#
|
||||
# Exit 0 with METRIC lines on success; non-zero on any failure (the window
|
||||
# script's EXIT trap restores production on every path).
|
||||
set -euo pipefail
|
||||
|
||||
REPO="$(cd "$(dirname "$0")" && pwd)"
|
||||
HOST=dankclaw
|
||||
RDIR=/home/dank/jobs/autoresearch
|
||||
HEALTH_TRIES=200 # x5s = ~17 min ceiling (BF16 boots ~230s; INT8/dequant may differ)
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
HOST="${FRANK_PRIMARY_AR_HOST:-dankserver}"
|
||||
STAGE="/home/dank/.autoresearch-frank-primary"
|
||||
|
||||
restore() {
|
||||
local rc=$?
|
||||
trap - EXIT
|
||||
echo "[harness] tearing down candidate, restoring deployed baseline..." >&2
|
||||
ssh "$HOST" 'sudo -n systemctl stop frank-large.service 2>/dev/null;
|
||||
podman rm -f frank-large-mimo 2>/dev/null;
|
||||
pkill -x llama-server 2>/dev/null; true' >/dev/null 2>&1 || true
|
||||
ssh "$HOST" 'sudo -n systemctl start frank-large.service' >/dev/null 2>&1 || true
|
||||
exit $rc
|
||||
ENV_FILE="$SCRIPT_DIR/scripts/frank-primary.env"
|
||||
SERVE="$SCRIPT_DIR/scripts/serve-frank-primary.sh"
|
||||
UNIT="$SCRIPT_DIR/scripts/frank-primary.service"
|
||||
MEASURE="$SCRIPT_DIR/scripts/autoresearch/frank-primary-measure.py"
|
||||
WINDOW="$SCRIPT_DIR/scripts/autoresearch/frank-primary-window.sh"
|
||||
|
||||
for f in "$ENV_FILE" "$SERVE" "$UNIT" "$MEASURE" "$WINDOW"; do
|
||||
[ -f "$f" ] || { echo "[autoresearch] missing harness file: $f" >&2; exit 1; }
|
||||
done
|
||||
command -v python3 >/dev/null 2>&1 || { echo "[autoresearch] python3 required on operator box" >&2; exit 1; }
|
||||
|
||||
# Fingerprint of everything that can change the engine's argv or environment.
|
||||
HASH="$(cat "$ENV_FILE" "$SERVE" "$UNIT" | sha256sum | cut -d' ' -f1)"
|
||||
echo "[autoresearch] candidate fingerprint $HASH" >&2
|
||||
|
||||
ssh_run() { ssh -o BatchMode=yes -o ConnectTimeout=15 "$HOST" "$@"; }
|
||||
|
||||
emit_metrics() { # $1 = run.json
|
||||
python3 -c '
|
||||
import json, sys
|
||||
run = json.loads(sys.argv[1])
|
||||
m = run["measure"]
|
||||
print("METRIC decode_tps=%s" % m["decode_tps"])
|
||||
for k in ("decode_tps_115", "decode_tps_1056", "decode_tps_4096"):
|
||||
print("METRIC %s=%s" % (k, m[k]))
|
||||
print("METRIC ttft_ms_4096=%s" % m["ttft_ms_4096"])
|
||||
print("METRIC kv_tokens=%s" % (run.get("kv_tokens") or 0))
|
||||
print("METRIC boot_seconds=%s" % (run.get("boot_seconds") or -1))
|
||||
print("METRIC completion_tokens_4096=%s" % m.get("completion_tokens_4096", 0))
|
||||
' "$1"
|
||||
}
|
||||
trap restore EXIT
|
||||
|
||||
# 1. Ship the candidate (env + serve + battery) outside the git checkout.
|
||||
ssh "$HOST" "mkdir -p $RDIR"
|
||||
rsync -a "$REPO/scripts/frank-large.env" "$REPO/scripts/serve-frank-large.sh" \
|
||||
"$REPO/scripts/autoresearch-bench.py" "$HOST:$RDIR/"
|
||||
# --- memo fast-path -------------------------------------------------------
|
||||
cached_hash="$(ssh_run "cat '$STAGE/last/hash' 2>/dev/null" 2>/dev/null || true)"
|
||||
if [ -n "$cached_hash" ] && [ "$cached_hash" = "$HASH" ]; then
|
||||
cached_run="$(ssh_run "cat '$STAGE/last/run.json' 2>/dev/null" || true)"
|
||||
if [ -n "$cached_run" ]; then
|
||||
echo "[autoresearch] memo hit — cached metrics for $HASH, fleet untouched" >&2
|
||||
emit_metrics "$cached_run"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# 2. Free the GPU and the port: stop the deployed unit (mask is unnecessary—
|
||||
# the unit only starts via systemd, and we never enable more of it).
|
||||
ssh "$HOST" 'sudo -n systemctl stop frank-large.service;
|
||||
podman rm -f frank-large-mimo 2>/dev/null;
|
||||
pkill -x llama-server 2>/dev/null; true' >/dev/null
|
||||
# --- stage the candidate ----------------------------------------------------
|
||||
echo "[autoresearch] staging candidate on $HOST:$STAGE" >&2
|
||||
ssh_run "mkdir -p '$STAGE'"
|
||||
scp -q "$ENV_FILE" "$HOST:$STAGE/frank-primary.env"
|
||||
scp -q "$SERVE" "$HOST:$STAGE/serve.sh"
|
||||
scp -q "$MEASURE" "$HOST:$STAGE/measure.py"
|
||||
scp -q "$WINDOW" "$HOST:$STAGE/window.sh"
|
||||
ssh_run "printf '%s\n' '$HASH' > '$STAGE/hash'"
|
||||
|
||||
# 3. Boot the candidate detached; the launcher's own output IS the serve
|
||||
# stage output (gate failures print there before any container exists).
|
||||
echo "[harness] starting candidate..." >&2
|
||||
ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \
|
||||
bash $RDIR/serve-frank-large.sh >$RDIR/serve.log 2>&1 </dev/null & echo ok" \
|
||||
| tee /tmp/autoresearch-launch.log >&2
|
||||
grep -q ok /tmp/autoresearch-launch.log || { echo "ERROR: launch failed" >&2; exit 1; }
|
||||
|
||||
# 4. Wait for health (503 while loading; anything else = boot failure).
|
||||
echo "[harness] waiting for health..." >&2
|
||||
if ! ssh "$HOST" "for i in \$(seq 1 $HEALTH_TRIES); do
|
||||
code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 \
|
||||
http://127.0.0.1:7770/health 2>/dev/null || echo 000)
|
||||
[ \"\$code\" = 200 ] && echo UP && exit 0
|
||||
st=\$(podman inspect -f '{{.State.Status}}' frank-large-mimo 2>/dev/null || echo gone)
|
||||
if [ "\$st" = exited ] || { [ "\$st" = gone ] && ! pgrep -f serve-frank-large >/dev/null && ! pgrep -x llama-server >/dev/null; }; then echo DOWN; exit 1; fi
|
||||
sleep 5
|
||||
done; exit 1" | grep -q UP; then
|
||||
echo "ERROR: candidate never became healthy; serve.log tail:" >&2
|
||||
ssh "$HOST" "tail -30 $RDIR/serve.log; echo '--- engine log ---';
|
||||
podman logs --tail 120 frank-large-mimo 2>&1 |
|
||||
grep -aiE 'error|raise|failed|traceback|assert|kernel' |
|
||||
tail -30" >&2 || true
|
||||
# --- the window -------------------------------------------------------------
|
||||
# Budgets: bench boot up to 20 min (warm boots measured ~7 min), measure up
|
||||
# to 30 min, restore boot up to 20 min — call run_experiment with
|
||||
# timeout_seconds >= 3600.
|
||||
echo "[autoresearch] opening bench window (stop+mask -> boot -> measure -> restore)" >&2
|
||||
set +e
|
||||
ssh_run "bash '$STAGE/window.sh'"
|
||||
rc=$?
|
||||
set -e
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "[autoresearch] window failed (rc=$rc); host-side evidence:" >&2
|
||||
ssh_run "tail -60 '$STAGE/fail.log' 2>/dev/null || tail -40 '$STAGE/serve.out' 2>/dev/null || tail -40 '$STAGE/measure.out' 2>/dev/null || true" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 5. Identity evidence for the loop's records (exact engine argv).
|
||||
ASI_ARGS="$(ssh "$HOST" 'podman inspect frank-large-mimo \
|
||||
--format "{{join .Config.Cmd \" \"}}" 2>/dev/null' | tr -d "\n" | head -c 400 || true)"
|
||||
if [ -z "$ASI_ARGS" ]; then
|
||||
ASI_ARGS="$(ssh "$HOST" "sh -c 'tr \"\\0\" \" \" < /proc/\\$(pgrep -x llama-server | head -1)/cmdline 2>/dev/null || true'" | head -c 400 || true)"
|
||||
fi
|
||||
echo "ASI engine_args=${ASI_ARGS:0:400}"
|
||||
|
||||
# 6. The battery, on the host, medians over 3 reps x 2 greedy prompts.
|
||||
ssh "$HOST" "python3 $RDIR/autoresearch-bench.py \
|
||||
--url http://127.0.0.1:7770 --model frank-large"
|
||||
run_json="$(ssh_run "cat '$STAGE/run.json'")" || {
|
||||
echo "[autoresearch] could not read $STAGE/run.json back" >&2; exit 1; }
|
||||
echo "[autoresearch] window result:" >&2
|
||||
printf '%s\n' "$run_json" >&2
|
||||
emit_metrics "$run_json"
|
||||
exit 0
|
||||
|
||||
+192
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
"""frank-primary decode measurement — runs INSIDE the engine container.
|
||||
|
||||
The method is the fleet's established one (the 2026-09-29 / 2026-10-01
|
||||
tuning campaigns, /home/dank/mtp_measure.py): SSE chunk counting is invalid
|
||||
under speculative decoding because rejected drafts change the accounting, so
|
||||
decode is measured from the server's own usage counters:
|
||||
|
||||
decode_tok_s = completion_tokens / (wall - ttft)
|
||||
|
||||
with ttft = wall time to the FIRST content-bearing chunk (thinking arrives in
|
||||
`reasoning`/`delta.reasoning` on this stack and is excluded from the decode
|
||||
window — and is deterministic per prompt at temperature 0, so the exclusion
|
||||
cancels in any A/B on the same workload).
|
||||
|
||||
Workload (fixed, deterministic, offline):
|
||||
* prompts built from a large fixed-seed shuffled word list so tokens rarely
|
||||
repeat (no n-gram/prefix predictability of the text itself), sized with
|
||||
the server's own /v1/messages/count_tokens counter
|
||||
* depths 115 / 1056 / 4096 target prompt tokens
|
||||
* one 64-token warm pass per depth (warms the expert arena), then
|
||||
max_tokens=256 at temperature 0, 3 runs, median
|
||||
Everything is greedy with a fixed seed — the same workload every run, no
|
||||
network, no time-of-day dependence.
|
||||
|
||||
Env:
|
||||
AR_BASE endpoint (default http://127.0.0.1:7770)
|
||||
AR_MODEL served name (default frank-primary)
|
||||
AR_PROMPT_CEILING hard prompt ceiling in tokens (default 100000)
|
||||
|
||||
Output: progress on stderr, ONE json object on stdout (the last line).
|
||||
Exit codes: 0 ok, 2 endpoint unhealthy / no usable completion, 3 bad usage.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
BASE = os.environ.get("AR_BASE", "http://127.0.0.1:7770")
|
||||
MODEL = os.environ.get("AR_MODEL", "frank-primary")
|
||||
PROMPT_CEILING = int(os.environ.get("AR_PROMPT_CEILING", "100000"))
|
||||
DEPTHS = (115, 1056, 4096)
|
||||
WARM_TOKENS = 64
|
||||
MAX_TOKENS = 256
|
||||
RUNS = 3
|
||||
|
||||
# 43-word x 12 = 516-word vocabulary, seed 1234 — the campaign's exact list
|
||||
# and seed, so prompts are byte-identical to the historical measurements.
|
||||
_VOCAB = ("alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima "
|
||||
"mike november oscar papa quebec romeo sierra tango uniform victor whiskey "
|
||||
"xray yankee zulu cardinal ordinal topology kernel scheduler resident "
|
||||
"unified aperture expert arena slab offload prefetch gemm triton quant "
|
||||
"tensor shard checkpoint manifest embed router cache page table window "
|
||||
"gradient sparse dense moe attention head dim block stream decode prefill "
|
||||
"bandwidth latency throughput residency eviction tier swap pin memlock").split()
|
||||
_R = random.Random(1234)
|
||||
_ORDER = list(range(len(_VOCAB)))
|
||||
_R.shuffle(_ORDER)
|
||||
|
||||
|
||||
def log(msg):
|
||||
print("[measure] %s" % msg, file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def post_json(path, body, timeout):
|
||||
req = urllib.request.Request(
|
||||
BASE + path, data=json.dumps(body).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return json.load(r)
|
||||
|
||||
|
||||
def count_tokens(text):
|
||||
"""Token count without a prefill, from the server's own counter."""
|
||||
body = {"model": MODEL,
|
||||
"messages": [{"role": "user", "content": text}]}
|
||||
r = post_json("/v1/messages/count_tokens", body, timeout=300)
|
||||
n = r.get("input_tokens") or r.get("count") or r.get("token_count")
|
||||
if not n:
|
||||
raise RuntimeError("count_tokens returned no count: %r" % (r,))
|
||||
return int(n)
|
||||
|
||||
|
||||
def words(n):
|
||||
return " ".join(_VOCAB[_ORDER[i % len(_VOCAB)]] for i in range(n))
|
||||
|
||||
|
||||
def calibrate():
|
||||
"""Measure words->tokens from the server's own counter."""
|
||||
return 2000.0 / count_tokens(words(2000))
|
||||
|
||||
|
||||
def prompt_for(target_tokens, wpt):
|
||||
n = max(1, int(target_tokens * wpt))
|
||||
text = words(n)
|
||||
pt = count_tokens(text)
|
||||
while pt > PROMPT_CEILING:
|
||||
n = max(1, int(n * (PROMPT_CEILING / pt) * 0.98))
|
||||
text = words(n)
|
||||
pt = count_tokens(text)
|
||||
return text, pt
|
||||
|
||||
|
||||
def measured(prompt, max_tokens):
|
||||
"""Return (ttft_s, decode_tok_s, completion_tokens) via usage counters."""
|
||||
body = {"model": MODEL,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens, "temperature": 0,
|
||||
"stream": True, "stream_options": {"include_usage": True}}
|
||||
req = urllib.request.Request(
|
||||
BASE + "/v1/chat/completions", data=json.dumps(body).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
t0 = time.time()
|
||||
ttft = None
|
||||
usage = {}
|
||||
with urllib.request.urlopen(req, timeout=1800) as r:
|
||||
for raw in r:
|
||||
line = raw.decode("utf-8", "replace").strip()
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
s = line[5:].strip()
|
||||
if s == "[DONE]":
|
||||
break
|
||||
try:
|
||||
ev = json.loads(s)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if ev.get("usage"):
|
||||
usage = ev["usage"]
|
||||
for ch in ev.get("choices", []):
|
||||
if (ch.get("delta") or {}).get("content"):
|
||||
if ttft is None:
|
||||
ttft = time.time() - t0
|
||||
wall = time.time() - t0
|
||||
ct = int(usage.get("completion_tokens") or 0)
|
||||
if ct <= 0:
|
||||
# No usage on the stream: the endpoint is unhealthy or the model
|
||||
# returned nothing (e.g. HTTP 200 with an error payload).
|
||||
raise RuntimeError("no completion_tokens in usage: %r" % (usage,))
|
||||
dec_window = max(wall - (ttft or 0.0), 1e-9)
|
||||
return (ttft or wall), (ct / dec_window), ct
|
||||
|
||||
|
||||
def main():
|
||||
log("base=%s model=%s ceiling=%d" % (BASE, MODEL, PROMPT_CEILING))
|
||||
try:
|
||||
wpt = calibrate()
|
||||
except Exception as e:
|
||||
log("endpoint unhealthy during calibration: %s" % e)
|
||||
return 2
|
||||
log("words/token: %.3f" % wpt)
|
||||
|
||||
result = {"words_per_token": round(wpt, 5)}
|
||||
for tgt in DEPTHS:
|
||||
try:
|
||||
p, pt = prompt_for(tgt, wpt)
|
||||
measured(p, WARM_TOKENS) # warm the arena
|
||||
except Exception as e:
|
||||
log("warm pass failed at depth %d: %s" % (tgt, e))
|
||||
return 2
|
||||
rates, tts, cts = [], [], []
|
||||
for i in range(RUNS):
|
||||
try:
|
||||
tt, r, ct = measured(p, MAX_TOKENS)
|
||||
except Exception as e:
|
||||
log("run %d/%d failed at depth %d: %s" % (i + 1, RUNS, tgt, e))
|
||||
continue
|
||||
rates.append(r)
|
||||
tts.append(tt)
|
||||
cts.append(ct)
|
||||
if len(rates) < 2:
|
||||
log("only %d usable run(s) at depth %d — refusing to median" % (len(rates), tgt))
|
||||
return 2
|
||||
result["decode_tps_%d" % tgt] = round(statistics.median(rates), 2)
|
||||
result["ttft_ms_%d" % tgt] = round(1000.0 * statistics.median(tts), 1)
|
||||
result["completion_tokens_%d" % tgt] = int(statistics.median(cts))
|
||||
result["prompt_tokens_%d" % tgt] = pt
|
||||
log("depth %5d tok decode %.2f t/s ttft %.3f s (runs=%d)"
|
||||
% (pt, result["decode_tps_%d" % tgt], result["ttft_ms_%d" % tgt] / 1000.0, len(rates)))
|
||||
|
||||
meds = [result["decode_tps_%d" % t] for t in DEPTHS]
|
||||
result["decode_tps"] = round(statistics.median(meds), 2)
|
||||
print(json.dumps(result))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+200
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env bash
|
||||
# frank-primary-window.sh — one full bench window for frank-primary, run ON
|
||||
# DANKSERVER as the deploy user (invoked over ssh by ./autoresearch.sh).
|
||||
#
|
||||
# Discipline (AGENTS.md Benchmarking Policy): the production unit is
|
||||
# STOPPED + MASKED first, the candidate config boots AS THE SERVICE itself
|
||||
# (same serve script, same container name, same port 7770 — no scratch
|
||||
# ports, no co-tenant GPU), the in-container measure runs, and the window
|
||||
# ALWAYS ends by restoring the production unit and verifying its health.
|
||||
# The EXIT trap makes the restore unconditional: a boot failure, a measure
|
||||
# failure, or a dead ssh session all end with production back up.
|
||||
#
|
||||
# Staging layout (created by the orchestrator, ephemeral — never a config
|
||||
# source; the repo checkout is):
|
||||
# /home/dank/.autoresearch-frank-primary/
|
||||
# serve.sh staged candidate serve script
|
||||
# frank-primary.env staged candidate env (the candidate knobs)
|
||||
# measure.py the in-container measure client
|
||||
# hash candidate fingerprint
|
||||
# run.json result (metrics + boot + kv + fingerprint)
|
||||
# serve.out/measure.out/fail.log
|
||||
# last/ memo: run.json + hash of the last FULLY
|
||||
# successful window (bench AND verified restore)
|
||||
#
|
||||
# This script logs to stderr and exits 0 only when the measure produced a
|
||||
# usable result (the restore happens in the trap regardless of the bench
|
||||
# outcome; a failed restore forces exit 1).
|
||||
set -uo pipefail
|
||||
|
||||
STAGE="${FRANK_PRIMARY_AR_STAGE:-/home/dank/.autoresearch-frank-primary}"
|
||||
UNIT="${FRANK_PRIMARY_AR_UNIT:-frank-primary}"
|
||||
CTR="${FRANK_PRIMARY_AR_CTR:-frank-primary-qwen}"
|
||||
HEALTH_URL="http://127.0.0.1:7770/health"
|
||||
MODELS_URL="http://127.0.0.1:7770/v1/models"
|
||||
BOOT_TIMEOUT="${FRANK_PRIMARY_AR_BOOT_TIMEOUT:-1200}"
|
||||
RESTORE_TIMEOUT="${FRANK_PRIMARY_AR_RESTORE_TIMEOUT:-1200}"
|
||||
MEASURE_TIMEOUT="${FRANK_PRIMARY_AR_MEASURE_TIMEOUT:-1800}"
|
||||
SERVED="$(sed -n 's/^FRANK_PRIMARY_SERVED_NAME=//p' "$STAGE/frank-primary.env" 2>/dev/null | head -1)"
|
||||
SERVED="${SERVED:-frank-primary}"
|
||||
|
||||
RESTORED=0
|
||||
|
||||
log() { printf '[window %s] %s\n' "$(date '+%H:%M:%S')" "$*" >&2; }
|
||||
|
||||
container_state() { podman inspect -f '{{.State.Status}}' "$CTR" 2>/dev/null || echo absent; }
|
||||
|
||||
wait_port_free() {
|
||||
local i
|
||||
for i in $(seq 1 90); do
|
||||
[ "$(container_state)" != "running" ] && return 0
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_health() { # $1 = timeout seconds
|
||||
local t0 code
|
||||
t0=$(date +%s)
|
||||
while :; do
|
||||
code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$HEALTH_URL" 2>/dev/null || echo 000)"
|
||||
[ "$code" = "200" ] && return 0
|
||||
[ "$(( $(date +%s) - t0 ))" -ge "$1" ] && return 1
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
capture_bench_failure() {
|
||||
log "capturing bench-boot failure evidence"
|
||||
{
|
||||
echo "=== podman logs (tail 400) ==="
|
||||
podman logs --tail 400 "$CTR" 2>&1 || true
|
||||
echo "=== staged serve.out (tail 60) ==="
|
||||
tail -60 "$STAGE/serve.out" 2>/dev/null || true
|
||||
echo "=== engine log (tail 60) ==="
|
||||
tail -60 /home/dank/logs/frank_primary.log 2>/dev/null || true
|
||||
} >"$STAGE/fail.log" 2>&1 || true
|
||||
grep -E 'OutOfMemory|Tried to allocate|Engine core initialization failed|Error|error' \
|
||||
"$STAGE/fail.log" 2>/dev/null | tail -25 >&2 || true
|
||||
}
|
||||
|
||||
restore() {
|
||||
[ "$RESTORED" = 1 ] && return 0
|
||||
RESTORED=1
|
||||
log "restoring production unit $UNIT"
|
||||
if [ -f "$STAGE/serve.pid" ]; then
|
||||
kill "$(cat "$STAGE/serve.pid" 2>/dev/null)" 2>/dev/null || true
|
||||
fi
|
||||
podman stop -t 120 "$CTR" >/dev/null 2>&1 || true
|
||||
wait_port_free || log "WARNING: bench container still draining after 180s"
|
||||
sudo systemctl unmask "$UNIT" 2>/dev/null || true
|
||||
if ! sudo systemctl start "$UNIT"; then
|
||||
log "ERROR: systemctl start $UNIT failed — operator intervention required"
|
||||
return 1
|
||||
fi
|
||||
if wait_health "$RESTORE_TIMEOUT"; then
|
||||
if curl -s --max-time 10 "$MODELS_URL" 2>/dev/null | grep -q "$SERVED"; then
|
||||
log "production restored and verified ($SERVED answering on :7770)"
|
||||
# Memo promotion lives HERE: a cached result is only ever the
|
||||
# product of a fully successful window (bench AND verified
|
||||
# restore), never of a bare bench success.
|
||||
cp "$STAGE/run.json" "$STAGE/last/" 2>/dev/null || true
|
||||
cp "$STAGE/hash" "$STAGE/last/" 2>/dev/null || true
|
||||
else
|
||||
log "ERROR: health 200 but /v1/models does not list $SERVED"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log "ERROR: production not healthy within ${RESTORE_TIMEOUT}s — unit left to systemd (Restart=always)"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
trap 'restore; rc=$?; [ "$rc" -ne 0 ] && exit 1' EXIT
|
||||
|
||||
main() {
|
||||
local f
|
||||
for f in serve.sh frank-primary.env measure.py hash; do
|
||||
[ -f "$STAGE/$f" ] || { log "staged file missing: $STAGE/$f"; exit 1; }
|
||||
done
|
||||
mkdir -p "$STAGE/last"
|
||||
rm -f "$STAGE/run.json" "$STAGE/fail.log" "$STAGE/measure.out"
|
||||
|
||||
local t_start t0 i kvline kv_tokens kv_conc measure_json boot_seconds
|
||||
t_start="$(date +%s)"
|
||||
|
||||
log "stopping and masking production unit $UNIT"
|
||||
sudo systemctl stop "$UNIT" 2>/dev/null || true
|
||||
# Explicit stop is not auto-restarted even with Restart=always; the mask
|
||||
# is belt-and-braces against a stray start during the window.
|
||||
sudo systemctl mask "$UNIT" 2>/dev/null || \
|
||||
log "WARNING: mask failed — relying on explicit-stop semantics"
|
||||
wait_port_free || log "note: production container still draining"
|
||||
|
||||
t0="$(date +%s)"
|
||||
log "starting bench instance from staged config (fingerprint $(tr -d '[:space:]' <"$STAGE/hash"))"
|
||||
FRANK_PRIMARY_ENV_FILE="$STAGE/frank-primary.env" \
|
||||
setsid nohup bash "$STAGE/serve.sh" >"$STAGE/serve.out" 2>&1 </dev/null &
|
||||
echo "$!" >"$STAGE/serve.pid"
|
||||
|
||||
for i in $(seq 1 45); do
|
||||
[ "$(container_state)" = "running" ] && break
|
||||
sleep 2
|
||||
done
|
||||
if [ "$(container_state)" != "running" ]; then
|
||||
log "bench container never reached running state"
|
||||
tail -40 "$STAGE/serve.out" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if wait_health "$BOOT_TIMEOUT"; then
|
||||
boot_seconds=$(( $(date +%s) - t0 ))
|
||||
log "bench healthy after ${boot_seconds}s"
|
||||
else
|
||||
if [ "$(container_state)" = "running" ]; then
|
||||
log "bench boot timed out after ${BOOT_TIMEOUT}s (still loading)"
|
||||
else
|
||||
log "bench container died during boot"
|
||||
capture_bench_failure
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "running measure in container (timeout ${MEASURE_TIMEOUT}s)"
|
||||
if ! podman cp "$STAGE/measure.py" "$CTR":/tmp/ar-measure.py; then
|
||||
log "podman cp of measure client failed"
|
||||
exit 1
|
||||
fi
|
||||
if ! timeout "$MEASURE_TIMEOUT" podman exec "$CTR" python3 /tmp/ar-measure.py \
|
||||
>"$STAGE/measure.out" 2>&1; then
|
||||
log "measure failed or timed out"
|
||||
tail -40 "$STAGE/measure.out" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Capacity facts for the run (context-size experiments change these).
|
||||
kvline="$(podman logs "$CTR" 2>&1 | grep -E 'GPU KV cache size' | tail -1 || true)"
|
||||
kv_tokens="$(printf '%s' "$kvline" | sed -E 's/.*GPU KV cache size: ([0-9,]+) tokens.*/\1/' | tr -d ',' || true)"
|
||||
kv_conc="$(printf '%s' "$kvline" | sed -E 's/.*per request: ([0-9.]+x).*/\1/' || true)"
|
||||
|
||||
measure_json="$(grep -E '^\{' "$STAGE/measure.out" | tail -1 || true)"
|
||||
if [ -z "$measure_json" ]; then
|
||||
log "measure produced no JSON result line"
|
||||
tail -40 "$STAGE/measure.out" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The measure line is already JSON, so the run object can be assembled
|
||||
# without any host-side interpreter.
|
||||
printf '{"hash":"%s","boot_seconds":%s,"kv_tokens":%s,"kv_concurrency":"%s","total_seconds":%s,"measure":%s}\n' \
|
||||
"$(tr -d '[:space:]' <"$STAGE/hash")" \
|
||||
"$boot_seconds" "${kv_tokens:-0}" "${kv_conc:-unknown}" \
|
||||
"$(( $(date +%s) - t_start ))" "$measure_json" >"$STAGE/run.json"
|
||||
|
||||
log "bench window complete in $(( $(date +%s) - t_start ))s"
|
||||
# Memo only after a FULL success; the restore below still has to verify.
|
||||
exit 0
|
||||
}
|
||||
|
||||
main "$@"
|
||||
exit "$?"
|
||||
Executable
+150
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: autoresearch measure client behavior.
|
||||
#
|
||||
# The measure client is the whole yardstick of the frank-primary speed
|
||||
# session, so its METRIC arithmetic gets a behavior test: against a
|
||||
# deterministic mock endpoint that emits a KNOWN number of completion
|
||||
# tokens at a KNOWN per-token delay, the reported decode rate must land
|
||||
# near the nominal rate, the usage accounting must come back exact, and an
|
||||
# unhealthy endpoint must fail loudly (no half-measured numbers).
|
||||
#
|
||||
# The mock implements only what the client calls: POST
|
||||
# /v1/messages/count_tokens (word count as the token count, which makes the
|
||||
# client's words/token calibration come out to exactly 1.0) and a streaming
|
||||
# POST /v1/chat/completions (one content chunk per completion token, then a
|
||||
# usage chunk, then [DONE]).
|
||||
|
||||
load helpers
|
||||
|
||||
MEASURE="$REPO_ROOT/scripts/autoresearch/frank-primary-measure.py"
|
||||
MOCK="$BATS_TEST_TMPDIR/mock.py"
|
||||
PORT=""
|
||||
MOCK_PID=""
|
||||
|
||||
setup() {
|
||||
# Deterministic mock; argv: <port> <delay-seconds>; env MOCK_FAIL=1
|
||||
# makes the chat endpoint answer 500.
|
||||
cat >"$MOCK" <<'PYEOF'
|
||||
import json, os, sys, time
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
PORT = int(sys.argv[1])
|
||||
DELAY = float(sys.argv[2])
|
||||
FAIL = os.environ.get("MOCK_FAIL", "0") == "1"
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *a):
|
||||
pass
|
||||
|
||||
def _send(self, obj, code=200):
|
||||
b = json.dumps(obj).encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(b)))
|
||||
self.end_headers()
|
||||
self.wfile.write(b)
|
||||
|
||||
def do_POST(self):
|
||||
n = int(self.headers.get("Content-Length", "0"))
|
||||
body = json.loads(self.rfile.read(n) or b"{}")
|
||||
if self.path == "/v1/messages/count_tokens":
|
||||
text = body["messages"][0]["content"]
|
||||
self._send({"input_tokens": max(1, len(text.split()))})
|
||||
elif self.path == "/v1/chat/completions":
|
||||
if FAIL:
|
||||
self._send({"error": "engine down"}, 500)
|
||||
return
|
||||
max_tokens = int(body.get("max_tokens", 64))
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/event-stream")
|
||||
self.end_headers()
|
||||
for i in range(max_tokens):
|
||||
ev = {"choices": [{"delta": {"content": "tok%d " % i}}]}
|
||||
self.wfile.write(("data: %s\n\n" % json.dumps(ev)).encode())
|
||||
self.wfile.flush()
|
||||
time.sleep(DELAY)
|
||||
prompt_words = max(1, len(body["messages"][0]["content"].split()))
|
||||
ev = {"choices": [{"delta": {}}],
|
||||
"usage": {"prompt_tokens": prompt_words,
|
||||
"completion_tokens": max_tokens}}
|
||||
self.wfile.write(("data: %s\n\n" % json.dumps(ev)).encode())
|
||||
self.wfile.write(b"data: [DONE]\n\n")
|
||||
self.wfile.flush()
|
||||
else:
|
||||
self._send({"error": "unknown path"}, 404)
|
||||
|
||||
|
||||
HTTPServer(("127.0.0.1", PORT), Handler).serve_forever()
|
||||
PYEOF
|
||||
}
|
||||
free_port() {
|
||||
python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()'
|
||||
}
|
||||
|
||||
start_mock() { # $1 = ok|fail
|
||||
PORT="$(free_port)"
|
||||
if [ "$1" = "fail" ]; then
|
||||
MOCK_FAIL=1 python3 "$MOCK" "$PORT" 0.005 &
|
||||
else
|
||||
python3 "$MOCK" "$PORT" 0.005 &
|
||||
fi
|
||||
MOCK_PID=$!
|
||||
local i
|
||||
for i in $(seq 1 100); do
|
||||
if python3 -c "import socket; socket.create_connection(('127.0.0.1', $PORT), 0.2).close()" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
sleep 0.1
|
||||
done
|
||||
echo "mock did not start" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
stop_mock() {
|
||||
if [ -n "$MOCK_PID" ]; then
|
||||
kill "$MOCK_PID" 2>/dev/null || true
|
||||
wait "$MOCK_PID" 2>/dev/null || true
|
||||
MOCK_PID=""
|
||||
fi
|
||||
}
|
||||
|
||||
@test "measure exists and compiles" {
|
||||
[ -f "$MEASURE" ]
|
||||
python3 -m py_compile "$MEASURE"
|
||||
}
|
||||
|
||||
@test "measure computes decode rate and exact usage from a controlled endpoint" {
|
||||
start_mock ok || return 1
|
||||
run env AR_BASE="http://127.0.0.1:$PORT" AR_MODEL=mock python3 "$MEASURE"
|
||||
stop_mock
|
||||
[ "$status" -eq 0 ] || { echo "$output"; return 1; }
|
||||
# run() merges stderr in; the contract is: exactly ONE json result line
|
||||
# (the window script parses the same way), progress on stderr.
|
||||
printf '%s\n' "$output" | python3 -c '
|
||||
import json, sys
|
||||
lines = [l for l in sys.stdin.read().splitlines() if l.startswith("{")]
|
||||
assert len(lines) == 1, "expected exactly one JSON line, got %d" % len(lines)
|
||||
d = json.loads(lines[0])
|
||||
# nominal 200 t/s (0.005 s/chunk) minus small per-chunk overhead
|
||||
for k in ("decode_tps", "decode_tps_115", "decode_tps_1056", "decode_tps_4096"):
|
||||
v = d[k]
|
||||
assert 60.0 <= v <= 320.0, "%s=%s outside [60, 320]" % (k, v)
|
||||
# the mock emits exactly max_tokens completion tokens; the accounting must be exact
|
||||
assert d["completion_tokens_4096"] == 256, d["completion_tokens_4096"]
|
||||
assert d["completion_tokens_115"] == 256
|
||||
# prompt sizing: calibration is exactly 1.0 words/token in the mock, so the
|
||||
# measured prompt lengths equal the targets
|
||||
assert d["prompt_tokens_115"] == 115
|
||||
assert d["prompt_tokens_4096"] == 4096
|
||||
print("measure claims verified")
|
||||
'
|
||||
}
|
||||
|
||||
@test "measure fails loudly on an unhealthy endpoint instead of half-measuring" {
|
||||
start_mock fail || return 1
|
||||
run env AR_BASE="http://127.0.0.1:$PORT" AR_MODEL=mock python3 "$MEASURE"
|
||||
stop_mock
|
||||
[ "$status" -ne 0 ]
|
||||
! printf '%s\n' "$output" | grep -qE '^\{'
|
||||
}
|
||||
Reference in New Issue
Block a user