autoresearch: harness setup
Benchmark entrypoint: bash autoresearch.sh Goal: Make frank-primary faster: raise decode tok/s on the tcclaviger vLLM endpoint (dankserver :7770, Qwen3.8-Flash-Next-Uncensored-FP8, 2x R9700, automatic tiered-expert offload). Context size may be altered (MAXLEN/SEQS/KV_BYTES and friends); quantization may NOT be altered (the 131-shard FP8 payload and the digest-pinned image are frozen).
This commit is contained in:
Frank committed
1 parent
934285c2b2
commit
f8f845d86b
4 files changed
+633
-67
No files matched your search
+91
-67
@@ -1,80 +1,104 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# autoresearch.sh — canonical speed benchmark for frank-large (Dankclaw iGPU).
|
# autoresearch.sh — canonical benchmark entrypoint: speed of frank-primary.
|
||||||
#
|
#
|
||||||
# Window semantics per the repo Benchmarking Policy: the deployed unit is
|
# Runs on the operator box and drives one bench window on DANKSERVER:
|
||||||
# STOPPED, the candidate (this worktree's scripts/frank-large.env +
|
# 1. fingerprint the candidate trio (env + serve script + unit)
|
||||||
# scripts/serve-frank-large.sh, rsynced to ~/jobs/autoresearch so the host's
|
# 2. memo fast-path: an identical fingerprint that already produced a FULLY
|
||||||
# git checkout stays clean) boots standalone on :7770, the fixed greedy
|
# successful window (bench + verified production restore) returns the
|
||||||
# battery runs against it ON the host, and a trap restores the deployed
|
# cached metrics without touching the fleet
|
||||||
# baseline unit whatever happens. The candidate never touches the host
|
# 3. stage the candidate trio plus the measure client into
|
||||||
# checkout; "keep" in the loop means a real deploy later.
|
# /home/dank/.autoresearch-frank-primary/ (ephemeral; the repo checkout
|
||||||
|
# on the host stays the production source of truth)
|
||||||
|
# 4. run the window (scripts/autoresearch/frank-primary-window.sh):
|
||||||
|
# stop + mask the production unit, boot the staged config AS THE SERVICE
|
||||||
|
# (same serve script, same container, same port 7770), wait for health,
|
||||||
|
# run the in-container usage-counter measure, then restore the production
|
||||||
|
# unit and verify its health — AGENTS.md Benchmarking Policy
|
||||||
|
# 5. print METRIC lines (parse by run_experiment)
|
||||||
#
|
#
|
||||||
# Emits: METRIC decode_tokens_per_sec (primary, higher better),
|
# Primary metric: decode_tps (HIGHER is better) — median across the three
|
||||||
# METRIC prefill_tokens_per_sec, METRIC ttft_ms, ASI identity lines.
|
# prompt depths (115/1056/4096) of per-depth medians of
|
||||||
|
# completion_tokens / (wall - ttft), 3 runs each after a warm pass,
|
||||||
|
# temperature 0, the campaign's deterministic word-seed workload.
|
||||||
|
#
|
||||||
|
# Exit 0 with METRIC lines on success; non-zero on any failure (the window
|
||||||
|
# script's EXIT trap restores production on every path).
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
REPO="$(cd "$(dirname "$0")" && pwd)"
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
HOST=dankclaw
|
HOST="${FRANK_PRIMARY_AR_HOST:-dankserver}"
|
||||||
RDIR=/home/dank/jobs/autoresearch
|
STAGE="/home/dank/.autoresearch-frank-primary"
|
||||||
HEALTH_TRIES=200 # x5s = ~17 min ceiling (BF16 boots ~230s; INT8/dequant may differ)
|
|
||||||
|
|
||||||
restore() {
|
ENV_FILE="$SCRIPT_DIR/scripts/frank-primary.env"
|
||||||
local rc=$?
|
SERVE="$SCRIPT_DIR/scripts/serve-frank-primary.sh"
|
||||||
trap - EXIT
|
UNIT="$SCRIPT_DIR/scripts/frank-primary.service"
|
||||||
echo "[harness] tearing down candidate, restoring deployed baseline..." >&2
|
MEASURE="$SCRIPT_DIR/scripts/autoresearch/frank-primary-measure.py"
|
||||||
ssh "$HOST" 'sudo -n systemctl stop frank-large.service 2>/dev/null;
|
WINDOW="$SCRIPT_DIR/scripts/autoresearch/frank-primary-window.sh"
|
||||||
podman rm -f frank-large-mimo 2>/dev/null;
|
|
||||||
pkill -x llama-server 2>/dev/null; true' >/dev/null 2>&1 || true
|
for f in "$ENV_FILE" "$SERVE" "$UNIT" "$MEASURE" "$WINDOW"; do
|
||||||
ssh "$HOST" 'sudo -n systemctl start frank-large.service' >/dev/null 2>&1 || true
|
[ -f "$f" ] || { echo "[autoresearch] missing harness file: $f" >&2; exit 1; }
|
||||||
exit $rc
|
done
|
||||||
|
command -v python3 >/dev/null 2>&1 || { echo "[autoresearch] python3 required on operator box" >&2; exit 1; }
|
||||||
|
|
||||||
|
# Fingerprint of everything that can change the engine's argv or environment.
|
||||||
|
HASH="$(cat "$ENV_FILE" "$SERVE" "$UNIT" | sha256sum | cut -d' ' -f1)"
|
||||||
|
echo "[autoresearch] candidate fingerprint $HASH" >&2
|
||||||
|
|
||||||
|
ssh_run() { ssh -o BatchMode=yes -o ConnectTimeout=15 "$HOST" "$@"; }
|
||||||
|
|
||||||
|
emit_metrics() { # $1 = run.json
|
||||||
|
python3 -c '
|
||||||
|
import json, sys
|
||||||
|
run = json.loads(sys.argv[1])
|
||||||
|
m = run["measure"]
|
||||||
|
print("METRIC decode_tps=%s" % m["decode_tps"])
|
||||||
|
for k in ("decode_tps_115", "decode_tps_1056", "decode_tps_4096"):
|
||||||
|
print("METRIC %s=%s" % (k, m[k]))
|
||||||
|
print("METRIC ttft_ms_4096=%s" % m["ttft_ms_4096"])
|
||||||
|
print("METRIC kv_tokens=%s" % (run.get("kv_tokens") or 0))
|
||||||
|
print("METRIC boot_seconds=%s" % (run.get("boot_seconds") or -1))
|
||||||
|
print("METRIC completion_tokens_4096=%s" % m.get("completion_tokens_4096", 0))
|
||||||
|
' "$1"
|
||||||
}
|
}
|
||||||
trap restore EXIT
|
|
||||||
|
|
||||||
# 1. Ship the candidate (env + serve + battery) outside the git checkout.
|
# --- memo fast-path -------------------------------------------------------
|
||||||
ssh "$HOST" "mkdir -p $RDIR"
|
cached_hash="$(ssh_run "cat '$STAGE/last/hash' 2>/dev/null" 2>/dev/null || true)"
|
||||||
rsync -a "$REPO/scripts/frank-large.env" "$REPO/scripts/serve-frank-large.sh" \
|
if [ -n "$cached_hash" ] && [ "$cached_hash" = "$HASH" ]; then
|
||||||
"$REPO/scripts/autoresearch-bench.py" "$HOST:$RDIR/"
|
cached_run="$(ssh_run "cat '$STAGE/last/run.json' 2>/dev/null" || true)"
|
||||||
|
if [ -n "$cached_run" ]; then
|
||||||
|
echo "[autoresearch] memo hit — cached metrics for $HASH, fleet untouched" >&2
|
||||||
|
emit_metrics "$cached_run"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# 2. Free the GPU and the port: stop the deployed unit (mask is unnecessary—
|
# --- stage the candidate ----------------------------------------------------
|
||||||
# the unit only starts via systemd, and we never enable more of it).
|
echo "[autoresearch] staging candidate on $HOST:$STAGE" >&2
|
||||||
ssh "$HOST" 'sudo -n systemctl stop frank-large.service;
|
ssh_run "mkdir -p '$STAGE'"
|
||||||
podman rm -f frank-large-mimo 2>/dev/null;
|
scp -q "$ENV_FILE" "$HOST:$STAGE/frank-primary.env"
|
||||||
pkill -x llama-server 2>/dev/null; true' >/dev/null
|
scp -q "$SERVE" "$HOST:$STAGE/serve.sh"
|
||||||
|
scp -q "$MEASURE" "$HOST:$STAGE/measure.py"
|
||||||
|
scp -q "$WINDOW" "$HOST:$STAGE/window.sh"
|
||||||
|
ssh_run "printf '%s\n' '$HASH' > '$STAGE/hash'"
|
||||||
|
|
||||||
# 3. Boot the candidate detached; the launcher's own output IS the serve
|
# --- the window -------------------------------------------------------------
|
||||||
# stage output (gate failures print there before any container exists).
|
# Budgets: bench boot up to 20 min (warm boots measured ~7 min), measure up
|
||||||
echo "[harness] starting candidate..." >&2
|
# to 30 min, restore boot up to 20 min — call run_experiment with
|
||||||
ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \
|
# timeout_seconds >= 3600.
|
||||||
bash $RDIR/serve-frank-large.sh >$RDIR/serve.log 2>&1 </dev/null & echo ok" \
|
echo "[autoresearch] opening bench window (stop+mask -> boot -> measure -> restore)" >&2
|
||||||
| tee /tmp/autoresearch-launch.log >&2
|
set +e
|
||||||
grep -q ok /tmp/autoresearch-launch.log || { echo "ERROR: launch failed" >&2; exit 1; }
|
ssh_run "bash '$STAGE/window.sh'"
|
||||||
|
rc=$?
|
||||||
# 4. Wait for health (503 while loading; anything else = boot failure).
|
set -e
|
||||||
echo "[harness] waiting for health..." >&2
|
if [ "$rc" -ne 0 ]; then
|
||||||
if ! ssh "$HOST" "for i in \$(seq 1 $HEALTH_TRIES); do
|
echo "[autoresearch] window failed (rc=$rc); host-side evidence:" >&2
|
||||||
code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 \
|
ssh_run "tail -60 '$STAGE/fail.log' 2>/dev/null || tail -40 '$STAGE/serve.out' 2>/dev/null || tail -40 '$STAGE/measure.out' 2>/dev/null || true" >&2 || true
|
||||||
http://127.0.0.1:7770/health 2>/dev/null || echo 000)
|
|
||||||
[ \"\$code\" = 200 ] && echo UP && exit 0
|
|
||||||
st=\$(podman inspect -f '{{.State.Status}}' frank-large-mimo 2>/dev/null || echo gone)
|
|
||||||
if [ "\$st" = exited ] || { [ "\$st" = gone ] && ! pgrep -f serve-frank-large >/dev/null && ! pgrep -x llama-server >/dev/null; }; then echo DOWN; exit 1; fi
|
|
||||||
sleep 5
|
|
||||||
done; exit 1" | grep -q UP; then
|
|
||||||
echo "ERROR: candidate never became healthy; serve.log tail:" >&2
|
|
||||||
ssh "$HOST" "tail -30 $RDIR/serve.log; echo '--- engine log ---';
|
|
||||||
podman logs --tail 120 frank-large-mimo 2>&1 |
|
|
||||||
grep -aiE 'error|raise|failed|traceback|assert|kernel' |
|
|
||||||
tail -30" >&2 || true
|
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 5. Identity evidence for the loop's records (exact engine argv).
|
run_json="$(ssh_run "cat '$STAGE/run.json'")" || {
|
||||||
ASI_ARGS="$(ssh "$HOST" 'podman inspect frank-large-mimo \
|
echo "[autoresearch] could not read $STAGE/run.json back" >&2; exit 1; }
|
||||||
--format "{{join .Config.Cmd \" \"}}" 2>/dev/null' | tr -d "\n" | head -c 400 || true)"
|
echo "[autoresearch] window result:" >&2
|
||||||
if [ -z "$ASI_ARGS" ]; then
|
printf '%s\n' "$run_json" >&2
|
||||||
ASI_ARGS="$(ssh "$HOST" "sh -c 'tr \"\\0\" \" \" < /proc/\\$(pgrep -x llama-server | head -1)/cmdline 2>/dev/null || true'" | head -c 400 || true)"
|
emit_metrics "$run_json"
|
||||||
fi
|
exit 0
|
||||||
echo "ASI engine_args=${ASI_ARGS:0:400}"
|
|
||||||
|
|
||||||
# 6. The battery, on the host, medians over 3 reps x 2 greedy prompts.
|
|
||||||
ssh "$HOST" "python3 $RDIR/autoresearch-bench.py \
|
|
||||||
--url http://127.0.0.1:7770 --model frank-large"
|
|
||||||
+192
@@ -0,0 +1,192 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""frank-primary decode measurement — runs INSIDE the engine container.
|
||||||
|
|
||||||
|
The method is the fleet's established one (the 2026-09-29 / 2026-10-01
|
||||||
|
tuning campaigns, /home/dank/mtp_measure.py): SSE chunk counting is invalid
|
||||||
|
under speculative decoding because rejected drafts change the accounting, so
|
||||||
|
decode is measured from the server's own usage counters:
|
||||||
|
|
||||||
|
decode_tok_s = completion_tokens / (wall - ttft)
|
||||||
|
|
||||||
|
with ttft = wall time to the FIRST content-bearing chunk (thinking arrives in
|
||||||
|
`reasoning`/`delta.reasoning` on this stack and is excluded from the decode
|
||||||
|
window — and is deterministic per prompt at temperature 0, so the exclusion
|
||||||
|
cancels in any A/B on the same workload).
|
||||||
|
|
||||||
|
Workload (fixed, deterministic, offline):
|
||||||
|
* prompts built from a large fixed-seed shuffled word list so tokens rarely
|
||||||
|
repeat (no n-gram/prefix predictability of the text itself), sized with
|
||||||
|
the server's own /v1/messages/count_tokens counter
|
||||||
|
* depths 115 / 1056 / 4096 target prompt tokens
|
||||||
|
* one 64-token warm pass per depth (warms the expert arena), then
|
||||||
|
max_tokens=256 at temperature 0, 3 runs, median
|
||||||
|
Everything is greedy with a fixed seed — the same workload every run, no
|
||||||
|
network, no time-of-day dependence.
|
||||||
|
|
||||||
|
Env:
|
||||||
|
AR_BASE endpoint (default http://127.0.0.1:7770)
|
||||||
|
AR_MODEL served name (default frank-primary)
|
||||||
|
AR_PROMPT_CEILING hard prompt ceiling in tokens (default 100000)
|
||||||
|
|
||||||
|
Output: progress on stderr, ONE json object on stdout (the last line).
|
||||||
|
Exit codes: 0 ok, 2 endpoint unhealthy / no usable completion, 3 bad usage.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
import statistics
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
BASE = os.environ.get("AR_BASE", "http://127.0.0.1:7770")
|
||||||
|
MODEL = os.environ.get("AR_MODEL", "frank-primary")
|
||||||
|
PROMPT_CEILING = int(os.environ.get("AR_PROMPT_CEILING", "100000"))
|
||||||
|
DEPTHS = (115, 1056, 4096)
|
||||||
|
WARM_TOKENS = 64
|
||||||
|
MAX_TOKENS = 256
|
||||||
|
RUNS = 3
|
||||||
|
|
||||||
|
# 43-word x 12 = 516-word vocabulary, seed 1234 — the campaign's exact list
|
||||||
|
# and seed, so prompts are byte-identical to the historical measurements.
|
||||||
|
_VOCAB = ("alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima "
|
||||||
|
"mike november oscar papa quebec romeo sierra tango uniform victor whiskey "
|
||||||
|
"xray yankee zulu cardinal ordinal topology kernel scheduler resident "
|
||||||
|
"unified aperture expert arena slab offload prefetch gemm triton quant "
|
||||||
|
"tensor shard checkpoint manifest embed router cache page table window "
|
||||||
|
"gradient sparse dense moe attention head dim block stream decode prefill "
|
||||||
|
"bandwidth latency throughput residency eviction tier swap pin memlock").split()
|
||||||
|
_R = random.Random(1234)
|
||||||
|
_ORDER = list(range(len(_VOCAB)))
|
||||||
|
_R.shuffle(_ORDER)
|
||||||
|
|
||||||
|
|
||||||
|
def log(msg):
|
||||||
|
print("[measure] %s" % msg, file=sys.stderr, flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def post_json(path, body, timeout):
|
||||||
|
req = urllib.request.Request(
|
||||||
|
BASE + path, data=json.dumps(body).encode(),
|
||||||
|
headers={"Content-Type": "application/json"})
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||||
|
return json.load(r)
|
||||||
|
|
||||||
|
|
||||||
|
def count_tokens(text):
|
||||||
|
"""Token count without a prefill, from the server's own counter."""
|
||||||
|
body = {"model": MODEL,
|
||||||
|
"messages": [{"role": "user", "content": text}]}
|
||||||
|
r = post_json("/v1/messages/count_tokens", body, timeout=300)
|
||||||
|
n = r.get("input_tokens") or r.get("count") or r.get("token_count")
|
||||||
|
if not n:
|
||||||
|
raise RuntimeError("count_tokens returned no count: %r" % (r,))
|
||||||
|
return int(n)
|
||||||
|
|
||||||
|
|
||||||
|
def words(n):
|
||||||
|
return " ".join(_VOCAB[_ORDER[i % len(_VOCAB)]] for i in range(n))
|
||||||
|
|
||||||
|
|
||||||
|
def calibrate():
|
||||||
|
"""Measure words->tokens from the server's own counter."""
|
||||||
|
return 2000.0 / count_tokens(words(2000))
|
||||||
|
|
||||||
|
|
||||||
|
def prompt_for(target_tokens, wpt):
|
||||||
|
n = max(1, int(target_tokens * wpt))
|
||||||
|
text = words(n)
|
||||||
|
pt = count_tokens(text)
|
||||||
|
while pt > PROMPT_CEILING:
|
||||||
|
n = max(1, int(n * (PROMPT_CEILING / pt) * 0.98))
|
||||||
|
text = words(n)
|
||||||
|
pt = count_tokens(text)
|
||||||
|
return text, pt
|
||||||
|
|
||||||
|
|
||||||
|
def measured(prompt, max_tokens):
|
||||||
|
"""Return (ttft_s, decode_tok_s, completion_tokens) via usage counters."""
|
||||||
|
body = {"model": MODEL,
|
||||||
|
"messages": [{"role": "user", "content": prompt}],
|
||||||
|
"max_tokens": max_tokens, "temperature": 0,
|
||||||
|
"stream": True, "stream_options": {"include_usage": True}}
|
||||||
|
req = urllib.request.Request(
|
||||||
|
BASE + "/v1/chat/completions", data=json.dumps(body).encode(),
|
||||||
|
headers={"Content-Type": "application/json"})
|
||||||
|
t0 = time.time()
|
||||||
|
ttft = None
|
||||||
|
usage = {}
|
||||||
|
with urllib.request.urlopen(req, timeout=1800) as r:
|
||||||
|
for raw in r:
|
||||||
|
line = raw.decode("utf-8", "replace").strip()
|
||||||
|
if not line.startswith("data:"):
|
||||||
|
continue
|
||||||
|
s = line[5:].strip()
|
||||||
|
if s == "[DONE]":
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
ev = json.loads(s)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if ev.get("usage"):
|
||||||
|
usage = ev["usage"]
|
||||||
|
for ch in ev.get("choices", []):
|
||||||
|
if (ch.get("delta") or {}).get("content"):
|
||||||
|
if ttft is None:
|
||||||
|
ttft = time.time() - t0
|
||||||
|
wall = time.time() - t0
|
||||||
|
ct = int(usage.get("completion_tokens") or 0)
|
||||||
|
if ct <= 0:
|
||||||
|
# No usage on the stream: the endpoint is unhealthy or the model
|
||||||
|
# returned nothing (e.g. HTTP 200 with an error payload).
|
||||||
|
raise RuntimeError("no completion_tokens in usage: %r" % (usage,))
|
||||||
|
dec_window = max(wall - (ttft or 0.0), 1e-9)
|
||||||
|
return (ttft or wall), (ct / dec_window), ct
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
log("base=%s model=%s ceiling=%d" % (BASE, MODEL, PROMPT_CEILING))
|
||||||
|
try:
|
||||||
|
wpt = calibrate()
|
||||||
|
except Exception as e:
|
||||||
|
log("endpoint unhealthy during calibration: %s" % e)
|
||||||
|
return 2
|
||||||
|
log("words/token: %.3f" % wpt)
|
||||||
|
|
||||||
|
result = {"words_per_token": round(wpt, 5)}
|
||||||
|
for tgt in DEPTHS:
|
||||||
|
try:
|
||||||
|
p, pt = prompt_for(tgt, wpt)
|
||||||
|
measured(p, WARM_TOKENS) # warm the arena
|
||||||
|
except Exception as e:
|
||||||
|
log("warm pass failed at depth %d: %s" % (tgt, e))
|
||||||
|
return 2
|
||||||
|
rates, tts, cts = [], [], []
|
||||||
|
for i in range(RUNS):
|
||||||
|
try:
|
||||||
|
tt, r, ct = measured(p, MAX_TOKENS)
|
||||||
|
except Exception as e:
|
||||||
|
log("run %d/%d failed at depth %d: %s" % (i + 1, RUNS, tgt, e))
|
||||||
|
continue
|
||||||
|
rates.append(r)
|
||||||
|
tts.append(tt)
|
||||||
|
cts.append(ct)
|
||||||
|
if len(rates) < 2:
|
||||||
|
log("only %d usable run(s) at depth %d — refusing to median" % (len(rates), tgt))
|
||||||
|
return 2
|
||||||
|
result["decode_tps_%d" % tgt] = round(statistics.median(rates), 2)
|
||||||
|
result["ttft_ms_%d" % tgt] = round(1000.0 * statistics.median(tts), 1)
|
||||||
|
result["completion_tokens_%d" % tgt] = int(statistics.median(cts))
|
||||||
|
result["prompt_tokens_%d" % tgt] = pt
|
||||||
|
log("depth %5d tok decode %.2f t/s ttft %.3f s (runs=%d)"
|
||||||
|
% (pt, result["decode_tps_%d" % tgt], result["ttft_ms_%d" % tgt] / 1000.0, len(rates)))
|
||||||
|
|
||||||
|
meds = [result["decode_tps_%d" % t] for t in DEPTHS]
|
||||||
|
result["decode_tps"] = round(statistics.median(meds), 2)
|
||||||
|
print(json.dumps(result))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
Executable
+200
@@ -0,0 +1,200 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# frank-primary-window.sh — one full bench window for frank-primary, run ON
|
||||||
|
# DANKSERVER as the deploy user (invoked over ssh by ./autoresearch.sh).
|
||||||
|
#
|
||||||
|
# Discipline (AGENTS.md Benchmarking Policy): the production unit is
|
||||||
|
# STOPPED + MASKED first, the candidate config boots AS THE SERVICE itself
|
||||||
|
# (same serve script, same container name, same port 7770 — no scratch
|
||||||
|
# ports, no co-tenant GPU), the in-container measure runs, and the window
|
||||||
|
# ALWAYS ends by restoring the production unit and verifying its health.
|
||||||
|
# The EXIT trap makes the restore unconditional: a boot failure, a measure
|
||||||
|
# failure, or a dead ssh session all end with production back up.
|
||||||
|
#
|
||||||
|
# Staging layout (created by the orchestrator, ephemeral — never a config
|
||||||
|
# source; the repo checkout is):
|
||||||
|
# /home/dank/.autoresearch-frank-primary/
|
||||||
|
# serve.sh staged candidate serve script
|
||||||
|
# frank-primary.env staged candidate env (the candidate knobs)
|
||||||
|
# measure.py the in-container measure client
|
||||||
|
# hash candidate fingerprint
|
||||||
|
# run.json result (metrics + boot + kv + fingerprint)
|
||||||
|
# serve.out/measure.out/fail.log
|
||||||
|
# last/ memo: run.json + hash of the last FULLY
|
||||||
|
# successful window (bench AND verified restore)
|
||||||
|
#
|
||||||
|
# This script logs to stderr and exits 0 only when the measure produced a
|
||||||
|
# usable result (the restore happens in the trap regardless of the bench
|
||||||
|
# outcome; a failed restore forces exit 1).
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
STAGE="${FRANK_PRIMARY_AR_STAGE:-/home/dank/.autoresearch-frank-primary}"
|
||||||
|
UNIT="${FRANK_PRIMARY_AR_UNIT:-frank-primary}"
|
||||||
|
CTR="${FRANK_PRIMARY_AR_CTR:-frank-primary-qwen}"
|
||||||
|
HEALTH_URL="http://127.0.0.1:7770/health"
|
||||||
|
MODELS_URL="http://127.0.0.1:7770/v1/models"
|
||||||
|
BOOT_TIMEOUT="${FRANK_PRIMARY_AR_BOOT_TIMEOUT:-1200}"
|
||||||
|
RESTORE_TIMEOUT="${FRANK_PRIMARY_AR_RESTORE_TIMEOUT:-1200}"
|
||||||
|
MEASURE_TIMEOUT="${FRANK_PRIMARY_AR_MEASURE_TIMEOUT:-1800}"
|
||||||
|
SERVED="$(sed -n 's/^FRANK_PRIMARY_SERVED_NAME=//p' "$STAGE/frank-primary.env" 2>/dev/null | head -1)"
|
||||||
|
SERVED="${SERVED:-frank-primary}"
|
||||||
|
|
||||||
|
RESTORED=0
|
||||||
|
|
||||||
|
log() { printf '[window %s] %s\n' "$(date '+%H:%M:%S')" "$*" >&2; }
|
||||||
|
|
||||||
|
container_state() { podman inspect -f '{{.State.Status}}' "$CTR" 2>/dev/null || echo absent; }
|
||||||
|
|
||||||
|
wait_port_free() {
|
||||||
|
local i
|
||||||
|
for i in $(seq 1 90); do
|
||||||
|
[ "$(container_state)" != "running" ] && return 0
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_health() { # $1 = timeout seconds
|
||||||
|
local t0 code
|
||||||
|
t0=$(date +%s)
|
||||||
|
while :; do
|
||||||
|
code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$HEALTH_URL" 2>/dev/null || echo 000)"
|
||||||
|
[ "$code" = "200" ] && return 0
|
||||||
|
[ "$(( $(date +%s) - t0 ))" -ge "$1" ] && return 1
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
capture_bench_failure() {
|
||||||
|
log "capturing bench-boot failure evidence"
|
||||||
|
{
|
||||||
|
echo "=== podman logs (tail 400) ==="
|
||||||
|
podman logs --tail 400 "$CTR" 2>&1 || true
|
||||||
|
echo "=== staged serve.out (tail 60) ==="
|
||||||
|
tail -60 "$STAGE/serve.out" 2>/dev/null || true
|
||||||
|
echo "=== engine log (tail 60) ==="
|
||||||
|
tail -60 /home/dank/logs/frank_primary.log 2>/dev/null || true
|
||||||
|
} >"$STAGE/fail.log" 2>&1 || true
|
||||||
|
grep -E 'OutOfMemory|Tried to allocate|Engine core initialization failed|Error|error' \
|
||||||
|
"$STAGE/fail.log" 2>/dev/null | tail -25 >&2 || true
|
||||||
|
}
|
||||||
|
|
||||||
|
restore() {
|
||||||
|
[ "$RESTORED" = 1 ] && return 0
|
||||||
|
RESTORED=1
|
||||||
|
log "restoring production unit $UNIT"
|
||||||
|
if [ -f "$STAGE/serve.pid" ]; then
|
||||||
|
kill "$(cat "$STAGE/serve.pid" 2>/dev/null)" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
podman stop -t 120 "$CTR" >/dev/null 2>&1 || true
|
||||||
|
wait_port_free || log "WARNING: bench container still draining after 180s"
|
||||||
|
sudo systemctl unmask "$UNIT" 2>/dev/null || true
|
||||||
|
if ! sudo systemctl start "$UNIT"; then
|
||||||
|
log "ERROR: systemctl start $UNIT failed — operator intervention required"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if wait_health "$RESTORE_TIMEOUT"; then
|
||||||
|
if curl -s --max-time 10 "$MODELS_URL" 2>/dev/null | grep -q "$SERVED"; then
|
||||||
|
log "production restored and verified ($SERVED answering on :7770)"
|
||||||
|
# Memo promotion lives HERE: a cached result is only ever the
|
||||||
|
# product of a fully successful window (bench AND verified
|
||||||
|
# restore), never of a bare bench success.
|
||||||
|
cp "$STAGE/run.json" "$STAGE/last/" 2>/dev/null || true
|
||||||
|
cp "$STAGE/hash" "$STAGE/last/" 2>/dev/null || true
|
||||||
|
else
|
||||||
|
log "ERROR: health 200 but /v1/models does not list $SERVED"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
log "ERROR: production not healthy within ${RESTORE_TIMEOUT}s — unit left to systemd (Restart=always)"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
trap 'restore; rc=$?; [ "$rc" -ne 0 ] && exit 1' EXIT
|
||||||
|
|
||||||
|
main() {
|
||||||
|
local f
|
||||||
|
for f in serve.sh frank-primary.env measure.py hash; do
|
||||||
|
[ -f "$STAGE/$f" ] || { log "staged file missing: $STAGE/$f"; exit 1; }
|
||||||
|
done
|
||||||
|
mkdir -p "$STAGE/last"
|
||||||
|
rm -f "$STAGE/run.json" "$STAGE/fail.log" "$STAGE/measure.out"
|
||||||
|
|
||||||
|
local t_start t0 i kvline kv_tokens kv_conc measure_json boot_seconds
|
||||||
|
t_start="$(date +%s)"
|
||||||
|
|
||||||
|
log "stopping and masking production unit $UNIT"
|
||||||
|
sudo systemctl stop "$UNIT" 2>/dev/null || true
|
||||||
|
# Explicit stop is not auto-restarted even with Restart=always; the mask
|
||||||
|
# is belt-and-braces against a stray start during the window.
|
||||||
|
sudo systemctl mask "$UNIT" 2>/dev/null || \
|
||||||
|
log "WARNING: mask failed — relying on explicit-stop semantics"
|
||||||
|
wait_port_free || log "note: production container still draining"
|
||||||
|
|
||||||
|
t0="$(date +%s)"
|
||||||
|
log "starting bench instance from staged config (fingerprint $(tr -d '[:space:]' <"$STAGE/hash"))"
|
||||||
|
FRANK_PRIMARY_ENV_FILE="$STAGE/frank-primary.env" \
|
||||||
|
setsid nohup bash "$STAGE/serve.sh" >"$STAGE/serve.out" 2>&1 </dev/null &
|
||||||
|
echo "$!" >"$STAGE/serve.pid"
|
||||||
|
|
||||||
|
for i in $(seq 1 45); do
|
||||||
|
[ "$(container_state)" = "running" ] && break
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
if [ "$(container_state)" != "running" ]; then
|
||||||
|
log "bench container never reached running state"
|
||||||
|
tail -40 "$STAGE/serve.out" >&2 || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if wait_health "$BOOT_TIMEOUT"; then
|
||||||
|
boot_seconds=$(( $(date +%s) - t0 ))
|
||||||
|
log "bench healthy after ${boot_seconds}s"
|
||||||
|
else
|
||||||
|
if [ "$(container_state)" = "running" ]; then
|
||||||
|
log "bench boot timed out after ${BOOT_TIMEOUT}s (still loading)"
|
||||||
|
else
|
||||||
|
log "bench container died during boot"
|
||||||
|
capture_bench_failure
|
||||||
|
fi
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "running measure in container (timeout ${MEASURE_TIMEOUT}s)"
|
||||||
|
if ! podman cp "$STAGE/measure.py" "$CTR":/tmp/ar-measure.py; then
|
||||||
|
log "podman cp of measure client failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! timeout "$MEASURE_TIMEOUT" podman exec "$CTR" python3 /tmp/ar-measure.py \
|
||||||
|
>"$STAGE/measure.out" 2>&1; then
|
||||||
|
log "measure failed or timed out"
|
||||||
|
tail -40 "$STAGE/measure.out" >&2 || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Capacity facts for the run (context-size experiments change these).
|
||||||
|
kvline="$(podman logs "$CTR" 2>&1 | grep -E 'GPU KV cache size' | tail -1 || true)"
|
||||||
|
kv_tokens="$(printf '%s' "$kvline" | sed -E 's/.*GPU KV cache size: ([0-9,]+) tokens.*/\1/' | tr -d ',' || true)"
|
||||||
|
kv_conc="$(printf '%s' "$kvline" | sed -E 's/.*per request: ([0-9.]+x).*/\1/' || true)"
|
||||||
|
|
||||||
|
measure_json="$(grep -E '^\{' "$STAGE/measure.out" | tail -1 || true)"
|
||||||
|
if [ -z "$measure_json" ]; then
|
||||||
|
log "measure produced no JSON result line"
|
||||||
|
tail -40 "$STAGE/measure.out" >&2 || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# The measure line is already JSON, so the run object can be assembled
|
||||||
|
# without any host-side interpreter.
|
||||||
|
printf '{"hash":"%s","boot_seconds":%s,"kv_tokens":%s,"kv_concurrency":"%s","total_seconds":%s,"measure":%s}\n' \
|
||||||
|
"$(tr -d '[:space:]' <"$STAGE/hash")" \
|
||||||
|
"$boot_seconds" "${kv_tokens:-0}" "${kv_conc:-unknown}" \
|
||||||
|
"$(( $(date +%s) - t_start ))" "$measure_json" >"$STAGE/run.json"
|
||||||
|
|
||||||
|
log "bench window complete in $(( $(date +%s) - t_start ))s"
|
||||||
|
# Memo only after a FULL success; the restore below still has to verify.
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
|
exit "$?"
|
||||||
Executable
+150
@@ -0,0 +1,150 @@
|
|||||||
|
#!/usr/bin/env bats
|
||||||
|
# Seam: autoresearch measure client behavior.
|
||||||
|
#
|
||||||
|
# The measure client is the whole yardstick of the frank-primary speed
|
||||||
|
# session, so its METRIC arithmetic gets a behavior test: against a
|
||||||
|
# deterministic mock endpoint that emits a KNOWN number of completion
|
||||||
|
# tokens at a KNOWN per-token delay, the reported decode rate must land
|
||||||
|
# near the nominal rate, the usage accounting must come back exact, and an
|
||||||
|
# unhealthy endpoint must fail loudly (no half-measured numbers).
|
||||||
|
#
|
||||||
|
# The mock implements only what the client calls: POST
|
||||||
|
# /v1/messages/count_tokens (word count as the token count, which makes the
|
||||||
|
# client's words/token calibration come out to exactly 1.0) and a streaming
|
||||||
|
# POST /v1/chat/completions (one content chunk per completion token, then a
|
||||||
|
# usage chunk, then [DONE]).
|
||||||
|
|
||||||
|
load helpers
|
||||||
|
|
||||||
|
MEASURE="$REPO_ROOT/scripts/autoresearch/frank-primary-measure.py"
|
||||||
|
MOCK="$BATS_TEST_TMPDIR/mock.py"
|
||||||
|
PORT=""
|
||||||
|
MOCK_PID=""
|
||||||
|
|
||||||
|
setup() {
|
||||||
|
# Deterministic mock; argv: <port> <delay-seconds>; env MOCK_FAIL=1
|
||||||
|
# makes the chat endpoint answer 500.
|
||||||
|
cat >"$MOCK" <<'PYEOF'
|
||||||
|
import json, os, sys, time
|
||||||
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||||
|
|
||||||
|
PORT = int(sys.argv[1])
|
||||||
|
DELAY = float(sys.argv[2])
|
||||||
|
FAIL = os.environ.get("MOCK_FAIL", "0") == "1"
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
def log_message(self, *a):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _send(self, obj, code=200):
|
||||||
|
b = json.dumps(obj).encode()
|
||||||
|
self.send_response(code)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(b)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(b)
|
||||||
|
|
||||||
|
def do_POST(self):
|
||||||
|
n = int(self.headers.get("Content-Length", "0"))
|
||||||
|
body = json.loads(self.rfile.read(n) or b"{}")
|
||||||
|
if self.path == "/v1/messages/count_tokens":
|
||||||
|
text = body["messages"][0]["content"]
|
||||||
|
self._send({"input_tokens": max(1, len(text.split()))})
|
||||||
|
elif self.path == "/v1/chat/completions":
|
||||||
|
if FAIL:
|
||||||
|
self._send({"error": "engine down"}, 500)
|
||||||
|
return
|
||||||
|
max_tokens = int(body.get("max_tokens", 64))
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "text/event-stream")
|
||||||
|
self.end_headers()
|
||||||
|
for i in range(max_tokens):
|
||||||
|
ev = {"choices": [{"delta": {"content": "tok%d " % i}}]}
|
||||||
|
self.wfile.write(("data: %s\n\n" % json.dumps(ev)).encode())
|
||||||
|
self.wfile.flush()
|
||||||
|
time.sleep(DELAY)
|
||||||
|
prompt_words = max(1, len(body["messages"][0]["content"].split()))
|
||||||
|
ev = {"choices": [{"delta": {}}],
|
||||||
|
"usage": {"prompt_tokens": prompt_words,
|
||||||
|
"completion_tokens": max_tokens}}
|
||||||
|
self.wfile.write(("data: %s\n\n" % json.dumps(ev)).encode())
|
||||||
|
self.wfile.write(b"data: [DONE]\n\n")
|
||||||
|
self.wfile.flush()
|
||||||
|
else:
|
||||||
|
self._send({"error": "unknown path"}, 404)
|
||||||
|
|
||||||
|
|
||||||
|
HTTPServer(("127.0.0.1", PORT), Handler).serve_forever()
|
||||||
|
PYEOF
|
||||||
|
}
|
||||||
|
free_port() {
|
||||||
|
python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()'
|
||||||
|
}
|
||||||
|
|
||||||
|
start_mock() { # $1 = ok|fail
|
||||||
|
PORT="$(free_port)"
|
||||||
|
if [ "$1" = "fail" ]; then
|
||||||
|
MOCK_FAIL=1 python3 "$MOCK" "$PORT" 0.005 &
|
||||||
|
else
|
||||||
|
python3 "$MOCK" "$PORT" 0.005 &
|
||||||
|
fi
|
||||||
|
MOCK_PID=$!
|
||||||
|
local i
|
||||||
|
for i in $(seq 1 100); do
|
||||||
|
if python3 -c "import socket; socket.create_connection(('127.0.0.1', $PORT), 0.2).close()" 2>/dev/null; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
sleep 0.1
|
||||||
|
done
|
||||||
|
echo "mock did not start" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
stop_mock() {
|
||||||
|
if [ -n "$MOCK_PID" ]; then
|
||||||
|
kill "$MOCK_PID" 2>/dev/null || true
|
||||||
|
wait "$MOCK_PID" 2>/dev/null || true
|
||||||
|
MOCK_PID=""
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "measure exists and compiles" {
|
||||||
|
[ -f "$MEASURE" ]
|
||||||
|
python3 -m py_compile "$MEASURE"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "measure computes decode rate and exact usage from a controlled endpoint" {
|
||||||
|
start_mock ok || return 1
|
||||||
|
run env AR_BASE="http://127.0.0.1:$PORT" AR_MODEL=mock python3 "$MEASURE"
|
||||||
|
stop_mock
|
||||||
|
[ "$status" -eq 0 ] || { echo "$output"; return 1; }
|
||||||
|
# run() merges stderr in; the contract is: exactly ONE json result line
|
||||||
|
# (the window script parses the same way), progress on stderr.
|
||||||
|
printf '%s\n' "$output" | python3 -c '
|
||||||
|
import json, sys
|
||||||
|
lines = [l for l in sys.stdin.read().splitlines() if l.startswith("{")]
|
||||||
|
assert len(lines) == 1, "expected exactly one JSON line, got %d" % len(lines)
|
||||||
|
d = json.loads(lines[0])
|
||||||
|
# nominal 200 t/s (0.005 s/chunk) minus small per-chunk overhead
|
||||||
|
for k in ("decode_tps", "decode_tps_115", "decode_tps_1056", "decode_tps_4096"):
|
||||||
|
v = d[k]
|
||||||
|
assert 60.0 <= v <= 320.0, "%s=%s outside [60, 320]" % (k, v)
|
||||||
|
# the mock emits exactly max_tokens completion tokens; the accounting must be exact
|
||||||
|
assert d["completion_tokens_4096"] == 256, d["completion_tokens_4096"]
|
||||||
|
assert d["completion_tokens_115"] == 256
|
||||||
|
# prompt sizing: calibration is exactly 1.0 words/token in the mock, so the
|
||||||
|
# measured prompt lengths equal the targets
|
||||||
|
assert d["prompt_tokens_115"] == 115
|
||||||
|
assert d["prompt_tokens_4096"] == 4096
|
||||||
|
print("measure claims verified")
|
||||||
|
'
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "measure fails loudly on an unhealthy endpoint instead of half-measuring" {
|
||||||
|
start_mock fail || return 1
|
||||||
|
run env AR_BASE="http://127.0.0.1:$PORT" AR_MODEL=mock python3 "$MEASURE"
|
||||||
|
stop_mock
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
! printf '%s\n' "$output" | grep -qE '^\{'
|
||||||
|
}
|
||||||
Reference in new issue
Block a user