#!/usr/bin/env bash
# autoresearch.sh — canonical benchmark entrypoint: speed of frank-primary.
#
# Runs on the operator box and drives one bench window on DANKSERVER:
#   1. fingerprint the candidate trio (env + serve script + unit)
#   2. memo fast-path: an identical fingerprint that already produced a FULLY
#      successful window (bench + verified production restore) returns the
#      cached metrics without touching the fleet
#   3. stage the candidate trio plus the measure client into
#      /home/dank/.autoresearch-frank-primary/ (ephemeral; the repo checkout
#      on the host stays the production source of truth)
#   4. run the window (scripts/autoresearch/frank-primary-window.sh):
#      stop + mask the production unit, boot the staged config AS THE SERVICE
#      (same serve script, same container, same port 7770), wait for health,
#      run the in-container usage-counter measure, then restore the production
#      unit and verify its health — AGENTS.md Benchmarking Policy
#   5. print METRIC lines (parse by run_experiment)
#
# Primary metric: decode_tps (HIGHER is better) — median across the three
# prompt depths (115/1056/4096) of per-depth medians of
# completion_tokens / (wall - ttft), 3 runs each after a warm pass,
# temperature 0, the campaign's deterministic word-seed workload.
#
# Exit 0 with METRIC lines on success; non-zero on any failure (the window
# script's EXIT trap restores production on every path).
set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
HOST="${FRANK_PRIMARY_AR_HOST:-dankserver}"
STAGE="/home/dank/.autoresearch-frank-primary"

# FRANK_PRIMARY_AR_ENV_FILE stages a DIFFERENT env than the repo's — that is how
# the arms driver A/Bs engine knobs without mutating the checkout. The
# fingerprint covers whatever file is staged, so the memo cannot cross arms.
ENV_FILE="${FRANK_PRIMARY_AR_ENV_FILE:-$SCRIPT_DIR/scripts/frank-primary.env}"
SERVE="$SCRIPT_DIR/scripts/serve-frank-primary.sh"
UNIT="$SCRIPT_DIR/scripts/frank-primary.service"
MEASURE="$SCRIPT_DIR/scripts/autoresearch/frank-primary-measure.py"
WINDOW="$SCRIPT_DIR/scripts/autoresearch/frank-primary-window.sh"

for f in "$ENV_FILE" "$SERVE" "$UNIT" "$MEASURE" "$WINDOW"; do
    [ -f "$f" ] || { echo "[autoresearch] missing harness file: $f" >&2; exit 1; }
done
command -v python3 >/dev/null 2>&1 || { echo "[autoresearch] python3 required on operator box" >&2; exit 1; }

# Fingerprint of everything that can change the engine's argv or environment.
HASH="$(cat "$ENV_FILE" "$SERVE" "$UNIT" | sha256sum | cut -d' ' -f1)"
echo "[autoresearch] candidate fingerprint $HASH" >&2

ssh_run() { ssh -o BatchMode=yes -o ConnectTimeout=15 "$HOST" "$@"; }

emit_metrics() { # $1 = run.json
    python3 -c '
import json, sys
run = json.loads(sys.argv[1])
m = run["measure"]
print("METRIC decode_tps=%s" % m["decode_tps"])
for k in ("decode_tps_115", "decode_tps_1056", "decode_tps_4096"):
    print("METRIC %s=%s" % (k, m[k]))
print("METRIC ttft_ms_4096=%s" % m["ttft_ms_4096"])
print("METRIC kv_tokens=%s" % (run.get("kv_tokens") or 0))
print("METRIC boot_seconds=%s" % (run.get("boot_seconds") or -1))
print("METRIC completion_tokens_4096=%s" % m.get("completion_tokens_4096", 0))
# Any extra sampling temperature the measure swept (AR_TEMPS), e.g. the 0.7
# arm — that is where speculative-decode acceptance shows up, since a greedy
# draft matches a greedy target exactly and is rejected more under real
# sampling. Suffixed keys only (_t0.7), so the historical lines are untouched.
for k in sorted(m):
    if "_t" in k and (k.startswith("decode_tps") or k.startswith("ttft_ms")):
        print("METRIC %s=%s" % (k, m[k]))
' "$1"
}

# --- memo fast-path -------------------------------------------------------
cached_hash="$(ssh_run "cat '$STAGE/last/hash' 2>/dev/null" 2>/dev/null || true)"
if [ -n "$cached_hash" ] && [ "$cached_hash" = "$HASH" ]; then
    cached_run="$(ssh_run "cat '$STAGE/last/run.json' 2>/dev/null" || true)"
    if [ -n "$cached_run" ]; then
        echo "[autoresearch] memo hit — cached metrics for $HASH, fleet untouched" >&2
        emit_metrics "$cached_run"
        exit 0
    fi
fi

# --- stage the candidate ----------------------------------------------------
echo "[autoresearch] staging candidate on $HOST:$STAGE" >&2
ssh_run "mkdir -p '$STAGE'"
scp -q "$ENV_FILE" "$HOST:$STAGE/frank-primary.env"
scp -q "$SERVE" "$HOST:$STAGE/serve.sh"
scp -q "$MEASURE" "$HOST:$STAGE/measure.py"
scp -q "$WINDOW" "$HOST:$STAGE/window.sh"
ssh_run "printf '%s\n' '$HASH' > '$STAGE/hash'"

# --- the window -------------------------------------------------------------
# Budgets: bench boot up to 20 min (warm boots measured ~7 min), measure up
# to 30 min, restore boot up to 20 min — call run_experiment with
# timeout_seconds >= 3600.
echo "[autoresearch] opening bench window (stop+mask -> boot -> measure -> restore)" >&2
set +e
ssh_run "bash '$STAGE/window.sh'"
rc=$?
set -e
if [ "$rc" -ne 0 ]; then
    echo "[autoresearch] window failed (rc=$rc); host-side evidence:" >&2
    ssh_run "tail -60 '$STAGE/fail.log' 2>/dev/null || tail -40 '$STAGE/serve.out' 2>/dev/null || tail -40 '$STAGE/measure.out' 2>/dev/null || true" >&2 || true
    exit 1
fi

run_json="$(ssh_run "cat '$STAGE/run.json'")" || {
    echo "[autoresearch] could not read $STAGE/run.json back" >&2; exit 1; }
echo "[autoresearch] window result:" >&2
printf '%s\n' "$run_json" >&2
emit_metrics "$run_json"

echo "[autoresearch] engine evidence (plan / compile path / spec counters):" >&2
ssh_run "cat '$STAGE/evidence.log' 2>/dev/null" >&2 || true
exit 0