#!/usr/bin/env bash
# autoresearch.sh — canonical speed benchmark for frank-large (Dankclaw iGPU).
#
# Window semantics per the repo Benchmarking Policy: the deployed unit is
# STOPPED, the candidate (this worktree's scripts/frank-large.env +
# scripts/serve-frank-large.sh, rsynced to ~/jobs/autoresearch so the host's
# git checkout stays clean) boots standalone on :7770, the fixed greedy
# battery runs against it ON the host, and a trap restores the deployed
# baseline unit whatever happens. The candidate never touches the host
# checkout; "keep" in the loop means a real deploy later.
#
# Emits: METRIC decode_tokens_per_sec (primary, higher better),
#        METRIC prefill_tokens_per_sec, METRIC ttft_ms, ASI identity lines.
set -euo pipefail

REPO="$(cd "$(dirname "$0")" && pwd)"
HOST=dankclaw
RDIR=/home/dank/jobs/autoresearch
HEALTH_TRIES=200   # x5s = ~17 min ceiling (BF16 boots ~230s; INT8/dequant may differ)

restore() {
    local rc=$?
    trap - EXIT
    echo "[harness] tearing down candidate, restoring deployed baseline..." >&2
    ssh "$HOST" 'sudo -n systemctl stop frank-large.service 2>/dev/null;
                 podman rm -f frank-large-mimo 2>/dev/null;
                 pkill -x llama-server 2>/dev/null; true' >/dev/null 2>&1 || true
    ssh "$HOST" 'sudo -n systemctl start frank-large.service' >/dev/null 2>&1 || true
    exit $rc
}
trap restore EXIT

# 1. Ship the candidate (env + serve + battery) outside the git checkout.
ssh "$HOST" "mkdir -p $RDIR"
rsync -a "$REPO/scripts/frank-large.env" "$REPO/scripts/serve-frank-large.sh" \
    "$REPO/scripts/autoresearch-bench.py" "$HOST:$RDIR/"

# 2. Free the GPU and the port: stop the deployed unit (mask is unnecessary—
#    the unit only starts via systemd, and we never enable more of it).
ssh "$HOST" 'sudo -n systemctl stop frank-large.service;
             podman rm -f frank-large-mimo 2>/dev/null;
             pkill -x llama-server 2>/dev/null; true' >/dev/null

# 3. Boot the candidate detached; the launcher's own output IS the serve
#    stage output (gate failures print there before any container exists).
echo "[harness] starting candidate..." >&2
ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \
        bash $RDIR/serve-frank-large.sh >$RDIR/serve.log 2>&1 </dev/null & echo ok" \
    | tee /tmp/autoresearch-launch.log >&2
grep -q ok /tmp/autoresearch-launch.log || { echo "ERROR: launch failed" >&2; exit 1; }

# 4. Wait for health (503 while loading; anything else = boot failure).
echo "[harness] waiting for health..." >&2
if ! ssh "$HOST" "for i in \$(seq 1 $HEALTH_TRIES); do
        code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 \
              http://127.0.0.1:7770/health 2>/dev/null || echo 000)
        [ \"\$code\" = 200 ] && echo UP && exit 0
        st=\$(podman inspect -f '{{.State.Status}}' frank-large-mimo 2>/dev/null || echo gone)
        if [ "\$st" = exited ] || { [ "\$st" = gone ] && ! pgrep -f serve-frank-large >/dev/null && ! pgrep -x llama-server >/dev/null; }; then echo DOWN; exit 1; fi
        sleep 5
    done; exit 1" | grep -q UP; then
    echo "ERROR: candidate never became healthy; serve.log tail:" >&2
    ssh "$HOST" "tail -30 $RDIR/serve.log; echo '--- engine log ---';
                 podman logs --tail 120 frank-large-mimo 2>&1 |
                 grep -aiE 'error|raise|failed|traceback|assert|kernel' |
                 tail -30" >&2 || true
    exit 1
fi

# 5. Identity evidence for the loop's records (exact engine argv).
ASI_ARGS="$(ssh "$HOST" 'podman inspect frank-large-mimo \
    --format "{{join .Config.Cmd \" \"}}" 2>/dev/null' | tr -d "\n" | head -c 400 || true)"
if [ -z "$ASI_ARGS" ]; then
    ASI_ARGS="$(ssh "$HOST" "sh -c 'tr \"\\0\" \" \" < /proc/\\$(pgrep -x llama-server | head -1)/cmdline 2>/dev/null || true'" | head -c 400 || true)"
fi
echo "ASI engine_args=${ASI_ARGS:0:400}"

# 6. The battery, on the host, medians over 3 reps x 2 greedy prompts.
ssh "$HOST" "python3 $RDIR/autoresearch-bench.py \
    --url http://127.0.0.1:7770 --model frank-large"