#!/usr/bin/env bash
# run-swift15-bf16.sh — RUNS ON DANKSERVER. Boot the no-recipe bf16 Swift1.5
# container for the two offline campaign phases. The engine owns the GPUs, so
# the production unit must be stopped first (this script does it and restores
# on exit, autoresearch pattern: stop, never mask — relinked unit).
#
#   kld    --kld-record the reference logits from the bf16 model
#          (runs the corpus and EXITS; engine writes DIR and leaves)
#   calib  serve with the calibration tap; a separate driver
#          (scripts/calib-push.py) pushes the corpus; the engine writes the
#          Hessians at RADIANCE_CALIB_TOKENS and the tap switches itself off.
#          Stop this script after calib-push prints TAP_DONE.
#
# Shape = the vendor's bf16 calibration lines (GUIDE.md §4.3:528 + MOE-W4.md
# "calibrate the checkpoint as it ships"): expert_tiered + weights-disk-tier,
# no speculation, deterministic. Large prefill steps because a disk-tier step
# costs the same at 1 token as at 32K. The container declared --max-ctx 4096;
# --max-model-len 16384 exceeds it on purpose (the KLD prompts run to 12,007
# tokens, calib to 8,242) — the boot ledger arbitrates the QSA buffers; if
# declare refuses, re-convert with a wider --max-ctx.
set -uo pipefail

MODE="${1:?usage: run-swift15-bf16.sh kld|calib}"
IMAGE="stilldeadcode/radiance@sha256:4c3f484564bb2872784e46b0a59cbd71becc9c1e746e301a3596ffae253c322e"
RAD=/home/dank/models/rad/qwen4exp-swift15-bf16.rad
C=swift15ref
LOG=/home/dank/logs/swift15_bf16.log
KLD_REF_DIR=/home/dank/corpora/kld-ref
CALIB_DIR=/home/dank/corpora/calib
KLD_CORPUS=/home/dank/corpora/out/kld.jsonl

restore() { sudo systemctl start frank-primary >/dev/null 2>&1 || true; }
trap restore EXIT
sudo systemctl stop frank-primary >/dev/null 2>&1 || true
podman rm -f "$C" >/dev/null 2>&1 || true

ARGS=(--model /models/rad/qwen4exp-swift15-bf16.rad
      --tp 2 --placement expert_tiered --weights-disk-tier
      --host-pool-mib 12288 --deterministic --gpu-headroom-mib 768
      --num-speculative-tokens 0
      --kv-cache-dtype fp8 --max-model-len 16384
      --ngram-placement ram
      --override-chat-template /models/tpl.jinja)

case "$MODE" in
  kld)
    mkdir -p "$KLD_REF_DIR"
    ARGS+=(--kld-record /data/kld-ref --kld-corpus /data/kld.jsonl
           --max-num-seqs 32 --max-num-batched-tokens 32768)
    ;;
  calib)
    mkdir -p "$CALIB_DIR"
    ARGS+=(--port 7770 --max-num-seqs 64 --max-num-batched-tokens 65536
           --prefix-cache-dir /data/calib-kvcache)
    ;;
  probe)
    ARGS+=(--port 7770 --max-num-seqs 8 --max-num-batched-tokens 4096)
    ;;
  *) echo "unknown mode: $MODE" >&2; exit 2 ;;
esac

# The prod template copy, readable from the container: mount the models tree
# ro plus the state dir rw (KLD record + prefix tier land there).
cp -f /home/dank/serverAppConfigs/scripts/qwen38-froggeric-v22.5.jinja \
      /home/dank/models/tpl.jinja 2>/dev/null || true

EXTRA_ENV=()
if [ "$MODE" = calib ]; then
    EXTRA_ENV=(-v "$CALIB_DIR:/data/calib"
               -e RADIANCE_CALIB_DIR=/data/calib
               -e RADIANCE_CALIB_TOKENS=10000000)
fi

podman run --rm --name "$C" \
    --network host --init \
    --security-opt seccomp=unconfined \
    --ulimit memlock=-1:-1 \
    --device /dev/kfd --device /dev/dri \
    -v /home/dank/models:/models:ro \
    -v /home/dank/radiance-state:/data \
    "${KLD_MOUNT[@]}" \
    "${EXTRA_ENV[@]}" \
    "$IMAGE" radiance "${ARGS[@]}" 2>&1 | tee "$LOG"
rc=${PIPESTATUS[0]}
echo "swift15-bf16 $MODE exited rc=$rc; log: $LOG"
exit "$rc"