#!/usr/bin/env bash
# kld-gate-swift15.sh — RUNS ON DANKSERVER. Boot the w4dn8 candidate on the
# serving pin (1.0.13) in --kld-ref mode: score it against the bf16 reference
# recorded by run-swift15-bf16.sh kld, on the held-out 1.8M-token corpus.
# Gate (settled design): mean KL <= 0.03 AND top-1 agreement >= 95%.
# On fail the escalation is gate_up w4nl -> w5nl (+3-4 h), not a re-tune.
#
# Shape: the candidate's experts are ~37.6 GiB a rank at w4dn8, under the
# 40960 MiB pool, so the tier is fully resident — no disk tier here; the
# record scored the bf16 model at its own (disk-tier) shape, which is a
# placement choice, not a numerics choice (weights resident are identical).
set -uo pipefail

PIN_1013=sha256:4c3f484564bb2872784e46b0a59cbd71becc9c1e746e301a3596ffae253c322e
IMAGE="stilldeadcode/radiance@$PIN_1013"
RAD=/home/dank/models/rad/qwen4exp-swift15-w4dn8-i8-hc8m.rad
C=swift15gate
LOG=/home/dank/logs/swift15_gate.log
REPORT=/home/dank/corpora/kld-cand.json
MAX_KL=${FRANK_KLD_GATE_MAX_KL:-0.03}
MIN_TOP1=${FRANK_KLD_GATE_MIN_TOP1:-95}

restore() { sudo systemctl start frank-primary >/dev/null 2>&1 || true; }
trap restore EXIT
sudo systemctl stop frank-primary >/dev/null 2>&1 || true
podman rm -f "$C" >/dev/null 2>&1 || true
[ -d /home/dank/corpora/kld-ref ] && [ -n "$(ls /home/dank/corpora/kld-ref)" ] || {
    echo "no kld reference recorded" >&2; exit 1; }

podman run --rm --name "$C" \
    --network host --init \
    --security-opt seccomp=unconfined \
    --ulimit memlock=-1:-1 \
    --device /dev/kfd --device /dev/dri \
    -v /home/dank/models:/models:ro \
    -v /home/dank/radiance-state:/data \
    -v /home/dank/corpora/kld-ref:/data/kld-ref:ro \
    -v /home/dank/corpora/out/kld.jsonl:/data/kld.jsonl:ro \
    -v /home/dank/corpora:/out \
    "$IMAGE" \
    --model /models/rad/$(basename "$RAD") \
    --tp 2 --placement expert_tiered --host-pool-mib 40960 \
    --deterministic --gpu-headroom-mib 768 \
    --num-speculative-tokens 0 \
    --kv-cache-dtype fp8 --max-model-len 16384 \
    --ngram-placement ram \
    --override-chat-template /models/tpl.jinja \
    --kld-ref /data/kld-ref --kld-corpus /data/kld.jsonl \
    --vram-kv-mib 3072 --max-num-seqs 4 --max-num-batched-tokens 32768 \
    --kld-out /out/kld-cand.json 2>&1 | tee "$LOG"
rc=${PIPESTATUS[0]}
[ "$rc" -eq 0 ] || { echo "kld run rc=$rc; log: $LOG" >&2; exit "$rc"; }

python3 - "$REPORT" "$MAX_KL" "$MIN_TOP1" <<'PYEOF'
import json, sys, math
report, max_kl, min_top1 = sys.argv[1], float(sys.argv[2]), float(sys.argv[3])
d = json.load(open(report))
print(json.dumps(d, indent=1)[:4000])
flat = {}
def walk(o, path=""):
    if isinstance(o, dict):
        for k, v in o.items(): walk(v, f"{path}.{k}" if path else k)
    elif isinstance(o, list):
        pass
    else:
        flat[path] = o
walk(d)
def pick(*pats):
    for p in pats:
        for k, v in flat.items():
            if all(s in k.lower() for s in p) and isinstance(v, (int, float)) \
               and not (isinstance(v, float) and math.isnan(v)):
                return k, v
    return None, None
klk, kl = pick(("mean", "kl"), ("kl", "mean"), ("mean",))
tk, t1 = pick(("top1",), ("top", "1"), ("agreement",), ("top-1",))
t1 = t1 * 100 if t1 is not None and t1 <= 1 else t1
print(f"GATE mean_kl: {kl!r} ({klk}) <= {max_kl}? top1: {t1!r} ({tk}) >= {min_top1}?")
if kl is None or t1 is None:
    print("GATE UNKNOWN — key names not recognised, inspect the report by hand")
    sys.exit(2)
ok = kl <= max_kl and t1 >= min_top1
print("GATE_PASS" if ok else "GATE_FAIL")
sys.exit(0 if ok else 1)
PYEOF
echo "report: $REPORT rows: $REPORT.rows"