Frank
f998a2d9f2
DFlash2 (100.0 t/s decode, 287.1 t/s burst-4) beats MTP self-speculation (85.9 t/s, 179.7 t/s) despite a lower accept length (4.24 vs 5.66): the small separate drafter steps faster than the target's own MTP layer applied autoregressively. DFlash2 is the deployed shape; both swap-script paths now default to dflash/7. Docs updated.
125 lines
5.6 KiB
Bash
Executable File
125 lines
5.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# swap-frank-medium.sh — orchestrate a frank-medium model swap.
|
|
#
|
|
# A swap is four steps, and this script drives all of them:
|
|
# 1. generate a manifest for the new HF repo (hf-manifest.py),
|
|
# 2. download it to the host, resumable + byte/ sha256-verified (hf-fetch.sh),
|
|
# 3. detect the checkpoint's weight quantization and point the quadlet at it
|
|
# (SNAP + the matching SPEC_METHOD/SPEC + KV_MEM),
|
|
# 4. hand off to deploy.sh (pull, restart, health gate).
|
|
#
|
|
# The entrypoint is QUANTIZATION-AWARE: it reads quant_method from the
|
|
# checkpoint's config.json and serves the plain FP8 path (no MXFP4 patches,
|
|
# --quantization=fp8, ROCM_AITER unified attention, MTP self-speculation) or
|
|
# the MXFP4 patch-and-serve path (DFlash2 external drafter). So the ONLY repo
|
|
# change a swap makes is the quadlet's four env vars — this script edits them.
|
|
#
|
|
# Usage:
|
|
# scripts/swap-frank-medium.sh --repo <owner/repo> --target <host-dir> \
|
|
# [--generate-manifest] [--manifest <path>] [--spec N] [--kv MB] \
|
|
# [--apply] [--deploy]
|
|
#
|
|
# --repo <owner/repo> the Hugging Face repo to serve
|
|
# --target <host-dir> the host dir the model lands in
|
|
# (/home/dank/models/<name>); the quadlet SNAP is
|
|
# derived as /models/<name>
|
|
# --generate-manifest write scripts/frank-medium-<name>.manifest from the
|
|
# live repo tree (needs the host's HF token)
|
|
# --manifest <path> use an existing manifest instead of generating one
|
|
# --spec N speculative tokens (default 8 for fp8, 7 for mxfp4)
|
|
# --kv MB KV pool pin in MiB (default 0 = auto-size)
|
|
# --apply edit the quadlet (SNAP + SPEC_METHOD + SPEC + KV_MEM)
|
|
# --deploy run deploy.sh after --apply
|
|
#
|
|
# Without --apply the script stops after the download + quantization
|
|
# detection and prints the quadlet lines to set. The download and the
|
|
# quantization detection run on the host (the model dir and the token are
|
|
# there); the quadlet edit runs in the repo.
|
|
|
|
set -euo pipefail
|
|
|
|
HOST="dank@192.168.0.235"
|
|
QUADLET="dankserver/frank-medium/quadlet/frank-medium.container"
|
|
MODELS_ROOT="/home/dank/models"
|
|
|
|
repo="" target="" manifest="" spec="" kv=0 gen=0 apply=0 deploy=0
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--repo) repo="$2"; shift 2 ;;
|
|
--target) target="$2"; shift 2 ;;
|
|
--manifest) manifest="$2"; shift 2 ;;
|
|
--spec) spec="$2"; shift 2 ;;
|
|
--kv) kv="$2"; shift 2 ;;
|
|
--generate-manifest) gen=1; shift ;;
|
|
--apply) apply=1; shift ;;
|
|
--deploy) deploy=1; shift ;;
|
|
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
[[ -n "$repo" && -n "$target" ]] || {
|
|
echo "usage: $0 --repo <owner/repo> --target <host-dir> [--generate-manifest|--manifest <path>] [--spec N] [--kv MB] [--apply] [--deploy]" >&2
|
|
exit 2;
|
|
}
|
|
|
|
name="$(basename "$target")"
|
|
snap_c="/models/$name"
|
|
|
|
# --- 1. manifest -------------------------------------------------------------
|
|
if [[ "$gen" = 1 ]]; then
|
|
manifest="scripts/frank-medium-${name}.manifest"
|
|
echo "==> Generating manifest -> $manifest"
|
|
scp -q scripts/hf-manifest.py "$HOST:/tmp/hf-manifest.py"
|
|
ssh "$HOST" "python3 /tmp/hf-manifest.py '$repo' /tmp/swap-manifest"
|
|
scp -q "$HOST:/tmp/swap-manifest" "$manifest"
|
|
elif [[ -z "$manifest" ]]; then
|
|
echo "ERROR: pass --generate-manifest or --manifest <path>" >&2
|
|
exit 2
|
|
fi
|
|
[[ -f "$manifest" ]] || { echo "ERROR: manifest not found: $manifest" >&2; exit 2; }
|
|
|
|
# --- 2. download (resumable, verified) ---------------------------------------
|
|
echo "==> Staging manifest + hf-fetch.sh on $HOST"
|
|
scp -q "$manifest" "$HOST:/tmp/swap-manifest"
|
|
scp -q scripts/hf-fetch.sh "$HOST:/tmp/swap-hf-fetch.sh"
|
|
echo "==> Downloading $repo -> $target"
|
|
ssh "$HOST" "bash /tmp/swap-hf-fetch.sh '$target' /tmp/swap-manifest '$repo'"
|
|
|
|
# --- 3. detect the quantization on the host ----------------------------------
|
|
quant="$(ssh "$HOST" "python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get(\"quantization_config\",{}).get(\"quant_method\",\"\"))' '$target/config.json' 2>/dev/null || true")"
|
|
echo "==> Detected quantization: ${quant:-<none>}"
|
|
|
|
case "$quant" in
|
|
fp8) spec_method=dflash; default_spec=7 ;;
|
|
mxfp4|quark|"") spec_method=dflash; default_spec=7 ;;
|
|
*) spec_method=dflash; default_spec=7
|
|
echo "WARNING: unknown quant '$quant'; defaulting to the DFlash2 path" >&2 ;;
|
|
esac
|
|
[[ -n "$spec" ]] || spec="$default_spec"
|
|
|
|
echo "==> Quadlet env vars for this checkpoint:"
|
|
echo " Environment=SNAP=$snap_c"
|
|
echo " Environment=SPEC_METHOD=$spec_method"
|
|
echo " Environment=SPEC=$spec"
|
|
echo " Environment=KV_MEM=$kv"
|
|
|
|
# --- 4. apply the quadlet edit (in the repo) ---------------------------------
|
|
if [[ "$apply" = 1 ]]; then
|
|
echo "==> Editing $QUADLET"
|
|
sed -i -E "s|^Environment=SNAP=.*|Environment=SNAP=$snap_c|" "$QUADLET"
|
|
sed -i -E "s|^Environment=SPEC_METHOD=.*|Environment=SPEC_METHOD=$spec_method|" "$QUADLET"
|
|
sed -i -E "s|^Environment=SPEC=.*|Environment=SPEC=$spec|" "$QUADLET"
|
|
sed -i -E "s|^Environment=KV_MEM=.*|Environment=KV_MEM=$kv|" "$QUADLET"
|
|
echo "==> Quadlet updated."
|
|
fi
|
|
|
|
# --- 5. deploy ----------------------------------------------------------------
|
|
if [[ "$deploy" = 1 ]]; then
|
|
[[ "$apply" = 1 ]] || { echo "ERROR: --deploy requires --apply" >&2; exit 2; }
|
|
echo "==> Deploying (pull, restart, health gate)"
|
|
./scripts/deploy.sh dankserver --service frank-medium --skip-e2e
|
|
echo "==> Next: run the capability battery"
|
|
echo " python3 scripts/bench-conversation.py http://192.168.0.235:7770 --model frank-medium --salt ${name} --max-tokens 512 --burst 4"
|
|
fi
|