feat(frank-medium): swap MXFP4 -> FP8; add the one-command swap toolchain
Checkpoint: just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq -> orcarouter/Qwen3.8-27B-Uncensored-FP8 (FP8 E4M3 block-wise, 30.9 GB / 7 shards, its own single-layer MTP head). The quantization-aware entrypoint serves the plain FP8 path: --quantization fp8, ROCM_AITER_UNIFIED_ATTN, MTP self-speculation (SPEC_METHOD=mtp, SPEC=8). Measured: MTP mean accept 5.66/8 (66.2% draft acceptance), KV pool auto-sized to 461,540 tokens (1.76x), battery 85.9 t/s decode / 0.06 s TTFT / 179.7 burst-4 (vs MXFP4 92.4/0.22/ 248.9). MXFP4 checkpoint stays on disk as the rollback target. Streamlining: scripts/hf-manifest.py (generate a <filename> <size> <sha256> manifest from the live repo tree), scripts/hf-fetch.sh (resumable, byte + sha256 verified, idempotent fetch), scripts/swap-frank-medium.sh (generate manifest -> download -> detect quantization -> edit the quadlet -> deploy). Docs: new frank-medium-model-swap runbook; radiance runbook, both indexes, AGENTS.md, and log.md flipped to the FP8 shape.
This commit is contained in:
Frank
Executable
+103
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""hf-manifest.py — generate an hf-fetch.sh manifest from a HuggingFace repo.
|
||||
|
||||
Usage: python3 scripts/hf-manifest.py <owner/repo> [outfile]
|
||||
|
||||
Lists the repo tree via the HF API and, for each file, records the size and
|
||||
the LFS sha256 (the X-Linked-ETag header on the resolve endpoint). Non-LFS
|
||||
files (config.json, tokenizer, ...) have no LFS sha256, so they get a blank
|
||||
hash field — hf-fetch.sh then checks size only for them.
|
||||
|
||||
Output: one line per file, three space-separated fields:
|
||||
<filename> <size_bytes> <sha256-or-blank>
|
||||
|
||||
Token: $HF_TOKEN, else ~/.cache/huggingface/token, else the llamaswap conf
|
||||
(grep hf_...). Gated repos need a token that has been granted access.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
|
||||
def get_token():
|
||||
t = os.environ.get("HF_TOKEN", "")
|
||||
if not t:
|
||||
p = os.path.expanduser("~/.cache/huggingface/token")
|
||||
if os.path.exists(p):
|
||||
with open(p) as f:
|
||||
t = f.read().strip()
|
||||
if not t:
|
||||
p = os.path.expanduser("~/.config/llamaswap.conf.yaml")
|
||||
if os.path.exists(p):
|
||||
with open(p) as f:
|
||||
m = re.search(r"hf_[a-zA-Z0-9]+", f.read())
|
||||
if m:
|
||||
t = m.group(0)
|
||||
return t
|
||||
|
||||
|
||||
def head(url, token):
|
||||
h = {"User-Agent": "hf-manifest/1.0"}
|
||||
if token:
|
||||
h["Authorization"] = f"Bearer {token}"
|
||||
req = urllib.request.Request(url, method="HEAD", headers=h)
|
||||
return urllib.request.urlopen(req, timeout=30)
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print(f"usage: {sys.argv[0]} <owner/repo> [outfile]", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
repo = sys.argv[1]
|
||||
outfile = sys.argv[2] if len(sys.argv) > 2 else None
|
||||
token = get_token()
|
||||
# 1. List the repo tree (siblings[].rfilename). Sizes and LFS sha256s
|
||||
# come from the resolve endpoint's HEAD (Content-Length + X-Linked-ETag);
|
||||
# the API's siblings carry only rfilename.
|
||||
api = f"https://huggingface.co/api/models/{repo}"
|
||||
req = urllib.request.Request(
|
||||
api, headers={"User-Agent": "hf-manifest/1.0",
|
||||
**({"Authorization": f"Bearer {token}"} if token else {})})
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
meta = json.load(r)
|
||||
siblings = meta.get("siblings", [])
|
||||
print(f"# {repo}: {len(siblings)} entries", file=sys.stderr)
|
||||
|
||||
lines = []
|
||||
for s in siblings:
|
||||
name = s.get("rfilename", "")
|
||||
if not name or name.endswith("/"):
|
||||
continue
|
||||
# 2. Size (Content-Length) + LFS sha256 (X-Linked-ETag) from the
|
||||
# resolve endpoint's HEAD. Non-LFS files have no 64-hex etag, so the
|
||||
# hash field stays blank (hf-fetch.sh checks size only for them).
|
||||
size = 0
|
||||
sha = ""
|
||||
resolve = f"https://huggingface.co/{repo}/resolve/main/{name}"
|
||||
try:
|
||||
with head(resolve, token) as r:
|
||||
size = int(r.headers.get("Content-Length") or 0)
|
||||
etag = (r.headers.get("X-Linked-ETag")
|
||||
or r.headers.get("ETag") or "").strip('"')
|
||||
if re.fullmatch(r"[0-9a-f]{64}", etag):
|
||||
sha = etag
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 404:
|
||||
raise
|
||||
lines.append(f"{name} {size} {sha}")
|
||||
print(f" {name} {size:>12} {sha or '(size only)'}", file=sys.stderr)
|
||||
|
||||
out = "\n".join(lines) + "\n"
|
||||
if outfile:
|
||||
with open(outfile, "w") as f:
|
||||
f.write(out)
|
||||
print(f"wrote {outfile}", file=sys.stderr)
|
||||
else:
|
||||
sys.stdout.write(out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+124
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env bash
|
||||
# swap-frank-medium.sh — orchestrate a frank-medium model swap.
|
||||
#
|
||||
# A swap is four steps, and this script drives all of them:
|
||||
# 1. generate a manifest for the new HF repo (hf-manifest.py),
|
||||
# 2. download it to the host, resumable + byte/ sha256-verified (hf-fetch.sh),
|
||||
# 3. detect the checkpoint's weight quantization and point the quadlet at it
|
||||
# (SNAP + the matching SPEC_METHOD/SPEC + KV_MEM),
|
||||
# 4. hand off to deploy.sh (pull, restart, health gate).
|
||||
#
|
||||
# The entrypoint is QUANTIZATION-AWARE: it reads quant_method from the
|
||||
# checkpoint's config.json and serves the plain FP8 path (no MXFP4 patches,
|
||||
# --quantization=fp8, ROCM_AITER unified attention, MTP self-speculation) or
|
||||
# the MXFP4 patch-and-serve path (DFlash2 external drafter). So the ONLY repo
|
||||
# change a swap makes is the quadlet's four env vars — this script edits them.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/swap-frank-medium.sh --repo <owner/repo> --target <host-dir> \
|
||||
# [--generate-manifest] [--manifest <path>] [--spec N] [--kv MB] \
|
||||
# [--apply] [--deploy]
|
||||
#
|
||||
# --repo <owner/repo> the Hugging Face repo to serve
|
||||
# --target <host-dir> the host dir the model lands in
|
||||
# (/home/dank/models/<name>); the quadlet SNAP is
|
||||
# derived as /models/<name>
|
||||
# --generate-manifest write scripts/frank-medium-<name>.manifest from the
|
||||
# live repo tree (needs the host's HF token)
|
||||
# --manifest <path> use an existing manifest instead of generating one
|
||||
# --spec N speculative tokens (default 8 for fp8, 7 for mxfp4)
|
||||
# --kv MB KV pool pin in MiB (default 0 = auto-size)
|
||||
# --apply edit the quadlet (SNAP + SPEC_METHOD + SPEC + KV_MEM)
|
||||
# --deploy run deploy.sh after --apply
|
||||
#
|
||||
# Without --apply the script stops after the download + quantization
|
||||
# detection and prints the quadlet lines to set. The download and the
|
||||
# quantization detection run on the host (the model dir and the token are
|
||||
# there); the quadlet edit runs in the repo.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
HOST="dank@192.168.0.235"
|
||||
QUADLET="dankserver/frank-medium/quadlet/frank-medium.container"
|
||||
MODELS_ROOT="/home/dank/models"
|
||||
|
||||
repo="" target="" manifest="" spec="" kv=0 gen=0 apply=0 deploy=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--repo) repo="$2"; shift 2 ;;
|
||||
--target) target="$2"; shift 2 ;;
|
||||
--manifest) manifest="$2"; shift 2 ;;
|
||||
--spec) spec="$2"; shift 2 ;;
|
||||
--kv) kv="$2"; shift 2 ;;
|
||||
--generate-manifest) gen=1; shift ;;
|
||||
--apply) apply=1; shift ;;
|
||||
--deploy) deploy=1; shift ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$repo" && -n "$target" ]] || {
|
||||
echo "usage: $0 --repo <owner/repo> --target <host-dir> [--generate-manifest|--manifest <path>] [--spec N] [--kv MB] [--apply] [--deploy]" >&2
|
||||
exit 2;
|
||||
}
|
||||
|
||||
name="$(basename "$target")"
|
||||
snap_c="/models/$name"
|
||||
|
||||
# --- 1. manifest -------------------------------------------------------------
|
||||
if [[ "$gen" = 1 ]]; then
|
||||
manifest="scripts/frank-medium-${name}.manifest"
|
||||
echo "==> Generating manifest -> $manifest"
|
||||
scp -q scripts/hf-manifest.py "$HOST:/tmp/hf-manifest.py"
|
||||
ssh "$HOST" "python3 /tmp/hf-manifest.py '$repo' /tmp/swap-manifest"
|
||||
scp -q "$HOST:/tmp/swap-manifest" "$manifest"
|
||||
elif [[ -z "$manifest" ]]; then
|
||||
echo "ERROR: pass --generate-manifest or --manifest <path>" >&2
|
||||
exit 2
|
||||
fi
|
||||
[[ -f "$manifest" ]] || { echo "ERROR: manifest not found: $manifest" >&2; exit 2; }
|
||||
|
||||
# --- 2. download (resumable, verified) ---------------------------------------
|
||||
echo "==> Staging manifest + hf-fetch.sh on $HOST"
|
||||
scp -q "$manifest" "$HOST:/tmp/swap-manifest"
|
||||
scp -q scripts/hf-fetch.sh "$HOST:/tmp/swap-hf-fetch.sh"
|
||||
echo "==> Downloading $repo -> $target"
|
||||
ssh "$HOST" "bash /tmp/swap-hf-fetch.sh '$target' /tmp/swap-manifest '$repo'"
|
||||
|
||||
# --- 3. detect the quantization on the host ----------------------------------
|
||||
quant="$(ssh "$HOST" "python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get(\"quantization_config\",{}).get(\"quant_method\",\"\"))' '$target/config.json' 2>/dev/null || true")"
|
||||
echo "==> Detected quantization: ${quant:-<none>}"
|
||||
|
||||
case "$quant" in
|
||||
fp8) spec_method=mtp; default_spec=8 ;;
|
||||
mxfp4|quark|"") spec_method=dflash; default_spec=7 ;;
|
||||
*) spec_method=dflash; default_spec=7
|
||||
echo "WARNING: unknown quant '$quant'; defaulting to the MXFP4 path" >&2 ;;
|
||||
esac
|
||||
[[ -n "$spec" ]] || spec="$default_spec"
|
||||
|
||||
echo "==> Quadlet env vars for this checkpoint:"
|
||||
echo " Environment=SNAP=$snap_c"
|
||||
echo " Environment=SPEC_METHOD=$spec_method"
|
||||
echo " Environment=SPEC=$spec"
|
||||
echo " Environment=KV_MEM=$kv"
|
||||
|
||||
# --- 4. apply the quadlet edit (in the repo) ---------------------------------
|
||||
if [[ "$apply" = 1 ]]; then
|
||||
echo "==> Editing $QUADLET"
|
||||
sed -i -E "s|^Environment=SNAP=.*|Environment=SNAP=$snap_c|" "$QUADLET"
|
||||
sed -i -E "s|^Environment=SPEC_METHOD=.*|Environment=SPEC_METHOD=$spec_method|" "$QUADLET"
|
||||
sed -i -E "s|^Environment=SPEC=.*|Environment=SPEC=$spec|" "$QUADLET"
|
||||
sed -i -E "s|^Environment=KV_MEM=.*|Environment=KV_MEM=$kv|" "$QUADLET"
|
||||
echo "==> Quadlet updated."
|
||||
fi
|
||||
|
||||
# --- 5. deploy ----------------------------------------------------------------
|
||||
if [[ "$deploy" = 1 ]]; then
|
||||
[[ "$apply" = 1 ]] || { echo "ERROR: --deploy requires --apply" >&2; exit 2; }
|
||||
echo "==> Deploying (pull, restart, health gate)"
|
||||
./scripts/deploy.sh dankserver --service frank-medium --skip-e2e
|
||||
echo "==> Next: run the capability battery"
|
||||
echo " python3 scripts/bench-conversation.py http://192.168.0.235:7770 --model frank-medium --salt ${name} --max-tokens 512 --burst 4"
|
||||
fi
|
||||
Reference in New Issue
Block a user