feat(frank-medium): quantization-aware entrypoint + FP8 model swap

The entrypoint now reads the model's weight quantization from config.json and serves the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP self-speculation, ROCM_AITER unified attention with the R4D all-reduce) or the MXFP4 patch-and-serve path. A model swap is now: change SNAP (+ SPEC_METHOD/SPEC), restart. - radiance-entrypoint.sh: quantization detection, gated MXFP4 env vars and patch block, --quantization fp8 flag - quadlet: SNAP -> Qwen3.8-27B-Uncensored-FP8, SPEC_METHOD=mtp, SPEC=8, KV_MEM=0 (profile the shape, the documented plain-FP8 default) - deploy-remote.sh: model-aware gates (read SNAP/SPEC_METHOD from the quadlet; drafter gate only for dflash, r4d.so gate only for mxfp4) - tests: quantization-aware drafter/spec and KV-pool tests - hf-fetch.sh: generic, resumable, verified, idempotent HF fetch tool - frank-medium-fp8.manifest: 17 files (10 config + 7 shards) with sizes and LFS sha256
2026-09-20 21:30:30 -07:00
parent c42780f9f3
commit d389d09d97
6 changed files with 251 additions and 63 deletions
@@ -1211,6 +1211,19 @@ deploy_llm_medium() {
    # missing never becomes healthy, and a green deploy would leave the endpoint
    # dark with nothing in the deploy log to say why.
    local missing=0
+    # Model-aware gates: read the serving shape from the quadlet (the source
+    # of truth), so a model swap is just "edit the quadlet + deploy". The
+    # container /models mount maps to /home/dank/models on the host. The
+    # drafter gate applies only to the dflash path; the patched libr4d gate
+    # applies only to the mxfp4/quark path (the plain FP8 path uses the
+    # image's built-in r4d.so and skips the MXFP4 patches).
+    local quadlet_file="dankserver/frank-medium/quadlet/frank-medium.container"
+    local snap_c snap spec_method quant
+    snap_c="$(grep -E '^Environment=SNAP=' "$quadlet_file" | head -1 | cut -d= -f2-)"
+    spec_method="$(grep -E '^Environment=SPEC_METHOD=' "$quadlet_file" | head -1 | cut -d= -f2-)"
+    snap="/home/dank/models${snap_c#/models}"
+    quant="$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("quantization_config",{}).get("quant_method",""))' "$snap/config.json" 2>/dev/null || true)"
+    [[ -n "$snap_c" ]] || { echo "ERROR: cannot read SNAP from $quadlet_file" >&2; return 1; }
    if ! podman image exists "$LLM_MEDIUM_IMAGE_LOCAL"; then
        if podman image exists "$LLM_MEDIUM_IMAGE"; then
            podman tag "$LLM_MEDIUM_IMAGE" "$LLM_MEDIUM_IMAGE_LOCAL"
@@ -1220,27 +1233,33 @@ deploy_llm_medium() {
            missing=1
        fi
    fi
-    if [[ ! -f "$LLM_MEDIUM_SNAP/config.json" ]]; then
-        echo "ERROR: MXFP4 checkpoint absent at $LLM_MEDIUM_SNAP" >&2
+    if [[ ! -f "$snap/config.json" ]]; then
+        echo "ERROR: model checkpoint absent at $snap" >&2
        missing=1
    fi
-    if [[ ! -f "$LLM_MEDIUM_DRAFTER/config.json" ]]; then
+    if [[ "$spec_method" == "dflash" && ! -f "$LLM_MEDIUM_DRAFTER/config.json" ]]; then
        echo "ERROR: DFlash2-FP8 drafter absent at $LLM_MEDIUM_DRAFTER" >&2
        missing=1
    fi
-    # The patches clone is mounted and its scripts run at EVERY container start;
-    # the chat template is the fleet's reasoning-format contract.
+    # The patches clone is mounted and its scripts run at EVERY container start
+    # on the MXFP4 path; the chat template is the fleet's reasoning-format
+    # contract on both paths.
    if [[ ! -f "$LLM_MEDIUM_PATCHES/qwen-fixed-v22.3.jinja" ]]; then
        echo "ERROR: pinned patch clone / chat template absent at $LLM_MEDIUM_PATCHES" >&2
        echo "frank-medium: git clone https://github.com/ggz14/radiance-vllm-mxfp4 $LLM_MEDIUM_PATCHES" >&2
        missing=1
    fi
    # The image's shipped r4d.so predates the gated-delta-net overflow fix and
-    # NaNs this model, so the patched build is not optional.
-    if [[ ! -f "$LLM_MEDIUM_R4D/r4d.so" ]]; then
-        echo "ERROR: patched libr4d absent at $LLM_MEDIUM_R4D" >&2
-        missing=1
-    fi
+    # NaNs the MXFP4 model, so the patched build is not optional there. The
+    # plain FP8 path uses the image's built-in r4d.so (no MXFP4 patches).
+    case "$quant" in
+      mxfp4|quark)
+        if [[ ! -f "$LLM_MEDIUM_R4D/r4d.so" ]]; then
+            echo "ERROR: patched libr4d absent at $LLM_MEDIUM_R4D" >&2
+            missing=1
+        fi
+        ;;
+    esac
    [[ "$missing" -eq 0 ]] || return 1
    mkdir -p "$LLM_MEDIUM_CACHE"

@@ -0,0 +1,20 @@
+# orcarouter/Qwen3.8-27B-Uncensored-FP8 — FP8 E4M3 block-wise (128x128), dynamic act-quant
+# 7 safetensors shards + config/tokenizer/processor. Sizes + LFS sha256 from the
+# HF tree API and X-Linked-ETag (2026-09-20).
+config.json 51350
+generation_config.json 202
+chat_template.jinja 8952
+merges.txt 3353259
+preprocessor_config.json 390
+tokenizer.json 12809320
+tokenizer_config.json 17928
+video_preprocessor_config.json 385
+vocab.json 6722759
+model.safetensors.index.json 156034
+model-00001-of-00007.safetensors 3969317448 141e200c1950c59bc42ae28744f50db8f6ea4c66ec3312b55f1ccad2b60af858
+model-00002-of-00007.safetensors 4939321704 e863e88f53ea5a737581638e41334e9f9871a93053c4251a2cebd4aa6aa1f7c0
+model-00003-of-00007.safetensors 4939863424 7cbf8db425114b9efcfae64364a60e15e223cc5cadb510b7a3a6d3b48c8617cf
+model-00004-of-00007.safetensors 4971328696 37338dc30b490663b641567a57565408fee5f1385e7860dff16dca7be07e1519
+model-00005-of-00007.safetensors 4955596064 fc6fb0dd3675e0cc7e83a454376d12b705a51000f6625cf13f85e384018fc5cb
+model-00006-of-00007.safetensors 4071440632 5fdcbaf80805a750970e076bcd37e6ebe4c3a3d93ce73c767d2854119947952a
+model-00007-of-00007.safetensors 3019999168 1bed352a0625d22dc606de47fd3563f24fd9fa73915c4ddffe7947c4b56c28bf
@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+# hf-fetch.sh — resumable, verified HuggingFace fetch (gated or public repos).
+#
+#   bash scripts/hf-fetch.sh <dest_dir> <manifest>
+#
+# Manifest: one line per file, three tab/space-separated fields:
+#   <filename> <size_bytes> <sha256>
+# A blank sha256 field means "check size only". Lines starting with '#' and
+# blank lines are ignored. The first field is a path relative to <dest_dir>.
+#
+# Token: $HF_TOKEN, else ~/.cache/huggingface/token, else the llamaswap conf
+# (grep hf_...). Passed out-of-band via a mode-600 curl -K file — a -H header
+# is world-readable in /proc/<pid>/cmdline (the 2026-09-16 HF-token leak).
+#
+# Idempotent: a file whose size AND sha256 already match is skipped, so a
+# re-run after an interruption only fetches what is missing or corrupt.
+set -euo pipefail
+
+dest="${1:?usage: hf-fetch.sh <dest_dir> <manifest>}"
+manifest="${2:?usage: hf-fetch.sh <dest_dir> <manifest>}"
+repo="${3:?usage: hf-fetch.sh <dest_dir> <manifest> <owner/repo>}"
+
+mkdir -p "$dest"
+
+# --- token resolution --------------------------------------------------------
+token="${HF_TOKEN:-}"
+if [[ -z "$token" && -r "$HOME/.cache/huggingface/token" ]]; then
+    token="$(tr -d '[:space:]' < "$HOME/.cache/huggingface/token")"
+fi
+if [[ -z "$token" && -r "$HOME/.config/llamaswap.conf.yaml" ]]; then
+    token="$(grep -oE 'hf_[a-zA-Z0-9]+' "$HOME/.config/llamaswap.conf.yaml" | head -1)"
+fi
+[[ -n "$token" ]] || { echo "hf-fetch: no HF token (set HF_TOKEN or ~/.cache/huggingface/token)" >&2; exit 3; }
+
+auth_file="$(mktemp)"
+chmod 600 "$auth_file"
+printf 'header = "Authorization: Bearer %s"\n' "$token" > "$auth_file"
+cleanup() { rm -f "$auth_file"; }
+trap cleanup EXIT
+
+base="https://huggingface.co/${repo}/resolve/main"
+
+# --- per-file fetch + verify -------------------------------------------------
+fetch_one() {
+    local name="$1" size="$2" sha="$3"
+    local url="$base/$name"
+    local path="$dest/$name"
+    mkdir -p "$(dirname "$path")"
+
+    # idempotent skip
+    if [[ -f "$path" ]]; then
+        local cur_size sha_ok
+        cur_size="$(wc -c < "$path")"
+        sha_ok=true
+        if [[ -n "$sha" && "$(sha256sum "$path" | cut -d' ' -f1)" != "$sha" ]]; then
+            sha_ok=false
+        fi
+        if [[ "$cur_size" == "$size" && "$sha_ok" == "true" ]]; then
+            echo "  ok   $name ($size bytes)"
+            return 0
+        fi
+        echo "  refetch $name (have $cur_size, want $size)"
+    fi
+
+    # resumable download; -f fails on HTTP error so a 401/403 is not written as a file
+    curl -fsSL -C - -K "$auth_file" -o "$path" "$url"
+
+    # verify byte count
+    local got
+    got="$(wc -c < "$path")"
+    if [[ "$got" != "$size" ]]; then
+        echo "  FAIL $name: size $got != $size" >&2
+        return 1
+    fi
+    # verify sha256
+    if [[ -n "$sha" ]]; then
+        local actual
+        actual="$(sha256sum "$path" | cut -d' ' -f1)"
+        if [[ "$actual" != "$sha" ]]; then
+            echo "  FAIL $name: sha256 $actual != $sha" >&2
+            return 1
+        fi
+    fi
+    echo "  done $name ($size bytes)"
+}
+
+echo "hf-fetch: $repo -> $dest"
+status=0
+while IFS= read -r line || [[ -n "$line" ]]; do
+    [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue
+    name="$(awk '{print $1}' <<<"$line")"
+    size="$(awk '{print $2}' <<<"$line")"
+    sha="$(awk '{print $3}' <<<"$line")"
+    fetch_one "$name" "$size" "$sha" || status=1
+done < "$manifest"
+
+if [[ "$status" -ne 0 ]]; then
+    echo "hf-fetch: FAILED (see above)" >&2
+    exit 1
+fi
+echo "hf-fetch: all files verified"