feat(frank-medium): quantization-aware entrypoint + FP8 model swap

The entrypoint now reads the model's weight quantization from config.json
and serves the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP
self-speculation, ROCM_AITER unified attention with the R4D all-reduce)
or the MXFP4 patch-and-serve path. A model swap is now: change SNAP
(+ SPEC_METHOD/SPEC), restart.

- radiance-entrypoint.sh: quantization detection, gated MXFP4 env vars
  and patch block, --quantization fp8 flag
- quadlet: SNAP -> Qwen3.8-27B-Uncensored-FP8, SPEC_METHOD=mtp, SPEC=8,
  KV_MEM=0 (profile the shape, the documented plain-FP8 default)
- deploy-remote.sh: model-aware gates (read SNAP/SPEC_METHOD from the
  quadlet; drafter gate only for dflash, r4d.so gate only for mxfp4)
- tests: quantization-aware drafter/spec and KV-pool tests
- hf-fetch.sh: generic, resumable, verified, idempotent HF fetch tool
- frank-medium-fp8.manifest: 17 files (10 config + 7 shards) with sizes
  and LFS sha256
This commit is contained in:
Frank committed 2026-09-20 21:30:30 -07:00
1 parent c42780f9f3
commit d389d09d97
6 files changed
+251 -63

No files matched your search

+26 -2
View File
@@ -42,7 +42,20 @@ if [ "$SPEC_METHOD" = dflash ]; then : "${SPEC:=7}"; else : "${SPEC:=4}"; fi
: "${R4D_ATTN:=1}"
: "${ASYNC:=0}"
if [ "$R4D_ATTN" = "1" ]; then ATTN=R4D; else ATTN=ROCM_AITER_UNIFIED_ATTN; fi
# Quantization-aware: read the model's weight quantization from config.json.
# fp8 -> the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP
# self-speculation, ROCM_AITER unified attention with the R4D all-reduce).
# mxfp4/quark (or unknown) -> the MXFP4 patch-and-serve path (the default).
QUANT="$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("quantization_config",{}).get("quant_method",""))' "$SNAP/config.json" 2>/dev/null || true)"
case "$QUANT" in
fp8) IS_MXFP4=0 ;;
mxfp4|quark|"") IS_MXFP4=1 ;;
*) IS_MXFP4=1; echo "[radiance] WARNING: quantization '$QUANT'; defaulting to the MXFP4 path" >&2 ;;
esac
if [ "$IS_MXFP4" = "0" ]; then ATTN=ROCM_AITER_UNIFIED_ATTN
elif [ "$R4D_ATTN" = "1" ]; then ATTN=R4D
else ATTN=ROCM_AITER_UNIFIED_ATTN; fi
if [ "$ASYNC" = 1 ]; then ASYNC_FLAG="--async-scheduling"; UNPAD=false; else
ASYNC_FLAG="--no-async-scheduling"; UNPAD=true; fi
@@ -82,7 +95,11 @@ export RADIANCE_DFLASH_SELECTOR_TOPK="${RADIANCE_DFLASH_SELECTOR_TOPK:-}"
export RADIANCE_R4D_REPORT=1
export RADIANCE_PRESHUFFLE=${RADIANCE_PRESHUFFLE:-1}
export RADIANCE_MXFP4_SANITIZE="${RADIANCE_MXFP4_SANITIZE:-0}"
export RADIANCE_MXFP4=1 RADIANCE_MXFP4_W4A8=1 RADIANCE_MXFP4_W4A8_MIN_M="$MIN_M"
if [ "$IS_MXFP4" = "1" ]; then
export RADIANCE_MXFP4=1 RADIANCE_MXFP4_W4A8=1 RADIANCE_MXFP4_W4A8_MIN_M="$MIN_M"
else
export RADIANCE_MXFP4=0 RADIANCE_MXFP4_W4A8=0 RADIANCE_USE_R4D_AR_QUANT=1
fi
export RADIANCE_FAST_DRAFT=${RADIANCE_FAST_DRAFT:-1}
export RADIANCE_DRAFT_TAU=${RADIANCE_DRAFT_TAU:-0.20}
export RADIANCE_DRAFT_RERANK
@@ -171,6 +188,7 @@ if [ -n "$CC_ITEMS" ]; then
EXTRA="$EXTRA --compilation-config {$CC_ITEMS}"
fi
if [ "$IS_MXFP4" = "1" ]; then
# ---------------------------------------------------------------- patch block
# Byte-for-byte the launcher's container-start block. The patches come from
# the /patches mount (the pinned radiance clone). SP must match the image's
@@ -214,9 +232,15 @@ fi
# Leave /patches before exec: a stale radiance_mxfp4_fp8.so there would shadow
# the one just compiled into site-packages (cwd precedes it on sys.path).
cd /
fi
# The weight-quantization flag. MXFP4 is auto-detected from the checkpoint;
# FP8 is passed explicitly (the documented plain-FP8 path).
QUANT_FLAG=""
if [ "$IS_MXFP4" = "0" ]; then QUANT_FLAG="--quantization fp8"; fi
exec /opt/radiance_entrypoint.sh "$SNAP" --served-model-name $SERVED_NAMES \
--host 0.0.0.0 --port "$PORT" \
--kv-cache-dtype fp8 --tensor-parallel-size "$TP" \
$QUANT_FLAG \
--gpu-memory-utilization "$GPU_UTIL" \
${KV_MEM:+--kv-cache-memory "$KV_MEM"} \
--max-model-len "$MAXLEN" --max-num-seqs "$MAXSEQS" --max-num-batched-tokens "$CHUNK" \