feat(frank-medium): quantization-aware entrypoint + FP8 model swap
The entrypoint now reads the model's weight quantization from config.json and serves the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP self-speculation, ROCM_AITER unified attention with the R4D all-reduce) or the MXFP4 patch-and-serve path. A model swap is now: change SNAP (+ SPEC_METHOD/SPEC), restart. - radiance-entrypoint.sh: quantization detection, gated MXFP4 env vars and patch block, --quantization fp8 flag - quadlet: SNAP -> Qwen3.8-27B-Uncensored-FP8, SPEC_METHOD=mtp, SPEC=8, KV_MEM=0 (profile the shape, the documented plain-FP8 default) - deploy-remote.sh: model-aware gates (read SNAP/SPEC_METHOD from the quadlet; drafter gate only for dflash, r4d.so gate only for mxfp4) - tests: quantization-aware drafter/spec and KV-pool tests - hf-fetch.sh: generic, resumable, verified, idempotent HF fetch tool - frank-medium-fp8.manifest: 17 files (10 config + 7 shards) with sizes and LFS sha256
This commit is contained in:
1 parent
c42780f9f3
commit
d389d09d97
6 files changed
+251
-63
No files matched your search
@@ -42,7 +42,20 @@ if [ "$SPEC_METHOD" = dflash ]; then : "${SPEC:=7}"; else : "${SPEC:=4}"; fi
|
||||
: "${R4D_ATTN:=1}"
|
||||
: "${ASYNC:=0}"
|
||||
|
||||
if [ "$R4D_ATTN" = "1" ]; then ATTN=R4D; else ATTN=ROCM_AITER_UNIFIED_ATTN; fi
|
||||
# Quantization-aware: read the model's weight quantization from config.json.
|
||||
# fp8 -> the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP
|
||||
# self-speculation, ROCM_AITER unified attention with the R4D all-reduce).
|
||||
# mxfp4/quark (or unknown) -> the MXFP4 patch-and-serve path (the default).
|
||||
QUANT="$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("quantization_config",{}).get("quant_method",""))' "$SNAP/config.json" 2>/dev/null || true)"
|
||||
case "$QUANT" in
|
||||
fp8) IS_MXFP4=0 ;;
|
||||
mxfp4|quark|"") IS_MXFP4=1 ;;
|
||||
*) IS_MXFP4=1; echo "[radiance] WARNING: quantization '$QUANT'; defaulting to the MXFP4 path" >&2 ;;
|
||||
esac
|
||||
|
||||
if [ "$IS_MXFP4" = "0" ]; then ATTN=ROCM_AITER_UNIFIED_ATTN
|
||||
elif [ "$R4D_ATTN" = "1" ]; then ATTN=R4D
|
||||
else ATTN=ROCM_AITER_UNIFIED_ATTN; fi
|
||||
if [ "$ASYNC" = 1 ]; then ASYNC_FLAG="--async-scheduling"; UNPAD=false; else
|
||||
ASYNC_FLAG="--no-async-scheduling"; UNPAD=true; fi
|
||||
|
||||
@@ -82,7 +95,11 @@ export RADIANCE_DFLASH_SELECTOR_TOPK="${RADIANCE_DFLASH_SELECTOR_TOPK:-}"
|
||||
export RADIANCE_R4D_REPORT=1
|
||||
export RADIANCE_PRESHUFFLE=${RADIANCE_PRESHUFFLE:-1}
|
||||
export RADIANCE_MXFP4_SANITIZE="${RADIANCE_MXFP4_SANITIZE:-0}"
|
||||
export RADIANCE_MXFP4=1 RADIANCE_MXFP4_W4A8=1 RADIANCE_MXFP4_W4A8_MIN_M="$MIN_M"
|
||||
if [ "$IS_MXFP4" = "1" ]; then
|
||||
export RADIANCE_MXFP4=1 RADIANCE_MXFP4_W4A8=1 RADIANCE_MXFP4_W4A8_MIN_M="$MIN_M"
|
||||
else
|
||||
export RADIANCE_MXFP4=0 RADIANCE_MXFP4_W4A8=0 RADIANCE_USE_R4D_AR_QUANT=1
|
||||
fi
|
||||
export RADIANCE_FAST_DRAFT=${RADIANCE_FAST_DRAFT:-1}
|
||||
export RADIANCE_DRAFT_TAU=${RADIANCE_DRAFT_TAU:-0.20}
|
||||
export RADIANCE_DRAFT_RERANK
|
||||
@@ -171,6 +188,7 @@ if [ -n "$CC_ITEMS" ]; then
|
||||
EXTRA="$EXTRA --compilation-config {$CC_ITEMS}"
|
||||
fi
|
||||
|
||||
if [ "$IS_MXFP4" = "1" ]; then
|
||||
# ---------------------------------------------------------------- patch block
|
||||
# Byte-for-byte the launcher's container-start block. The patches come from
|
||||
# the /patches mount (the pinned radiance clone). SP must match the image's
|
||||
@@ -214,9 +232,15 @@ fi
|
||||
# Leave /patches before exec: a stale radiance_mxfp4_fp8.so there would shadow
|
||||
# the one just compiled into site-packages (cwd precedes it on sys.path).
|
||||
cd /
|
||||
fi
|
||||
# The weight-quantization flag. MXFP4 is auto-detected from the checkpoint;
|
||||
# FP8 is passed explicitly (the documented plain-FP8 path).
|
||||
QUANT_FLAG=""
|
||||
if [ "$IS_MXFP4" = "0" ]; then QUANT_FLAG="--quantization fp8"; fi
|
||||
exec /opt/radiance_entrypoint.sh "$SNAP" --served-model-name $SERVED_NAMES \
|
||||
--host 0.0.0.0 --port "$PORT" \
|
||||
--kv-cache-dtype fp8 --tensor-parallel-size "$TP" \
|
||||
$QUANT_FLAG \
|
||||
--gpu-memory-utilization "$GPU_UTIL" \
|
||||
${KV_MEM:+--kv-cache-memory "$KV_MEM"} \
|
||||
--max-model-len "$MAXLEN" --max-num-seqs "$MAXSEQS" --max-num-batched-tokens "$CHUNK" \
|
||||
|
||||
Reference in new issue
Block a user