frank-medium: swap to ParoQuant MXFP6 (Swift Qwen3.8-27B) + Qwen-Sharp v22.5.0
- Model: hugypufy/Swift-Qwen3.8-27B-PARO-MXFP6 (25.16 GiB, 18 shards, quant_method=paroquant_mxfp6, bits=6, group_size=128, krot=8, rotation on) - Image: ggz14/vllm-radiance-mxfp4:0.13.0-e060895 (PR #51 launch6_p kernel, built on the published 0.9.3 base; docker->podman via save/load) - GPU_UTIL 0.98 -> 0.99 (user request); KV_DTYPE=auto (unquantized bf16) - Entrypoint: mxfp6 branch (17 patches + paroquant kernel compile + sitecustomize registration); GPU_UTIL default for set -u robustness - Quadlet: SNAP/GPU_UTIL/Image/cache updated; new mxfp6 cache dir - deploy-remote.sh: R4D gate + paroquant_mxfp6; new image/SNAP/cache defaults - Chat template: Qwen-Sharp v22.5.0 (peculiar-ragdoll); v22.4.0 kept as rollback - Test 168: dflash branch comment quant-aware (mxfp4 + paroquant_mxfp6) - Runbook: MXFP6 active shape, MXFP4 rollback MXFP4 checkpoint remains on disk as the rollback target.
This commit is contained in:
Frank committed
1 parent
600812d459
commit
e9a01cd013
6 files changed
+709
-111
No files matched your search
@@ -1,6 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
# frank-medium container entrypoint: vllm-radiance MXFP4 on 2x Radeon AI PRO
|
||||
# R9700 (gfx1201), serving just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq.
|
||||
# frank-medium container entrypoint: vllm-radiance on 2x Radeon AI PRO R9700
|
||||
# (gfx1201). Quantization-aware (config.json quant_method): fp8 -> the plain
|
||||
# FP8 path; mxfp4/quark -> the MXFP4 patch-and-serve path; paroquant_mxfp6 ->
|
||||
# the ParoQuant W4A8 stack + the MXFP6 kernels (PR #51's launch6_p path).
|
||||
#
|
||||
# This is a fork of the patch-and-serve block inside radiance-vllm-mxfp4's
|
||||
# serve-mxfp4.sh (clone at /home/dank/GitPackages/radiance-vllm-mxfp4, pinned
|
||||
@@ -42,17 +44,25 @@ if [ "$SPEC_METHOD" = dflash ]; then : "${SPEC:=7}"; else : "${SPEC:=4}"; fi
|
||||
: "${MIN_M:=0}" # 0 = the W4A8 kernel serves every M (MIN_M=1 still routes M=1 to aiter)
|
||||
: "${R4D_ATTN:=1}"
|
||||
: "${ASYNC:=0}"
|
||||
: "${GPU_UTIL:=0.99}"
|
||||
|
||||
# Quantization-aware: read the model's weight quantization from config.json.
|
||||
# fp8 -> the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP
|
||||
# self-speculation, ROCM_AITER unified attention with the R4D all-reduce).
|
||||
# mxfp4/quark (or unknown) -> the MXFP4 patch-and-serve path (the default).
|
||||
QUANT="$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("quantization_config",{}).get("quant_method",""))' "$SNAP/config.json" 2>/dev/null || true)"
|
||||
IS_MXFP6=0
|
||||
case "$QUANT" in
|
||||
fp8) IS_MXFP4=0 ;;
|
||||
paroquant_mxfp6) IS_MXFP4=1; IS_MXFP6=1 ;;
|
||||
mxfp4|quark|"") IS_MXFP4=1 ;;
|
||||
*) IS_MXFP4=1; echo "[radiance] WARNING: quantization '$QUANT'; defaulting to the MXFP4 path" >&2 ;;
|
||||
esac
|
||||
# MXFP6 (ParoQuant) requires TP>=2 (rad_require_tp2 in the canonical launcher);
|
||||
# ${TP:-1} keeps the check set -u safe when the quadlet omits TP.
|
||||
if [ "$IS_MXFP6" = "1" ]; then
|
||||
[ "${TP:-1}" -ge 2 ] || { echo "frank-medium: paroquant_mxfp6 requires TP>=2 (got ${TP:-1})" >&2; exit 1; }
|
||||
fi
|
||||
|
||||
if [ "$IS_MXFP4" = "0" ]; then ATTN=ROCM_AITER_UNIFIED_ATTN
|
||||
elif [ "$R4D_ATTN" = "1" ]; then ATTN=R4D
|
||||
@@ -96,6 +106,47 @@ export RADIANCE_DFLASH_SELECTOR_TOPK="${RADIANCE_DFLASH_SELECTOR_TOPK:-}"
|
||||
export RADIANCE_R4D_REPORT=1
|
||||
export RADIANCE_PRESHUFFLE=${RADIANCE_PRESHUFFLE:-1}
|
||||
export RADIANCE_MXFP4_SANITIZE="${RADIANCE_MXFP4_SANITIZE:-0}"
|
||||
# ---------------------------------------------------------------- mxfp6 profile
|
||||
# The ParoQuant W4A8 stack (PR #51, the launch6_p kernel). Mirrors
|
||||
# paroquant/run_paroquant.sh's env. Runs BEFORE the MXFP4 base block below so
|
||||
# the base block's ${VAR:-default} guards preserve these values where they
|
||||
# conflict (GDN_MERGE_INPROJ=0, MXFP4_DECODE_MAX_M=64); the base block fills
|
||||
# the rest. Gated on IS_MXFP6 so the mxfp4/fp8 paths are untouched.
|
||||
if [ "$IS_MXFP6" = "1" ]; then
|
||||
export RADIANCE_PAROQUANT=1
|
||||
export RADIANCE_GDN_MERGE_INPROJ=${RADIANCE_GDN_MERGE_INPROJ:-0}
|
||||
export RADIANCE_GDN_FUSED_MAX_ITEMS=${RADIANCE_GDN_FUSED_MAX_ITEMS:-32}
|
||||
export RADIANCE_USE_R4D_AR_QUANT=${RADIANCE_USE_R4D_AR_QUANT:-1}
|
||||
export RADIANCE_FUSE_RMS_QUANT=${RADIANCE_FUSE_RMS_QUANT:-1}
|
||||
export RADIANCE_KV_GROUP_OPT=${RADIANCE_KV_GROUP_OPT:-1}
|
||||
export RADIANCE_MXFP4_DECODE_MAX_M=${RADIANCE_MXFP4_DECODE_MAX_M:-64}
|
||||
export RADIANCE_PQ_CHECKALL=${RADIANCE_PQ_CHECKALL:-}
|
||||
export RADIANCE_PQM_CHECKALL=${RADIANCE_PQM_CHECKALL:-}
|
||||
export RADIANCE_PQM_CHECK_MAX_M=${RADIANCE_PQM_CHECK_MAX_M:-128}
|
||||
export RADIANCE_PQ_CHECK_MAX_M=${RADIANCE_PQ_CHECK_MAX_M:-128}
|
||||
export RADIANCE_PQ_DECODE_MAX_M=${RADIANCE_PQ_DECODE_MAX_M:-64}
|
||||
export RADIANCE_PQ_WPERM=${RADIANCE_PQ_WPERM:-1}
|
||||
export RADIANCE_PQ_DECODE_NT=${RADIANCE_PQ_DECODE_NT:-1}
|
||||
export RADIANCE_PQ_ATILED=${RADIANCE_PQ_ATILED:-1}
|
||||
export RADIANCE_PQ_AT_LBK=${RADIANCE_PQ_AT_LBK:-128}
|
||||
export RADIANCE_PQ_AT_HOIST=${RADIANCE_PQ_AT_HOIST:-1}
|
||||
export RADIANCE_PQ_PTOK=${RADIANCE_PQ_PTOK:-1}
|
||||
export RADIANCE_PQ_FUSED_TOKQ=${RADIANCE_PQ_FUSED_TOKQ:-1}
|
||||
export RADIANCE_PQ_I8=${RADIANCE_PQ_I8:-0}
|
||||
export RADIANCE_PQ_PG=${RADIANCE_PQ_PG:-0}
|
||||
export RADIANCE_PQ_ZPE=${RADIANCE_PQ_ZPE:-0}
|
||||
export RADIANCE_PQ_PG_PRODUCER=${RADIANCE_PQ_PG_PRODUCER:-3}
|
||||
export RADIANCE_PQ_ROT_STREAM=${RADIANCE_PQ_ROT_STREAM:-1}
|
||||
export RADIANCE_PQ_ROT_STREAM2=${RADIANCE_PQ_ROT_STREAM2:-1}
|
||||
export RADIANCE_PQ_ROT_STREAM3=${RADIANCE_PQ_ROT_STREAM3:-0}
|
||||
export RADIANCE_PQ_AR_CHECK=${RADIANCE_PQ_AR_CHECK:-0}
|
||||
export RADIANCE_PQ_AR_FALLBACK=${RADIANCE_PQ_AR_FALLBACK:-0}
|
||||
export RADIANCE_PQ_ROT_V2=${RADIANCE_PQ_ROT_V2:-1}
|
||||
export RADIANCE_PQ_HIPCC_FLAGS=${RADIANCE_PQ_HIPCC_FLAGS:-}
|
||||
export RADIANCE_PQM_FUSED_TOKQ=${RADIANCE_PQM_FUSED_TOKQ:-1}
|
||||
export RADIANCE_PQM_SINGLE_LAUNCH=${RADIANCE_PQM_SINGLE_LAUNCH:-1}
|
||||
export RADIANCE_DFLASH_CAPTURE_DIR=${RADIANCE_DFLASH_CAPTURE_DIR:-}
|
||||
fi
|
||||
if [ "$IS_MXFP4" = "1" ]; then
|
||||
export RADIANCE_MXFP4=1 RADIANCE_MXFP4_W4A8=1 RADIANCE_MXFP4_W4A8_MIN_M="$MIN_M"
|
||||
else
|
||||
@@ -193,39 +244,95 @@ if [ "$IS_MXFP4" = "1" ]; then
|
||||
# ---------------------------------------------------------------- patch block
|
||||
# Byte-for-byte the launcher's container-start block. The patches come from
|
||||
# the /patches mount (the pinned radiance clone). SP must match the image's
|
||||
# venv layout; the image tag is pinned in the quadlet.
|
||||
# venv layout; the image tag is pinned in the quadlet. The mxfp6 branch
|
||||
# mirrors paroquant/run_paroquant.sh (17 patches, 10 modules, + the paroquant
|
||||
# kernel module + the sitecustomize registration); the mxfp4 branch keeps the
|
||||
# existing 20-patch / 11-module list untouched.
|
||||
SP=/opt/vllm/lib/python3.12/site-packages
|
||||
cd /patches
|
||||
python3 patch_quark_mxfp4.py
|
||||
python3 patch_nvfp4_mxfp4.py
|
||||
python3 patch_tp3_pad.py
|
||||
python3 patch_ar_maxbytes.py
|
||||
python3 patch_topk_triton_rows.py
|
||||
python3 patch_dflash_calib.py
|
||||
python3 patch_dflash_mxfp4_kv.py
|
||||
python3 patch_rmsquant_fusion.py
|
||||
python3 patch_verify_head.py
|
||||
python3 patch_kv_group_size.py
|
||||
python3 patch_topk_composite.py
|
||||
python3 patch_gdn_shared_build.py
|
||||
python3 patch_dflash_selector_topk.py
|
||||
python3 patch_gdn_merge_inproj.py
|
||||
python3 patch_dynwidth.py
|
||||
python3 patch_async_dynwidth.py
|
||||
python3 patch_step_trace.py
|
||||
python3 patch_ar_geometry.py
|
||||
python3 patch_ar_3rank.py
|
||||
python3 patch_gdn_glue.py
|
||||
# Non-fatal: fixes content=null on thinking-off requests; not required to serve.
|
||||
python3 patch_qwen3_thinkoff.py \
|
||||
|| echo "[radiance] WARNING: thinkoff patch did not apply; thinking-off requests will return empty content"
|
||||
cp mxfp4-configs/*.json "$SP"/aiter/ops/triton/configs/gemm/
|
||||
cp radiance_preamble.py /opt/radiance_preamble.py
|
||||
cp radiance_nvfp4.py radiance_mxfp4.py radiance_gdn.py radiance_rmsquant.py radiance_drafthead.py \
|
||||
radiance_verifyhead.py radiance_gdnmerge.py radiance_aroverlap.py radiance_topk.py \
|
||||
radiance_arnq.py radiance_tp3pad.py "$SP"/
|
||||
hipcc -O3 -w -std=c++17 -fPIC -shared --offload-arch=gfx1201 $([ "${MXFP4_CUMODE:-0}" = 1 ] && echo -mcumode) \
|
||||
$(python3 -m pybind11 --includes) radiance_mxfp4_fp8.hip -o "$SP"/radiance_mxfp4_fp8.so
|
||||
if [ "$IS_MXFP6" = "1" ]; then
|
||||
python3 patch_quark_mxfp4.py
|
||||
python3 patch_ar_maxbytes.py
|
||||
python3 patch_topk_triton_rows.py
|
||||
python3 patch_dflash_calib.py
|
||||
python3 patch_dflash_mxfp4_kv.py
|
||||
python3 patch_rmsquant_fusion.py
|
||||
python3 patch_verify_head.py
|
||||
python3 patch_kv_group_size.py
|
||||
python3 patch_topk_composite.py
|
||||
python3 patch_gdn_shared_build.py
|
||||
python3 patch_async_dynwidth.py
|
||||
python3 patch_step_trace.py
|
||||
python3 patch_skinny_gemm.py
|
||||
python3 patch_dflash_selector_topk.py
|
||||
python3 patch_dynwidth.py
|
||||
python3 patch_ar_geometry.py
|
||||
python3 patch_gdn_merge_inproj.py
|
||||
# Non-fatal: fixes content=null on thinking-off requests; not required to serve.
|
||||
python3 patch_qwen3_thinkoff.py \
|
||||
|| echo "[radiance] WARNING: thinkoff patch did not apply; thinking-off requests will return empty content"
|
||||
cp mxfp4-configs/*.json "$SP"/aiter/ops/triton/configs/gemm/
|
||||
cp radiance_preamble.py /opt/radiance_preamble.py
|
||||
cp radiance_mxfp4.py radiance_gemm.py radiance_gdn.py radiance_gdnmerge.py radiance_rmsquant.py \
|
||||
radiance_drafthead.py radiance_verifyhead.py radiance_aroverlap.py radiance_topk.py \
|
||||
radiance_arnq.py "$SP"/
|
||||
hipcc -O3 -w -std=c++17 -fPIC -shared --offload-arch=gfx1201 $(python3 -m pybind11 --includes) \
|
||||
radiance_mxfp4_fp8.hip -o "$SP"/radiance_mxfp4_fp8.so
|
||||
# ---- paroquant: build the MXFP6 kernel module and register the quant method ----
|
||||
# RADIANCE_PQ_HIPCC_FLAGS: extra compile flags for A/B builds of the kernel
|
||||
# module only (e.g. -DPQ_HW_CVT=0). Empty in prod.
|
||||
hipcc -O3 -w -std=c++17 -fPIC -shared --offload-arch=gfx1201 $(python3 -m pybind11 --includes) \
|
||||
${RADIANCE_PQ_HIPCC_FLAGS:-} paroquant/radiance_paroquant.hip -o "$SP"/radiance_paroquant_kernel.so
|
||||
cp paroquant/radiance_paroquant.py paroquant/radiance_paroquant_mxfp4.py "$SP"/
|
||||
cp radiance_dflash_capture.py "$SP"/ 2>/dev/null || true
|
||||
# NB: appended to the STDLIB sitecustomize, not site-packages -- Ubuntu ships
|
||||
# /usr/lib/python3.12/sitecustomize.py and it shadows any site-packages one, so
|
||||
# a file dropped there is silently never imported. Guarded: an unguarded >>
|
||||
# appends another copy on every restart of an existing container.
|
||||
if ! grep -q radiance_paroquant /usr/lib/python3.12/sitecustomize.py; then
|
||||
printf "%s\n" \
|
||||
"try:" \
|
||||
" import radiance_paroquant # registers the paroquant quantization config" \
|
||||
" import radiance_paroquant_mxfp4 # and the MXFP4-weights variant (paroquant_mxfp4)" \
|
||||
" import radiance_dflash_capture # drafter training-data capture (inert unless RADIANCE_DFLASH_CAPTURE_DIR)" \
|
||||
"except Exception as e:" \
|
||||
" import sys" \
|
||||
" sys.stderr.write(\"[radiance.paroquant] registration failed: %r\\n\" % (e,))" \
|
||||
>> /usr/lib/python3.12/sitecustomize.py
|
||||
fi
|
||||
else
|
||||
python3 patch_quark_mxfp4.py
|
||||
python3 patch_nvfp4_mxfp4.py
|
||||
python3 patch_tp3_pad.py
|
||||
python3 patch_ar_maxbytes.py
|
||||
python3 patch_topk_triton_rows.py
|
||||
python3 patch_dflash_calib.py
|
||||
python3 patch_dflash_mxfp4_kv.py
|
||||
python3 patch_rmsquant_fusion.py
|
||||
python3 patch_verify_head.py
|
||||
python3 patch_kv_group_size.py
|
||||
python3 patch_topk_composite.py
|
||||
python3 patch_gdn_shared_build.py
|
||||
python3 patch_dflash_selector_topk.py
|
||||
python3 patch_gdn_merge_inproj.py
|
||||
python3 patch_dynwidth.py
|
||||
python3 patch_async_dynwidth.py
|
||||
python3 patch_step_trace.py
|
||||
python3 patch_ar_geometry.py
|
||||
python3 patch_ar_3rank.py
|
||||
python3 patch_gdn_glue.py
|
||||
# Non-fatal: fixes content=null on thinking-off requests; not required to serve.
|
||||
python3 patch_qwen3_thinkoff.py \
|
||||
|| echo "[radiance] WARNING: thinkoff patch did not apply; thinking-off requests will return empty content"
|
||||
cp mxfp4-configs/*.json "$SP"/aiter/ops/triton/configs/gemm/
|
||||
cp radiance_preamble.py /opt/radiance_preamble.py
|
||||
cp radiance_nvfp4.py radiance_mxfp4.py radiance_gdn.py radiance_rmsquant.py radiance_drafthead.py \
|
||||
radiance_verifyhead.py radiance_gdnmerge.py radiance_aroverlap.py radiance_topk.py \
|
||||
radiance_arnq.py radiance_tp3pad.py "$SP"/
|
||||
cp radiance_dflash_capture.py "$SP"/ 2>/dev/null || true
|
||||
hipcc -O3 -w -std=c++17 -fPIC -shared --offload-arch=gfx1201 $([ "${MXFP4_CUMODE:-0}" = 1 ] && echo -mcumode) \
|
||||
$(python3 -m pybind11 --includes) radiance_mxfp4_fp8.hip -o "$SP"/radiance_mxfp4_fp8.so
|
||||
fi
|
||||
if [ -n "${R4D_SO:-}" ] && [ -f /r4d/r4d.so ]; then
|
||||
cp /r4d/r4d.so "$SP"/r4d.so
|
||||
echo "[radiance] using patched r4d.so from $R4D_SO"
|
||||
@@ -234,8 +341,9 @@ fi
|
||||
# the one just compiled into site-packages (cwd precedes it on sys.path).
|
||||
cd /
|
||||
fi
|
||||
# The weight-quantization flag. MXFP4 is auto-detected from the checkpoint;
|
||||
# FP8 is passed explicitly (the documented plain-FP8 path).
|
||||
# The weight-quantization flag. MXFP4 and paroquant_mxfp6 are auto-detected
|
||||
# from the checkpoint's quant_method; FP8 is passed explicitly (the documented
|
||||
# plain-FP8 path).
|
||||
QUANT_FLAG=""
|
||||
if [ "$IS_MXFP4" = "0" ]; then QUANT_FLAG="--quantization fp8"; fi
|
||||
exec /opt/radiance_entrypoint.sh "$SNAP" --served-model-name $SERVED_NAMES \
|
||||
|
||||
Reference in new issue
Block a user