dank/serverAppConfigs
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(frank-medium): cut to MXFP4 checkpoint with unquantized bf16 KV

Browse Source 
Swap the serving checkpoint back to
just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq (quark/W4A8) and serve it with
KV_DTYPE=auto (unquantized bf16 KV) instead of the hardcoded fp8. fp8 KV
stacked on MXFP4's already-quantized W4A8 weights compounds attention-path
precision loss; clean KV tests whether that was hurting output quality. The
freed MXFP4 weights (19.37 vs 30.9 GB) buy the KV pool back, so the full
262,144 native window fits unquantized - MAXLEN returns to 262144.

The entrypoint gains a KV_DTYPE tunable (default fp8, the historical behavior),
shared across both quantization paths; the quadlet sets auto. Rollback: SNAP
back to the FP8 checkpoint with KV_DTYPE=fp8 and MAXLEN=192000.
This commit is contained in:
Frank
2026-09-21 16:32:19 -07:00
parent 0f203dfa41
commit a45f396c39
3 changed files with 29 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dankserver/frank-medium/quadlet/frank-medium.container
+21 -15
View File
@@ -3,19 +3,19 @@
# Install path on dankserver: ~/.config/containers/systemd/frank-medium.container
# Unit name: frank-medium.service (container name: frank-medium)
#
# The dankserver LLM endpoint (:7770, alias frank-medium) since 2026-09-20:
# the vllm-radiance stack (stilldeadcode/vllm-radiance:0.9.3 image) serving
# orcarouter/Qwen3.8-27B-Uncensored-FP8 (FP8 E4M3 block-wise, 128x128,
# dynamic activation quantization) on the 2x Radeon AI PRO R9700 (gfx1201)
# as ONE tensor-parallel process (TP=2, R4D one-shot all-reduce) with MTP
# self-speculation (SPEC=8, the model's own single-layer MTP head).
# The dankserver LLM endpoint (:7770, alias frank-medium): the vllm-radiance
# stack (stilldeadcode/vllm-radiance:0.9.3 image) serving
# just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq (MXFP4 / quark, W4A8, 19.37
# GiB) on the 2x Radeon AI PRO R9700 (gfx1201) as ONE tensor-parallel process
# (TP=2, R4D one-shot all-reduce) with the z-lab DFlash2 external drafter
# (SPEC=7) and UNQUANTIZED bf16 KV (KV_DTYPE=auto).
# The entrypoint is QUANTIZATION-AWARE: it reads the model's weight
# quantization from config.json and serves the plain FP8 path (no MXFP4
# patches, --quantization=fp8, ROCM_AITER unified attention) or the MXFP4
# patch-and-serve path (the DFlash2 drafter). A model swap is now: change
# SNAP (+ SPEC_METHOD/SPEC), restart. The MXFP4 checkpoint
# (just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq) remains on disk as the
# rollback target.
# SNAP (+ SPEC_METHOD/SPEC), restart. The FP8 checkpoint
# (orcarouter/Qwen3.8-27B-Uncensored-FP8) is the on-disk rollback target — the
# 2026-09-20 to 2026-09-21 serving shape, before this MXFP4 + clean-KV cut.
#
# SHAPE. The launcher the upstream repo ships binds the server with
# --network=host (vLLM takes --host 0.0.0.0 --port 7770 directly), which is
@@ -65,7 +65,7 @@
# DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the
# HIP runtime reads bare TEMP for its JIT cache path (2026-08-13 incident).
[Unit]
Description=frank-medium — vllm-radiance FP8 Qwen3.8-27B-Uncensored on 2x R9700 (gfx1201)
Description=frank-medium — vllm-radiance MXFP4 Qwen3.8-27B-Uncensored on 2x R9700 (gfx1201)
After=network-online.target
[Container]
@@ -82,15 +82,15 @@ PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add S
# memory is not a compute target.
Environment=ROCR_VISIBLE_DEVICES=0,1
Environment=HIP_VISIBLE_DEVICES=0,1
# The FP8 checkpoint carries its own single-layer MTP head (mtp.layers.0),
# so MTP self-speculation needs no external drafter and no fp8_mtp.py
# rewrite step. The MXFP4 checkpoint (rollback) ships its fp8 MTP head too.
# Drafting: the external z-lab DFlash2-FP8 drafter (SPEC_METHOD=dflash). The
# plain-FP8 checkpoint (rollback) carries its own single-layer MTP head, but
# DFlash2 beat MTP self-speculation on decode, so the drafter serves either.
# Chat template: the Qwen-Sharp variant (froggeric v22.4.0) — the terse
# system prompt (opt out per-request with chat_template_kwargs {"terse":
# false}), message.reasoning support (vLLM's thinking field), and the
# tightened tool-call rendering. The predecessor (qwen-fixed-v22.3.jinja in
# the /patches clone) remains there as the rollback target: flip CHAT_TEMPLATE.
Environment=SNAP=/models/Qwen3.8-27B-Uncensored-FP8
Environment=SNAP=/models/Qwen3.8-27B-Uncensored-MXFP4-awq
Environment=DRAFTER=/models/Qwen3.8-27B-DFlash2-FP8
Environment=CHAT_TEMPLATE=/templates/qwen-sharp-v22.4.0.jinja
# Fleet identity: the id at /v1/models and in every response body. Every
@@ -101,7 +101,7 @@ Environment=PORT=7770
# repo; the R4D one-shot all-reduce kernel serves it natively.
Environment=TP=2
# Measured production batch shape (upstream README): MAXSEQS 8 peaks
# aggregate throughput at 573 t/s; MAXLEN is the model's native context.
# aggregate throughput at 573 t/s. MAXLEN is the served window (see KV_DTYPE).
Environment=MAXSEQS=8
Environment=CHUNK=8192
Environment=MAXLEN=262144
@@ -128,6 +128,12 @@ Environment=SPEC=7
# the first healthy start, recalibrate with calibrate-kv.sh to use the extra
# VRAM on the 34.2 GiB cards.
Environment=KV_MEM=0
# KV cache dtype: auto = UNQUANTIZED bf16 KV. The experiment: fp8 KV stacked on
# MXFP4's already-quantized W4A8 weights compounds attention precision loss, so
# clean KV should recover output quality. MXFP4's freed weights (19.37 vs 30.9
# GB) buy the pool back — the full 262,144 native window still fits unquantized
# (~290k-token f16 pool > 262,144). Flip to fp8 for the measured 943k-token pool.
Environment=KV_DTYPE=auto
# Checkpoints (rw: nothing downloads at runtime, but vLLM mmaps the shards;
# keep parity with the launcher's rw mount).
Volume=/home/dank/models:/models
dankserver/frank-medium/radiance-entrypoint.sh
+2 -1
View File
@@ -36,6 +36,7 @@ set -euo pipefail
# after the weights load (2.16 GiB requested, 116 MiB free). Do not "fix"
# this into an empty expansion.
: "${KV_MEM:=18563072000}" # shipped kv-profiles.tsv row 2x7551-32624 @ 8/8192/262144/dflash
: "${KV_DTYPE:=fp8}" # fp8 = quantized KV (2x the tokens); auto = bf16 KV (half the pool)
: "${SPEC_METHOD:=dflash}"
if [ "$SPEC_METHOD" = dflash ]; then : "${SPEC:=7}"; else : "${SPEC:=4}"; fi
: "${MIN_M:=0}" # 0 = the W4A8 kernel serves every M (MIN_M=1 still routes M=1 to aiter)
@@ -239,7 +240,7 @@ QUANT_FLAG=""
if [ "$IS_MXFP4" = "0" ]; then QUANT_FLAG="--quantization fp8"; fi
exec /opt/radiance_entrypoint.sh "$SNAP" --served-model-name $SERVED_NAMES \
--host 0.0.0.0 --port "$PORT" \
--kv-cache-dtype fp8 --tensor-parallel-size "$TP" \
--kv-cache-dtype "$KV_DTYPE" --tensor-parallel-size "$TP" \
$QUANT_FLAG \
--gpu-memory-utilization "$GPU_UTIL" \
${KV_MEM:+--kv-cache-memory "$KV_MEM"} \
tests/frank_medium_unit.bats
+6 -1
View File
@@ -114,19 +114,24 @@ env_str() {
}
@test "the KV pool holds a full window and fits beside the weights" {
local kv_mem maxseqs chunk maxlen spec
local kv_mem maxseqs chunk maxlen spec kv_dtype
kv_mem="$(env_value KV_MEM)"
maxseqs="$(env_value MAXSEQS)"
chunk="$(env_value CHUNK)"
maxlen="$(env_value MAXLEN)"
spec="$(env_value SPEC)"
kv_dtype="$(env_str KV_DTYPE)"
[ -n "$kv_mem" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ] && [ -n "$maxlen" ]
# KV_MEM=0 = profile the shape (the documented plain-FP8 path's default;
# the upstream docker-compose leaves the pin commented out). The pool is
# auto-sized to fit, so there is no pin to verify — only that the shape
# is set and the model's weights fit beside an auto-sized pool.
if [ "$kv_mem" -eq 0 ]; then
# MXFP4 + unquantized (KV_DTYPE=auto -> bf16) KV: MXFP4's freed weights
# (19.37 vs 30.9 GB) buy the pool back, so the full 262,144 native window
# still fits clean (~290k f16 pool); vLLM boots or the pool is too small.
[ "$maxseqs" -eq 8 ] && [ "$chunk" -eq 8192 ] && [ "$maxlen" -eq 262144 ]
[ "$kv_dtype" = "auto" ]
return 0
fi
# Pinned pool: the pin must hold at least ONE full-window request or the