From a45f396c39f23545d8461e8a9a1aa5fb267aa1ba Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 21 Sep 2026 16:32:19 -0700 Subject: [PATCH] feat(frank-medium): cut to MXFP4 checkpoint with unquantized bf16 KV Swap the serving checkpoint back to just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq (quark/W4A8) and serve it with KV_DTYPE=auto (unquantized bf16 KV) instead of the hardcoded fp8. fp8 KV stacked on MXFP4's already-quantized W4A8 weights compounds attention-path precision loss; clean KV tests whether that was hurting output quality. The freed MXFP4 weights (19.37 vs 30.9 GB) buy the KV pool back, so the full 262,144 native window fits unquantized - MAXLEN returns to 262144. The entrypoint gains a KV_DTYPE tunable (default fp8, the historical behavior), shared across both quantization paths; the quadlet sets auto. Rollback: SNAP back to the FP8 checkpoint with KV_DTYPE=fp8 and MAXLEN=192000. --- .../quadlet/frank-medium.container | 36 +++++++++++-------- .../frank-medium/radiance-entrypoint.sh | 3 +- tests/frank_medium_unit.bats | 7 +++- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/dankserver/frank-medium/quadlet/frank-medium.container b/dankserver/frank-medium/quadlet/frank-medium.container index dc9dc9d..35ee1c9 100644 --- a/dankserver/frank-medium/quadlet/frank-medium.container +++ b/dankserver/frank-medium/quadlet/frank-medium.container @@ -3,19 +3,19 @@ # Install path on dankserver: ~/.config/containers/systemd/frank-medium.container # Unit name: frank-medium.service (container name: frank-medium) # -# The dankserver LLM endpoint (:7770, alias frank-medium) since 2026-09-20: -# the vllm-radiance stack (stilldeadcode/vllm-radiance:0.9.3 image) serving -# orcarouter/Qwen3.8-27B-Uncensored-FP8 (FP8 E4M3 block-wise, 128x128, -# dynamic activation quantization) on the 2x Radeon AI PRO R9700 (gfx1201) -# as ONE tensor-parallel process (TP=2, R4D one-shot all-reduce) with MTP -# self-speculation (SPEC=8, the model's own single-layer MTP head). +# The dankserver LLM endpoint (:7770, alias frank-medium): the vllm-radiance +# stack (stilldeadcode/vllm-radiance:0.9.3 image) serving +# just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq (MXFP4 / quark, W4A8, 19.37 +# GiB) on the 2x Radeon AI PRO R9700 (gfx1201) as ONE tensor-parallel process +# (TP=2, R4D one-shot all-reduce) with the z-lab DFlash2 external drafter +# (SPEC=7) and UNQUANTIZED bf16 KV (KV_DTYPE=auto). # The entrypoint is QUANTIZATION-AWARE: it reads the model's weight # quantization from config.json and serves the plain FP8 path (no MXFP4 # patches, --quantization=fp8, ROCM_AITER unified attention) or the MXFP4 # patch-and-serve path (the DFlash2 drafter). A model swap is now: change -# SNAP (+ SPEC_METHOD/SPEC), restart. The MXFP4 checkpoint -# (just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq) remains on disk as the -# rollback target. +# SNAP (+ SPEC_METHOD/SPEC), restart. The FP8 checkpoint +# (orcarouter/Qwen3.8-27B-Uncensored-FP8) is the on-disk rollback target — the +# 2026-09-20 to 2026-09-21 serving shape, before this MXFP4 + clean-KV cut. # # SHAPE. The launcher the upstream repo ships binds the server with # --network=host (vLLM takes --host 0.0.0.0 --port 7770 directly), which is @@ -65,7 +65,7 @@ # DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the # HIP runtime reads bare TEMP for its JIT cache path (2026-08-13 incident). [Unit] -Description=frank-medium — vllm-radiance FP8 Qwen3.8-27B-Uncensored on 2x R9700 (gfx1201) +Description=frank-medium — vllm-radiance MXFP4 Qwen3.8-27B-Uncensored on 2x R9700 (gfx1201) After=network-online.target [Container] @@ -82,15 +82,15 @@ PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add S # memory is not a compute target. Environment=ROCR_VISIBLE_DEVICES=0,1 Environment=HIP_VISIBLE_DEVICES=0,1 -# The FP8 checkpoint carries its own single-layer MTP head (mtp.layers.0), -# so MTP self-speculation needs no external drafter and no fp8_mtp.py -# rewrite step. The MXFP4 checkpoint (rollback) ships its fp8 MTP head too. +# Drafting: the external z-lab DFlash2-FP8 drafter (SPEC_METHOD=dflash). The +# plain-FP8 checkpoint (rollback) carries its own single-layer MTP head, but +# DFlash2 beat MTP self-speculation on decode, so the drafter serves either. # Chat template: the Qwen-Sharp variant (froggeric v22.4.0) — the terse # system prompt (opt out per-request with chat_template_kwargs {"terse": # false}), message.reasoning support (vLLM's thinking field), and the # tightened tool-call rendering. The predecessor (qwen-fixed-v22.3.jinja in # the /patches clone) remains there as the rollback target: flip CHAT_TEMPLATE. -Environment=SNAP=/models/Qwen3.8-27B-Uncensored-FP8 +Environment=SNAP=/models/Qwen3.8-27B-Uncensored-MXFP4-awq Environment=DRAFTER=/models/Qwen3.8-27B-DFlash2-FP8 Environment=CHAT_TEMPLATE=/templates/qwen-sharp-v22.4.0.jinja # Fleet identity: the id at /v1/models and in every response body. Every @@ -101,7 +101,7 @@ Environment=PORT=7770 # repo; the R4D one-shot all-reduce kernel serves it natively. Environment=TP=2 # Measured production batch shape (upstream README): MAXSEQS 8 peaks -# aggregate throughput at 573 t/s; MAXLEN is the model's native context. +# aggregate throughput at 573 t/s. MAXLEN is the served window (see KV_DTYPE). Environment=MAXSEQS=8 Environment=CHUNK=8192 Environment=MAXLEN=262144 @@ -128,6 +128,12 @@ Environment=SPEC=7 # the first healthy start, recalibrate with calibrate-kv.sh to use the extra # VRAM on the 34.2 GiB cards. Environment=KV_MEM=0 +# KV cache dtype: auto = UNQUANTIZED bf16 KV. The experiment: fp8 KV stacked on +# MXFP4's already-quantized W4A8 weights compounds attention precision loss, so +# clean KV should recover output quality. MXFP4's freed weights (19.37 vs 30.9 +# GB) buy the pool back — the full 262,144 native window still fits unquantized +# (~290k-token f16 pool > 262,144). Flip to fp8 for the measured 943k-token pool. +Environment=KV_DTYPE=auto # Checkpoints (rw: nothing downloads at runtime, but vLLM mmaps the shards; # keep parity with the launcher's rw mount). Volume=/home/dank/models:/models diff --git a/dankserver/frank-medium/radiance-entrypoint.sh b/dankserver/frank-medium/radiance-entrypoint.sh index 89b6fe7..e543176 100755 --- a/dankserver/frank-medium/radiance-entrypoint.sh +++ b/dankserver/frank-medium/radiance-entrypoint.sh @@ -36,6 +36,7 @@ set -euo pipefail # after the weights load (2.16 GiB requested, 116 MiB free). Do not "fix" # this into an empty expansion. : "${KV_MEM:=18563072000}" # shipped kv-profiles.tsv row 2x7551-32624 @ 8/8192/262144/dflash +: "${KV_DTYPE:=fp8}" # fp8 = quantized KV (2x the tokens); auto = bf16 KV (half the pool) : "${SPEC_METHOD:=dflash}" if [ "$SPEC_METHOD" = dflash ]; then : "${SPEC:=7}"; else : "${SPEC:=4}"; fi : "${MIN_M:=0}" # 0 = the W4A8 kernel serves every M (MIN_M=1 still routes M=1 to aiter) @@ -239,7 +240,7 @@ QUANT_FLAG="" if [ "$IS_MXFP4" = "0" ]; then QUANT_FLAG="--quantization fp8"; fi exec /opt/radiance_entrypoint.sh "$SNAP" --served-model-name $SERVED_NAMES \ --host 0.0.0.0 --port "$PORT" \ - --kv-cache-dtype fp8 --tensor-parallel-size "$TP" \ + --kv-cache-dtype "$KV_DTYPE" --tensor-parallel-size "$TP" \ $QUANT_FLAG \ --gpu-memory-utilization "$GPU_UTIL" \ ${KV_MEM:+--kv-cache-memory "$KV_MEM"} \ diff --git a/tests/frank_medium_unit.bats b/tests/frank_medium_unit.bats index 3dcf8d6..21a40d5 100755 --- a/tests/frank_medium_unit.bats +++ b/tests/frank_medium_unit.bats @@ -114,19 +114,24 @@ env_str() { } @test "the KV pool holds a full window and fits beside the weights" { - local kv_mem maxseqs chunk maxlen spec + local kv_mem maxseqs chunk maxlen spec kv_dtype kv_mem="$(env_value KV_MEM)" maxseqs="$(env_value MAXSEQS)" chunk="$(env_value CHUNK)" maxlen="$(env_value MAXLEN)" spec="$(env_value SPEC)" + kv_dtype="$(env_str KV_DTYPE)" [ -n "$kv_mem" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ] && [ -n "$maxlen" ] # KV_MEM=0 = profile the shape (the documented plain-FP8 path's default; # the upstream docker-compose leaves the pin commented out). The pool is # auto-sized to fit, so there is no pin to verify — only that the shape # is set and the model's weights fit beside an auto-sized pool. if [ "$kv_mem" -eq 0 ]; then + # MXFP4 + unquantized (KV_DTYPE=auto -> bf16) KV: MXFP4's freed weights + # (19.37 vs 30.9 GB) buy the pool back, so the full 262,144 native window + # still fits clean (~290k f16 pool); vLLM boots or the pool is too small. [ "$maxseqs" -eq 8 ] && [ "$chunk" -eq 8192 ] && [ "$maxlen" -eq 262144 ] + [ "$kv_dtype" = "auto" ] return 0 fi # Pinned pool: the pin must hold at least ONE full-window request or the