feat(frank-medium): quantization-aware entrypoint + FP8 model swap
The entrypoint now reads the model's weight quantization from config.json and serves the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP self-speculation, ROCM_AITER unified attention with the R4D all-reduce) or the MXFP4 patch-and-serve path. A model swap is now: change SNAP (+ SPEC_METHOD/SPEC), restart. - radiance-entrypoint.sh: quantization detection, gated MXFP4 env vars and patch block, --quantization fp8 flag - quadlet: SNAP -> Qwen3.8-27B-Uncensored-FP8, SPEC_METHOD=mtp, SPEC=8, KV_MEM=0 (profile the shape, the documented plain-FP8 default) - deploy-remote.sh: model-aware gates (read SNAP/SPEC_METHOD from the quadlet; drafter gate only for dflash, r4d.so gate only for mxfp4) - tests: quantization-aware drafter/spec and KV-pool tests - hf-fetch.sh: generic, resumable, verified, idempotent HF fetch tool - frank-medium-fp8.manifest: 17 files (10 config + 7 shards) with sizes and LFS sha256
This commit is contained in:
1 parent
c42780f9f3
commit
d389d09d97
6 files changed
+251
-63
No files matched your search
@@ -29,6 +29,10 @@ env_value() {
|
||||
env_ratio() {
|
||||
sed -nE "s/^Environment=$1=([0-9.]+)$/\1/p" "$UNIT" | head -n1
|
||||
}
|
||||
# SPEC_METHOD is a string (dflash|mtp), not a count; the integer reader drops it.
|
||||
env_str() {
|
||||
sed -nE "s/^Environment=$1=(.+)$/\1/p" "$UNIT" | head -n1
|
||||
}
|
||||
|
||||
@test "frank-medium's unit, entrypoint and healthcheck exist" {
|
||||
[ -f "$UNIT" ]
|
||||
@@ -76,32 +80,40 @@ env_ratio() {
|
||||
! grep -qE '^Environment=(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the drafter is DFlash2 at the measured depth and the kernel band covers the batch" {
|
||||
grep -q '^Environment=SPEC_METHOD=dflash$' "$UNIT"
|
||||
local maxseqs spec chunk
|
||||
@test "the speculative drafter matches the serving shape" {
|
||||
local spec_method spec maxseqs chunk
|
||||
spec_method="$(env_str SPEC_METHOD)"
|
||||
spec="$(env_value SPEC)"
|
||||
maxseqs="$(env_value MAXSEQS)"
|
||||
chunk="$(env_value CHUNK)"
|
||||
[ -n "$spec" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ]
|
||||
# DFlash2 drafts at most 7 tokens per step; deeper is a different drafter's
|
||||
# contract and costs acceptance, not just VRAM.
|
||||
[ "$spec" -eq 7 ]
|
||||
# The MXFP4 decode kernel band covers MAXSEQS x (SPEC+1) rows, and the
|
||||
# shipped default covers 64. Exceeding it without the entrypoint's >8 bump
|
||||
# is a launch failure at the worst moment (first concurrent request).
|
||||
[ $(( maxseqs * (spec + 1) )) -le 64 ]
|
||||
grep -q 'RADIANCE_MXFP4_DECODE_MAX_M' "$ENTRY"
|
||||
[ -n "$spec_method" ] && [ -n "$spec" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ]
|
||||
# The chunk is the AR size gate's input (CHUNK x hidden x 2 bytes); a chunk
|
||||
# larger than this row's silently changes which all-reduce path runs.
|
||||
[ "$chunk" -le 8192 ]
|
||||
case "$spec_method" in
|
||||
dflash)
|
||||
# DFlash2 drafts at most 7 tokens per step; deeper is a different
|
||||
# drafter's contract and costs acceptance, not just VRAM.
|
||||
[ "$spec" -eq 7 ]
|
||||
# The MXFP4 decode kernel band covers MAXSEQS x (SPEC+1) rows, and the
|
||||
# shipped default covers 64. Exceeding it without the entrypoint's >8
|
||||
# bump is a launch failure at the worst moment (first concurrent
|
||||
# request).
|
||||
[ $(( maxseqs * (spec + 1) )) -le 64 ]
|
||||
grep -q 'RADIANCE_MXFP4_DECODE_MAX_M' "$ENTRY"
|
||||
;;
|
||||
mtp)
|
||||
# MTP self-speculation at depth 8 (the documented plain-FP8 path; the
|
||||
# model's own single-layer MTP head, applied autoregressively).
|
||||
[ "$spec" -eq 8 ]
|
||||
;;
|
||||
*)
|
||||
false "unknown SPEC_METHOD: $spec_method"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
@test "the KV pin is valid only at the shape it was calibrated for" {
|
||||
# The pin is the upstream kv-profiles.tsv row for sig 2x7551-32624 at
|
||||
# maxseqs=8 chunk=8192 maxlen=262144 spec=dflash — exactly this host. It is
|
||||
# not a general-purpose budget: changing MAXSEQS, CHUNK or SPEC_METHOD
|
||||
# invalidates it, so the shape has to move with it or the pin goes to 0
|
||||
# (let the image profile the shape).
|
||||
@test "the KV pool holds a full window and fits beside the weights" {
|
||||
local kv_mem maxseqs chunk maxlen spec
|
||||
kv_mem="$(env_value KV_MEM)"
|
||||
maxseqs="$(env_value MAXSEQS)"
|
||||
@@ -109,17 +121,25 @@ env_ratio() {
|
||||
maxlen="$(env_value MAXLEN)"
|
||||
spec="$(env_value SPEC)"
|
||||
[ -n "$kv_mem" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ] && [ -n "$maxlen" ]
|
||||
# KV_MEM=0 = profile the shape (the documented plain-FP8 path's default;
|
||||
# the upstream docker-compose leaves the pin commented out). The pool is
|
||||
# auto-sized to fit, so there is no pin to verify — only that the shape
|
||||
# is set and the model's weights fit beside an auto-sized pool.
|
||||
if [ "$kv_mem" -eq 0 ]; then
|
||||
[ "$maxseqs" -eq 8 ] && [ "$chunk" -eq 8192 ] && [ "$maxlen" -eq 262144 ]
|
||||
return 0
|
||||
fi
|
||||
# Pinned pool: the pin must hold at least ONE full-window request or the
|
||||
# endpoint refuses its own --max-model-len, and it must fit on one card
|
||||
# beside the weights.
|
||||
grep -q 'kv-profiles.tsv' "$UNIT"
|
||||
[ "$maxseqs" -eq 8 ] && [ "$chunk" -eq 8192 ] && [ "$spec" -eq 7 ]
|
||||
grep -q '^Environment=SPEC_METHOD=dflash$' "$UNIT"
|
||||
# Pool arithmetic, from the engine's own boot line: 18563072000 bytes
|
||||
# reserved => "GPU KV cache size: 262,720 tokens". The pool must hold at
|
||||
# least ONE full-window request or the endpoint refuses its own
|
||||
# --max-model-len, and it must fit on one card beside the weights.
|
||||
[ $(( kv_mem / KV_BYTES_PER_TOKEN )) -ge "$maxlen" ]
|
||||
# 9.5 GiB of MXFP4 weights per card + the KV pool, under the 0.98 util cap.
|
||||
local weights=10200547328 # ~9.5 GiB per card of the 19 GB MXFP4 checkpoint
|
||||
local util util_pct floor
|
||||
# Weights per card: MXFP4 ~9.5 GiB (19 GB / 2), FP8 ~15.5 GiB (30.9 GB / 2).
|
||||
local weights util util_pct floor
|
||||
case "$(env_str SPEC_METHOD)" in
|
||||
dflash) weights=10200547328 ;;
|
||||
*) weights=16646128640 ;;
|
||||
esac
|
||||
util="$(env_ratio GPU_UTIL)"
|
||||
[ -n "$util" ]
|
||||
util_pct="${util#0.}"
|
||||
|
||||
Reference in new issue
Block a user