feat(frank-medium): quantization-aware entrypoint + FP8 model swap

The entrypoint now reads the model's weight quantization from config.json
and serves the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP
self-speculation, ROCM_AITER unified attention with the R4D all-reduce)
or the MXFP4 patch-and-serve path. A model swap is now: change SNAP
(+ SPEC_METHOD/SPEC), restart.

- radiance-entrypoint.sh: quantization detection, gated MXFP4 env vars
  and patch block, --quantization fp8 flag
- quadlet: SNAP -> Qwen3.8-27B-Uncensored-FP8, SPEC_METHOD=mtp, SPEC=8,
  KV_MEM=0 (profile the shape, the documented plain-FP8 default)
- deploy-remote.sh: model-aware gates (read SNAP/SPEC_METHOD from the
  quadlet; drafter gate only for dflash, r4d.so gate only for mxfp4)
- tests: quantization-aware drafter/spec and KV-pool tests
- hf-fetch.sh: generic, resumable, verified, idempotent HF fetch tool
- frank-medium-fp8.manifest: 17 files (10 config + 7 shards) with sizes
  and LFS sha256
This commit is contained in:
Frank committed 2026-09-20 21:30:30 -07:00
1 parent c42780f9f3
commit d389d09d97
6 files changed
+251 -63

No files matched your search

+47 -27
View File
@@ -29,6 +29,10 @@ env_value() {
env_ratio() {
sed -nE "s/^Environment=$1=([0-9.]+)$/\1/p" "$UNIT" | head -n1
}
# SPEC_METHOD is a string (dflash|mtp), not a count; the integer reader drops it.
env_str() {
sed -nE "s/^Environment=$1=(.+)$/\1/p" "$UNIT" | head -n1
}
@test "frank-medium's unit, entrypoint and healthcheck exist" {
[ -f "$UNIT" ]
@@ -76,32 +80,40 @@ env_ratio() {
! grep -qE '^Environment=(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$UNIT"
}
@test "the drafter is DFlash2 at the measured depth and the kernel band covers the batch" {
grep -q '^Environment=SPEC_METHOD=dflash$' "$UNIT"
local maxseqs spec chunk
@test "the speculative drafter matches the serving shape" {
local spec_method spec maxseqs chunk
spec_method="$(env_str SPEC_METHOD)"
spec="$(env_value SPEC)"
maxseqs="$(env_value MAXSEQS)"
chunk="$(env_value CHUNK)"
[ -n "$spec" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ]
# DFlash2 drafts at most 7 tokens per step; deeper is a different drafter's
# contract and costs acceptance, not just VRAM.
[ "$spec" -eq 7 ]
# The MXFP4 decode kernel band covers MAXSEQS x (SPEC+1) rows, and the
# shipped default covers 64. Exceeding it without the entrypoint's >8 bump
# is a launch failure at the worst moment (first concurrent request).
[ $(( maxseqs * (spec + 1) )) -le 64 ]
grep -q 'RADIANCE_MXFP4_DECODE_MAX_M' "$ENTRY"
[ -n "$spec_method" ] && [ -n "$spec" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ]
# The chunk is the AR size gate's input (CHUNK x hidden x 2 bytes); a chunk
# larger than this row's silently changes which all-reduce path runs.
[ "$chunk" -le 8192 ]
case "$spec_method" in
dflash)
# DFlash2 drafts at most 7 tokens per step; deeper is a different
# drafter's contract and costs acceptance, not just VRAM.
[ "$spec" -eq 7 ]
# The MXFP4 decode kernel band covers MAXSEQS x (SPEC+1) rows, and the
# shipped default covers 64. Exceeding it without the entrypoint's >8
# bump is a launch failure at the worst moment (first concurrent
# request).
[ $(( maxseqs * (spec + 1) )) -le 64 ]
grep -q 'RADIANCE_MXFP4_DECODE_MAX_M' "$ENTRY"
;;
mtp)
# MTP self-speculation at depth 8 (the documented plain-FP8 path; the
# model's own single-layer MTP head, applied autoregressively).
[ "$spec" -eq 8 ]
;;
*)
false "unknown SPEC_METHOD: $spec_method"
;;
esac
}
@test "the KV pin is valid only at the shape it was calibrated for" {
# The pin is the upstream kv-profiles.tsv row for sig 2x7551-32624 at
# maxseqs=8 chunk=8192 maxlen=262144 spec=dflash — exactly this host. It is
# not a general-purpose budget: changing MAXSEQS, CHUNK or SPEC_METHOD
# invalidates it, so the shape has to move with it or the pin goes to 0
# (let the image profile the shape).
@test "the KV pool holds a full window and fits beside the weights" {
local kv_mem maxseqs chunk maxlen spec
kv_mem="$(env_value KV_MEM)"
maxseqs="$(env_value MAXSEQS)"
@@ -109,17 +121,25 @@ env_ratio() {
maxlen="$(env_value MAXLEN)"
spec="$(env_value SPEC)"
[ -n "$kv_mem" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ] && [ -n "$maxlen" ]
# KV_MEM=0 = profile the shape (the documented plain-FP8 path's default;
# the upstream docker-compose leaves the pin commented out). The pool is
# auto-sized to fit, so there is no pin to verify — only that the shape
# is set and the model's weights fit beside an auto-sized pool.
if [ "$kv_mem" -eq 0 ]; then
[ "$maxseqs" -eq 8 ] && [ "$chunk" -eq 8192 ] && [ "$maxlen" -eq 262144 ]
return 0
fi
# Pinned pool: the pin must hold at least ONE full-window request or the
# endpoint refuses its own --max-model-len, and it must fit on one card
# beside the weights.
grep -q 'kv-profiles.tsv' "$UNIT"
[ "$maxseqs" -eq 8 ] && [ "$chunk" -eq 8192 ] && [ "$spec" -eq 7 ]
grep -q '^Environment=SPEC_METHOD=dflash$' "$UNIT"
# Pool arithmetic, from the engine's own boot line: 18563072000 bytes
# reserved => "GPU KV cache size: 262,720 tokens". The pool must hold at
# least ONE full-window request or the endpoint refuses its own
# --max-model-len, and it must fit on one card beside the weights.
[ $(( kv_mem / KV_BYTES_PER_TOKEN )) -ge "$maxlen" ]
# 9.5 GiB of MXFP4 weights per card + the KV pool, under the 0.98 util cap.
local weights=10200547328 # ~9.5 GiB per card of the 19 GB MXFP4 checkpoint
local util util_pct floor
# Weights per card: MXFP4 ~9.5 GiB (19 GB / 2), FP8 ~15.5 GiB (30.9 GB / 2).
local weights util util_pct floor
case "$(env_str SPEC_METHOD)" in
dflash) weights=10200547328 ;;
*) weights=16646128640 ;;
esac
util="$(env_ratio GPU_UTIL)"
[ -n "$util" ]
util_pct="${util#0.}"