From d389d09d9773209670d5ee28d7f9ad59f76e2707 Mon Sep 17 00:00:00 2001 From: Frank Date: Sun, 20 Sep 2026 21:30:30 -0700 Subject: [PATCH] feat(frank-medium): quantization-aware entrypoint + FP8 model swap The entrypoint now reads the model's weight quantization from config.json and serves the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP self-speculation, ROCM_AITER unified attention with the R4D all-reduce) or the MXFP4 patch-and-serve path. A model swap is now: change SNAP (+ SPEC_METHOD/SPEC), restart. - radiance-entrypoint.sh: quantization detection, gated MXFP4 env vars and patch block, --quantization fp8 flag - quadlet: SNAP -> Qwen3.8-27B-Uncensored-FP8, SPEC_METHOD=mtp, SPEC=8, KV_MEM=0 (profile the shape, the documented plain-FP8 default) - deploy-remote.sh: model-aware gates (read SNAP/SPEC_METHOD from the quadlet; drafter gate only for dflash, r4d.so gate only for mxfp4) - tests: quantization-aware drafter/spec and KV-pool tests - hf-fetch.sh: generic, resumable, verified, idempotent HF fetch tool - frank-medium-fp8.manifest: 17 files (10 config + 7 shards) with sizes and LFS sha256 --- .../quadlet/frank-medium.container | 52 ++++----- .../frank-medium/radiance-entrypoint.sh | 28 ++++- scripts/deploy-remote.sh | 39 +++++-- scripts/frank-medium-fp8.manifest | 20 ++++ scripts/hf-fetch.sh | 101 ++++++++++++++++++ tests/frank_medium_unit.bats | 74 ++++++++----- 6 files changed, 251 insertions(+), 63 deletions(-) create mode 100644 scripts/frank-medium-fp8.manifest create mode 100755 scripts/hf-fetch.sh diff --git a/dankserver/frank-medium/quadlet/frank-medium.container b/dankserver/frank-medium/quadlet/frank-medium.container index 48afd90..e8704ca 100644 --- a/dankserver/frank-medium/quadlet/frank-medium.container +++ b/dankserver/frank-medium/quadlet/frank-medium.container @@ -4,13 +4,18 @@ # Unit name: frank-medium.service (container name: frank-medium) # # The dankserver LLM endpoint (:7770, alias frank-medium) since 2026-09-20: -# the vllm-radiance MXFP4 stack (stilldeadcode/vllm-radiance:0.9.3 image + -# the ggz14/radiance-vllm-mxfp4 patches) serving -# just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq on the 2x Radeon AI PRO -# R9700 (gfx1201) as ONE tensor-parallel process (TP=2, R4D one-shot -# all-reduce) with the DFlash2-FP8 block-diffusion drafter (SPEC=7). -# It replaced the mainline llama.cpp layer-split deployment, which served the -# same alias from the orcarouter Q8_0 GGUF across the same two cards. +# the vllm-radiance stack (stilldeadcode/vllm-radiance:0.9.3 image) serving +# orcarouter/Qwen3.8-27B-Uncensored-FP8 (FP8 E4M3 block-wise, 128x128, +# dynamic activation quantization) on the 2x Radeon AI PRO R9700 (gfx1201) +# as ONE tensor-parallel process (TP=2, R4D one-shot all-reduce) with MTP +# self-speculation (SPEC=8, the model's own single-layer MTP head). +# The entrypoint is QUANTIZATION-AWARE: it reads the model's weight +# quantization from config.json and serves the plain FP8 path (no MXFP4 +# patches, --quantization=fp8, ROCM_AITER unified attention) or the MXFP4 +# patch-and-serve path (the DFlash2 drafter). A model swap is now: change +# SNAP (+ SPEC_METHOD/SPEC), restart. The MXFP4 checkpoint +# (just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq) remains on disk as the +# rollback target. # # SHAPE. The launcher the upstream repo ships binds the server with # --network=host (vLLM takes --host 0.0.0.0 --port 7770 directly), which is @@ -60,7 +65,7 @@ # DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the # HIP runtime reads bare TEMP for its JIT cache path (2026-08-13 incident). [Unit] -Description=frank-medium — vllm-radiance MXFP4 Qwen3.8-27B-Uncensored on 2x R9700 (gfx1201) +Description=frank-medium — vllm-radiance FP8 Qwen3.8-27B-Uncensored on 2x R9700 (gfx1201) After=network-online.target [Container] @@ -77,9 +82,10 @@ PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add S # memory is not a compute target. Environment=ROCR_VISIBLE_DEVICES=0,1 Environment=HIP_VISIBLE_DEVICES=0,1 -# The uncensored MXFP4 checkpoint ships its fp8 MTP head already, so no -# fp8_mtp.py rewrite step exists or is needed (unlike amd/Qwen3.8-27B-Quark-AWQ-MXFP4). -Environment=SNAP=/models/Qwen3.8-27B-Uncensored-MXFP4-awq +# The FP8 checkpoint carries its own single-layer MTP head (mtp.layers.0), +# so MTP self-speculation needs no external drafter and no fp8_mtp.py +# rewrite step. The MXFP4 checkpoint (rollback) ships its fp8 MTP head too. +Environment=SNAP=/models/Qwen3.8-27B-Uncensored-FP8 Environment=DRAFTER=/models/Qwen3.8-27B-DFlash2-FP8 Environment=CHAT_TEMPLATE=/patches/qwen-fixed-v22.3.jinja # Fleet identity: the id at /v1/models and in every response body. Every @@ -104,19 +110,17 @@ Environment=MAXLEN=262144 # 400 here. Vision traffic routes to frank-small on dankclaw (harness # modelRoles vision -> frank-small). Environment=GPU_UTIL=0.98 -# Speculative decoding: the DFlash2 drafter at depth 7 is the configuration -# every published number (224 t/s single-stream decode, 4749-4810 t/s -# prefill) was measured with. -Environment=SPEC_METHOD=dflash -Environment=SPEC=7 -# KV cache pin, bytes. From the upstream repo's shipped kv-profiles.tsv row -# sig=2x7551-32624 maxseqs=8 chunk=8192 maxlen=262144 spec=dflash -# which is EXACTLY this host (2x R9700 DID 0x7551, 32624 MiB each). Pinned -# explicitly because this unit does not run the launcher's lookup; valid -# only at this batch shape (change MAXSEQS/CHUNK/SPEC_METHOD => recalibrate -# with calibrate-kv.sh or set KV_MEM=0 to let vLLM profile). Worth -# 943,581 KV tokens vs ~892,799 profiled (~5.7%). -Environment=KV_MEM=18563072000 +# Speculative decoding: MTP self-speculation at depth 8 (the model's own +# single-layer MTP head, applied autoregressively). The documented plain-FP8 +# path (upstream docker-compose) uses num_speculative_tokens=8. +Environment=SPEC_METHOD=mtp +Environment=SPEC=8 +# KV cache: 0 = profile the shape (the documented plain-FP8 path's default; +# the upstream docker-compose leaves the pin commented out). The measured pin +# for this shape is 15989735424 (14.89 GiB, upstream, 2x R9700 32GB); after +# the first healthy start, recalibrate with calibrate-kv.sh to use the extra +# VRAM on the 34.2 GiB cards. +Environment=KV_MEM=0 # Checkpoints (rw: nothing downloads at runtime, but vLLM mmaps the shards; # keep parity with the launcher's rw mount). Volume=/home/dank/models:/models diff --git a/dankserver/frank-medium/radiance-entrypoint.sh b/dankserver/frank-medium/radiance-entrypoint.sh index e6e60b2..89b6fe7 100755 --- a/dankserver/frank-medium/radiance-entrypoint.sh +++ b/dankserver/frank-medium/radiance-entrypoint.sh @@ -42,7 +42,20 @@ if [ "$SPEC_METHOD" = dflash ]; then : "${SPEC:=7}"; else : "${SPEC:=4}"; fi : "${R4D_ATTN:=1}" : "${ASYNC:=0}" -if [ "$R4D_ATTN" = "1" ]; then ATTN=R4D; else ATTN=ROCM_AITER_UNIFIED_ATTN; fi +# Quantization-aware: read the model's weight quantization from config.json. +# fp8 -> the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP +# self-speculation, ROCM_AITER unified attention with the R4D all-reduce). +# mxfp4/quark (or unknown) -> the MXFP4 patch-and-serve path (the default). +QUANT="$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("quantization_config",{}).get("quant_method",""))' "$SNAP/config.json" 2>/dev/null || true)" +case "$QUANT" in + fp8) IS_MXFP4=0 ;; + mxfp4|quark|"") IS_MXFP4=1 ;; + *) IS_MXFP4=1; echo "[radiance] WARNING: quantization '$QUANT'; defaulting to the MXFP4 path" >&2 ;; +esac + +if [ "$IS_MXFP4" = "0" ]; then ATTN=ROCM_AITER_UNIFIED_ATTN +elif [ "$R4D_ATTN" = "1" ]; then ATTN=R4D +else ATTN=ROCM_AITER_UNIFIED_ATTN; fi if [ "$ASYNC" = 1 ]; then ASYNC_FLAG="--async-scheduling"; UNPAD=false; else ASYNC_FLAG="--no-async-scheduling"; UNPAD=true; fi @@ -82,7 +95,11 @@ export RADIANCE_DFLASH_SELECTOR_TOPK="${RADIANCE_DFLASH_SELECTOR_TOPK:-}" export RADIANCE_R4D_REPORT=1 export RADIANCE_PRESHUFFLE=${RADIANCE_PRESHUFFLE:-1} export RADIANCE_MXFP4_SANITIZE="${RADIANCE_MXFP4_SANITIZE:-0}" -export RADIANCE_MXFP4=1 RADIANCE_MXFP4_W4A8=1 RADIANCE_MXFP4_W4A8_MIN_M="$MIN_M" +if [ "$IS_MXFP4" = "1" ]; then + export RADIANCE_MXFP4=1 RADIANCE_MXFP4_W4A8=1 RADIANCE_MXFP4_W4A8_MIN_M="$MIN_M" +else + export RADIANCE_MXFP4=0 RADIANCE_MXFP4_W4A8=0 RADIANCE_USE_R4D_AR_QUANT=1 +fi export RADIANCE_FAST_DRAFT=${RADIANCE_FAST_DRAFT:-1} export RADIANCE_DRAFT_TAU=${RADIANCE_DRAFT_TAU:-0.20} export RADIANCE_DRAFT_RERANK @@ -171,6 +188,7 @@ if [ -n "$CC_ITEMS" ]; then EXTRA="$EXTRA --compilation-config {$CC_ITEMS}" fi +if [ "$IS_MXFP4" = "1" ]; then # ---------------------------------------------------------------- patch block # Byte-for-byte the launcher's container-start block. The patches come from # the /patches mount (the pinned radiance clone). SP must match the image's @@ -214,9 +232,15 @@ fi # Leave /patches before exec: a stale radiance_mxfp4_fp8.so there would shadow # the one just compiled into site-packages (cwd precedes it on sys.path). cd / +fi +# The weight-quantization flag. MXFP4 is auto-detected from the checkpoint; +# FP8 is passed explicitly (the documented plain-FP8 path). +QUANT_FLAG="" +if [ "$IS_MXFP4" = "0" ]; then QUANT_FLAG="--quantization fp8"; fi exec /opt/radiance_entrypoint.sh "$SNAP" --served-model-name $SERVED_NAMES \ --host 0.0.0.0 --port "$PORT" \ --kv-cache-dtype fp8 --tensor-parallel-size "$TP" \ + $QUANT_FLAG \ --gpu-memory-utilization "$GPU_UTIL" \ ${KV_MEM:+--kv-cache-memory "$KV_MEM"} \ --max-model-len "$MAXLEN" --max-num-seqs "$MAXSEQS" --max-num-batched-tokens "$CHUNK" \ diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 5ba06d1..3d97196 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1211,6 +1211,19 @@ deploy_llm_medium() { # missing never becomes healthy, and a green deploy would leave the endpoint # dark with nothing in the deploy log to say why. local missing=0 + # Model-aware gates: read the serving shape from the quadlet (the source + # of truth), so a model swap is just "edit the quadlet + deploy". The + # container /models mount maps to /home/dank/models on the host. The + # drafter gate applies only to the dflash path; the patched libr4d gate + # applies only to the mxfp4/quark path (the plain FP8 path uses the + # image's built-in r4d.so and skips the MXFP4 patches). + local quadlet_file="dankserver/frank-medium/quadlet/frank-medium.container" + local snap_c snap spec_method quant + snap_c="$(grep -E '^Environment=SNAP=' "$quadlet_file" | head -1 | cut -d= -f2-)" + spec_method="$(grep -E '^Environment=SPEC_METHOD=' "$quadlet_file" | head -1 | cut -d= -f2-)" + snap="/home/dank/models${snap_c#/models}" + quant="$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("quantization_config",{}).get("quant_method",""))' "$snap/config.json" 2>/dev/null || true)" + [[ -n "$snap_c" ]] || { echo "ERROR: cannot read SNAP from $quadlet_file" >&2; return 1; } if ! podman image exists "$LLM_MEDIUM_IMAGE_LOCAL"; then if podman image exists "$LLM_MEDIUM_IMAGE"; then podman tag "$LLM_MEDIUM_IMAGE" "$LLM_MEDIUM_IMAGE_LOCAL" @@ -1220,27 +1233,33 @@ deploy_llm_medium() { missing=1 fi fi - if [[ ! -f "$LLM_MEDIUM_SNAP/config.json" ]]; then - echo "ERROR: MXFP4 checkpoint absent at $LLM_MEDIUM_SNAP" >&2 + if [[ ! -f "$snap/config.json" ]]; then + echo "ERROR: model checkpoint absent at $snap" >&2 missing=1 fi - if [[ ! -f "$LLM_MEDIUM_DRAFTER/config.json" ]]; then + if [[ "$spec_method" == "dflash" && ! -f "$LLM_MEDIUM_DRAFTER/config.json" ]]; then echo "ERROR: DFlash2-FP8 drafter absent at $LLM_MEDIUM_DRAFTER" >&2 missing=1 fi - # The patches clone is mounted and its scripts run at EVERY container start; - # the chat template is the fleet's reasoning-format contract. + # The patches clone is mounted and its scripts run at EVERY container start + # on the MXFP4 path; the chat template is the fleet's reasoning-format + # contract on both paths. if [[ ! -f "$LLM_MEDIUM_PATCHES/qwen-fixed-v22.3.jinja" ]]; then echo "ERROR: pinned patch clone / chat template absent at $LLM_MEDIUM_PATCHES" >&2 echo "frank-medium: git clone https://github.com/ggz14/radiance-vllm-mxfp4 $LLM_MEDIUM_PATCHES" >&2 missing=1 fi # The image's shipped r4d.so predates the gated-delta-net overflow fix and - # NaNs this model, so the patched build is not optional. - if [[ ! -f "$LLM_MEDIUM_R4D/r4d.so" ]]; then - echo "ERROR: patched libr4d absent at $LLM_MEDIUM_R4D" >&2 - missing=1 - fi + # NaNs the MXFP4 model, so the patched build is not optional there. The + # plain FP8 path uses the image's built-in r4d.so (no MXFP4 patches). + case "$quant" in + mxfp4|quark) + if [[ ! -f "$LLM_MEDIUM_R4D/r4d.so" ]]; then + echo "ERROR: patched libr4d absent at $LLM_MEDIUM_R4D" >&2 + missing=1 + fi + ;; + esac [[ "$missing" -eq 0 ]] || return 1 mkdir -p "$LLM_MEDIUM_CACHE" diff --git a/scripts/frank-medium-fp8.manifest b/scripts/frank-medium-fp8.manifest new file mode 100644 index 0000000..d08843d --- /dev/null +++ b/scripts/frank-medium-fp8.manifest @@ -0,0 +1,20 @@ +# orcarouter/Qwen3.8-27B-Uncensored-FP8 — FP8 E4M3 block-wise (128x128), dynamic act-quant +# 7 safetensors shards + config/tokenizer/processor. Sizes + LFS sha256 from the +# HF tree API and X-Linked-ETag (2026-09-20). +config.json 51350 +generation_config.json 202 +chat_template.jinja 8952 +merges.txt 3353259 +preprocessor_config.json 390 +tokenizer.json 12809320 +tokenizer_config.json 17928 +video_preprocessor_config.json 385 +vocab.json 6722759 +model.safetensors.index.json 156034 +model-00001-of-00007.safetensors 3969317448 141e200c1950c59bc42ae28744f50db8f6ea4c66ec3312b55f1ccad2b60af858 +model-00002-of-00007.safetensors 4939321704 e863e88f53ea5a737581638e41334e9f9871a93053c4251a2cebd4aa6aa1f7c0 +model-00003-of-00007.safetensors 4939863424 7cbf8db425114b9efcfae64364a60e15e223cc5cadb510b7a3a6d3b48c8617cf +model-00004-of-00007.safetensors 4971328696 37338dc30b490663b641567a57565408fee5f1385e7860dff16dca7be07e1519 +model-00005-of-00007.safetensors 4955596064 fc6fb0dd3675e0cc7e83a454376d12b705a51000f6625cf13f85e384018fc5cb +model-00006-of-00007.safetensors 4071440632 5fdcbaf80805a750970e076bcd37e6ebe4c3a3d93ce73c767d2854119947952a +model-00007-of-00007.safetensors 3019999168 1bed352a0625d22dc606de47fd3563f24fd9fa73915c4ddffe7947c4b56c28bf diff --git a/scripts/hf-fetch.sh b/scripts/hf-fetch.sh new file mode 100755 index 0000000..62ad2d3 --- /dev/null +++ b/scripts/hf-fetch.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# hf-fetch.sh — resumable, verified HuggingFace fetch (gated or public repos). +# +# bash scripts/hf-fetch.sh +# +# Manifest: one line per file, three tab/space-separated fields: +# +# A blank sha256 field means "check size only". Lines starting with '#' and +# blank lines are ignored. The first field is a path relative to . +# +# Token: $HF_TOKEN, else ~/.cache/huggingface/token, else the llamaswap conf +# (grep hf_...). Passed out-of-band via a mode-600 curl -K file — a -H header +# is world-readable in /proc//cmdline (the 2026-09-16 HF-token leak). +# +# Idempotent: a file whose size AND sha256 already match is skipped, so a +# re-run after an interruption only fetches what is missing or corrupt. +set -euo pipefail + +dest="${1:?usage: hf-fetch.sh }" +manifest="${2:?usage: hf-fetch.sh }" +repo="${3:?usage: hf-fetch.sh }" + +mkdir -p "$dest" + +# --- token resolution -------------------------------------------------------- +token="${HF_TOKEN:-}" +if [[ -z "$token" && -r "$HOME/.cache/huggingface/token" ]]; then + token="$(tr -d '[:space:]' < "$HOME/.cache/huggingface/token")" +fi +if [[ -z "$token" && -r "$HOME/.config/llamaswap.conf.yaml" ]]; then + token="$(grep -oE 'hf_[a-zA-Z0-9]+' "$HOME/.config/llamaswap.conf.yaml" | head -1)" +fi +[[ -n "$token" ]] || { echo "hf-fetch: no HF token (set HF_TOKEN or ~/.cache/huggingface/token)" >&2; exit 3; } + +auth_file="$(mktemp)" +chmod 600 "$auth_file" +printf 'header = "Authorization: Bearer %s"\n' "$token" > "$auth_file" +cleanup() { rm -f "$auth_file"; } +trap cleanup EXIT + +base="https://huggingface.co/${repo}/resolve/main" + +# --- per-file fetch + verify ------------------------------------------------- +fetch_one() { + local name="$1" size="$2" sha="$3" + local url="$base/$name" + local path="$dest/$name" + mkdir -p "$(dirname "$path")" + + # idempotent skip + if [[ -f "$path" ]]; then + local cur_size sha_ok + cur_size="$(wc -c < "$path")" + sha_ok=true + if [[ -n "$sha" && "$(sha256sum "$path" | cut -d' ' -f1)" != "$sha" ]]; then + sha_ok=false + fi + if [[ "$cur_size" == "$size" && "$sha_ok" == "true" ]]; then + echo " ok $name ($size bytes)" + return 0 + fi + echo " refetch $name (have $cur_size, want $size)" + fi + + # resumable download; -f fails on HTTP error so a 401/403 is not written as a file + curl -fsSL -C - -K "$auth_file" -o "$path" "$url" + + # verify byte count + local got + got="$(wc -c < "$path")" + if [[ "$got" != "$size" ]]; then + echo " FAIL $name: size $got != $size" >&2 + return 1 + fi + # verify sha256 + if [[ -n "$sha" ]]; then + local actual + actual="$(sha256sum "$path" | cut -d' ' -f1)" + if [[ "$actual" != "$sha" ]]; then + echo " FAIL $name: sha256 $actual != $sha" >&2 + return 1 + fi + fi + echo " done $name ($size bytes)" +} + +echo "hf-fetch: $repo -> $dest" +status=0 +while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue + name="$(awk '{print $1}' <<<"$line")" + size="$(awk '{print $2}' <<<"$line")" + sha="$(awk '{print $3}' <<<"$line")" + fetch_one "$name" "$size" "$sha" || status=1 +done < "$manifest" + +if [[ "$status" -ne 0 ]]; then + echo "hf-fetch: FAILED (see above)" >&2 + exit 1 +fi +echo "hf-fetch: all files verified" diff --git a/tests/frank_medium_unit.bats b/tests/frank_medium_unit.bats index 8a099b8..3dcf8d6 100755 --- a/tests/frank_medium_unit.bats +++ b/tests/frank_medium_unit.bats @@ -29,6 +29,10 @@ env_value() { env_ratio() { sed -nE "s/^Environment=$1=([0-9.]+)$/\1/p" "$UNIT" | head -n1 } +# SPEC_METHOD is a string (dflash|mtp), not a count; the integer reader drops it. +env_str() { + sed -nE "s/^Environment=$1=(.+)$/\1/p" "$UNIT" | head -n1 +} @test "frank-medium's unit, entrypoint and healthcheck exist" { [ -f "$UNIT" ] @@ -76,32 +80,40 @@ env_ratio() { ! grep -qE '^Environment=(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$UNIT" } -@test "the drafter is DFlash2 at the measured depth and the kernel band covers the batch" { - grep -q '^Environment=SPEC_METHOD=dflash$' "$UNIT" - local maxseqs spec chunk +@test "the speculative drafter matches the serving shape" { + local spec_method spec maxseqs chunk + spec_method="$(env_str SPEC_METHOD)" spec="$(env_value SPEC)" maxseqs="$(env_value MAXSEQS)" chunk="$(env_value CHUNK)" - [ -n "$spec" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ] - # DFlash2 drafts at most 7 tokens per step; deeper is a different drafter's - # contract and costs acceptance, not just VRAM. - [ "$spec" -eq 7 ] - # The MXFP4 decode kernel band covers MAXSEQS x (SPEC+1) rows, and the - # shipped default covers 64. Exceeding it without the entrypoint's >8 bump - # is a launch failure at the worst moment (first concurrent request). - [ $(( maxseqs * (spec + 1) )) -le 64 ] - grep -q 'RADIANCE_MXFP4_DECODE_MAX_M' "$ENTRY" + [ -n "$spec_method" ] && [ -n "$spec" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ] # The chunk is the AR size gate's input (CHUNK x hidden x 2 bytes); a chunk # larger than this row's silently changes which all-reduce path runs. [ "$chunk" -le 8192 ] + case "$spec_method" in + dflash) + # DFlash2 drafts at most 7 tokens per step; deeper is a different + # drafter's contract and costs acceptance, not just VRAM. + [ "$spec" -eq 7 ] + # The MXFP4 decode kernel band covers MAXSEQS x (SPEC+1) rows, and the + # shipped default covers 64. Exceeding it without the entrypoint's >8 + # bump is a launch failure at the worst moment (first concurrent + # request). + [ $(( maxseqs * (spec + 1) )) -le 64 ] + grep -q 'RADIANCE_MXFP4_DECODE_MAX_M' "$ENTRY" + ;; + mtp) + # MTP self-speculation at depth 8 (the documented plain-FP8 path; the + # model's own single-layer MTP head, applied autoregressively). + [ "$spec" -eq 8 ] + ;; + *) + false "unknown SPEC_METHOD: $spec_method" + ;; + esac } -@test "the KV pin is valid only at the shape it was calibrated for" { - # The pin is the upstream kv-profiles.tsv row for sig 2x7551-32624 at - # maxseqs=8 chunk=8192 maxlen=262144 spec=dflash — exactly this host. It is - # not a general-purpose budget: changing MAXSEQS, CHUNK or SPEC_METHOD - # invalidates it, so the shape has to move with it or the pin goes to 0 - # (let the image profile the shape). +@test "the KV pool holds a full window and fits beside the weights" { local kv_mem maxseqs chunk maxlen spec kv_mem="$(env_value KV_MEM)" maxseqs="$(env_value MAXSEQS)" @@ -109,17 +121,25 @@ env_ratio() { maxlen="$(env_value MAXLEN)" spec="$(env_value SPEC)" [ -n "$kv_mem" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ] && [ -n "$maxlen" ] + # KV_MEM=0 = profile the shape (the documented plain-FP8 path's default; + # the upstream docker-compose leaves the pin commented out). The pool is + # auto-sized to fit, so there is no pin to verify — only that the shape + # is set and the model's weights fit beside an auto-sized pool. + if [ "$kv_mem" -eq 0 ]; then + [ "$maxseqs" -eq 8 ] && [ "$chunk" -eq 8192 ] && [ "$maxlen" -eq 262144 ] + return 0 + fi + # Pinned pool: the pin must hold at least ONE full-window request or the + # endpoint refuses its own --max-model-len, and it must fit on one card + # beside the weights. grep -q 'kv-profiles.tsv' "$UNIT" - [ "$maxseqs" -eq 8 ] && [ "$chunk" -eq 8192 ] && [ "$spec" -eq 7 ] - grep -q '^Environment=SPEC_METHOD=dflash$' "$UNIT" - # Pool arithmetic, from the engine's own boot line: 18563072000 bytes - # reserved => "GPU KV cache size: 262,720 tokens". The pool must hold at - # least ONE full-window request or the endpoint refuses its own - # --max-model-len, and it must fit on one card beside the weights. [ $(( kv_mem / KV_BYTES_PER_TOKEN )) -ge "$maxlen" ] - # 9.5 GiB of MXFP4 weights per card + the KV pool, under the 0.98 util cap. - local weights=10200547328 # ~9.5 GiB per card of the 19 GB MXFP4 checkpoint - local util util_pct floor + # Weights per card: MXFP4 ~9.5 GiB (19 GB / 2), FP8 ~15.5 GiB (30.9 GB / 2). + local weights util util_pct floor + case "$(env_str SPEC_METHOD)" in + dflash) weights=10200547328 ;; + *) weights=16646128640 ;; + esac util="$(env_ratio GPU_UTIL)" [ -n "$util" ] util_pct="${util#0.}"