feat(frank-medium): quantization-aware entrypoint + FP8 model swap

The entrypoint now reads the model's weight quantization from config.json
and serves the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP
self-speculation, ROCM_AITER unified attention with the R4D all-reduce)
or the MXFP4 patch-and-serve path. A model swap is now: change SNAP
(+ SPEC_METHOD/SPEC), restart.

- radiance-entrypoint.sh: quantization detection, gated MXFP4 env vars
  and patch block, --quantization fp8 flag
- quadlet: SNAP -> Qwen3.8-27B-Uncensored-FP8, SPEC_METHOD=mtp, SPEC=8,
  KV_MEM=0 (profile the shape, the documented plain-FP8 default)
- deploy-remote.sh: model-aware gates (read SNAP/SPEC_METHOD from the
  quadlet; drafter gate only for dflash, r4d.so gate only for mxfp4)
- tests: quantization-aware drafter/spec and KV-pool tests
- hf-fetch.sh: generic, resumable, verified, idempotent HF fetch tool
- frank-medium-fp8.manifest: 17 files (10 config + 7 shards) with sizes
  and LFS sha256
This commit is contained in:
Frank
2026-09-20 21:30:30 -07:00
parent c42780f9f3
commit d389d09d97
6 changed files with 251 additions and 63 deletions
+29 -10
View File
@@ -1211,6 +1211,19 @@ deploy_llm_medium() {
# missing never becomes healthy, and a green deploy would leave the endpoint
# dark with nothing in the deploy log to say why.
local missing=0
# Model-aware gates: read the serving shape from the quadlet (the source
# of truth), so a model swap is just "edit the quadlet + deploy". The
# container /models mount maps to /home/dank/models on the host. The
# drafter gate applies only to the dflash path; the patched libr4d gate
# applies only to the mxfp4/quark path (the plain FP8 path uses the
# image's built-in r4d.so and skips the MXFP4 patches).
local quadlet_file="dankserver/frank-medium/quadlet/frank-medium.container"
local snap_c snap spec_method quant
snap_c="$(grep -E '^Environment=SNAP=' "$quadlet_file" | head -1 | cut -d= -f2-)"
spec_method="$(grep -E '^Environment=SPEC_METHOD=' "$quadlet_file" | head -1 | cut -d= -f2-)"
snap="/home/dank/models${snap_c#/models}"
quant="$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("quantization_config",{}).get("quant_method",""))' "$snap/config.json" 2>/dev/null || true)"
[[ -n "$snap_c" ]] || { echo "ERROR: cannot read SNAP from $quadlet_file" >&2; return 1; }
if ! podman image exists "$LLM_MEDIUM_IMAGE_LOCAL"; then
if podman image exists "$LLM_MEDIUM_IMAGE"; then
podman tag "$LLM_MEDIUM_IMAGE" "$LLM_MEDIUM_IMAGE_LOCAL"
@@ -1220,27 +1233,33 @@ deploy_llm_medium() {
missing=1
fi
fi
if [[ ! -f "$LLM_MEDIUM_SNAP/config.json" ]]; then
echo "ERROR: MXFP4 checkpoint absent at $LLM_MEDIUM_SNAP" >&2
if [[ ! -f "$snap/config.json" ]]; then
echo "ERROR: model checkpoint absent at $snap" >&2
missing=1
fi
if [[ ! -f "$LLM_MEDIUM_DRAFTER/config.json" ]]; then
if [[ "$spec_method" == "dflash" && ! -f "$LLM_MEDIUM_DRAFTER/config.json" ]]; then
echo "ERROR: DFlash2-FP8 drafter absent at $LLM_MEDIUM_DRAFTER" >&2
missing=1
fi
# The patches clone is mounted and its scripts run at EVERY container start;
# the chat template is the fleet's reasoning-format contract.
# The patches clone is mounted and its scripts run at EVERY container start
# on the MXFP4 path; the chat template is the fleet's reasoning-format
# contract on both paths.
if [[ ! -f "$LLM_MEDIUM_PATCHES/qwen-fixed-v22.3.jinja" ]]; then
echo "ERROR: pinned patch clone / chat template absent at $LLM_MEDIUM_PATCHES" >&2
echo "frank-medium: git clone https://github.com/ggz14/radiance-vllm-mxfp4 $LLM_MEDIUM_PATCHES" >&2
missing=1
fi
# The image's shipped r4d.so predates the gated-delta-net overflow fix and
# NaNs this model, so the patched build is not optional.
if [[ ! -f "$LLM_MEDIUM_R4D/r4d.so" ]]; then
echo "ERROR: patched libr4d absent at $LLM_MEDIUM_R4D" >&2
missing=1
fi
# NaNs the MXFP4 model, so the patched build is not optional there. The
# plain FP8 path uses the image's built-in r4d.so (no MXFP4 patches).
case "$quant" in
mxfp4|quark)
if [[ ! -f "$LLM_MEDIUM_R4D/r4d.so" ]]; then
echo "ERROR: patched libr4d absent at $LLM_MEDIUM_R4D" >&2
missing=1
fi
;;
esac
[[ "$missing" -eq 0 ]] || return 1
mkdir -p "$LLM_MEDIUM_CACHE"