feat(frank-medium): quantization-aware entrypoint + FP8 model swap
The entrypoint now reads the model's weight quantization from config.json and serves the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP self-speculation, ROCM_AITER unified attention with the R4D all-reduce) or the MXFP4 patch-and-serve path. A model swap is now: change SNAP (+ SPEC_METHOD/SPEC), restart. - radiance-entrypoint.sh: quantization detection, gated MXFP4 env vars and patch block, --quantization fp8 flag - quadlet: SNAP -> Qwen3.8-27B-Uncensored-FP8, SPEC_METHOD=mtp, SPEC=8, KV_MEM=0 (profile the shape, the documented plain-FP8 default) - deploy-remote.sh: model-aware gates (read SNAP/SPEC_METHOD from the quadlet; drafter gate only for dflash, r4d.so gate only for mxfp4) - tests: quantization-aware drafter/spec and KV-pool tests - hf-fetch.sh: generic, resumable, verified, idempotent HF fetch tool - frank-medium-fp8.manifest: 17 files (10 config + 7 shards) with sizes and LFS sha256
This commit is contained in:
+29
-10
@@ -1211,6 +1211,19 @@ deploy_llm_medium() {
|
||||
# missing never becomes healthy, and a green deploy would leave the endpoint
|
||||
# dark with nothing in the deploy log to say why.
|
||||
local missing=0
|
||||
# Model-aware gates: read the serving shape from the quadlet (the source
|
||||
# of truth), so a model swap is just "edit the quadlet + deploy". The
|
||||
# container /models mount maps to /home/dank/models on the host. The
|
||||
# drafter gate applies only to the dflash path; the patched libr4d gate
|
||||
# applies only to the mxfp4/quark path (the plain FP8 path uses the
|
||||
# image's built-in r4d.so and skips the MXFP4 patches).
|
||||
local quadlet_file="dankserver/frank-medium/quadlet/frank-medium.container"
|
||||
local snap_c snap spec_method quant
|
||||
snap_c="$(grep -E '^Environment=SNAP=' "$quadlet_file" | head -1 | cut -d= -f2-)"
|
||||
spec_method="$(grep -E '^Environment=SPEC_METHOD=' "$quadlet_file" | head -1 | cut -d= -f2-)"
|
||||
snap="/home/dank/models${snap_c#/models}"
|
||||
quant="$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("quantization_config",{}).get("quant_method",""))' "$snap/config.json" 2>/dev/null || true)"
|
||||
[[ -n "$snap_c" ]] || { echo "ERROR: cannot read SNAP from $quadlet_file" >&2; return 1; }
|
||||
if ! podman image exists "$LLM_MEDIUM_IMAGE_LOCAL"; then
|
||||
if podman image exists "$LLM_MEDIUM_IMAGE"; then
|
||||
podman tag "$LLM_MEDIUM_IMAGE" "$LLM_MEDIUM_IMAGE_LOCAL"
|
||||
@@ -1220,27 +1233,33 @@ deploy_llm_medium() {
|
||||
missing=1
|
||||
fi
|
||||
fi
|
||||
if [[ ! -f "$LLM_MEDIUM_SNAP/config.json" ]]; then
|
||||
echo "ERROR: MXFP4 checkpoint absent at $LLM_MEDIUM_SNAP" >&2
|
||||
if [[ ! -f "$snap/config.json" ]]; then
|
||||
echo "ERROR: model checkpoint absent at $snap" >&2
|
||||
missing=1
|
||||
fi
|
||||
if [[ ! -f "$LLM_MEDIUM_DRAFTER/config.json" ]]; then
|
||||
if [[ "$spec_method" == "dflash" && ! -f "$LLM_MEDIUM_DRAFTER/config.json" ]]; then
|
||||
echo "ERROR: DFlash2-FP8 drafter absent at $LLM_MEDIUM_DRAFTER" >&2
|
||||
missing=1
|
||||
fi
|
||||
# The patches clone is mounted and its scripts run at EVERY container start;
|
||||
# the chat template is the fleet's reasoning-format contract.
|
||||
# The patches clone is mounted and its scripts run at EVERY container start
|
||||
# on the MXFP4 path; the chat template is the fleet's reasoning-format
|
||||
# contract on both paths.
|
||||
if [[ ! -f "$LLM_MEDIUM_PATCHES/qwen-fixed-v22.3.jinja" ]]; then
|
||||
echo "ERROR: pinned patch clone / chat template absent at $LLM_MEDIUM_PATCHES" >&2
|
||||
echo "frank-medium: git clone https://github.com/ggz14/radiance-vllm-mxfp4 $LLM_MEDIUM_PATCHES" >&2
|
||||
missing=1
|
||||
fi
|
||||
# The image's shipped r4d.so predates the gated-delta-net overflow fix and
|
||||
# NaNs this model, so the patched build is not optional.
|
||||
if [[ ! -f "$LLM_MEDIUM_R4D/r4d.so" ]]; then
|
||||
echo "ERROR: patched libr4d absent at $LLM_MEDIUM_R4D" >&2
|
||||
missing=1
|
||||
fi
|
||||
# NaNs the MXFP4 model, so the patched build is not optional there. The
|
||||
# plain FP8 path uses the image's built-in r4d.so (no MXFP4 patches).
|
||||
case "$quant" in
|
||||
mxfp4|quark)
|
||||
if [[ ! -f "$LLM_MEDIUM_R4D/r4d.so" ]]; then
|
||||
echo "ERROR: patched libr4d absent at $LLM_MEDIUM_R4D" >&2
|
||||
missing=1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
[[ "$missing" -eq 0 ]] || return 1
|
||||
mkdir -p "$LLM_MEDIUM_CACHE"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user