feat(qwen38): the w5nl64 heavier-quant campaign toolchain (fetch, BF16 reference, arms, FP8-anchored gate) #76
No files matched your search
Executable
+221
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env bash
|
||||
# convert-qwen38-w5nl64.sh — RUNS ON DANKSERVER. The CPU half of the Qwen3.8-Flash-Next five-bit
|
||||
# campaign: convert the vendor BF16 checkpoint into the campaign's arms, each one the whole
|
||||
# `w5nl64a8h` rung on BOTH expert projections with the protected experts kept bf16.
|
||||
#
|
||||
# Nothing here touches a GPU: the container is started without /dev/kfd or /dev/dri, so the
|
||||
# converter runs on the host backend and the fleet can keep serving through the whole script.
|
||||
# The engine pin is the digest that serves frank-archon, so a winning arm crosses no version
|
||||
# boundary between measurement and serving.
|
||||
#
|
||||
# Three arms, because the campaign's question is which search knob pays for its hours
|
||||
# (recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe):
|
||||
#
|
||||
# cd3 the recipe as written — coordinate descent three sweeps. The control.
|
||||
# cd6 six sweeps: the same search, twice the passes.
|
||||
# ao1 act_order=1 instead of cd: GPTQ sweeps columns in activation-importance order and
|
||||
# stores them permuted. This is a SEPARATE arm, not a fourth sweep — libquant refuses
|
||||
# cd= together with act_order=1 ("cd= sweeps the columns in their stored order, and
|
||||
# act_order=1 stores them permuted; one or the other", libquant/lq_registry.cpp), so
|
||||
# the two knobs cannot be combined into a best-of-both container.
|
||||
#
|
||||
# The arms are generated from ONE template with `--quant` overrides, not three near-duplicate
|
||||
# recipes: --quant rules are inserted BEFORE the recipe file and the first rule whose glob matches
|
||||
# a weight decides it (core/format/recipe.cpp: Recipe::match), so an override cleanly replaces the
|
||||
# expert line and the other 60 lines cannot drift between arms.
|
||||
#
|
||||
# Protected experts: apply-protected-cast.sh derives the bf16 set from THIS campaign's grams
|
||||
# (--target-count 48, two per layer so each rank's 256-expert half stays even) and splices it into
|
||||
# a per-run COPY of the template; the committed recipe keeps its markers and stays reviewable.
|
||||
#
|
||||
# Gates, in the order they bite — each one is a lesson the swift15 campaign paid for:
|
||||
# inputs checkpoint (dir + index), recipe, grams, imatrix, tokenizer. A missing grams
|
||||
# directory converts UNCALIBRATED with one stderr note, which is silently worse
|
||||
# than failing here.
|
||||
# reference the 335.3 GiB BF16 reference container must be GONE before the converts start.
|
||||
# It and the checkpoint together eat the volume; its job (the KLD recording and
|
||||
# the calibration grams) is finished by then, and both survive on disk without the
|
||||
# container beside it. The FP8-native anchor is FETCHED after the converts (disk
|
||||
# sequencing), so nothing here waits on it.
|
||||
# plan --plan-only runs FIRST, per arm, and prints "about X would be written". The
|
||||
# number is the bill: free space must cover it plus the 128k-prompt overshoot
|
||||
# margin, and the finished artifact is checked against it.
|
||||
# outputs no arm overwrites anything; move the old one to trash.
|
||||
#
|
||||
# Usage: convert-qwen38-w5nl64.sh [--arm cd3|cd6|ao1]... [--dry-run] [--plan-only]
|
||||
# [--target-count N] [--keep-recipe]
|
||||
set -u -o pipefail
|
||||
|
||||
SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
IMAGE=${IMAGE:-stilldeadcode/radiance@sha256:d590e300234bf6d2f942a0b8d456d183153fd808f28820e6c2fa69c9389178ed}
|
||||
CKPT=${CKPT:-/home/dank/models/Qwen3.8-Flash-Next}
|
||||
CAMPAIGN=${CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}
|
||||
RECIPE=${RECIPE:-$SELF/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe}
|
||||
CALIB=${CALIB:-$CAMPAIGN/calib}
|
||||
IMATRIX=${IMATRIX:-$CAMPAIGN/imatrix.gguf}
|
||||
OUTDIR=${OUTDIR:-/home/dank/models/rad}
|
||||
BF16_RAD=${BF16_RAD:-$OUTDIR/qwen38-bf16.rad} # the reference container, not an arm output
|
||||
STATE=${STATE:-$CAMPAIGN} # the dir the container mounts read-only: the
|
||||
# spliced recipe and the imatrix live here
|
||||
TARGET_COUNT=${TARGET_COUNT:-48}
|
||||
MIN_GRAMS=${MIN_GRAMS:-24576} # per-expert DOWN grams: 48 recorded layers x 512
|
||||
# experts. The checkpoint has 49 MoE layers, but the
|
||||
# MTP layer is never recorded, so its 512 experts are
|
||||
# absent from the tap's output by design
|
||||
MARGIN_MIB=${MARGIN_MIB:-40960} # the long-prompt overshoot, off the plan number
|
||||
TS=$(date +%Y%m%d-%H%M%S)
|
||||
|
||||
ARMS=()
|
||||
DRY=0
|
||||
PLAN_ONLY=0
|
||||
KEEP_RECIPE=0
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--arm) ARMS+=("$2"); shift 2 ;;
|
||||
--dry-run) DRY=1; shift ;;
|
||||
--plan-only) PLAN_ONLY=1; shift ;;
|
||||
--target-count) TARGET_COUNT="$2"; shift 2 ;;
|
||||
--keep-recipe) KEEP_RECIPE=1; shift ;;
|
||||
*) echo "unknown argument: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
[ "${#ARMS[@]}" -gt 0 ] || ARMS=(cd3 cd6 ao1)
|
||||
for a in "${ARMS[@]}"; do
|
||||
case "$a" in cd3|cd6|ao1) ;; *) echo "unknown arm: $a (cd3|cd6|ao1)" >&2; exit 2 ;; esac
|
||||
done
|
||||
|
||||
# The expert rule's option set, spelled once. --quant replaces the template's line wholesale, so
|
||||
# every option has to be repeated here; `calib=$CALIB` is expanded by the recipe reader from the
|
||||
# container environment (core/format/recipe.cpp: expand refuses an unset name rather than taking
|
||||
# it for empty).
|
||||
expert_opts='table=w5nl,group=64,scale=fp8_e4m3,scale2=f32,block2=*x*,scale2_value=0.0001220703125,transform=fwht128,rule=search,calib=$CALIB'
|
||||
arm_quant() {
|
||||
case "$1" in
|
||||
# The control is the template itself: no override, so the recipe's own line decides.
|
||||
cd3) return 0 ;;
|
||||
cd6) echo "blk.*.ffn_*_exps.*.weight=gptq:${expert_opts},cd=6" ;;
|
||||
# act_order=1 WITHOUT cd: libquant refuses the pair (lq_registry.cpp:105, "cd= sweeps the
|
||||
# columns in their stored order, and act_order=1 stores them permuted; one or the other"), so
|
||||
# this arm is the interleave knob alone, not cd=3 plus it.
|
||||
ao1) echo "blk.*.ffn_*_exps.*.weight=gptq:${expert_opts},act_order=1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# --- gates ------------------------------------------------------------------------------------
|
||||
[ -d "$CKPT" ] || { echo "no checkpoint at $CKPT; run scripts/download-qwen38-bf16.sh --fetch" >&2; exit 2; }
|
||||
[ -f "$CKPT/model.safetensors.index.json" ] || { echo "checkpoint at $CKPT has no safetensors index — a torn fetch" >&2; exit 2; }
|
||||
[ -f "$CKPT/tokenizer.json" ] || { echo "no tokenizer.json in $CKPT; the .rad cannot carry one without it" >&2; exit 2; }
|
||||
[ -f "$RECIPE" ] || { echo "no recipe at $RECIPE" >&2; exit 2; }
|
||||
GRAMS=$(find "$CALIB" -maxdepth 1 -name 'gram.r*.bin' 2>/dev/null | wc -l)
|
||||
[ "$GRAMS" -ge "$MIN_GRAMS" ] || {
|
||||
echo "calib dir $CALIB holds $GRAMS gram files (<$MIN_GRAMS); run $0 after run-qwen38-bf16.sh calib" >&2; exit 2; }
|
||||
[ -f "$IMATRIX" ] || { echo "no imatrix at $IMATRIX; run scripts/grams-to-imatrix.py over $CALIB" >&2; exit 2; }
|
||||
# The container sees the campaign dir at one fixed mount point, so an imatrix kept elsewhere has
|
||||
# no path inside it: copy it in (or point IMATRIX at the campaign dir) rather than let the
|
||||
# converter fail halfway through a planned arm.
|
||||
[ "$(dirname "$IMATRIX")" = "$STATE" ] || {
|
||||
echo "imatrix $IMATRIX is not in the campaign dir $STATE the container mounts; move it there" >&2; exit 2; }
|
||||
if [ -e "$BF16_RAD" ]; then
|
||||
echo "the 335.3 GiB BF16 reference container $BF16_RAD is still on disk; its recording and" \
|
||||
"the calibration grams are already taken, so move it to trash — the converts need its" \
|
||||
"space (peak-disk law)" >&2
|
||||
exit 2
|
||||
fi
|
||||
for a in "${ARMS[@]}"; do
|
||||
[ -e "$OUTDIR/qwen38-w5nl64-$a.rad" ] && {
|
||||
echo "$OUTDIR/qwen38-w5nl64-$a.rad already exists; move it to trash first" >&2; exit 2; }
|
||||
done
|
||||
|
||||
# --- the protected plane, spliced into a per-run copy of the template --------------------------
|
||||
mkdir -p "$STATE"
|
||||
WORK="$STATE/recipe-$TS.recipe"
|
||||
cp "$RECIPE" "$WORK"
|
||||
if [ "$DRY" = 1 ]; then
|
||||
echo "would run: $SELF/apply-protected-cast.sh $CALIB $WORK $TARGET_COUNT"
|
||||
else
|
||||
# Positional, because the tool reads its recipe and calib dir by position: passing them as
|
||||
# environment variables would be silently ignored and it would rewrite the COMMITTED
|
||||
# template in place instead of this run's copy.
|
||||
"$SELF/apply-protected-cast.sh" "$CALIB" "$WORK" "$TARGET_COUNT" \
|
||||
|| { echo "the protected-expert derivation refused; the arms would convert unprotected" >&2; exit 1; }
|
||||
fi
|
||||
|
||||
run_convert() { # run_convert ARM [extra flags...]
|
||||
local arm="$1"; shift
|
||||
local out="$OUTDIR/qwen38-w5nl64-$arm.rad" log="$STATE/convert-$arm-$TS.log"
|
||||
local argv=(--rm --name "qwen38-convert-$arm"
|
||||
-e "CALIB=/data/calib" -e "OMP_NUM_THREADS=$(nproc)"
|
||||
-v "$(dirname "$CKPT"):/models:ro"
|
||||
-v "$CALIB:/data/calib:ro"
|
||||
-v "$STATE:/campaign:ro"
|
||||
-v "$OUTDIR:/out")
|
||||
local conv=(/models/"${CKPT##*/}" --recipe "/campaign/$(basename "$WORK")"
|
||||
--tokenizer "/models/${CKPT##*/}/tokenizer.json"
|
||||
--imatrix /campaign/"${IMATRIX##*/}"
|
||||
# No --max-ctx: the context declare then runs at the checkpoint's training context
|
||||
# (262144), which is what the shipped container did — rad-info --meta on it carries
|
||||
# no override. A container declared at a smaller bound cannot be expected to serve
|
||||
# the 200k window D2 holds, and the convert is not where to discover that.
|
||||
# --kv-cache-dtype changes nothing in the artifact; it picks the kernels the declare
|
||||
# checks, and the fleet serves fp8 KV, so an unresolvable fp8 kernel must be said no
|
||||
# to by the convert, not by a boot on a drained window.
|
||||
--kv-cache-dtype fp8 -o "/out/$(basename "$out")")
|
||||
local q; q=$(arm_quant "$arm")
|
||||
[ -n "$q" ] && conv+=(--quant "$q")
|
||||
if [ "$DRY" = 1 ]; then
|
||||
echo "podman run ${argv[*]} \$IMAGE --entrypoint /opt/radiance/bin/rad-convert ${conv[*]} $*"
|
||||
return 0
|
||||
fi
|
||||
podman run "${argv[@]}" --entrypoint /opt/radiance/bin/rad-convert "$IMAGE" "${conv[@]}" "$@" \
|
||||
2>&1 | tee "$log"
|
||||
return "${PIPESTATUS[0]}"
|
||||
}
|
||||
|
||||
# --- plan, then bill, then convert -------------------------------------------------------------
|
||||
overall=0
|
||||
for arm in "${ARMS[@]}"; do
|
||||
echo "== arm $arm =="
|
||||
if [ "$DRY" = 1 ]; then
|
||||
# Both lines, in order: the plan runs before the convert, and the convert is what the
|
||||
# operator is about to pay an afternoon for.
|
||||
run_convert "$arm" --plan-only
|
||||
run_convert "$arm"
|
||||
continue
|
||||
fi
|
||||
rc=0; plan_log=$(run_convert "$arm" --plan-only) || rc=$?
|
||||
[ "$rc" -eq 0 ] || { echo "arm $arm: --plan-only failed (rc=$rc); nothing was written" >&2; overall=1; continue; }
|
||||
echo "$plan_log"
|
||||
# "about 141.2 GiB would be written" is the converter's own bill; parse it rather than
|
||||
# hardcoding a size, so the gate still works after any recipe edit.
|
||||
want=$(printf '%s' "$plan_log" | sed -n 's/.*about \([0-9.]*\) \([KMGT]iB\) would be written.*/\1 \2/p' | tail -1)
|
||||
[ -n "$want" ] || { echo "arm $arm: the plan printed no size; refusing to convert blind" >&2; overall=1; continue; }
|
||||
need_mib=$(awk -v n="${want% *}" -v u="${want#* }" \
|
||||
'BEGIN{ f = (u=="KiB") ? 2^-10 : (u=="MiB") ? 1 : (u=="GiB") ? 1024 : (u=="TiB") ? 1048576 : 0
|
||||
if (f == 0) { exit 1 }; printf "%d", n*f + 0.5 }') || {
|
||||
echo "arm $arm: the plan printed an unknown unit in '$want'" >&2; overall=1; continue; }
|
||||
free_mib=$(df --output=avail -m "$OUTDIR" | tail -1 | tr -d ' ')
|
||||
echo "plan: $want ($need_mib MiB); free ${free_mib} MiB on $OUTDIR (margin $MARGIN_MIB MiB)"
|
||||
if [ "$free_mib" -lt $((need_mib + MARGIN_MIB)) ]; then
|
||||
echo "arm $arm: needs ${need_mib} MiB + ${MARGIN_MIB} margin, ${free_mib} free; free space and re-run" >&2
|
||||
overall=1; continue
|
||||
fi
|
||||
if [ "$PLAN_ONLY" = 1 ]; then echo "planned, not converted (--plan-only)"; continue; fi
|
||||
rc=0; run_convert "$arm" || rc=$?
|
||||
[ "$rc" -eq 0 ] || { echo "arm $arm: rad-convert rc=$rc; log: $STATE/convert-$arm-$TS.log" >&2; overall=1; continue; }
|
||||
out="$OUTDIR/qwen38-w5nl64-$arm.rad"
|
||||
got_mib=$(( $(stat -c %s "$out") / 1048576 ))
|
||||
# The receipt: the artifact is what the plan predicted, within the metadata the plan does not
|
||||
# count. An arm that came out materially smaller lost a plane somewhere.
|
||||
lo=$(( need_mib * 98 / 100 )); hi=$(( need_mib * 103 / 100 ))
|
||||
if [ "$got_mib" -lt "$lo" ] || [ "$got_mib" -gt "$hi" ]; then
|
||||
echo "arm $arm: $out is ${got_mib} MiB, the plan said ${need_mib} MiB (allowed ${lo}-${hi})" >&2
|
||||
overall=1; continue
|
||||
fi
|
||||
echo "CONVERT_DONE $arm ${got_mib} MiB -> $out"
|
||||
done
|
||||
|
||||
if [ "$KEEP_RECIPE" = 0 ] && [ "$DRY" = 0 ]; then
|
||||
mkdir -p "$CAMPAIGN/.AGENTTRASH"
|
||||
mv "$WORK" "$CAMPAIGN/.AGENTTRASH/$(basename "$WORK")" 2>/dev/null || true
|
||||
fi
|
||||
exit "$overall"
|
||||
@@ -0,0 +1,80 @@
|
||||
# qwen4exp-qwen38-w5nl64-bf16trunk.recipe — every routed expert, BOTH projections, at five bits:
|
||||
# u5 codes into libquant's 32-level w5nl table, an E4M3 scale a group of 64 under the fixed 2^-13
|
||||
# second level, rotated by the 128-wide Hadamard, chosen by GPTQ against $CALIB and swept three
|
||||
# times by coordinate descent. This is `w5nl64a8h` in the engine's served-encoding table — the top of
|
||||
# the four-bit class — applied wider than any published radiance container of this checkpoint: the
|
||||
# author's own w5 rung (`qwen4exp-w5dn64-i8-hc8m.recipe`) moves only `down`, because that rung was
|
||||
# priced as a byte-for-error trade against prefill; this campaign buys the whole rung and pays for
|
||||
# it out of the pinned host pool (+15.3 GiB of expert plane, `--host-pool-mib 61440` per rank), not
|
||||
# out of the KV pool.
|
||||
#
|
||||
# Every other line is the served container's recipe (dump it with
|
||||
# `rad-info --recipe /home/dank/models/rad/qwen3.8-next-flash-fp8-iq4r-moe.rad`), minus the rows this
|
||||
# campaign leaves at the checkpoint:
|
||||
#
|
||||
# * the trunk's block linears, the shared expert and the hyper-connection mixers are NOT listed —
|
||||
# dropping their i8/fp8 lines leaves them bf16, which is what the checkpoint holds. Measured bill
|
||||
# +1.978 GiB of always-resident weight = ~131k tokens of KV off the pool.
|
||||
# * `ssm_inz`/`ssm_out` STAY int8: the delta-net gates are the largest trunk rows (+1.9 GiB, 126k
|
||||
# tokens) and the least sensitive per byte.
|
||||
# * `mtp.draft_head` STAYS 2-bit and the PLE n-gram table STAYS one fixed E4M3 scale: the draft
|
||||
# head's rung is cheap and its acceptance cost is a decode question this campaign does not touch,
|
||||
# and bf16 for the n-gram table would pin +47.69 GiB of host RAM for a lookup.
|
||||
#
|
||||
# Protected experts: none in this file. `scripts/apply-protected-cast.sh` appends the derived set
|
||||
# (48, two per layer so each rank's 256-expert half stays even — the four-bit GEMM takes the rest of
|
||||
# a half as two equal tables by parity) from the fresh 12M-token energy/routing census, and they must
|
||||
# sit ABOVE the expert row because the first rule that matches a weight decides it.
|
||||
#
|
||||
# CALIB=/home/dank/campaign/qwen38-w5nl64/calib \
|
||||
# rad-convert /home/dank/models/Qwen3.8-Flash-Next \
|
||||
# --recipe this.file --imatrix .../imatrix.gguf -o out.rad
|
||||
#
|
||||
# The arm driver (`scripts/convert-qwen38-w5nl64.sh`) generates the per-arm variants of the expert
|
||||
# row from this template — cd=3 (control), cd=6, and `act_order=1` with NO cd= (libquant refuses the
|
||||
# pair, lq_registry.cpp:105) — rather than keeping three near-duplicate recipes that can drift apart.
|
||||
#
|
||||
# The whole campaign, in the order that fits on one disk (every step runs on dankserver):
|
||||
#
|
||||
# 1. scripts/download-qwen38-bf16.sh --fetch the vendor BF16 checkpoint, 335.3 GiB
|
||||
# 2. scripts/build-qwen38-bf16-rad.sh the BF16 reference container, another 335.3 GiB
|
||||
# 3. one drained window, two boots:
|
||||
# scripts/run-qwen38-bf16.sh kldref the KLD reference recording (the zero of scale)
|
||||
# scripts/run-qwen38-bf16.sh calib the 12M-position calibration tap -> grams
|
||||
# 4. trash qwen38-bf16.rad the peak-disk law: the converts need its room
|
||||
# 5. scripts/grams-to-imatrix.py grams -> imatrix.gguf in the campaign dir
|
||||
# 6. scripts/convert-qwen38-w5nl64.sh the arms, planned and billed before each one runs
|
||||
# 7. fetch the published FP8-native release, then per container:
|
||||
# scripts/run-qwen38-bf16.sh score RAD OUT fp8 anchor, served iq4r, each arm
|
||||
# then scripts/kld-gate-qwen38.sh fp8.json iq4r.json cand.json
|
||||
# 8. a winner is a trio edit plus `scripts/deploy.sh dankserver --service frank-archon`; the
|
||||
# rollback is the commit before it. If the win needs the bigger pinned plane, raise
|
||||
# FRANK_ARCHON_HOST_POOL_MIB and FRANK_ARCHON_TTM_TARGET_MIB in the SAME commit: the preflight
|
||||
# floor is 2xHOST_POOL + 48,824 + prefix cache + MNBT + the store, and a pool raised without
|
||||
# the TTM target is a boot refusal that Restart=always repeats forever, invisible to deploy.sh.
|
||||
|
||||
# PROTECTED-BEGIN — scripts/apply-protected-cast.sh splices the derived cast lines in here.
|
||||
# They MUST sit above the expert rule: the first rule whose glob matches a weight decides it, so a
|
||||
# cast line below the five-bit rule would never win. The set is derived from the campaign's own
|
||||
# 12M-token recording (`derive-protected-experts.py --target-count 48` over the fresh grams), and the
|
||||
# driver splices it into a per-arm COPY so this template stays the clean, reviewable baseline.
|
||||
# PROTECTED-END
|
||||
|
||||
# Every routed expert, trunk and MTP layer alike, both projections, at five bits. The experts' scales
|
||||
# sit between 2^-15 and 2^-10.6 under w4nl's top level of 22 and half a binade lower under w5nl's 30
|
||||
# — inside E4M3's normal range under 2^-13 either way. The MTP layer has no recording and is rounded
|
||||
# to nearest, which libquant says on stderr; it is one layer of 49 and only ever changes drafts.
|
||||
blk.*.ffn_*_exps.*.weight gptq table=w5nl group=64 scale=fp8_e4m3 scale2=f32 block2=*x* scale2_value=0.0001220703125 transform=fwht128 rule=search cd=3 calib=$CALIB
|
||||
|
||||
# The delta-net's in/out gates, int8 with a bf16 scale a group of 128 searched for least error — the
|
||||
# served recipe's lines, unchanged.
|
||||
blk.*.ssm_inz.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.ssm_out.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
|
||||
# The MTP head's two mixing rows, E4M3 as served. They only ever change the drafts.
|
||||
mtp.fc_hidden.weight rtn codes=fp8_e4m3 group=128 scale=f32
|
||||
mtp.fc_embedding.weight rtn codes=fp8_e4m3 group=128 scale=f32
|
||||
|
||||
# The n-gram table at one fixed E4M3 scale, and the 2-bit draft head — both the served lines.
|
||||
blk.*.ple_ngram.weight rtn codes=fp8_e4m3 block=*x* scale=bf16 rule=fixed scale_value=1.99317932128906e-4
|
||||
mtp.draft_head.weight rtn codes=u2 zero=u8 group=128 scale=f16
|
||||
Reference in new issue
Block a user