diff --git a/scripts/convert-qwen38-w5nl64.sh b/scripts/convert-qwen38-w5nl64.sh new file mode 100755 index 0000000..e95813e --- /dev/null +++ b/scripts/convert-qwen38-w5nl64.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# convert-qwen38-w5nl64.sh — RUNS ON DANKSERVER. The CPU half of the Qwen3.8-Flash-Next five-bit +# campaign: convert the vendor BF16 checkpoint into the campaign's arms, each one the whole +# `w5nl64a8h` rung on BOTH expert projections with the protected experts kept bf16. +# +# Nothing here touches a GPU: the container is started without /dev/kfd or /dev/dri, so the +# converter runs on the host backend and the fleet can keep serving through the whole script. +# The engine pin is the digest that serves frank-archon, so a winning arm crosses no version +# boundary between measurement and serving. +# +# Three arms, because the campaign's question is which search knob pays for its hours +# (recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe): +# +# cd3 the recipe as written — coordinate descent three sweeps. The control. +# cd6 six sweeps: the same search, twice the passes. +# ao1 act_order=1 instead of cd: GPTQ sweeps columns in activation-importance order and +# stores them permuted. This is a SEPARATE arm, not a fourth sweep — libquant refuses +# cd= together with act_order=1 ("cd= sweeps the columns in their stored order, and +# act_order=1 stores them permuted; one or the other", libquant/lq_registry.cpp), so +# the two knobs cannot be combined into a best-of-both container. +# +# The arms are generated from ONE template with `--quant` overrides, not three near-duplicate +# recipes: --quant rules are inserted BEFORE the recipe file and the first rule whose glob matches +# a weight decides it (core/format/recipe.cpp: Recipe::match), so an override cleanly replaces the +# expert line and the other 60 lines cannot drift between arms. +# +# Protected experts: apply-protected-cast.sh derives the bf16 set from THIS campaign's grams +# (--target-count 48, two per layer so each rank's 256-expert half stays even) and splices it into +# a per-run COPY of the template; the committed recipe keeps its markers and stays reviewable. +# +# Gates, in the order they bite — each one is a lesson the swift15 campaign paid for: +# inputs checkpoint (dir + index), recipe, grams, imatrix, tokenizer. A missing grams +# directory converts UNCALIBRATED with one stderr note, which is silently worse +# than failing here. +# reference the 335.3 GiB BF16 reference container must be GONE before the converts start. +# It and the checkpoint together eat the volume; its job (the KLD recording and +# the calibration grams) is finished by then, and both survive on disk without the +# container beside it. The FP8-native anchor is FETCHED after the converts (disk +# sequencing), so nothing here waits on it. +# plan --plan-only runs FIRST, per arm, and prints "about X would be written". The +# number is the bill: free space must cover it plus the 128k-prompt overshoot +# margin, and the finished artifact is checked against it. +# outputs no arm overwrites anything; move the old one to trash. +# +# Usage: convert-qwen38-w5nl64.sh [--arm cd3|cd6|ao1]... [--dry-run] [--plan-only] +# [--target-count N] [--keep-recipe] +set -u -o pipefail + +SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +IMAGE=${IMAGE:-stilldeadcode/radiance@sha256:d590e300234bf6d2f942a0b8d456d183153fd808f28820e6c2fa69c9389178ed} +CKPT=${CKPT:-/home/dank/models/Qwen3.8-Flash-Next} +CAMPAIGN=${CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64} +RECIPE=${RECIPE:-$SELF/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe} +CALIB=${CALIB:-$CAMPAIGN/calib} +IMATRIX=${IMATRIX:-$CAMPAIGN/imatrix.gguf} +OUTDIR=${OUTDIR:-/home/dank/models/rad} +BF16_RAD=${BF16_RAD:-$OUTDIR/qwen38-bf16.rad} # the reference container, not an arm output +STATE=${STATE:-$CAMPAIGN} # the dir the container mounts read-only: the + # spliced recipe and the imatrix live here +TARGET_COUNT=${TARGET_COUNT:-48} +MIN_GRAMS=${MIN_GRAMS:-24576} # per-expert DOWN grams: 48 recorded layers x 512 + # experts. The checkpoint has 49 MoE layers, but the + # MTP layer is never recorded, so its 512 experts are + # absent from the tap's output by design +MARGIN_MIB=${MARGIN_MIB:-40960} # the long-prompt overshoot, off the plan number +TS=$(date +%Y%m%d-%H%M%S) + +ARMS=() +DRY=0 +PLAN_ONLY=0 +KEEP_RECIPE=0 +while [ "$#" -gt 0 ]; do + case "$1" in + --arm) ARMS+=("$2"); shift 2 ;; + --dry-run) DRY=1; shift ;; + --plan-only) PLAN_ONLY=1; shift ;; + --target-count) TARGET_COUNT="$2"; shift 2 ;; + --keep-recipe) KEEP_RECIPE=1; shift ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done +[ "${#ARMS[@]}" -gt 0 ] || ARMS=(cd3 cd6 ao1) +for a in "${ARMS[@]}"; do + case "$a" in cd3|cd6|ao1) ;; *) echo "unknown arm: $a (cd3|cd6|ao1)" >&2; exit 2 ;; esac +done + +# The expert rule's option set, spelled once. --quant replaces the template's line wholesale, so +# every option has to be repeated here; `calib=$CALIB` is expanded by the recipe reader from the +# container environment (core/format/recipe.cpp: expand refuses an unset name rather than taking +# it for empty). +expert_opts='table=w5nl,group=64,scale=fp8_e4m3,scale2=f32,block2=*x*,scale2_value=0.0001220703125,transform=fwht128,rule=search,calib=$CALIB' +arm_quant() { + case "$1" in + # The control is the template itself: no override, so the recipe's own line decides. + cd3) return 0 ;; + cd6) echo "blk.*.ffn_*_exps.*.weight=gptq:${expert_opts},cd=6" ;; + # act_order=1 WITHOUT cd: libquant refuses the pair (lq_registry.cpp:105, "cd= sweeps the + # columns in their stored order, and act_order=1 stores them permuted; one or the other"), so + # this arm is the interleave knob alone, not cd=3 plus it. + ao1) echo "blk.*.ffn_*_exps.*.weight=gptq:${expert_opts},act_order=1" ;; + esac +} + +# --- gates ------------------------------------------------------------------------------------ +[ -d "$CKPT" ] || { echo "no checkpoint at $CKPT; run scripts/download-qwen38-bf16.sh --fetch" >&2; exit 2; } +[ -f "$CKPT/model.safetensors.index.json" ] || { echo "checkpoint at $CKPT has no safetensors index — a torn fetch" >&2; exit 2; } +[ -f "$CKPT/tokenizer.json" ] || { echo "no tokenizer.json in $CKPT; the .rad cannot carry one without it" >&2; exit 2; } +[ -f "$RECIPE" ] || { echo "no recipe at $RECIPE" >&2; exit 2; } +GRAMS=$(find "$CALIB" -maxdepth 1 -name 'gram.r*.bin' 2>/dev/null | wc -l) +[ "$GRAMS" -ge "$MIN_GRAMS" ] || { + echo "calib dir $CALIB holds $GRAMS gram files (<$MIN_GRAMS); run $0 after run-qwen38-bf16.sh calib" >&2; exit 2; } +[ -f "$IMATRIX" ] || { echo "no imatrix at $IMATRIX; run scripts/grams-to-imatrix.py over $CALIB" >&2; exit 2; } +# The container sees the campaign dir at one fixed mount point, so an imatrix kept elsewhere has +# no path inside it: copy it in (or point IMATRIX at the campaign dir) rather than let the +# converter fail halfway through a planned arm. +[ "$(dirname "$IMATRIX")" = "$STATE" ] || { + echo "imatrix $IMATRIX is not in the campaign dir $STATE the container mounts; move it there" >&2; exit 2; } +if [ -e "$BF16_RAD" ]; then + echo "the 335.3 GiB BF16 reference container $BF16_RAD is still on disk; its recording and" \ + "the calibration grams are already taken, so move it to trash — the converts need its" \ + "space (peak-disk law)" >&2 + exit 2 +fi +for a in "${ARMS[@]}"; do + [ -e "$OUTDIR/qwen38-w5nl64-$a.rad" ] && { + echo "$OUTDIR/qwen38-w5nl64-$a.rad already exists; move it to trash first" >&2; exit 2; } +done + +# --- the protected plane, spliced into a per-run copy of the template -------------------------- +mkdir -p "$STATE" +WORK="$STATE/recipe-$TS.recipe" +cp "$RECIPE" "$WORK" +if [ "$DRY" = 1 ]; then + echo "would run: $SELF/apply-protected-cast.sh $CALIB $WORK $TARGET_COUNT" +else + # Positional, because the tool reads its recipe and calib dir by position: passing them as + # environment variables would be silently ignored and it would rewrite the COMMITTED + # template in place instead of this run's copy. + "$SELF/apply-protected-cast.sh" "$CALIB" "$WORK" "$TARGET_COUNT" \ + || { echo "the protected-expert derivation refused; the arms would convert unprotected" >&2; exit 1; } +fi + +run_convert() { # run_convert ARM [extra flags...] + local arm="$1"; shift + local out="$OUTDIR/qwen38-w5nl64-$arm.rad" log="$STATE/convert-$arm-$TS.log" + local argv=(--rm --name "qwen38-convert-$arm" + -e "CALIB=/data/calib" -e "OMP_NUM_THREADS=$(nproc)" + -v "$(dirname "$CKPT"):/models:ro" + -v "$CALIB:/data/calib:ro" + -v "$STATE:/campaign:ro" + -v "$OUTDIR:/out") + local conv=(/models/"${CKPT##*/}" --recipe "/campaign/$(basename "$WORK")" + --tokenizer "/models/${CKPT##*/}/tokenizer.json" + --imatrix /campaign/"${IMATRIX##*/}" + # No --max-ctx: the context declare then runs at the checkpoint's training context + # (262144), which is what the shipped container did — rad-info --meta on it carries + # no override. A container declared at a smaller bound cannot be expected to serve + # the 200k window D2 holds, and the convert is not where to discover that. + # --kv-cache-dtype changes nothing in the artifact; it picks the kernels the declare + # checks, and the fleet serves fp8 KV, so an unresolvable fp8 kernel must be said no + # to by the convert, not by a boot on a drained window. + --kv-cache-dtype fp8 -o "/out/$(basename "$out")") + local q; q=$(arm_quant "$arm") + [ -n "$q" ] && conv+=(--quant "$q") + if [ "$DRY" = 1 ]; then + echo "podman run ${argv[*]} \$IMAGE --entrypoint /opt/radiance/bin/rad-convert ${conv[*]} $*" + return 0 + fi + podman run "${argv[@]}" --entrypoint /opt/radiance/bin/rad-convert "$IMAGE" "${conv[@]}" "$@" \ + 2>&1 | tee "$log" + return "${PIPESTATUS[0]}" +} + +# --- plan, then bill, then convert ------------------------------------------------------------- +overall=0 +for arm in "${ARMS[@]}"; do + echo "== arm $arm ==" + if [ "$DRY" = 1 ]; then + # Both lines, in order: the plan runs before the convert, and the convert is what the + # operator is about to pay an afternoon for. + run_convert "$arm" --plan-only + run_convert "$arm" + continue + fi + rc=0; plan_log=$(run_convert "$arm" --plan-only) || rc=$? + [ "$rc" -eq 0 ] || { echo "arm $arm: --plan-only failed (rc=$rc); nothing was written" >&2; overall=1; continue; } + echo "$plan_log" + # "about 141.2 GiB would be written" is the converter's own bill; parse it rather than + # hardcoding a size, so the gate still works after any recipe edit. + want=$(printf '%s' "$plan_log" | sed -n 's/.*about \([0-9.]*\) \([KMGT]iB\) would be written.*/\1 \2/p' | tail -1) + [ -n "$want" ] || { echo "arm $arm: the plan printed no size; refusing to convert blind" >&2; overall=1; continue; } + need_mib=$(awk -v n="${want% *}" -v u="${want#* }" \ + 'BEGIN{ f = (u=="KiB") ? 2^-10 : (u=="MiB") ? 1 : (u=="GiB") ? 1024 : (u=="TiB") ? 1048576 : 0 + if (f == 0) { exit 1 }; printf "%d", n*f + 0.5 }') || { + echo "arm $arm: the plan printed an unknown unit in '$want'" >&2; overall=1; continue; } + free_mib=$(df --output=avail -m "$OUTDIR" | tail -1 | tr -d ' ') + echo "plan: $want ($need_mib MiB); free ${free_mib} MiB on $OUTDIR (margin $MARGIN_MIB MiB)" + if [ "$free_mib" -lt $((need_mib + MARGIN_MIB)) ]; then + echo "arm $arm: needs ${need_mib} MiB + ${MARGIN_MIB} margin, ${free_mib} free; free space and re-run" >&2 + overall=1; continue + fi + if [ "$PLAN_ONLY" = 1 ]; then echo "planned, not converted (--plan-only)"; continue; fi + rc=0; run_convert "$arm" || rc=$? + [ "$rc" -eq 0 ] || { echo "arm $arm: rad-convert rc=$rc; log: $STATE/convert-$arm-$TS.log" >&2; overall=1; continue; } + out="$OUTDIR/qwen38-w5nl64-$arm.rad" + got_mib=$(( $(stat -c %s "$out") / 1048576 )) + # The receipt: the artifact is what the plan predicted, within the metadata the plan does not + # count. An arm that came out materially smaller lost a plane somewhere. + lo=$(( need_mib * 98 / 100 )); hi=$(( need_mib * 103 / 100 )) + if [ "$got_mib" -lt "$lo" ] || [ "$got_mib" -gt "$hi" ]; then + echo "arm $arm: $out is ${got_mib} MiB, the plan said ${need_mib} MiB (allowed ${lo}-${hi})" >&2 + overall=1; continue + fi + echo "CONVERT_DONE $arm ${got_mib} MiB -> $out" +done + +if [ "$KEEP_RECIPE" = 0 ] && [ "$DRY" = 0 ]; then + mkdir -p "$CAMPAIGN/.AGENTTRASH" + mv "$WORK" "$CAMPAIGN/.AGENTTRASH/$(basename "$WORK")" 2>/dev/null || true +fi +exit "$overall" diff --git a/scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe b/scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe new file mode 100644 index 0000000..6dedd31 --- /dev/null +++ b/scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe @@ -0,0 +1,80 @@ +# qwen4exp-qwen38-w5nl64-bf16trunk.recipe — every routed expert, BOTH projections, at five bits: +# u5 codes into libquant's 32-level w5nl table, an E4M3 scale a group of 64 under the fixed 2^-13 +# second level, rotated by the 128-wide Hadamard, chosen by GPTQ against $CALIB and swept three +# times by coordinate descent. This is `w5nl64a8h` in the engine's served-encoding table — the top of +# the four-bit class — applied wider than any published radiance container of this checkpoint: the +# author's own w5 rung (`qwen4exp-w5dn64-i8-hc8m.recipe`) moves only `down`, because that rung was +# priced as a byte-for-error trade against prefill; this campaign buys the whole rung and pays for +# it out of the pinned host pool (+15.3 GiB of expert plane, `--host-pool-mib 61440` per rank), not +# out of the KV pool. +# +# Every other line is the served container's recipe (dump it with +# `rad-info --recipe /home/dank/models/rad/qwen3.8-next-flash-fp8-iq4r-moe.rad`), minus the rows this +# campaign leaves at the checkpoint: +# +# * the trunk's block linears, the shared expert and the hyper-connection mixers are NOT listed — +# dropping their i8/fp8 lines leaves them bf16, which is what the checkpoint holds. Measured bill +# +1.978 GiB of always-resident weight = ~131k tokens of KV off the pool. +# * `ssm_inz`/`ssm_out` STAY int8: the delta-net gates are the largest trunk rows (+1.9 GiB, 126k +# tokens) and the least sensitive per byte. +# * `mtp.draft_head` STAYS 2-bit and the PLE n-gram table STAYS one fixed E4M3 scale: the draft +# head's rung is cheap and its acceptance cost is a decode question this campaign does not touch, +# and bf16 for the n-gram table would pin +47.69 GiB of host RAM for a lookup. +# +# Protected experts: none in this file. `scripts/apply-protected-cast.sh` appends the derived set +# (48, two per layer so each rank's 256-expert half stays even — the four-bit GEMM takes the rest of +# a half as two equal tables by parity) from the fresh 12M-token energy/routing census, and they must +# sit ABOVE the expert row because the first rule that matches a weight decides it. +# +# CALIB=/home/dank/campaign/qwen38-w5nl64/calib \ +# rad-convert /home/dank/models/Qwen3.8-Flash-Next \ +# --recipe this.file --imatrix .../imatrix.gguf -o out.rad +# +# The arm driver (`scripts/convert-qwen38-w5nl64.sh`) generates the per-arm variants of the expert +# row from this template — cd=3 (control), cd=6, and `act_order=1` with NO cd= (libquant refuses the +# pair, lq_registry.cpp:105) — rather than keeping three near-duplicate recipes that can drift apart. +# +# The whole campaign, in the order that fits on one disk (every step runs on dankserver): +# +# 1. scripts/download-qwen38-bf16.sh --fetch the vendor BF16 checkpoint, 335.3 GiB +# 2. scripts/build-qwen38-bf16-rad.sh the BF16 reference container, another 335.3 GiB +# 3. one drained window, two boots: +# scripts/run-qwen38-bf16.sh kldref the KLD reference recording (the zero of scale) +# scripts/run-qwen38-bf16.sh calib the 12M-position calibration tap -> grams +# 4. trash qwen38-bf16.rad the peak-disk law: the converts need its room +# 5. scripts/grams-to-imatrix.py grams -> imatrix.gguf in the campaign dir +# 6. scripts/convert-qwen38-w5nl64.sh the arms, planned and billed before each one runs +# 7. fetch the published FP8-native release, then per container: +# scripts/run-qwen38-bf16.sh score RAD OUT fp8 anchor, served iq4r, each arm +# then scripts/kld-gate-qwen38.sh fp8.json iq4r.json cand.json +# 8. a winner is a trio edit plus `scripts/deploy.sh dankserver --service frank-archon`; the +# rollback is the commit before it. If the win needs the bigger pinned plane, raise +# FRANK_ARCHON_HOST_POOL_MIB and FRANK_ARCHON_TTM_TARGET_MIB in the SAME commit: the preflight +# floor is 2xHOST_POOL + 48,824 + prefix cache + MNBT + the store, and a pool raised without +# the TTM target is a boot refusal that Restart=always repeats forever, invisible to deploy.sh. + +# PROTECTED-BEGIN — scripts/apply-protected-cast.sh splices the derived cast lines in here. +# They MUST sit above the expert rule: the first rule whose glob matches a weight decides it, so a +# cast line below the five-bit rule would never win. The set is derived from the campaign's own +# 12M-token recording (`derive-protected-experts.py --target-count 48` over the fresh grams), and the +# driver splices it into a per-arm COPY so this template stays the clean, reviewable baseline. +# PROTECTED-END + +# Every routed expert, trunk and MTP layer alike, both projections, at five bits. The experts' scales +# sit between 2^-15 and 2^-10.6 under w4nl's top level of 22 and half a binade lower under w5nl's 30 +# — inside E4M3's normal range under 2^-13 either way. The MTP layer has no recording and is rounded +# to nearest, which libquant says on stderr; it is one layer of 49 and only ever changes drafts. +blk.*.ffn_*_exps.*.weight gptq table=w5nl group=64 scale=fp8_e4m3 scale2=f32 block2=*x* scale2_value=0.0001220703125 transform=fwht128 rule=search cd=3 calib=$CALIB + +# The delta-net's in/out gates, int8 with a bf16 scale a group of 128 searched for least error — the +# served recipe's lines, unchanged. +blk.*.ssm_inz.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search +blk.*.ssm_out.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search + +# The MTP head's two mixing rows, E4M3 as served. They only ever change the drafts. +mtp.fc_hidden.weight rtn codes=fp8_e4m3 group=128 scale=f32 +mtp.fc_embedding.weight rtn codes=fp8_e4m3 group=128 scale=f32 + +# The n-gram table at one fixed E4M3 scale, and the 2-bit draft head — both the served lines. +blk.*.ple_ngram.weight rtn codes=fp8_e4m3 block=*x* scale=bf16 rule=fixed scale_value=1.99317932128906e-4 +mtp.draft_head.weight rtn codes=u2 zero=u8 group=128 scale=f16