diff --git a/scripts/grams-to-imatrix.py b/scripts/grams-to-imatrix.py new file mode 100755 index 0000000..12f6d48 --- /dev/null +++ b/scripts/grams-to-imatrix.py @@ -0,0 +1,418 @@ +#!/usr/bin/env python3 +"""grams-to-imatrix.py — the engine's own calibration recording, re-voiced as a +llama.cpp importance matrix for `rad-convert --imatrix F`. RUNS ON DANKSERVER +beside the recording. Python 3 stdlib only: numpy must NOT be required, because +this runs in the plain container next to a 335 GiB checkpoint, not in a venv. + +WHY. `rad-convert` takes its per-column activation second moments and its +expert-popularity profile from ONE input, an imatrix (core/format/imatrix.h). +Only the GGUF form carries a per-expert dimension — with the legacy flat form, +or with none, every expert in a layer is weighted by the layer's aggregate, +which is exactly the wrong thing on a 512-expert MoE. The calibration tap has +already measured everything; this tool only re-voices it, so the imatrix and the +protected-expert census (derive-protected-experts.py) are the SAME recording: +one measurement, two consumers, neither able to disagree about which experts are +hot. + +INPUT. RADGRAM2 files in a RADIANCE_CALIB_DIR output directory +(arch/common/rad_arch.h, calib_write). Header law, same parse as +scripts/derive-protected-experts.py — 32 bytes, `<8sIIQI`: magic `RADGRAM2`, +u32 k, u32 rank, u64 rows, u32 domain, a pad word, then k*k f32 row-major. Two +kinds of name, both GGUF-style, both with the checkpoint's GLOBAL expert +numbering (rad_block_moe_fp8.h: ge = expert_base + e): + + gram.r.blk.L.ffn_down_exps.bin the layer's pooled gram + gram.r.blk.L.ffn_down_exps..bin expert e's own gram + +`rows` is that file's routing/token count and the matrix is Σ x xᵀ over those +rows, so **the diagonal is exactly the per-column activation second moment an +imatrix wants**, and `rows` is the count the reader hands back from `count()`. +Every off-diagonal term is dropped on purpose: the GGUF format has nowhere to +put a full matrix, and the quantiser asks for columns. + +OUTPUT, per layer L (core/format/imatrix.h: names WITHOUT `.weight`, a MoE +layer's experts folded into ONE entry, so expert 9 is slice 9 of +`blk.L.ffn_down_exps`): + + blk.L.ffn_down_exps in_sum2 [k, n_expert] expert-major, counts [1, n_expert] + blk.L.ffn_gate_up_exps the SAME matrices at the SAME offsets. A routed expert's + gate_up input is the same hidden-state rows as its down + input, so the second moment over them is identical and + so is the routing count. Two tensor infos over one + payload: the reader maps each half independently + (imatrix.cpp joins `.in_sum2` with `.counts` + by base name), and copying ~200 MB to state one fact + twice buys no correctness. + blk.L.ssm_inz [k] and [1], from the pooled gram: that row's input is + the same hidden state, so its columns take the pooled + diagonal and its count is the layer's pooled rows. + + Then STOP. Nothing whose input width is not the gram's k — no ssm_out, no + attn_output, no mtp.*, no ple_ngram. importance_for() answers false when the + entry's cols do not equal the weight's width and the convert proceeds without + it: an entry the reader silently ignores is a lie in the artifact, worse than + an absent one. The corollary to check before trusting the two hidden-width + entries: the tap writes the gate_up gram at g.n_embd and the down gram at the + expert width, so the gram's k must equal the checkpoint's hidden width (its + config.json). Where they differ, gate_up/ssm_inz are inert, not wrong. + +INHERITANCE. An expert with no own gram carries the layer's POOLED diagonal, and +its count stays its own routing rows — the rule the engine itself applies when it +picks a Hessian (libquant/lq_hess.h, calib_resolve: an expert's own gram only +once its rank files hold kOwnRowsPerK * K rows; below that a fit to so few rows +is worse than round-to-nearest, and those experts carry a fraction of a percent +of the routing). The count never inherits: popularity is `count`'s job. The count +of inherited experts is reported on stderr. + +RANK FOLDING. TP=2 writes gram.r0.* and gram.r1.* with GLOBAL expert numbering, +so the ranks describe DISJOINT experts and the fold is a UNION into the one +[k, n_expert] entry. A pooled gram is the opposite case: every rank summed the +same stem over its own rows, so the fold is a SUM, rows included (calib_sum, +calib_rows). The same global expert in two per-expert files is legal only when +the numbers agree exactly; otherwise the recording is corrupt and the entry would +be whichever file sorted last, so refuse. + +GGUF v3 (the only version imatrix.cpp accepts — version != 3 is a refusal): +`general.alignment` u32, `imatrix.chunk_size` u32, `imatrix.chunk_count` u32, +`imatrix.datasets` array of strings; tensors f32 (ggml type 0) with offsets +relative to the aligned data section. Every tensor offset is padded up to +`general.alignment` and the data section itself starts aligned. That is the +classic hand-rolled-writer bug: an unaligned offset makes the reader's extent +arithmetic point into the middle of the previous tensor's floats, and it reads +them without complaint. + +chunk_count DERIVATION. The corpus row count is the LARGEST single-layer pooled +row total, not a sum over layers: every layer's tap accumulated over the whole +corpus, so adding the layers counts each token once per layer and inflates the +figure by the layer count. chunk_count = that row total // chunk_size, the +"chunks of chunk_size" line imatrix.cpp prints for itself. + +REFUSALS, each exit 2 with a message naming what is missing: + NO_GRAMS the directory holds no RADGRAM2 down-projection gram + NOT_RADGRAM2 a gram whose magic is not RADGRAM2 + SHORT a gram whose length is not 32 + k*k*4 + NO_POOLED a layer with expert grams and no pooled gram to inherit from + K_MISMATCH a layer whose files disagree about k (rank pooled grams, or an + expert gram against its layer's pooled gram) + EXPERT_CONFLICT a global expert recorded twice with different numbers + EXISTS the -o path already holds a file (the trash law: move it) + CHUNK_COUNT the derived chunk count does not fit the u32 metadata field + +USAGE. grams-to-imatrix.py -o + [--datasets NAME,...] [--chunk-size N] [--min-rows N] [-v] +""" +import argparse +import os +import re +import struct +import sys + +MAGIC = b"RADGRAM2" +HDR = 32 # 8s I I Q I + pad, rad_arch.h +KOWN_ROWS_PER_K = 4 # libquant/lq_hess.h kOwnRowsPerK +ALIGNMENT = 32 # general.alignment, and the pad unit + +POOLED_RE = re.compile(r"^gram\.r(\d+)\.blk\.(\d+)\.ffn_down_exps\.bin$") +EXPERT_RE = re.compile(r"^gram\.r(\d+)\.blk\.(\d+)\.ffn_down_exps\.(\d+)\.bin$") + +GT_U32, GT_STR, GT_ARR = 4, 8, 9 +GGML_F32 = 0 + + +def refuse(msg): + sys.stderr.write("REFUSED %s\n" % msg) + sys.exit(2) + + +def read_gram(path): + """(k, rank, rows, domain, diagonal) from one RADGRAM2 file.""" + try: + with open(path, "rb") as f: + raw = f.read() + except OSError as e: + refuse("UNREADABLE %s: %s" % (path, e)) + if len(raw) < HDR: + refuse("SHORT %s: %d bytes, under the %d-byte RADGRAM2 header" + % (path, len(raw), HDR)) + magic, k, rank, rows, domain = struct.unpack_from("<8sIIQI", raw, 0) + if magic != MAGIC: + refuse("NOT_RADGRAM2 %s: magic %r, expected b'RADGRAM2'" % (path, magic)) + if k <= 0: + refuse("SHORT %s: header claims k=%d, which describes no matrix" % (path, k)) + need = HDR + k * k * 4 + if len(raw) != need: + refuse("SHORT %s: %d bytes, expected %d (32 + k*k*4 for k=%d)" + % (path, len(raw), need, k)) + # The diagonal only, read by stride out of the row-major matrix. + diag = [struct.unpack_from(".blk..ffn_down_exps" + "[.].bin files in the directory" % d) + return layers, rotated + + +def build_entries(layers, min_rows, verbose): + """([ (name, dims, payload|None, key) ], census, layers emitted).""" + entries = [] + census = {"inherited": 0, "experts": 0, "routing": 0, "rows": 0, "layers": 0} + for layer in sorted(layers): + P, E = layers[layer]["pooled"], layers[layer]["experts"] + if E and P is None: + refuse("NO_POOLED layer %d: %d expert gram(s) but no pooled gram (an" + " expected gram.r.blk.%d.ffn_down_exps.bin), so an expert" + " under the row threshold would have nothing to inherit" + % (layer, len(E), layer)) + if P is None: + continue + k = P["k"] + census["layers"] += 1 + census["rows"] = max(census["rows"], P["rows"]) + dn = "blk.%d.ffn_down_exps" % layer + # A staging key names ONE payload. A gate_up entry reuses the down + # layer's bytes by carrying the same key with no payload of its own; + # in_sum2 and counts are different bytes and must never share a key. + ksum, kcnt = (layer, "sum2"), (layer, "counts") + kpgm, kpcnt = (layer, "pooled_sum2"), (layer, "pooled_counts") + # One entry holds one width: an expert gram recorded at another k would + # inherit (or be inherited by) a diagonal of the wrong length, and the + # reader would then answer `cols` from whichever file set dims[0]. + for e in sorted(E): + if E[e]["k"] != k: + refuse("K_MISMATCH layer %d expert %d: %s holds k=%d and this" + " layer's pooled gram is k=%d" % (layer, e, E[e]["path"], + E[e]["k"], k)) + + if not E: + # No per-expert recording for this layer: emit the dense form + # ([k], [1]) rather than 512 identical slices, which would claim a + # per-expert measurement that was never taken. + entries.append((dn + ".in_sum2", [k], + struct.pack("<%df" % k, *P["diag"]), ksum)) + entries.append((dn + ".counts", [1], + struct.pack(" 0 else KOWN_ROWS_PER_K * k + n_expert = max(E) + 1 + sum2, counts = bytearray(), bytearray() + for e in range(n_expert): + own = E.get(e) + if own is not None and own["rows"] >= floor: + src = own["diag"] + rows = own["rows"] + else: + src = P["diag"] + rows = own["rows"] if own is not None else 0 + census["inherited"] += 1 + sum2 += struct.pack("<%df" % k, *src) + counts += struct.pack(" 0xFFFFFFFF: + # imatrix.chunk_count is a u32 in the reader's grammar, so there is no + # legal way to record a bigger one. Reaching this means either a chunk + # size picked for a far larger run or a row total read out of the wrong + # header offset — rows is a u64 and a u32 parse is off by 2**32 — and + # the reader would trust whatever number landed in the field. + refuse("CHUNK_COUNT %d = %d rows // %d does not fit the u32" + " imatrix.chunk_count field; raise --chunk-size" + % (chunk_count, census["rows"], a.chunk_size)) + nbytes = write_gguf(a.out, entries, [d for d in a.datasets.split(",") if d], + a.chunk_size, chunk_count) + sys.stderr.write("IMATRIX_OK entries=%d layers=%d experts=%d inherited=%d" + " rows=%d routing=%d chunks=%d/%d bytes=%d out=%s\n" + % (len(entries), census["layers"], census["experts"], + census["inherited"], census["rows"], census["routing"], + chunk_count, a.chunk_size, nbytes, a.out)) + + +if __name__ == "__main__": + main() diff --git a/tests/qwen38_grams.bats b/tests/qwen38_grams.bats new file mode 100755 index 0000000..cd2bd80 --- /dev/null +++ b/tests/qwen38_grams.bats @@ -0,0 +1,445 @@ +#!/usr/bin/env bats +# Seam: scripts/grams-to-imatrix.py — the calibration recording (RADGRAM2 grams) +# folded into the GGUF v3 imatrix `rad-convert --imatrix F` reads. +# +# The only contract that matters is the reader's: core/format/imatrix.cpp joins +# `.in_sum2` with `.counts`, takes cols from in_sum2 dims[0] and +# n_expert from dims[1], demands ggml f32 tensors and GGUF version 3, and +# silently IGNORES an entry whose width does not match the weight it is asked +# about. So every test here parses the artifact back with its own tiny GGUF +# reader and asserts bytes — a file that merely exists and exits 0 proves +# nothing about a quantiser that will read it hours later on another machine. +# +# Fixtures are synthetic: k=4 grams with real RADGRAM2 headers (rows is a u64 +# at offset 16 — pack it as u32 and 100 rows read back as 4294967296000) and +# real f32 payloads whose OFF-DIAGONAL is filled with 99.0, so an entry built +# from anything but the diagonal is instantly visible. Expected numbers are +# worked examples from the fixture tables below, never recomputed the way the +# tool computes them. + +load helpers + +TOOL="$REPO_ROOT/scripts/grams-to-imatrix.py" + +# make_gram +# A RADGRAM2 file: 32-byte header <8sIIQI (magic, k, rank, rows, domain, pad) +# then k*k f32 row-major. Off-diagonals are 99.0 — a sentinel, not data. +make_gram() { + python3 - "$@" <<'PY' +import struct, sys +path, k, rank, rows, domain = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), \ + int(sys.argv[4]), int(sys.argv[5]) +diag = [float(x) for x in sys.argv[6].split(",")] +assert len(diag) == k, "diagonal must hold k values" +vals = [99.0] * (k * k) +for i, d in enumerate(diag): + vals[i * k + i] = d +hdr = struct.pack("<8sIIQI", b"RADGRAM2", k, rank, rows, domain) + struct.pack(" — the fixture every happy-path test reads. +# +# layer 0 (k=4, own-gram floor 4*k = 16 rows): +# pooled r0 rows 40 diag 1,2,3,4 + r1 rows 8 diag .5,.5,.5,.5 +# -> summed gram (the engine's calib_sum law): diag 1.5,2.5,3.5,4.5, +# pooled rows 48 +# expert 0 r0 rows 100 diag 10,20,30,40 (>= 16: own gram kept) +# expert 1 r0 rows 3 diag 7,7,7,7 (< 16: inherits pooled) +# expert 2 r1 rows 64 diag 1,1,1,2 (>= 16: own gram kept) +# expert 3 no file at all (inherits pooled, count 0) +# expert 4 r1 rows 100 diag 5,6,7,8 (>= 16: own gram kept) +# -> n_expert 5, counts 100 3 64 0 100, inherited 2, routing 267 +# layer 1: pooled r0 rows 48 diag 2,2,2,2 ; expert 0 on r1 rows 48 diag 1,1,1,1 +# -> n_expert 1, counts 48, inherited 0, routing 48 +# Corpus rows = the largest single-layer pooled rows = 48. +full_recording() { + local d="$1" + mkdir -p "$d" + make_gram "$d/gram.r0.blk.0.ffn_down_exps.bin" 4 0 40 0 "1,2,3,4" + make_gram "$d/gram.r1.blk.0.ffn_down_exps.bin" 4 1 8 0 "0.5,0.5,0.5,0.5" + make_gram "$d/gram.r0.blk.0.ffn_down_exps.0.bin" 4 0 100 0 "10,20,30,40" + make_gram "$d/gram.r0.blk.0.ffn_down_exps.1.bin" 4 0 3 0 "7,7,7,7" + make_gram "$d/gram.r1.blk.0.ffn_down_exps.2.bin" 4 1 64 0 "1,1,1,2" + make_gram "$d/gram.r1.blk.0.ffn_down_exps.4.bin" 4 1 100 0 "5,6,7,8" + make_gram "$d/gram.r0.blk.1.ffn_down_exps.bin" 4 0 48 0 "2,2,2,2" + make_gram "$d/gram.r1.blk.1.ffn_down_exps.0.bin" 4 1 48 0 "1,1,1,1" +} + +# gguf_read [vals ] — an independent GGUF +# v3 reader: KV block, tensor infos, and (on request) the f32 values behind a +# tensor's own declared offset. Prints KV/HDR/TENSOR lines. Nothing here shares +# code with the writer, which is the only reason the byte assertions mean +# anything: imatrix.cpp walks the same three structures. +gguf_read() { + python3 - "$@" <<'PY' +import struct, sys +d = open(sys.argv[1], "rb").read() +def u32(o): return struct.unpack_from(" 2 and sys.argv[2] == "vals": + name, i0, n = sys.argv[3], int(sys.argv[4]), int(sys.argv[5]) + for nm, dims, ty, off in tens: + if nm == name: + base = ds + off + vals = [("%d" % v) if float(v).is_integer() else repr(v) + for v in (struct.unpack_from(" — exit 0 iff the data section starts aligned AND every +# tensor's absolute data offset (data_start + t.off) is a multiple of +# general.alignment. This is the writer bug that no value assertion can see: an +# unaligned offset makes the reader's extent arithmetic land mid-float in the +# previous tensor, and it reads the result without complaining. +aligned_offsets() { + python3 - "$1" <<'PY' +import struct, sys +d = open(sys.argv[1], "rb").read() +def u32(o): return struct.unpack_from(""$BATS_TEST_TMPDIR/err" + [ -s "$BATS_TEST_TMPDIR/err" ] + grep -q "inherited=2" "$BATS_TEST_TMPDIR/err" +} + +@test "the hidden-width entries are the same bytes, and nothing else is emitted" { + C="$BATS_TEST_TMPDIR/full" + full_recording "$C" + run python3 "$TOOL" "$C" -o "$BATS_TEST_TMPDIR/im.gguf" + [ "$status" -eq 0 ] + run gguf_read "$BATS_TEST_TMPDIR/im.gguf" + # gate_up shares the down matrices (same hidden-state rows), so its slice 4 + # must read 5 6 7 8 at its own tensor info's offset. + [[ "$output" == *"TENSOR blk.0.ffn_gate_up_exps.in_sum2 4x5 0 "* ]] + [[ "$output" == *"TENSOR blk.0.ssm_inz.in_sum2 4 0 "* ]] + [[ "$output" == *"TENSOR blk.0.ssm_inz.counts 1 0 "* ]] + run gguf_read "$BATS_TEST_TMPDIR/im.gguf" vals blk.0.ffn_gate_up_exps.in_sum2 16 4 + [[ "$output" == *"VALS blk.0.ffn_gate_up_exps.in_sum2 5 6 7 8"* ]] + run gguf_read "$BATS_TEST_TMPDIR/im.gguf" vals blk.0.ssm_inz.in_sum2 0 4 + [[ "$output" == *"VALS blk.0.ssm_inz.in_sum2 1.5 2.5 3.5 4.5"* ]] + run gguf_read "$BATS_TEST_TMPDIR/im.gguf" vals blk.0.ssm_inz.counts 0 1 + [[ "$output" == *"VALS blk.0.ssm_inz.counts 48"* ]] + # The stop rule: importance_for() false-on-a-width-mismatch means an entry + # for a projection whose input is NOT the gram's width is a silent lie. + [[ "$output" != *"ssm_out"* ]] + [[ "$output" != *"attn"* ]] + [[ "$output" != *"mtp."* ]] + [[ "$output" != *"ple_ngram"* ]] + [[ "$output" != *"output.weight"* ]] +} + +@test "rows is a u64: a corpus too big for u32 survives the header" { + # The trap this fixture law exists for: pack rows as u32 at offset 16 and a + # 100-row file reads back as 4294967296000. 2**32 is exactly representable + # in f32, so the count round-trips only if the tool read 8 bytes. + C="$BATS_TEST_TMPDIR/wide" + mkdir -p "$C" + make_gram "$C/gram.r0.blk.0.ffn_down_exps.bin" 4 0 4294967296 0 "1,1,1,1" + make_gram "$C/gram.r0.blk.0.ffn_down_exps.0.bin" 4 0 4294967296 0 "3,3,3,3" + run python3 "$TOOL" "$C" -o "$BATS_TEST_TMPDIR/wide.gguf" --chunk-size 4096 + [ "$status" -eq 0 ] + run gguf_read "$BATS_TEST_TMPDIR/wide.gguf" vals blk.0.ffn_down_exps.counts 0 1 + [[ "$output" == *"VALS blk.0.ffn_down_exps.counts 4294967296"* ]] + [[ "$output" != *"4294967296000"* ]] + run gguf_read "$BATS_TEST_TMPDIR/wide.gguf" + [[ "$output" == *"KV imatrix.chunk_count 1048576"* ]] + # imatrix.chunk_count is a u32 field, so 2**32 rows in 1-row chunks has no + # legal encoding: wrapping it would put a number downstream believes. + run python3 "$TOOL" "$C" -o "$BATS_TEST_TMPDIR/wide2.gguf" --chunk-size 1 + [ "$status" -eq 2 ] + [[ "$output" == *"REFUSED"*"CHUNK_COUNT"* ]] + [ ! -e "$BATS_TEST_TMPDIR/wide2.gguf" ] +} + +@test "every tensor's data offset is a multiple of general.alignment" { + C="$BATS_TEST_TMPDIR/full" + full_recording "$C" + run python3 "$TOOL" "$C" -o "$BATS_TEST_TMPDIR/im.gguf" + [ "$status" -eq 0 ] + run aligned_offsets "$BATS_TEST_TMPDIR/im.gguf" + [ "$status" -eq 0 ] + [[ "$output" == "ALIGNED 12 tensor(s),"*"alignment=32" ]] + + # Red proof, in the same test so the assertion cannot rot: the same file + # with one tensor's declared offset nudged by 4. A counts tensor is 4 bytes, + # so a writer that forgets to pad between tensors produces exactly this + # shape of file — and every value assertion above still passes on it. + python3 - "$BATS_TEST_TMPDIR/im.gguf" "$BATS_TEST_TMPDIR/unaligned.gguf" <<'PY' +import struct, sys +src, dst = sys.argv[1], sys.argv[2] +d = bytearray(open(src, "rb").read()) +def u32(o): return struct.unpack_from(".blk.3.ffn_down_exps.bin"* ]] +} + +@test "the same global expert recorded twice with different numbers is refused" { + # Rank files carry GLOBAL expert numbering and so describe disjoint experts; + # a collision means the recording is corrupt, and whichever file sorted last + # would win. + C="$BATS_TEST_TMPDIR/conflict" + full_recording "$C" + make_gram "$C/gram.r1.blk.0.ffn_down_exps.0.bin" 4 1 77 0 "10,20,30,40" + run python3 "$TOOL" "$C" -o "$BATS_TEST_TMPDIR/no.gguf" + [ "$status" -eq 2 ] + [[ "$output" == *"REFUSED"*"EXPERT_CONFLICT"*"expert 0"* ]] + [[ "$output" == *"gram.r0.blk.0.ffn_down_exps.0.bin"* ]] + [[ "$output" == *"gram.r1.blk.0.ffn_down_exps.0.bin"* ]] +} + +@test "an existing output is refused by name rather than overwritten" { + # The trash law: a re-run must not destroy the imatrix a convert is reading. + C="$BATS_TEST_TMPDIR/full" + full_recording "$C" + : > "$BATS_TEST_TMPDIR/keep.gguf" + printf 'do not clobber me\n' > "$BATS_TEST_TMPDIR/keep.gguf" + run python3 "$TOOL" "$C" -o "$BATS_TEST_TMPDIR/keep.gguf" + [ "$status" -eq 2 ] + [[ "$output" == *"REFUSED"*"EXISTS"*"keep.gguf"* ]] + run cat "$BATS_TEST_TMPDIR/keep.gguf" + [[ "$output" == *"do not clobber me"* ]] +} + +@test "--min-rows moves the threshold and the bytes follow it" { + # The default threshold is the engine's own law (4*k rows, lq_hess.h), but + # the operator has to be able to raise it when a corpus is thin. Asserted on + # the slices, not on a report line: a flag that only changes prose is a + # flag that changes nothing. + C="$BATS_TEST_TMPDIR/floor" + full_recording "$C" + run python3 "$TOOL" "$C" -o "$BATS_TEST_TMPDIR/far.gguf" --min-rows 200 + [ "$status" -eq 0 ] + # 200 is above every expert's routing (100, 3, 64, 0, 100 and layer 1's 48), + # so all six slices now inherit. + [[ "$output" == *"inherited=6"* ]] + run gguf_read "$BATS_TEST_TMPDIR/far.gguf" vals blk.0.ffn_down_exps.in_sum2 0 4 + [[ "$output" == *"VALS blk.0.ffn_down_exps.in_sum2 1.5 2.5 3.5 4.5"* ]] + # The counts do not move with the threshold: routing is a measurement, not a + # judgement about how much data is enough. + run gguf_read "$BATS_TEST_TMPDIR/far.gguf" vals blk.0.ffn_down_exps.counts 0 5 + [[ "$output" == *"VALS blk.0.ffn_down_exps.counts 100 3 64 0 100"* ]] +}