From 9c386d4f898633216375f5c2eb70ac97c43529bd Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 28 Oct 2026 00:26:28 -0700 Subject: [PATCH] feat(qwen38): score the inherited corpus at the window the container really has The reference rig declared a 32,768-token window while the boot ledger hands it 7,436, so 197 of the 198 inherited documents were too long to score. The engine warns about the real pool; the flag was the only cited source. Add scripts/rechunk-kld-corpus.py, which moves document boundaries without moving tokens: each record is re-cut into windows of at most --max-doc-tokens (7,168 = 7x1024, the largest whole kilopage under 7,436), and each window gets its own score_from = len - --score-tail. The reference scores the same token series at more positions and at a shorter context; the calibration corpus is untouched, so no quant moves. Writes go through a temp file and the token counts must match before the output is published. The rig header now says what the pool actually is, and scripts/test-changed.sh routes the new script to its own seam instead of the generic syntax fallback. --- scripts/rechunk-kld-corpus.py | 178 ++++++++++++++++++++++++++++++++++ scripts/run-qwen38-bf16.sh | 10 +- scripts/test-changed.sh | 7 ++ tests/qwen38_rechunk.bats | 138 ++++++++++++++++++++++++++ 4 files changed, 328 insertions(+), 5 deletions(-) create mode 100755 scripts/rechunk-kld-corpus.py create mode 100755 tests/qwen38_rechunk.bats diff --git a/scripts/rechunk-kld-corpus.py b/scripts/rechunk-kld-corpus.py new file mode 100755 index 0000000..bfefad0 --- /dev/null +++ b/scripts/rechunk-kld-corpus.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Re-chunk a KLD corpus so every document fits the container that scores it. + +A KLD reference is recorded by booting one container and prefilling every corpus +document through it. radiance refuses a document the KV pool cannot hold on its +own (core/mem/kv.cpp `can_ever_fit`: a paged group holds `n_blocks * +block_size` tokens), and the number that matters is printed by the boot ledger, +not by a flag: + + W kv group 'kv_attn' holds 7436 tokens but the context limit is 32768, so + a prompt longer than 7436 tokens is refused (400) ... or lower + --max-model-len to 7436 + +The BF16 qwen4exp container at the campaign's shape gets 270.00 MiB of cache -- +the expert plane owns both cards -- so its real window is 7,436 tokens however +`--max-model-len` is spelled. The corpus the campaign inherited from the swift15 +era is 197 documents of 8,193-11,970 tokens, because make-calib-corpus.py's +`--kld-min-prompt 8192` sets a floor and no ceiling: every one of them is +unscoreable there, and three drained windows have proved it. + +This script moves the document boundaries and nothing else. Each record's token +ids -- counted with the same tokenizer the generator used -- are cut into +windows of at most `--max-doc-tokens`, so a record's windows concatenate back to +that record's own token sequence, unchanged: no token dropped, rewritten or +reworded. Each window gets its own `score_from = len - --score-tail`, the same +relation the generator wrote for the parent, so the reference scores 256 +positions per window instead of per document: 100,864 measured positions over +the same text against the 50,432 the inherited corpus scored, at a shorter +context. The calibration corpus is untouched, so the quantization itself does +not move. + +Usage: + rechunk-kld-corpus.py --corpus kld.jsonl --out kld.jsonl.new \ + [--tokenizer-json PATH] [--max-doc-tokens 7168] [--score-tail 256] [--force] + +--out must not exist and be non-empty without --force: the same +one-artifact-one-name law run-qwen38-bf16.sh applies to a score report. The file +is written through a temporary name and renamed, so an interrupted run leaves no +half corpus behind. A JSON summary of what was written goes to stdout. +""" + +import argparse +import json +import os +import sys +import tempfile + + +def parse_args() -> argparse.Namespace: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--corpus", required=True, + help="the kld.jsonl to re-chunk (records: prompt, score_from, source)") + ap.add_argument("--out", required=True, + help="file to write; refuse if it exists and is non-empty without --force") + ap.add_argument("--tokenizer-json", + default=os.environ.get("FRANK_QWEN38_TOKENIZER", + "/home/dank/models/Qwen3.8-Flash-Next/tokenizer.json"), + help="the tokenizer the corpus was authored with (the same one the " + "manifest in the corpus output dir names)") + ap.add_argument("--max-doc-tokens", type=int, default=7168, + help="the widest document the scoring container can hold; 7,168 is " + "7x1024, the largest whole kilopage under the 7,436 the bf16 " + "container's ledger reports") + ap.add_argument("--score-tail", type=int, default=256, + help="positions scored at the end of every document, as the generator " + "recorded it") + ap.add_argument("--force", action="store_true", + help="overwrite a non-empty --out (after moving the old file to trash)") + return ap.parse_args() + + +def load_tokenizer(path: str): + try: + from tokenizers import Tokenizer + except ImportError as exc: # the corpus scripts carry their own venv + raise SystemExit( + f"REFUSED: tokenizers is not importable by {sys.executable} — run the " + f"corpus venv's python (on dankserver: /home/dank/venvs/radcorpus/bin/python) " + f"or set FRANK_QWEN38_TOKENIZER: {exc}") from None + if not os.path.isfile(path): + raise SystemExit(f"REFUSED: no tokenizer at {path}") + return Tokenizer.from_file(path) + + +def refuse(msg: str) -> None: + print(f"REFUSED: {msg}", file=sys.stderr) + sys.exit(2) # 2 is the code argparse uses for unusable arguments + + +def main() -> None: + a = parse_args() + if a.max_doc_tokens <= 0: + refuse(f"--max-doc-tokens must be positive (got {a.max_doc_tokens})") + if a.score_tail <= 0: + refuse(f"--score-tail must be positive (got {a.score_tail})") + if a.max_doc_tokens <= a.score_tail: + refuse(f"--max-doc-tokens {a.max_doc_tokens} cannot hold a {a.score_tail}-position " + "score tail: every window would be all tail and no context") + if not os.path.isfile(a.corpus): + refuse(f"no corpus at {a.corpus}") + if os.path.realpath(a.out) == os.path.realpath(a.corpus): + refuse(f"--out {a.out} is the corpus itself; move it to trash first — the trash law " + "means the old segmentation stays recoverable") + if os.path.exists(a.out) and os.path.getsize(a.out) > 0 and not a.force: + refuse(f"{a.out} exists and is not empty; pass --force after moving it to trash") + + tok = load_tokenizer(a.tokenizer_json) + + docs_in = docs_out = tokens_in = tokens_out = 0 + longest_child = 0 + clamped = scored = 0 + sources: dict[str, int] = {} + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(os.path.abspath(a.out)) or ".", + prefix=".rechunk-") + try: + with os.fdopen(fd, "w", encoding="utf-8") as out: + with open(a.corpus, encoding="utf-8") as src: + for lineno, line in enumerate(src, 1): + if not line.strip(): + continue + try: + rec = json.loads(line) + except json.JSONDecodeError as exc: + refuse(f"{a.corpus}:{lineno} is not JSON: {exc}") + prompt = rec.get("prompt") + if not isinstance(prompt, str): + refuse(f"{a.corpus}:{lineno} has no string 'prompt' field") + ids = tok.encode(prompt, add_special_tokens=False).ids + docs_in += 1 + tokens_in += len(ids) + for start in range(0, max(len(ids), 1), a.max_doc_tokens): + window = ids[start:start + a.max_doc_tokens] + if not window: + continue + score_from = max(len(window) - a.score_tail, 0) + if len(window) < a.score_tail: + clamped += 1 + scored += len(window) - score_from + child = {"prompt": tok.decode(window), + "score_from": score_from} + if "source" in rec: + child["source"] = rec["source"] + out.write(json.dumps(child, ensure_ascii=False) + "\n") + docs_out += 1 + tokens_out += len(window) + longest_child = max(longest_child, len(window)) + if "source" in rec: + for cls in str(rec["source"]).split("+"): + sources[cls] = sources.get(cls, 0) + 1 + if tokens_out != tokens_in: + raise SystemExit(f"REFUSED: the windows hold {tokens_out} tokens and the corpus " + f"has {tokens_in} — re-chunking lost or invented tokens") + os.replace(tmp, a.out) + except BaseException: + if os.path.exists(tmp): + os.unlink(tmp) + raise + os.chmod(a.out, 0o644) + + print(json.dumps({ + "corpus": a.corpus, + "out": a.out, + "tokenizer": os.path.basename(a.tokenizer_json), + "documents_in": docs_in, + "documents_out": docs_out, + "tokens_in": tokens_in, + "tokens_out": tokens_out, + "longest_document": longest_child, + "max_doc_tokens": a.max_doc_tokens, + "score_tail": a.score_tail, + "scored_positions": scored, + "clamped_score_from": clamped, + "source_documents": sources, + }, indent=1)) + + +if __name__ == "__main__": + main() diff --git a/scripts/run-qwen38-bf16.sh b/scripts/run-qwen38-bf16.sh index eafd40d..ca64cb9 100755 --- a/scripts/run-qwen38-bf16.sh +++ b/scripts/run-qwen38-bf16.sh @@ -39,11 +39,11 @@ # speculation, fp8 KV, a 32,768-token window, the PLE n-gram table pinned to RAM, # and the production chat template rather than the one baked into the container — # every scored container is asked the same tokenised questions the reference -# answered. The window is 32,768: kld.jsonl is 197 records with a longest of -# 38,263 characters (longest prompt FIELD 36,855, median record 27,228). Its -# token length — 10,554 for that longest record, counted by the engine itself — -# is in the funding paragraph below; the kldref gate re-checks the file against -# the window before it boots anything. +# answered. The window the flag spells is 32,768; the pool this payload's boot +# actually gets is 7,436 tokens, so kld.jsonl is re-chunked by +# scripts/rechunk-kld-corpus.py into documents of at most 7,168 tokens — the +# widest one the reference can hold. The kldref gate below re-checks the file +# against the declared window, and the re-chunker is what keeps that honest. # # NO --vram-kv-mib: it was REMOVED at 1.3.0 and naming it is a boot refusal, not # a no-op — verified on the pinned image 2026-10-27: "E --vram-kv-mib was diff --git a/scripts/test-changed.sh b/scripts/test-changed.sh index bbad299..4c5b9aa 100755 --- a/scripts/test-changed.sh +++ b/scripts/test-changed.sh @@ -265,6 +265,13 @@ for path in "${CHANGED[@]}"; do add_test tests/voice_pack_unit.bats add_test tests/scripts_syntax.bats ;; + # The KLD corpus re-chunker: its seam is its own (the campaign's + # one-token-series invariant is asserted there), so a change to it does + # not land on the generic scripts/* fallback below. + scripts/rechunk-kld-corpus.py) + add_test tests/qwen38_rechunk.bats + add_test tests/scripts_syntax.bats + ;; scripts/*) add_test tests/scripts_syntax.bats ;; diff --git a/tests/qwen38_rechunk.bats b/tests/qwen38_rechunk.bats new file mode 100755 index 0000000..59dde9c --- /dev/null +++ b/tests/qwen38_rechunk.bats @@ -0,0 +1,138 @@ +#!/usr/bin/env bats +# Seam: scripts/rechunk-kld-corpus.py — the tool that moves KLD document +# boundaries so every document fits the KV pool of the container that has to +# score it. The campaign's whole value is one comparable token series, so this +# seam asserts the invariant that keeps a re-chunk honest — a record's windows +# concatenate back to that record's own tokens — plus the refusal set. Offline: +# no engine, no GPU, no network; the tokenizer is a fixture built in the test's +# own tmpdir. +# +# A python that can import `tokenizers` is required, the way the corpus scripts +# require their venv: QWEN38_PYTHON names it, else python3, else the corpus +# venv; the tests skip with a reason when none answers. + +load helpers + +SCRIPT="$REPO_ROOT/scripts/rechunk-kld-corpus.py" + +# py — the first interpreter that can import tokenizers, or failure. +py() { + local p + for p in "${QWEN38_PYTHON:-}" python3 "${HOME}/.venvs/corpus/bin/python" \ + /home/dank/venvs/radcorpus/bin/python; do + [ -n "$p" ] && [ -x "$p" ] || continue + "$p" -c 'import tokenizers' >/dev/null 2>&1 && { printf '%s' "$p"; return 0; } + done + return 1 +} + +# words N — N space-separated words drawn from the fixture vocabulary. +words() { + local n="$1" i out="" + local vocab=(alpha beta gamma delta epsilon) + for ((i = 0; i < n; i++)); do + [ -n "$out" ] && out+=" " + out+="${vocab[$((i % 5))]}" + done + printf '%s' "$out" +} + +setup() { + PY="$(py)" || skip "no python can import tokenizers (set QWEN38_PYTHON)" + TOK="$BATS_TEST_TMPDIR/tokenizer.json" + "$PY" -c ' +import sys +from tokenizers import Tokenizer, models, pre_tokenizers +vocab = {w: i for i, w in enumerate(["alpha", "beta", "gamma", "delta", "epsilon"])} +tok = Tokenizer(models.WordLevel(vocab, unk_token="[UNK]")) +tok.pre_tokenizer = pre_tokenizers.Whitespace() +tok.save(sys.argv[1]) +' "$TOK" + # Three parents with distinct sources, so every child can be traced back to + # the record it came from: 300, 130 and 100 tokens. + CORPUS="$BATS_TEST_TMPDIR/kld.jsonl" + { + printf '{"prompt":"%s","score_from":284,"source":"code+prose"}\n' "$(words 300)" + printf '{"prompt":"%s","score_from":114,"source":"tools"}\n' "$(words 130)" + printf '{"prompt":"%s","score_from":84,"source":"prose"}\n' "$(words 100)" + } > "$CORPUS" +} + +@test "a record's windows are that record's own tokens, cut at the ceiling the container holds" { + OUT="$BATS_TEST_TMPDIR/kld-7168.jsonl" + run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \ + --max-doc-tokens 128 --score-tail 16 + [ "$status" -eq 0 ] + "$PY" - "$CORPUS" "$OUT" "$TOK" <<'PY' +import json, sys +from tokenizers import Tokenizer +corpus, out, tok_path = sys.argv[1:4] +tok = Tokenizer.from_file(tok_path) +ids_of = lambda text: tok.encode(text, add_special_tokens=False).ids +parents = [json.loads(l) for l in open(corpus)] +kids = [json.loads(l) for l in open(out)] +family = {} +for k in kids: + family.setdefault(k["source"], []).append(k) +assert len(kids) == 6, f"six windows expected, got {len(kids)}" +for p in parents: + mine = family[p["source"]] + joined = [t for k in mine for t in ids_of(k["prompt"])] + assert joined == ids_of(p["prompt"]), f"{p['source']}: the windows are not the parent's tokens" + for k in mine: + n = len(ids_of(k["prompt"])) + assert n <= 128, f"{p['source']}: a {n}-token window over the 128 ceiling" + assert k["score_from"] == max(n - 16, 0), f"{p['source']}: {k['score_from']} for {n} tokens" +PY +} + +@test "the summary reports exactly the documents and tokens it wrote" { + OUT="$BATS_TEST_TMPDIR/kld-7168.jsonl" + run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \ + --max-doc-tokens 128 --score-tail 16 + [ "$status" -eq 0 ] + printf '%s' "$output" | "$PY" -c ' +import json, sys +s = json.load(sys.stdin) +assert (s["documents_in"], s["documents_out"]) == (3, 6), s +assert (s["tokens_in"], s["tokens_out"]) == (530, 530), s +assert s["longest_document"] == 128 and s["max_doc_tokens"] == 128, s +assert s["score_tail"] == 16 and s["clamped_score_from"] == 1, s +assert s["scored_positions"] == 82, s +assert s["source_documents"] == {"code": 3, "prose": 4, "tools": 2}, s +' +} + +@test "an existing output is refused without --force and replaced with it" { + OUT="$BATS_TEST_TMPDIR/kld-7168.jsonl" + printf '{"prompt":"keepme","score_from":0}\n' > "$OUT" + run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \ + --max-doc-tokens 128 --score-tail 16 + [ "$status" -eq 2 ] + [[ "$output" == *"--force"* ]] + [ "$(head -1 "$OUT")" = '{"prompt":"keepme","score_from":0}' ] + run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \ + --max-doc-tokens 128 --force --score-tail 16 + [ "$status" -eq 0 ] + run grep -c keepme "$OUT" + [ "$status" -eq 1 ] +} + +@test "a corpus or ceiling it cannot honour is refused before anything is written" { + # writing the corpus onto itself is the one move the trash law forbids + run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$CORPUS" --tokenizer-json "$TOK" \ + --max-doc-tokens 128 --score-tail 16 + [ "$status" -eq 2 ] + [[ "$output" == *"trash"* ]] + OUT="$BATS_TEST_TMPDIR/o.jsonl" + run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \ + --max-doc-tokens 16 --score-tail 16 + [ "$status" -eq 2 ] + [[ "$output" == *"--max-doc-tokens 16 cannot hold a 16-position score tail"* ]] + [ ! -e "$OUT" ] + run "$PY" "$SCRIPT" --corpus "$BATS_TEST_TMPDIR/gone.jsonl" --out "$OUT" \ + --tokenizer-json "$TOK" + [ "$status" -eq 2 ] + [[ "$output" == *"no corpus at"* ]] + [ ! -e "$OUT" ] +} -- 2.54.0