Merge pull request 'feat(qwen38): score the inherited corpus at the window the container really has' (#78) from feat/qwen38-w5nl64 into main

scripts/rechunk-kld-corpus.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""Re-chunk a KLD corpus so every document fits the container that scores it.
+
+A KLD reference is recorded by booting one container and prefilling every corpus
+document through it. radiance refuses a document the KV pool cannot hold on its
+own (core/mem/kv.cpp `can_ever_fit`: a paged group holds `n_blocks *
+block_size` tokens), and the number that matters is printed by the boot ledger,
+not by a flag:
+
+    W kv group 'kv_attn' holds 7436 tokens but the context limit is 32768, so
+    a prompt longer than 7436 tokens is refused (400) ... or lower
+    --max-model-len to 7436
+
+The BF16 qwen4exp container at the campaign's shape gets 270.00 MiB of cache --
+the expert plane owns both cards -- so its real window is 7,436 tokens however
+`--max-model-len` is spelled. The corpus the campaign inherited from the swift15
+era is 197 documents of 8,193-11,970 tokens, because make-calib-corpus.py's
+`--kld-min-prompt 8192` sets a floor and no ceiling: every one of them is
+unscoreable there, and three drained windows have proved it.
+
+This script moves the document boundaries and nothing else. Each record's token
+ids -- counted with the same tokenizer the generator used -- are cut into
+windows of at most `--max-doc-tokens`, so a record's windows concatenate back to
+that record's own token sequence, unchanged: no token dropped, rewritten or
+reworded. Each window gets its own `score_from = len - --score-tail`, the same
+relation the generator wrote for the parent, so the reference scores 256
+positions per window instead of per document: 100,864 measured positions over
+the same text against the 50,432 the inherited corpus scored, at a shorter
+context. The calibration corpus is untouched, so the quantization itself does
+not move.
+
+Usage:
+  rechunk-kld-corpus.py --corpus kld.jsonl --out kld.jsonl.new \
+      [--tokenizer-json PATH] [--max-doc-tokens 7168] [--score-tail 256] [--force]
+
+--out must not exist and be non-empty without --force: the same
+one-artifact-one-name law run-qwen38-bf16.sh applies to a score report. The file
+is written through a temporary name and renamed, so an interrupted run leaves no
+half corpus behind. A JSON summary of what was written goes to stdout.
+"""
+
+import argparse
+import json
+import os
+import sys
+import tempfile
+
+
+def parse_args() -> argparse.Namespace:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--corpus", required=True,
+                    help="the kld.jsonl to re-chunk (records: prompt, score_from, source)")
+    ap.add_argument("--out", required=True,
+                    help="file to write; refuse if it exists and is non-empty without --force")
+    ap.add_argument("--tokenizer-json",
+                    default=os.environ.get("FRANK_QWEN38_TOKENIZER",
+                                           "/home/dank/models/Qwen3.8-Flash-Next/tokenizer.json"),
+                    help="the tokenizer the corpus was authored with (the same one the "
+                         "manifest in the corpus output dir names)")
+    ap.add_argument("--max-doc-tokens", type=int, default=7168,
+                    help="the widest document the scoring container can hold; 7,168 is "
+                         "7x1024, the largest whole kilopage under the 7,436 the bf16 "
+                         "container's ledger reports")
+    ap.add_argument("--score-tail", type=int, default=256,
+                    help="positions scored at the end of every document, as the generator "
+                         "recorded it")
+    ap.add_argument("--force", action="store_true",
+                    help="overwrite a non-empty --out (after moving the old file to trash)")
+    return ap.parse_args()
+
+
+def load_tokenizer(path: str):
+    try:
+        from tokenizers import Tokenizer
+    except ImportError as exc:  # the corpus scripts carry their own venv
+        raise SystemExit(
+            f"REFUSED: tokenizers is not importable by {sys.executable} — run the "
+            f"corpus venv's python (on dankserver: /home/dank/venvs/radcorpus/bin/python) "
+            f"or set FRANK_QWEN38_TOKENIZER: {exc}") from None
+    if not os.path.isfile(path):
+        raise SystemExit(f"REFUSED: no tokenizer at {path}")
+    return Tokenizer.from_file(path)
+
+
+def refuse(msg: str) -> None:
+    print(f"REFUSED: {msg}", file=sys.stderr)
+    sys.exit(2)  # 2 is the code argparse uses for unusable arguments
+
+
+def main() -> None:
+    a = parse_args()
+    if a.max_doc_tokens <= 0:
+        refuse(f"--max-doc-tokens must be positive (got {a.max_doc_tokens})")
+    if a.score_tail <= 0:
+        refuse(f"--score-tail must be positive (got {a.score_tail})")
+    if a.max_doc_tokens <= a.score_tail:
+        refuse(f"--max-doc-tokens {a.max_doc_tokens} cannot hold a {a.score_tail}-position "
+               "score tail: every window would be all tail and no context")
+    if not os.path.isfile(a.corpus):
+        refuse(f"no corpus at {a.corpus}")
+    if os.path.realpath(a.out) == os.path.realpath(a.corpus):
+        refuse(f"--out {a.out} is the corpus itself; move it to trash first — the trash law "
+               "means the old segmentation stays recoverable")
+    if os.path.exists(a.out) and os.path.getsize(a.out) > 0 and not a.force:
+        refuse(f"{a.out} exists and is not empty; pass --force after moving it to trash")
+
+    tok = load_tokenizer(a.tokenizer_json)
+
+    docs_in = docs_out = tokens_in = tokens_out = 0
+    longest_child = 0
+    clamped = scored = 0
+    sources: dict[str, int] = {}
+    fd, tmp = tempfile.mkstemp(dir=os.path.dirname(os.path.abspath(a.out)) or ".",
+                              prefix=".rechunk-")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as out:
+            with open(a.corpus, encoding="utf-8") as src:
+                for lineno, line in enumerate(src, 1):
+                    if not line.strip():
+                        continue
+                    try:
+                        rec = json.loads(line)
+                    except json.JSONDecodeError as exc:
+                        refuse(f"{a.corpus}:{lineno} is not JSON: {exc}")
+                    prompt = rec.get("prompt")
+                    if not isinstance(prompt, str):
+                        refuse(f"{a.corpus}:{lineno} has no string 'prompt' field")
+                    ids = tok.encode(prompt, add_special_tokens=False).ids
+                    docs_in += 1
+                    tokens_in += len(ids)
+                    for start in range(0, max(len(ids), 1), a.max_doc_tokens):
+                        window = ids[start:start + a.max_doc_tokens]
+                        if not window:
+                            continue
+                        score_from = max(len(window) - a.score_tail, 0)
+                        if len(window) < a.score_tail:
+                            clamped += 1
+                        scored += len(window) - score_from
+                        child = {"prompt": tok.decode(window),
+                                 "score_from": score_from}
+                        if "source" in rec:
+                            child["source"] = rec["source"]
+                        out.write(json.dumps(child, ensure_ascii=False) + "\n")
+                        docs_out += 1
+                        tokens_out += len(window)
+                        longest_child = max(longest_child, len(window))
+                        if "source" in rec:
+                            for cls in str(rec["source"]).split("+"):
+                                sources[cls] = sources.get(cls, 0) + 1
+        if tokens_out != tokens_in:
+            raise SystemExit(f"REFUSED: the windows hold {tokens_out} tokens and the corpus "
+                             f"has {tokens_in} — re-chunking lost or invented tokens")
+        os.replace(tmp, a.out)
+    except BaseException:
+        if os.path.exists(tmp):
+            os.unlink(tmp)
+        raise
+    os.chmod(a.out, 0o644)
+
+    print(json.dumps({
+        "corpus": a.corpus,
+        "out": a.out,
+        "tokenizer": os.path.basename(a.tokenizer_json),
+        "documents_in": docs_in,
+        "documents_out": docs_out,
+        "tokens_in": tokens_in,
+        "tokens_out": tokens_out,
+        "longest_document": longest_child,
+        "max_doc_tokens": a.max_doc_tokens,
+        "score_tail": a.score_tail,
+        "scored_positions": scored,
+        "clamped_score_from": clamped,
+        "source_documents": sources,
+    }, indent=1))
+
+
+if __name__ == "__main__":
+    main()

scripts/run-qwen38-bf16.sh

@@ -39,11 +39,11 @@
 # speculation, fp8 KV, a 32,768-token window, the PLE n-gram table pinned to RAM,
 # and the production chat template rather than the one baked into the container —
 # every scored container is asked the same tokenised questions the reference
-# answered. The window is 32,768: kld.jsonl is 197 records with a longest of
+# answered. The window the flag spells is 32,768; the pool this payload's boot
-# 38,263 characters (longest prompt FIELD 36,855, median record 27,228). Its
+# actually gets is 7,436 tokens, so kld.jsonl is re-chunked by
-# token length — 10,554 for that longest record, counted by the engine itself —
+# scripts/rechunk-kld-corpus.py into documents of at most 7,168 tokens — the
-# is in the funding paragraph below; the kldref gate re-checks the file against
+# widest one the reference can hold. The kldref gate below re-checks the file
-# the window before it boots anything.
+# against the declared window, and the re-chunker is what keeps that honest.
 #
 # NO --vram-kv-mib: it was REMOVED at 1.3.0 and naming it is a boot refusal, not
 # a no-op — verified on the pinned image 2026-10-27: "E --vram-kv-mib was

scripts/test-changed.sh

@@ -265,6 +265,13 @@ for path in "${CHANGED[@]}"; do
             add_test tests/voice_pack_unit.bats
             add_test tests/scripts_syntax.bats
             ;;
+        # The KLD corpus re-chunker: its seam is its own (the campaign's
+        # one-token-series invariant is asserted there), so a change to it does
+        # not land on the generic scripts/* fallback below.
+        scripts/rechunk-kld-corpus.py)
+            add_test tests/qwen38_rechunk.bats
+            add_test tests/scripts_syntax.bats
+            ;;
         scripts/*)
             add_test tests/scripts_syntax.bats
             ;;

tests/qwen38_rechunk.bats

@@ -0,0 +1,138 @@
+#!/usr/bin/env bats
+# Seam: scripts/rechunk-kld-corpus.py — the tool that moves KLD document
+# boundaries so every document fits the KV pool of the container that has to
+# score it. The campaign's whole value is one comparable token series, so this
+# seam asserts the invariant that keeps a re-chunk honest — a record's windows
+# concatenate back to that record's own tokens — plus the refusal set. Offline:
+# no engine, no GPU, no network; the tokenizer is a fixture built in the test's
+# own tmpdir.
+#
+# A python that can import `tokenizers` is required, the way the corpus scripts
+# require their venv: QWEN38_PYTHON names it, else python3, else the corpus
+# venv; the tests skip with a reason when none answers.
+
+load helpers
+
+SCRIPT="$REPO_ROOT/scripts/rechunk-kld-corpus.py"
+
+# py — the first interpreter that can import tokenizers, or failure.
+py() {
+    local p
+    for p in "${QWEN38_PYTHON:-}" python3 "${HOME}/.venvs/corpus/bin/python" \
+             /home/dank/venvs/radcorpus/bin/python; do
+        [ -n "$p" ] && [ -x "$p" ] || continue
+        "$p" -c 'import tokenizers' >/dev/null 2>&1 && { printf '%s' "$p"; return 0; }
+    done
+    return 1
+}
+
+# words N — N space-separated words drawn from the fixture vocabulary.
+words() {
+    local n="$1" i out=""
+    local vocab=(alpha beta gamma delta epsilon)
+    for ((i = 0; i < n; i++)); do
+        [ -n "$out" ] && out+=" "
+        out+="${vocab[$((i % 5))]}"
+    done
+    printf '%s' "$out"
+}
+
+setup() {
+    PY="$(py)" || skip "no python can import tokenizers (set QWEN38_PYTHON)"
+    TOK="$BATS_TEST_TMPDIR/tokenizer.json"
+    "$PY" -c '
+import sys
+from tokenizers import Tokenizer, models, pre_tokenizers
+vocab = {w: i for i, w in enumerate(["alpha", "beta", "gamma", "delta", "epsilon"])}
+tok = Tokenizer(models.WordLevel(vocab, unk_token="[UNK]"))
+tok.pre_tokenizer = pre_tokenizers.Whitespace()
+tok.save(sys.argv[1])
+' "$TOK"
+    # Three parents with distinct sources, so every child can be traced back to
+    # the record it came from: 300, 130 and 100 tokens.
+    CORPUS="$BATS_TEST_TMPDIR/kld.jsonl"
+    {
+        printf '{"prompt":"%s","score_from":284,"source":"code+prose"}\n' "$(words 300)"
+        printf '{"prompt":"%s","score_from":114,"source":"tools"}\n' "$(words 130)"
+        printf '{"prompt":"%s","score_from":84,"source":"prose"}\n' "$(words 100)"
+    } > "$CORPUS"
+}
+
+@test "a record's windows are that record's own tokens, cut at the ceiling the container holds" {
+    OUT="$BATS_TEST_TMPDIR/kld-7168.jsonl"
+    run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
+        --max-doc-tokens 128 --score-tail 16
+    [ "$status" -eq 0 ]
+    "$PY" - "$CORPUS" "$OUT" "$TOK" <<'PY'
+import json, sys
+from tokenizers import Tokenizer
+corpus, out, tok_path = sys.argv[1:4]
+tok = Tokenizer.from_file(tok_path)
+ids_of = lambda text: tok.encode(text, add_special_tokens=False).ids
+parents = [json.loads(l) for l in open(corpus)]
+kids = [json.loads(l) for l in open(out)]
+family = {}
+for k in kids:
+    family.setdefault(k["source"], []).append(k)
+assert len(kids) == 6, f"six windows expected, got {len(kids)}"
+for p in parents:
+    mine = family[p["source"]]
+    joined = [t for k in mine for t in ids_of(k["prompt"])]
+    assert joined == ids_of(p["prompt"]), f"{p['source']}: the windows are not the parent's tokens"
+    for k in mine:
+        n = len(ids_of(k["prompt"]))
+        assert n <= 128, f"{p['source']}: a {n}-token window over the 128 ceiling"
+        assert k["score_from"] == max(n - 16, 0), f"{p['source']}: {k['score_from']} for {n} tokens"
+PY
+}
+
+@test "the summary reports exactly the documents and tokens it wrote" {
+    OUT="$BATS_TEST_TMPDIR/kld-7168.jsonl"
+    run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
+        --max-doc-tokens 128 --score-tail 16
+    [ "$status" -eq 0 ]
+    printf '%s' "$output" | "$PY" -c '
+import json, sys
+s = json.load(sys.stdin)
+assert (s["documents_in"], s["documents_out"]) == (3, 6), s
+assert (s["tokens_in"], s["tokens_out"]) == (530, 530), s
+assert s["longest_document"] == 128 and s["max_doc_tokens"] == 128, s
+assert s["score_tail"] == 16 and s["clamped_score_from"] == 1, s
+assert s["scored_positions"] == 82, s
+assert s["source_documents"] == {"code": 3, "prose": 4, "tools": 2}, s
+'
+}
+
+@test "an existing output is refused without --force and replaced with it" {
+    OUT="$BATS_TEST_TMPDIR/kld-7168.jsonl"
+    printf '{"prompt":"keepme","score_from":0}\n' > "$OUT"
+    run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
+        --max-doc-tokens 128 --score-tail 16
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"--force"* ]]
+    [ "$(head -1 "$OUT")" = '{"prompt":"keepme","score_from":0}' ]
+    run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
+        --max-doc-tokens 128 --force --score-tail 16
+    [ "$status" -eq 0 ]
+    run grep -c keepme "$OUT"
+    [ "$status" -eq 1 ]
+}
+
+@test "a corpus or ceiling it cannot honour is refused before anything is written" {
+    # writing the corpus onto itself is the one move the trash law forbids
+    run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$CORPUS" --tokenizer-json "$TOK" \
+        --max-doc-tokens 128 --score-tail 16
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"trash"* ]]
+    OUT="$BATS_TEST_TMPDIR/o.jsonl"
+    run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
+        --max-doc-tokens 16 --score-tail 16
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"--max-doc-tokens 16 cannot hold a 16-position score tail"* ]]
+    [ ! -e "$OUT" ]
+    run "$PY" "$SCRIPT" --corpus "$BATS_TEST_TMPDIR/gone.jsonl" --out "$OUT" \
+        --tokenizer-json "$TOK"
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"no corpus at"* ]]
+    [ ! -e "$OUT" ]
+}