scripts/calib-push.py

@@ -22,6 +22,9 @@ Mechanics notes:
   prompts do not bias the Hessians (the corpus is drawn round-robin over
   source classes, so loss is class-neutral as long as failures are rare;
   a systematic 4xx is loud in the log and must be fixed, not ignored).
+- a round that seals no tap bytes is a FAILURE, not a retry: the engine
+  answering 200 while the tap directory is wrong, full or read-only is
+  how a corpus gets pushed four times for nothing.
 
 Usage: calib-push.py [--port 7770] [--parallel 64] [--container arbench]
        [--corpus /home/dank/corpora/out/calib.jsonl] [--max-rounds 4]
@@ -35,6 +38,12 @@ import sys
 import time
 import urllib.request
 
+_HERE = os.path.dirname(os.path.abspath(__file__)) if "__file__" in globals() else os.getcwd()
+if _HERE not in sys.path:
+    sys.path.insert(0, _HERE)
+
+import qwen38_common as qc  # noqa: E402 - sibling module, same directory
+
 DONE_MARK = "calibration wrote"
 
 
@@ -69,6 +78,15 @@ def push_one(url, prompt, model):
         return False
 
 
+def push_round(url, prompts, model, parallel):
+    """Push the whole corpus once; returns how many requests were accepted."""
+    ok = 0
+    with concurrent.futures.ThreadPoolExecutor(parallel) as ex:
+        for good in ex.map(lambda p: push_one(url, p, model), prompts):
+            ok += 1 if good else 0
+    return ok
+
+
 def tap_bytes(path):
     """Bytes the tap has sealed so far; 0 when the directory is absent.
 
@@ -89,7 +107,7 @@ def tap_bytes(path):
     return total
 
 
-def main():
+def parse_args():
     ap = argparse.ArgumentParser()
     ap.add_argument("--port", type=int, default=7770)
     ap.add_argument("--parallel", type=int, default=64)
@@ -98,31 +116,36 @@ def main():
     ap.add_argument("--tap-dir", default="/home/dank/campaign/qwen38-w5nl64/calib")
     ap.add_argument("--corpus", default="/home/dank/corpora/out/calib.jsonl")
     ap.add_argument("--max-rounds", type=int, default=4)
-    a = ap.parse_args()
+    return ap.parse_args()
-    url = f"http://127.0.0.1:{a.port}/v1/completions"
 
+
+def load_prompts(a) -> list[str]:
+    if not os.path.isfile(a.corpus):
+        qc.refuse(f"no corpus at {a.corpus}")
     prompts = []
-    with open(a.corpus) as f:
+    for lineno, rec in qc.iter_jsonl(a.corpus):
-        for line in f:
+        if not isinstance(rec.get("prompt"), str):
-            line = line.strip()
+            qc.refuse(f"{a.corpus}:{lineno} has no string 'prompt' field")
-            if line:
+        prompts.append(rec["prompt"])
-                prompts.append(json.loads(line)["prompt"])
+    if not prompts:
+        qc.refuse(f"the corpus at {a.corpus} holds no prompts")
+    return prompts
+
+
+def main():
+    a = parse_args()
+    url = f"http://127.0.0.1:{a.port}/v1/completions"
+    prompts = load_prompts(a)
     chars = sum(len(p) for p in prompts)
     print(f"corpus: {len(prompts)} prompts, ~{chars // 4} tokens (chars/4)",
           flush=True)
-    if not prompts:
-        print("REFUSED: the corpus holds no prompts", flush=True)
-        sys.exit(2)
 
     for rnd in range(1, a.max_rounds + 1):
         if tap_done(a.container):
             break
         before = tap_bytes(a.tap_dir)
         t0 = time.time()
-        ok = 0
+        ok = push_round(url, prompts, a.model, a.parallel)
-        with concurrent.futures.ThreadPoolExecutor(a.parallel) as ex:
-            for good in ex.map(lambda p: push_one(url, p, a.model), prompts):
-                ok += 1 if good else 0
         dt = max(time.time() - t0, 1e-9)
         print(f"round {rnd}: {ok}/{len(prompts)} ok in {dt:.0f}s "
               f"({chars / 4 / dt:.0f} tokens/s est)", flush=True)

scripts/make-calib-corpus.py

@@ -34,14 +34,25 @@ import json
 import os
 import random
 import re
+import sys
 
 from jinja2 import Environment, FileSystemLoader
-from tokenizers import Tokenizer
+
+_HERE = os.path.dirname(os.path.abspath(__file__)) if "__file__" in globals() else os.getcwd()
+if _HERE not in sys.path:
+    sys.path.insert(0, _HERE)
+
+import qwen38_common as qc  # noqa: E402 - sibling module, same directory
 
 CODE_EXT = (".sh", ".py", ".jinja", ".recipe", ".env", ".example",
             ".yaml", ".yml", ".bats", ".service", ".container", ".toml")
 JSONL_EXTS = (".jsonl", ".json")
 DOC_DIR = "serverAppConfigs-docs"
+FRAGMENT_TOKENS = 2400   # what one fragment may contribute
+CLASS_MIX = {"code": .45, "prose": .30, "tools": .25}
+HF_CAP = {"code": 40000, "prose": 60000, "tools": 0}
+POOL_CYCLES = 40         # how many times a class pool may repeat before giving up
+KLD_GUARD = 5000         # rendered conversations tried before declaring the pool short
 
 FLEET_CMDS = [
     "systemctl status frank-archon --no-pager",
@@ -142,7 +153,7 @@ def load_parquet_texts(path: str, kind: str, rng: random.Random,
     return out[:cap]
 
 
-def main() -> None:
+def parse_args() -> argparse.Namespace:
     ap = argparse.ArgumentParser(description=__doc__)
     ap.add_argument("--repo", required=True)
     ap.add_argument("--tokenizer-json", required=True)
@@ -156,22 +167,26 @@ def main() -> None:
     ap.add_argument("--hf-code-parquet", default="")
     ap.add_argument("--hf-prose-parquet", default="")
     ap.add_argument("--seed", type=int, default=20261006)
-    a = ap.parse_args()
+    return ap.parse_args()
 
-    rng = random.Random(a.seed)
+
-    tokenizer = Tokenizer.from_file(a.tokenizer_json)
+def load_template(template: str) -> tuple[Environment, str]:
-    env = Environment(loader=FileSystemLoader(os.path.dirname(a.template)),
+    """The chat template as the engine will render it, plus its short name."""
+    env = Environment(loader=FileSystemLoader(os.path.dirname(template)),
                       trim_blocks=True, lstrip_blocks=True)
     env.filters["tojson"] = lambda v: json.dumps(v, ensure_ascii=False)
     env.filters["raise_exception"] = lambda m: (_ for _ in ()).throw(
         ValueError(m))
-    tname = os.path.basename(a.template)
+    return env, os.path.basename(template)
 
+
+def classify(repo: str) -> dict[str, list[str]]:
+    """Walk the checkout into the three source classes the mix draws from."""
     code, prose, jsonl = [], [], []
-    for root, dirs, files in os.walk(a.repo):
+    for root, dirs, files in os.walk(repo):
         dirs[:] = [d for d in dirs if not d.startswith(".")]
         for f in sorted(files):
-            p = os.path.relpath(os.path.join(root, f), a.repo)
+            p = os.path.relpath(os.path.join(root, f), repo)
             if DOC_DIR in p and f.endswith(".md"):
                 prose.append(p)
             elif p.startswith(DOC_DIR):
@@ -180,153 +195,185 @@ def main() -> None:
                 code.append(p)
             elif f.endswith(JSONL_EXTS) and f != "package-lock.json":
                 jsonl.append(p)
-    # file-level split: bucket 0 -> KLD pool, 1..4 -> calibration pool
+    return {"code": code, "prose": prose, "tools": jsonl}
-    pools: dict[str, tuple[list, list]] = {}
-    for cls, lst in (("code", code), ("prose", prose), ("tools", jsonl)):
-        pools[cls] = ([p for p in lst if fnv(p) % 5 == 0],
-                      [p for p in lst if fnv(p) % 5 != 0])
 
-    def text_of(p: str) -> str:
-        path = os.path.join(a.repo, p)
-        raw = open(path, encoding="utf-8", errors="replace").read()
-        if p.endswith(JSONL_EXTS):
-            # A .json in this repo is not necessarily a content-keyed JSONL: the
-            # substring test passes on any line that merely mentions "content",
-            # and a list-shaped document raises TypeError — both crashed the whole
-            # build. Skip what does not fit instead.
-            lines = []
-            for line in raw.splitlines():
-                if not line.strip() or "content" not in line:
-                    continue
-                try:
-                    rec = json.loads(line)
-                except json.JSONDecodeError:
-                    continue
-                if isinstance(rec, dict) and isinstance(rec.get("content"), str):
-                    lines.append(rec["content"])
-            raw = "\n".join(lines)
-        toks = tokenizer.encode(raw, add_special_tokens=False).ids
-        return tokenizer.decode(toks[:2400], skip_special_tokens=True)
 
+def split_pools(classes: dict[str, list[str]]) -> dict[str, tuple[list, list]]:
+    """File-level split: bucket 0 -> KLD pool, everything else -> calibration."""
+    return {cls: ([p for p in lst if fnv(p) % 5 == 0],
+                  [p for p in lst if fnv(p) % 5 != 0])
+            for cls, lst in classes.items()}
+
+
+def jsonl_content(raw: str) -> str:
+    """The content fields of a content-keyed JSONL, or "" for anything else.
+
+    A .json in this repo is not necessarily a content-keyed JSONL: the substring
+    test passes on any line that merely mentions "content", and a list-shaped
+    document raises TypeError — both used to crash the whole build. Skip what
+    does not fit instead.
+    """
+    lines = []
+    for line in raw.splitlines():
+        if not line.strip() or "content" not in line:
+            continue
+        try:
+            rec = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(rec, dict) and isinstance(rec.get("content"), str):
+            lines.append(rec["content"])
+    return "\n".join(lines)
+
+
+def fragment_text(repo: str, rel: str, tok) -> str:
+    """One file as a fragment: its first FRAGMENT_TOKENS tokens, decoded."""
+    raw = open(os.path.join(repo, rel), encoding="utf-8",
+               errors="replace").read()
+    if rel.endswith(JSONL_EXTS):
+        raw = jsonl_content(raw)
+    ids = tok.encode(raw, add_special_tokens=False).ids
+    return tok.decode(ids[:FRAGMENT_TOKENS], skip_special_tokens=True)
+
+
+def build_frags(a: argparse.Namespace, tok,
+                rng: random.Random) -> dict[str, dict[str, list[str]]]:
+    """Both fragment pools per class, seeded so the same seed rebuilds the corpus."""
+    pools = split_pools(classify(a.repo))
     cmds = harvest_cmds(a.repo, rng)
-    # fragment pools per class, corpus-split first
     frags: dict[str, dict[str, list[str]]] = {}
-    for cls in pools:
+    for cls, (kld_files, cal_files) in pools.items():
-        kld_files, cal_files = pools[cls]
-        fr = {}
-        src = {"kld": kld_files, "cal": cal_files}
         if cls == "tools":
-            fr["kld"] = [tool_call(rng, cmds) for _ in range(600)]
+            frags[cls] = {"kld": [tool_call(rng, cmds) for _ in range(600)],
-            fr["cal"] = [tool_call(rng, cmds) for _ in range(40000)]
+                          "cal": [tool_call(rng, cmds) for _ in range(40000)]}
         else:
-            fr["kld"] = [text_of(p) for p in src["kld"]]
+            frags[cls] = {"kld": [fragment_text(a.repo, p, tok) for p in kld_files],
-            fr["cal"] = [text_of(p) for p in src["cal"]]
+                          "cal": [fragment_text(a.repo, p, tok) for p in cal_files]}
-        frags[cls] = fr
     # per-class fragment caps: prose rows are short (~66 tok) so need many;
     # codeparrot files are long (~2.6k tok) so a small cap suffices
-    hfcap = {"code": 40000, "prose": 60000, "tools": 0}
     for cls, spec in (("code", a.hf_code_parquet),
                       ("prose", a.hf_prose_parquet)):
         for path in [p for p in spec.split(",") if p]:
             frags[cls]["cal"].extend(
-                load_parquet_texts(path, cls, rng, cap=hfcap[cls]))
+                load_parquet_texts(path, cls, rng, cap=HF_CAP[cls]))
     for cls in frags:
         rng.shuffle(frags[cls]["cal"])
+    return frags
 
-    def toks(s: str) -> int:
-        return len(tokenizer.encode(s, add_special_tokens=False).ids)
 
-    def emit_calib() -> tuple[int, list[dict]]:
+def pack_class(cls: str, pool: list[str], want: int, tok,
-        targets = {"code": .45, "prose": .30, "tools": .25}
+               chunk_tokens: int) -> list[dict]:
-        rows: list[dict] = []
+    """Greedily pack `pool` into chunk_tokens-sized calibration prompts."""
-        for cls, frac in targets.items():
+    if not pool:
-            want = int(a.calib_tokens * frac)
+        return []
-            pool = frags[cls]["cal"]
+    rows: list[dict] = []
+    # Count every fragment once: the loop cycles the pool up to POOL_CYCLES
+    # times, and re-tokenizing a 2.4k-token fragment on each pass dominated the
+    # build's wall time.
+    counts = [qc.n_tokens(tok, f) for f in pool]
+    got = 0
+    i = 0
+    buf: list[str] = []
+    nbuf = 0
+    while got < want:
+        j = i % len(pool)
+        frag = pool[j]
+        i += 1
+        n = counts[j]
+        if nbuf + n > chunk_tokens and buf:
+            rows.append({"prompt": "\n".join(buf), "cls": cls})
+            got += nbuf
+            buf, nbuf = [], 0
+            if got >= want:
+                break
+        buf.append(frag)
+        nbuf += n
+        if i > len(pool) * POOL_CYCLES:
+            break  # pool exhausted with repeats; stop
+    if buf and got < want:
+        rows.append({"prompt": "\n".join(buf), "cls": cls})
+        got += nbuf
+    print(f"calib {cls}: {got} tokens, {i} fragments")
+    return rows
+
+
+def emit_calib(a: argparse.Namespace, frags, tok,
+               rng: random.Random) -> tuple[int, list[dict]]:
+    """calib.jsonl rows: the class mix, shuffled, prompts only."""
+    rows: list[dict] = []
+    for cls, frac in CLASS_MIX.items():
+        rows.extend(pack_class(cls, frags[cls]["cal"], int(a.calib_tokens * frac),
+                               tok, a.chunk_tokens))
+    rng.shuffle(rows)
+    return len(rows), [{"prompt": r["prompt"]} for r in rows]
+
+
+def emit_kld(a: argparse.Namespace, frags, tok, rng: random.Random,
+             env: Environment, tname: str) -> tuple[int, list[dict], int]:
+    """kld.jsonl rows: multi-turn conversations, each with a scored tail."""
+    template = env.get_template(tname)
+    rows: list[dict] = []
+    total = scored_total = 0
+    classes = ["code", "prose", "tools"]
+    cursor = {c: 0 for c in classes}
+    guard = 0
+    while total < a.kld_tokens and guard < KLD_GUARD:
+        guard += 1
+        msgs, srcs = [], []
+        for t in range(rng.choice([2, 3, 4])):
+            cls = classes[(len(rows) + t) % 3]
+            pool = frags[cls]["kld"]
             if not pool:
                 continue
-            got, i, buf, nbuf = 0, 0, [], 0
+            frag = pool[cursor[cls] % len(pool)]
-            # Count every fragment once: the loop cycles the pool up to 40 times,
+            cursor[cls] += 1
-            # and re-tokenizing a 2.4k-token fragment on each pass dominated the
+            msgs.append({"role": "user",
-            # build's wall time.
+                         "content": "Summarize and explain this "
-            counts = [toks(f) for f in pool]
+                                    f"engineering material:\n{frag}"})
-            while got < want:
+            msgs.append({"role": "assistant",
-                j = i % len(pool)
+                         "content": frag[:4000]})
-                frag = pool[j]
+            srcs.append(cls)
-                i += 1
+        rendered = template.render(messages=msgs, add_generation_prompt=False)
-                n = counts[j]
+        n = qc.n_tokens(tok, rendered)
-                if nbuf + n > a.chunk_tokens and buf:
+        if n < a.kld_min_prompt:
-                    rows.append({"prompt": "\n".join(buf), "cls": cls})
+            continue
-                    got += nbuf
+        rows.append({"prompt": rendered,
-                    buf, nbuf = [], 0
+                     "score_from": n - a.score_tail,
-                    if got >= want:
+                     "source": "+".join(sorted(set(srcs)))})
-                        break
+        total += n
-                buf.append(frag)
+        scored_total += a.score_tail
-                nbuf += n
+    return len(rows), rows, scored_total
-                if i > len(pool) * 40:
-                    break  # pool exhausted with repeats; stop
-            if buf and got < want:
-                rows.append({"prompt": "\n".join(buf), "cls": cls})
-                got += nbuf
-            print(f"calib {cls}: {got} tokens, {i} fragments")
-        rng.shuffle(rows)
-        return len(rows), [{k: v for k, v in r.items() if k == "prompt"}
-                           for r in rows]
 
-    def emit_kld() -> tuple[int, list[dict], int]:
-        rows, total, scored_total = [], 0, 0
-        classes = ["code", "prose", "tools"]
-        i = {c: 0 for c in classes}
-        guard = 0
-        while total < a.kld_tokens and guard < 5000:
-            guard += 1
-            msgs, srcs = [], []
-            for t in range(rng.choice([2, 3, 4])):
-                cls = classes[(len(rows) + t) % 3]
-                pool = frags[cls]["kld"]
-                if not pool:
-                    continue
-                frag = pool[i[cls] % len(pool)]
-                i[cls] += 1
-                msgs.append({"role": "user",
-                             "content": "Summarize and explain this "
-                                        f"engineering material:\n{frag}"})
-                msgs.append({"role": "assistant",
-                             "content": frag[:4000]})
-                srcs.append(cls)
-            rendered = env.get_template(tname).render(
-                messages=msgs, add_generation_prompt=False)
-            n = toks(rendered)
-            if n < a.kld_min_prompt:
-                continue
-            rows.append({"prompt": rendered,
-                         "score_from": n - a.score_tail,
-                         "source": "+".join(sorted(set(srcs)))})
-            total += n
-            scored_total += a.score_tail
-        return len(rows), rows, scored_total
 
-    os.makedirs(a.out_dir, exist_ok=True)
+def manifest(a: argparse.Namespace, tname: str, nc: int, nk: int,
-    nc, rows = emit_calib()
+             scored: int) -> dict:
-    with open(os.path.join(a.out_dir, "calib.jsonl"), "w") as f:
+    return {
-        for r in rows:
-            f.write(json.dumps(r) + "\n")
-    nk, krows, scored = emit_kld()
-    with open(os.path.join(a.out_dir, "kld.jsonl"), "w") as f:
-        for r in krows:
-            f.write(json.dumps(r) + "\n")
-    manifest = {
         "seed": a.seed, "template": tname,
         "tokenizer": os.path.basename(a.tokenizer_json),
         "calib_prompts": nc, "kld_prompts": nk,
         "kld_scored_positions": scored,
         "kld_score_tail": a.score_tail,
         "disjoint_file_pools": "local KLD material only; HF code feeds calib",
-        "calib_mix": {"code": .45, "prose": .30, "tools": .25},
+        "calib_mix": dict(CLASS_MIX),
         "hf_code": a.hf_code_parquet, "hf_prose": a.hf_prose_parquet,
     }
-    with open(os.path.join(a.out_dir, "manifest.json"), "w") as f:
+
-        json.dump(manifest, f, indent=1)
+
-    print(json.dumps(manifest, indent=1))
+def main() -> None:
+    a = parse_args()
+    rng = random.Random(a.seed)
+    tok = qc.load_tokenizer(a.tokenizer_json)
+    env, tname = load_template(a.template)
+
+    frags = build_frags(a, tok, rng)
+    os.makedirs(a.out_dir, exist_ok=True)
+    nc, calib_rows = emit_calib(a, frags, tok, rng)
+    qc.write_jsonl(os.path.join(a.out_dir, "calib.jsonl"), calib_rows)
+    nk, kld_rows, scored = emit_kld(a, frags, tok, rng, env, tname)
+    qc.write_jsonl(os.path.join(a.out_dir, "kld.jsonl"), kld_rows)
+    man = manifest(a, tname, nc, nk, scored)
+    qc.write_json(os.path.join(a.out_dir, "manifest.json"), man)
+    print(json.dumps(man, indent=1))
 
 
 if __name__ == "__main__":

scripts/qwen38_common.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Shared plumbing for the qwen38 campaign corpus tools.
+
+The three tools move the same two shapes — a JSONL of prompt records and a
+one-artifact JSON manifest — and they all tokenize with the SAME tokenizer. That
+last part is not a detail: a KLD reference is only meaningful against the exact
+token sequence the corpus recorded, so `n_tokens` exists once, here, and every
+tool counts the same way. A window that decode/encode does not round-trip is a
+refusal, never a warning.
+
+Loaded as a sibling module. Each tool puts its own directory on `sys.path`
+first, so `python3 scripts/<tool>.py` and the driver's container mount
+(`runpy.run_path('/work/<tool>.py')`, `-v scripts:/work:ro`) both resolve it —
+run the tools by path, not piped through stdin.
+"""
+import contextlib
+import json
+import os
+import sys
+import tempfile
+
+
+def refuse(msg: str) -> None:
+    """Exit 2 with a REFUSED line — the code argparse uses for unusable input."""
+    print(f"REFUSED: {msg}", file=sys.stderr)
+    sys.exit(2)
+
+
+def load_tokenizer(path: str):
+    try:
+        from tokenizers import Tokenizer
+    except ImportError as exc:  # the corpus tools carry their own venv
+        refuse(f"tokenizers is not importable by {sys.executable} — run the "
+               f"corpus venv's python (on dankserver: "
+               f"/home/dank/venvs/radcorpus/bin/python) or set "
+               f"FRANK_QWEN38_TOKENIZER: {exc}")
+    if not os.path.isfile(path):
+        refuse(f"no tokenizer at {path}")
+    return Tokenizer.from_file(path)
+
+
+def n_tokens(tok, text: str) -> int:
+    """The corpus's own count: no special tokens added, prompt side only."""
+    return len(tok.encode(text, add_special_tokens=False).ids)
+
+
+def iter_jsonl(path: str):
+    """Yield `(lineno, record)` for every non-blank line; refuse on a bad line.
+
+    Field validation stays with the caller: the three corpora carry different
+    fields (`prompt` alone, `prompt`+`cls`, `prompt`+`score_from`+`source`), and
+    a record missing the one the tool needs is that tool's refusal to make.
+    """
+    with open(path, encoding="utf-8", errors="replace") as fh:
+        for lineno, line in enumerate(fh, 1):
+            if not line.strip():
+                continue
+            try:
+                rec = json.loads(line)
+            except json.JSONDecodeError as exc:
+                refuse(f"{path}:{lineno} is not JSON: {exc}")
+            yield lineno, rec
+
+
+def write_jsonl(path: str, rows) -> None:
+    with open(path, "w", encoding="utf-8") as fh:
+        for row in rows:
+            fh.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+@contextlib.contextmanager
+def atomic_writer(path: str):
+    """Write `path` through a temporary sibling and rename on success.
+
+    An interrupted run then leaves no half corpus behind, and the temporary name
+    keeps the one-artifact-one-name law the driver applies to a score report.
+    """
+    d = os.path.dirname(os.path.abspath(path)) or "."
+    fd, tmp = tempfile.mkstemp(dir=d, prefix="." + os.path.basename(path) + "-")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            yield fh
+        os.replace(tmp, path)
+    except BaseException:
+        if os.path.exists(tmp):
+            os.unlink(tmp)
+        raise
+    os.chmod(path, 0o644)
+
+
+def write_json(path: str, obj) -> None:
+    with atomic_writer(path) as fh:
+        json.dump(obj, fh, indent=1)
+        fh.write("\n")

scripts/rechunk-kld-corpus.py

@@ -38,12 +38,19 @@ one-artifact-one-name law run-qwen38-bf16.sh applies to a score report. The file
 is written through a temporary name and renamed, so an interrupted run leaves no
 half corpus behind. A JSON summary of what was written goes to stdout.
 """
-
 import argparse
+import dataclasses
 import json
 import os
 import sys
-import tempfile
+
+_HERE = os.path.dirname(os.path.abspath(__file__)) if "__file__" in globals() else os.getcwd()
+if _HERE not in sys.path:
+    sys.path.insert(0, _HERE)
+
+import qwen38_common as qc
+
+MISSING = object()  # "the parent record carried no source field"
 
 
 def parse_args() -> argparse.Namespace:
@@ -69,100 +76,96 @@ def parse_args() -> argparse.Namespace:
     return ap.parse_args()
 
 
-def load_tokenizer(path: str):
+@dataclasses.dataclass(frozen=True)
-    try:
+class Window:
-        from tokenizers import Tokenizer
+    """One scoring document cut out of a parent record."""
-    except ImportError as exc:  # the corpus scripts carry their own venv
+    prompt: str
-        raise SystemExit(
+    score_from: int
-            f"REFUSED: tokenizers is not importable by {sys.executable} — run the "
+    n_tokens: int
-            f"corpus venv's python (on dankserver: /home/dank/venvs/radcorpus/bin/python) "
+    clamped: bool
-            f"or set FRANK_QWEN38_TOKENIZER: {exc}") from None
+    source: object = MISSING
-    if not os.path.isfile(path):
+
-        raise SystemExit(f"REFUSED: no tokenizer at {path}")
+    def record(self) -> dict:
-    return Tokenizer.from_file(path)
+        rec = {"prompt": self.prompt, "score_from": self.score_from}
+        if self.source is not MISSING:
+            rec["source"] = self.source
+        return rec
 
 
-def refuse(msg: str) -> None:
+def validate(a: argparse.Namespace) -> None:
-    print(f"REFUSED: {msg}", file=sys.stderr)
+    """Everything that makes this run unusable, named before a token is cut."""
-    sys.exit(2)  # 2 is the code argparse uses for unusable arguments
-
-
-def main() -> None:
-    a = parse_args()
     if a.max_doc_tokens <= 0:
-        refuse(f"--max-doc-tokens must be positive (got {a.max_doc_tokens})")
+        qc.refuse(f"--max-doc-tokens must be positive (got {a.max_doc_tokens})")
     if a.score_tail <= 0:
-        refuse(f"--score-tail must be positive (got {a.score_tail})")
+        qc.refuse(f"--score-tail must be positive (got {a.score_tail})")
     if a.max_doc_tokens <= a.score_tail:
-        refuse(f"--max-doc-tokens {a.max_doc_tokens} cannot hold a {a.score_tail}-position "
+        qc.refuse(f"--max-doc-tokens {a.max_doc_tokens} cannot hold a "
-               "score tail: every window would be all tail and no context")
+                  f"{a.score_tail}-position score tail: every window would be all "
+                  "tail and no context")
     if not os.path.isfile(a.corpus):
-        refuse(f"no corpus at {a.corpus}")
+        qc.refuse(f"no corpus at {a.corpus}")
     if os.path.realpath(a.out) == os.path.realpath(a.corpus):
-        refuse(f"--out {a.out} is the corpus itself; move it to trash first — the trash law "
+        qc.refuse(f"--out {a.out} is the corpus itself; move it to trash first — the "
-               "means the old segmentation stays recoverable")
+                  "trash law means the old segmentation stays recoverable")
     if os.path.exists(a.out) and os.path.getsize(a.out) > 0 and not a.force:
-        refuse(f"{a.out} exists and is not empty; pass --force after moving it to trash")
+        qc.refuse(f"{a.out} exists and is not empty; pass --force after moving it to trash")
 
-    tok = load_tokenizer(a.tokenizer_json)
 
+def cut(rec: dict, tok, a: argparse.Namespace,
+        where: str) -> tuple[list[Window], int]:
+    """Cut one record's token ids into windows, proving each one round-trips.
+
+    The round-trip encode is the whole reason this tool may claim the reference
+    scores the corpus's own token sequence: decoding a window to text and
+    encoding it back must return the identical ids, or the score is of a
+    different document than the one on disk.
+    """
+    ids = tok.encode(rec["prompt"], add_special_tokens=False).ids
+    windows: list[Window] = []
+    for start in range(0, max(len(ids), 1), a.max_doc_tokens):
+        window = ids[start:start + a.max_doc_tokens]
+        if not window:
+            continue
+        text = tok.decode(window)
+        if tok.encode(text, add_special_tokens=False).ids != window:
+            qc.refuse(f"{where} window at token {start} does not round-trip through "
+                      "text — the reference would score a different token sequence "
+                      "than the corpus holds")
+        windows.append(Window(prompt=text,
+                              score_from=max(len(window) - a.score_tail, 0),
+                              n_tokens=len(window),
+                              clamped=len(window) < a.score_tail,
+                              source=rec["source"] if "source" in rec else MISSING))
+    return windows, len(ids)
+
+
+def rechunk(a: argparse.Namespace, tok) -> dict:
+    """Write the re-chunked corpus and return the summary it printed."""
     docs_in = docs_out = tokens_in = tokens_out = 0
-    longest_child = 0
+    longest_child = clamped = scored = 0
-    clamped = scored = 0
     sources: dict[str, int] = {}
-    fd, tmp = tempfile.mkstemp(dir=os.path.dirname(os.path.abspath(a.out)) or ".",
+    with qc.atomic_writer(a.out) as out:
-                              prefix=".rechunk-")
+        for lineno, rec in qc.iter_jsonl(a.corpus):
-    try:
+            where = f"{a.corpus}:{lineno}"
-        with os.fdopen(fd, "w", encoding="utf-8") as out:
+            if not isinstance(rec.get("prompt"), str):
-            with open(a.corpus, encoding="utf-8") as src:
+                qc.refuse(f"{where} has no string 'prompt' field")
-                for lineno, line in enumerate(src, 1):
+            windows, n_parent = cut(rec, tok, a, where)
-                    if not line.strip():
+            docs_in += 1
-                        continue
+            tokens_in += n_parent
-                    try:
+            for w in windows:
-                        rec = json.loads(line)
+                out.write(json.dumps(w.record(), ensure_ascii=False) + "\n")
-                    except json.JSONDecodeError as exc:
+                docs_out += 1
-                        refuse(f"{a.corpus}:{lineno} is not JSON: {exc}")
+                tokens_out += w.n_tokens
-                    prompt = rec.get("prompt")
+                longest_child = max(longest_child, w.n_tokens)
-                    if not isinstance(prompt, str):
+                clamped += 1 if w.clamped else 0
-                        refuse(f"{a.corpus}:{lineno} has no string 'prompt' field")
+                scored += w.n_tokens - w.score_from
-                    ids = tok.encode(prompt, add_special_tokens=False).ids
+                if "source" in rec:
-                    docs_in += 1
+                    for cls in str(rec["source"]).split("+"):
-                    tokens_in += len(ids)
+                        sources[cls] = sources.get(cls, 0) + 1
-                    for start in range(0, max(len(ids), 1), a.max_doc_tokens):
+    if tokens_out != tokens_in:
-                        window = ids[start:start + a.max_doc_tokens]
+        raise SystemExit(f"REFUSED: the windows hold {tokens_out} tokens and the corpus "
-                        if not window:
+                         f"has {tokens_in} — re-chunking lost or invented tokens")
-                            continue
+    return {
-                        score_from = max(len(window) - a.score_tail, 0)
-                        if len(window) < a.score_tail:
-                            clamped += 1
-                        scored += len(window) - score_from
-                        text = tok.decode(window)
-                        if tok.encode(text, add_special_tokens=False).ids != window:
-                            refuse(f"{a.corpus}:{lineno} window at token {start} does not "
-                                   "round-trip through text — the reference would score a "
-                                   "different token sequence than the corpus holds")
-                        child = {"prompt": text,
-                                 "score_from": score_from}
-                        if "source" in rec:
-                            child["source"] = rec["source"]
-                        out.write(json.dumps(child, ensure_ascii=False) + "\n")
-                        docs_out += 1
-                        tokens_out += len(window)
-                        longest_child = max(longest_child, len(window))
-                        if "source" in rec:
-                            for cls in str(rec["source"]).split("+"):
-                                sources[cls] = sources.get(cls, 0) + 1
-        if tokens_out != tokens_in:
-            raise SystemExit(f"REFUSED: the windows hold {tokens_out} tokens and the corpus "
-                             f"has {tokens_in} — re-chunking lost or invented tokens")
-        os.replace(tmp, a.out)
-    except BaseException:
-        if os.path.exists(tmp):
-            os.unlink(tmp)
-        raise
-    os.chmod(a.out, 0o644)
-
-    print(json.dumps({
         "corpus": a.corpus,
         "out": a.out,
         "tokenizer": os.path.basename(a.tokenizer_json),
@@ -176,7 +179,13 @@ def main() -> None:
         "scored_positions": scored,
         "clamped_score_from": clamped,
         "source_documents": sources,
-    }, indent=1))
+    }
+
+
+def main() -> None:
+    a = parse_args()
+    validate(a)
+    print(json.dumps(rechunk(a, qc.load_tokenizer(a.tokenizer_json)), indent=1))
 
 
 if __name__ == "__main__":

serverAppConfigs-docs/LLM/frank-archon.md

@@ -641,7 +641,10 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
 - Convert campaign (dormant, kept for a future quant): `scripts/convert-swift15-w4dn8.sh`
   `scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe`, `scripts/run-swift15-bf16.sh`
   `scripts/kld-gate-swift15.sh`, `scripts/kld-then-calib.sh`, `scripts/calib-push.py`
-  `scripts/make-calib-corpus.py`, `scripts/derive-protected-experts.py`
+  `scripts/make-calib-corpus.py`, `scripts/rechunk-kld-corpus.py`,
+  `scripts/qwen38_common.py` (the tokenizer, JSONL and refusal plumbing those three
+  corpus tools share — one `n_tokens`, so every tool counts the same way),
+  `scripts/derive-protected-experts.py`
   `scripts/apply-protected-cast.sh`, `scripts/verify-swift15-ple.py`.
 - Probes: `scripts/probe-frank-archon-kv.py` (reads slots/window/dtype from the deployed env so
   it cannot drift; leg D = decode), `scripts/frank-archon-expert-activity.py`