feat(qwen38): score the inherited corpus at the window the container really has
The reference rig declared a 32,768-token window while the boot ledger hands it 7,436, so 197 of the 198 inherited documents were too long to score. The engine warns about the real pool; the flag was the only cited source. Add scripts/rechunk-kld-corpus.py, which moves document boundaries without moving tokens: each record is re-cut into windows of at most --max-doc-tokens (7,168 = 7x1024, the largest whole kilopage under 7,436), and each window gets its own score_from = len - --score-tail. The reference scores the same token series at more positions and at a shorter context; the calibration corpus is untouched, so no quant moves. Writes go through a temp file and the token counts must match before the output is published. The rig header now says what the pool actually is, and scripts/test-changed.sh routes the new script to its own seam instead of the generic syntax fallback.
This commit is contained in:
Frank committed
1 parent
1c56dea67a
commit
9c386d4f89
4 files changed
+328
-5
No files matched your search
Executable
+138
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: scripts/rechunk-kld-corpus.py — the tool that moves KLD document
|
||||
# boundaries so every document fits the KV pool of the container that has to
|
||||
# score it. The campaign's whole value is one comparable token series, so this
|
||||
# seam asserts the invariant that keeps a re-chunk honest — a record's windows
|
||||
# concatenate back to that record's own tokens — plus the refusal set. Offline:
|
||||
# no engine, no GPU, no network; the tokenizer is a fixture built in the test's
|
||||
# own tmpdir.
|
||||
#
|
||||
# A python that can import `tokenizers` is required, the way the corpus scripts
|
||||
# require their venv: QWEN38_PYTHON names it, else python3, else the corpus
|
||||
# venv; the tests skip with a reason when none answers.
|
||||
|
||||
load helpers
|
||||
|
||||
SCRIPT="$REPO_ROOT/scripts/rechunk-kld-corpus.py"
|
||||
|
||||
# py — the first interpreter that can import tokenizers, or failure.
|
||||
py() {
|
||||
local p
|
||||
for p in "${QWEN38_PYTHON:-}" python3 "${HOME}/.venvs/corpus/bin/python" \
|
||||
/home/dank/venvs/radcorpus/bin/python; do
|
||||
[ -n "$p" ] && [ -x "$p" ] || continue
|
||||
"$p" -c 'import tokenizers' >/dev/null 2>&1 && { printf '%s' "$p"; return 0; }
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# words N — N space-separated words drawn from the fixture vocabulary.
|
||||
words() {
|
||||
local n="$1" i out=""
|
||||
local vocab=(alpha beta gamma delta epsilon)
|
||||
for ((i = 0; i < n; i++)); do
|
||||
[ -n "$out" ] && out+=" "
|
||||
out+="${vocab[$((i % 5))]}"
|
||||
done
|
||||
printf '%s' "$out"
|
||||
}
|
||||
|
||||
setup() {
|
||||
PY="$(py)" || skip "no python can import tokenizers (set QWEN38_PYTHON)"
|
||||
TOK="$BATS_TEST_TMPDIR/tokenizer.json"
|
||||
"$PY" -c '
|
||||
import sys
|
||||
from tokenizers import Tokenizer, models, pre_tokenizers
|
||||
vocab = {w: i for i, w in enumerate(["alpha", "beta", "gamma", "delta", "epsilon"])}
|
||||
tok = Tokenizer(models.WordLevel(vocab, unk_token="[UNK]"))
|
||||
tok.pre_tokenizer = pre_tokenizers.Whitespace()
|
||||
tok.save(sys.argv[1])
|
||||
' "$TOK"
|
||||
# Three parents with distinct sources, so every child can be traced back to
|
||||
# the record it came from: 300, 130 and 100 tokens.
|
||||
CORPUS="$BATS_TEST_TMPDIR/kld.jsonl"
|
||||
{
|
||||
printf '{"prompt":"%s","score_from":284,"source":"code+prose"}\n' "$(words 300)"
|
||||
printf '{"prompt":"%s","score_from":114,"source":"tools"}\n' "$(words 130)"
|
||||
printf '{"prompt":"%s","score_from":84,"source":"prose"}\n' "$(words 100)"
|
||||
} > "$CORPUS"
|
||||
}
|
||||
|
||||
@test "a record's windows are that record's own tokens, cut at the ceiling the container holds" {
|
||||
OUT="$BATS_TEST_TMPDIR/kld-7168.jsonl"
|
||||
run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
|
||||
--max-doc-tokens 128 --score-tail 16
|
||||
[ "$status" -eq 0 ]
|
||||
"$PY" - "$CORPUS" "$OUT" "$TOK" <<'PY'
|
||||
import json, sys
|
||||
from tokenizers import Tokenizer
|
||||
corpus, out, tok_path = sys.argv[1:4]
|
||||
tok = Tokenizer.from_file(tok_path)
|
||||
ids_of = lambda text: tok.encode(text, add_special_tokens=False).ids
|
||||
parents = [json.loads(l) for l in open(corpus)]
|
||||
kids = [json.loads(l) for l in open(out)]
|
||||
family = {}
|
||||
for k in kids:
|
||||
family.setdefault(k["source"], []).append(k)
|
||||
assert len(kids) == 6, f"six windows expected, got {len(kids)}"
|
||||
for p in parents:
|
||||
mine = family[p["source"]]
|
||||
joined = [t for k in mine for t in ids_of(k["prompt"])]
|
||||
assert joined == ids_of(p["prompt"]), f"{p['source']}: the windows are not the parent's tokens"
|
||||
for k in mine:
|
||||
n = len(ids_of(k["prompt"]))
|
||||
assert n <= 128, f"{p['source']}: a {n}-token window over the 128 ceiling"
|
||||
assert k["score_from"] == max(n - 16, 0), f"{p['source']}: {k['score_from']} for {n} tokens"
|
||||
PY
|
||||
}
|
||||
|
||||
@test "the summary reports exactly the documents and tokens it wrote" {
|
||||
OUT="$BATS_TEST_TMPDIR/kld-7168.jsonl"
|
||||
run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
|
||||
--max-doc-tokens 128 --score-tail 16
|
||||
[ "$status" -eq 0 ]
|
||||
printf '%s' "$output" | "$PY" -c '
|
||||
import json, sys
|
||||
s = json.load(sys.stdin)
|
||||
assert (s["documents_in"], s["documents_out"]) == (3, 6), s
|
||||
assert (s["tokens_in"], s["tokens_out"]) == (530, 530), s
|
||||
assert s["longest_document"] == 128 and s["max_doc_tokens"] == 128, s
|
||||
assert s["score_tail"] == 16 and s["clamped_score_from"] == 1, s
|
||||
assert s["scored_positions"] == 82, s
|
||||
assert s["source_documents"] == {"code": 3, "prose": 4, "tools": 2}, s
|
||||
'
|
||||
}
|
||||
|
||||
@test "an existing output is refused without --force and replaced with it" {
|
||||
OUT="$BATS_TEST_TMPDIR/kld-7168.jsonl"
|
||||
printf '{"prompt":"keepme","score_from":0}\n' > "$OUT"
|
||||
run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
|
||||
--max-doc-tokens 128 --score-tail 16
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"--force"* ]]
|
||||
[ "$(head -1 "$OUT")" = '{"prompt":"keepme","score_from":0}' ]
|
||||
run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
|
||||
--max-doc-tokens 128 --force --score-tail 16
|
||||
[ "$status" -eq 0 ]
|
||||
run grep -c keepme "$OUT"
|
||||
[ "$status" -eq 1 ]
|
||||
}
|
||||
|
||||
@test "a corpus or ceiling it cannot honour is refused before anything is written" {
|
||||
# writing the corpus onto itself is the one move the trash law forbids
|
||||
run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$CORPUS" --tokenizer-json "$TOK" \
|
||||
--max-doc-tokens 128 --score-tail 16
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"trash"* ]]
|
||||
OUT="$BATS_TEST_TMPDIR/o.jsonl"
|
||||
run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
|
||||
--max-doc-tokens 16 --score-tail 16
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"--max-doc-tokens 16 cannot hold a 16-position score tail"* ]]
|
||||
[ ! -e "$OUT" ]
|
||||
run "$PY" "$SCRIPT" --corpus "$BATS_TEST_TMPDIR/gone.jsonl" --out "$OUT" \
|
||||
--tokenizer-json "$TOK"
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"no corpus at"* ]]
|
||||
[ ! -e "$OUT" ]
|
||||
}
|
||||
Reference in new issue
Block a user