serverAppConfigs/tests/qwen38_rechunk.bats

#!/usr/bin/env bats
# Seam: scripts/rechunk-kld-corpus.py — the tool that moves KLD document
# boundaries so every document fits the KV pool of the container that has to
# score it. The campaign's whole value is one comparable token series, so this
# seam asserts the invariant that keeps a re-chunk honest — a record's windows
# concatenate back to that record's own tokens — plus the refusal set. Offline:
# no engine, no GPU, no network; the tokenizer is a fixture built in the test's
# own tmpdir.
#
# A python that can import `tokenizers` is required, the way the corpus scripts
# require their venv: QWEN38_PYTHON names it, else python3, else the corpus
# venv; the tests skip with a reason when none answers.

load helpers

SCRIPT="$REPO_ROOT/scripts/rechunk-kld-corpus.py"

# py — the first interpreter that can import tokenizers, or failure.
py() {
    local p
    for p in "${QWEN38_PYTHON:-}" python3 "${HOME}/.venvs/corpus/bin/python" \
             /home/dank/venvs/radcorpus/bin/python; do
        [ -n "$p" ] && [ -x "$p" ] || continue
        "$p" -c 'import tokenizers' >/dev/null 2>&1 && { printf '%s' "$p"; return 0; }
    done
    return 1
}

# words N — N space-separated words drawn from the fixture vocabulary.
words() {
    local n="$1" i out=""
    local vocab=(alpha beta gamma delta epsilon)
    for ((i = 0; i < n; i++)); do
        [ -n "$out" ] && out+=" "
        out+="${vocab[$((i % 5))]}"
    done
    printf '%s' "$out"
}

setup() {
    PY="$(py)" || skip "no python can import tokenizers (set QWEN38_PYTHON)"
    TOK="$BATS_TEST_TMPDIR/tokenizer.json"
    "$PY" -c '
import sys
from tokenizers import Tokenizer, models, pre_tokenizers
vocab = {w: i for i, w in enumerate(["alpha", "beta", "gamma", "delta", "epsilon"])}
tok = Tokenizer(models.WordLevel(vocab, unk_token="[UNK]"))
tok.pre_tokenizer = pre_tokenizers.Whitespace()
tok.save(sys.argv[1])
' "$TOK"
    # Three parents with distinct sources, so every child can be traced back to
    # the record it came from: 300, 130 and 100 tokens.
    CORPUS="$BATS_TEST_TMPDIR/kld.jsonl"
    {
        printf '{"prompt":"%s","score_from":284,"source":"code+prose"}\n' "$(words 300)"
        printf '{"prompt":"%s","score_from":114,"source":"tools"}\n' "$(words 130)"
        printf '{"prompt":"%s","score_from":84,"source":"prose"}\n' "$(words 100)"
    } > "$CORPUS"
}

@test "a record's windows are that record's own tokens, cut at the ceiling the container holds" {
    OUT="$BATS_TEST_TMPDIR/kld-7168.jsonl"
    run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
        --max-doc-tokens 128 --score-tail 16
    [ "$status" -eq 0 ]
    "$PY" - "$CORPUS" "$OUT" "$TOK" <<'PY'
import json, sys
from tokenizers import Tokenizer
corpus, out, tok_path = sys.argv[1:4]
tok = Tokenizer.from_file(tok_path)
ids_of = lambda text: tok.encode(text, add_special_tokens=False).ids
parents = [json.loads(l) for l in open(corpus)]
kids = [json.loads(l) for l in open(out)]
family = {}
for k in kids:
    family.setdefault(k["source"], []).append(k)
assert len(kids) == 6, f"six windows expected, got {len(kids)}"
for p in parents:
    mine = family[p["source"]]
    joined = [t for k in mine for t in ids_of(k["prompt"])]
    assert joined == ids_of(p["prompt"]), f"{p['source']}: the windows are not the parent's tokens"
    for k in mine:
        n = len(ids_of(k["prompt"]))
        assert n <= 128, f"{p['source']}: a {n}-token window over the 128 ceiling"
        assert k["score_from"] == max(n - 16, 0), f"{p['source']}: {k['score_from']} for {n} tokens"
PY
}

@test "the summary reports exactly the documents and tokens it wrote" {
    OUT="$BATS_TEST_TMPDIR/kld-7168.jsonl"
    run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
        --max-doc-tokens 128 --score-tail 16
    [ "$status" -eq 0 ]
    printf '%s' "$output" | "$PY" -c '
import json, sys
s = json.load(sys.stdin)
assert (s["documents_in"], s["documents_out"]) == (3, 6), s
assert (s["tokens_in"], s["tokens_out"]) == (530, 530), s
assert s["longest_document"] == 128 and s["max_doc_tokens"] == 128, s
assert s["score_tail"] == 16 and s["clamped_score_from"] == 1, s
assert s["scored_positions"] == 82, s
assert s["source_documents"] == {"code": 3, "prose": 4, "tools": 2}, s
'
}

@test "an existing output is refused without --force and replaced with it" {
    OUT="$BATS_TEST_TMPDIR/kld-7168.jsonl"
    printf '{"prompt":"keepme","score_from":0}\n' > "$OUT"
    run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
        --max-doc-tokens 128 --score-tail 16
    [ "$status" -eq 2 ]
    [[ "$output" == *"--force"* ]]
    [ "$(head -1 "$OUT")" = '{"prompt":"keepme","score_from":0}' ]
    run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
        --max-doc-tokens 128 --force --score-tail 16
    [ "$status" -eq 0 ]
    run grep -c keepme "$OUT"
    [ "$status" -eq 1 ]
}

@test "a corpus or ceiling it cannot honour is refused before anything is written" {
    # writing the corpus onto itself is the one move the trash law forbids
    run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$CORPUS" --tokenizer-json "$TOK" \
        --max-doc-tokens 128 --score-tail 16
    [ "$status" -eq 2 ]
    [[ "$output" == *"trash"* ]]
    OUT="$BATS_TEST_TMPDIR/o.jsonl"
    run "$PY" "$SCRIPT" --corpus "$CORPUS" --out "$OUT" --tokenizer-json "$TOK" \
        --max-doc-tokens 16 --score-tail 16
    [ "$status" -eq 2 ]
    [[ "$output" == *"--max-doc-tokens 16 cannot hold a 16-position score tail"* ]]
    [ ! -e "$OUT" ]
    run "$PY" "$SCRIPT" --corpus "$BATS_TEST_TMPDIR/gone.jsonl" --out "$OUT" \
        --tokenizer-json "$TOK"
    [ "$status" -eq 2 ]
    [[ "$output" == *"no corpus at"* ]]
    [ ! -e "$OUT" ]
}