fix(quant): prove window token identity, gate the tap on progress, kill the runner's orphan race
Adversarial review of the campaign pipeline. Correctness: rechunk-kld-corpus.py wrote each window as text and asserted only lengths, so a KLD reference could score a different token sequence than the corpus holds — every window is now re-encoded and the tool refuses on mismatch; make-calib-corpus.py no longer crashes on a .json that is not a content-keyed JSONL, and its KLD cursor advanced by 2 per fragment (half the pool unused). Silent waste: calib-push.py now aborts when the tap sealed nothing or no prompt was accepted, instead of re-pushing 10M tokens per round; div-by-zero gone, served model id parameterised. Races: run-qwen38-bf16.sh restore() removes the rig container before starting production (today's orphan held the GPUs while the service reported active), the calib tap dir is proven writable, the score ro-guard resolves paths; convert-qwen38-w5nl64.sh refuses a missing --arm value, an existing qwen38-convert-<arm> container, and a zero-exit convert that wrote no artifact. stack-qwen38-official-fp8.py: bare asserts in verify() (skipped under -O) and a non-128-lane PLE HD that only warned are hard failures. Speed: fragment token counts computed once, KLD scored positions accumulated instead of a second full re-encode. Tests: 576 ok / 0 fail.
This commit is contained in:
1 parent
e6fba5d97e
commit
1e4fa14772
6 files changed
+127
-32
No files matched your search
@@ -190,8 +190,20 @@ def main() -> None:
|
||||
path = os.path.join(a.repo, p)
|
||||
raw = open(path, encoding="utf-8", errors="replace").read()
|
||||
if p.endswith(JSONL_EXTS):
|
||||
lines = [json.loads(l)["content"] for l in raw.splitlines()
|
||||
if l.strip() and "content" in l]
|
||||
# A .json in this repo is not necessarily a content-keyed JSONL: the
|
||||
# substring test passes on any line that merely mentions "content",
|
||||
# and a list-shaped document raises TypeError — both crashed the whole
|
||||
# build. Skip what does not fit instead.
|
||||
lines = []
|
||||
for line in raw.splitlines():
|
||||
if not line.strip() or "content" not in line:
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(rec, dict) and isinstance(rec.get("content"), str):
|
||||
lines.append(rec["content"])
|
||||
raw = "\n".join(lines)
|
||||
toks = tokenizer.encode(raw, add_special_tokens=False).ids
|
||||
return tokenizer.decode(toks[:2400], skip_special_tokens=True)
|
||||
@@ -233,10 +245,15 @@ def main() -> None:
|
||||
if not pool:
|
||||
continue
|
||||
got, i, buf, nbuf = 0, 0, [], 0
|
||||
# Count every fragment once: the loop cycles the pool up to 40 times,
|
||||
# and re-tokenizing a 2.4k-token fragment on each pass dominated the
|
||||
# build's wall time.
|
||||
counts = [toks(f) for f in pool]
|
||||
while got < want:
|
||||
frag = pool[i % len(pool)]
|
||||
j = i % len(pool)
|
||||
frag = pool[j]
|
||||
i += 1
|
||||
n = toks(frag)
|
||||
n = counts[j]
|
||||
if nbuf + n > a.chunk_tokens and buf:
|
||||
rows.append({"prompt": "\n".join(buf), "cls": cls})
|
||||
got += nbuf
|
||||
@@ -255,8 +272,8 @@ def main() -> None:
|
||||
return len(rows), [{k: v for k, v in r.items() if k == "prompt"}
|
||||
for r in rows]
|
||||
|
||||
def emit_kld() -> tuple[int, list[dict]]:
|
||||
rows, total = [], 0
|
||||
def emit_kld() -> tuple[int, list[dict], int]:
|
||||
rows, total, scored_total = [], 0, 0
|
||||
classes = ["code", "prose", "tools"]
|
||||
i = {c: 0 for c in classes}
|
||||
guard = 0
|
||||
@@ -269,7 +286,7 @@ def main() -> None:
|
||||
if not pool:
|
||||
continue
|
||||
frag = pool[i[cls] % len(pool)]
|
||||
i[cls] += 2
|
||||
i[cls] += 1
|
||||
msgs.append({"role": "user",
|
||||
"content": "Summarize and explain this "
|
||||
f"engineering material:\n{frag}"})
|
||||
@@ -285,19 +302,18 @@ def main() -> None:
|
||||
"score_from": n - a.score_tail,
|
||||
"source": "+".join(sorted(set(srcs)))})
|
||||
total += n
|
||||
return len(rows), rows
|
||||
scored_total += a.score_tail
|
||||
return len(rows), rows, scored_total
|
||||
|
||||
os.makedirs(a.out_dir, exist_ok=True)
|
||||
nc, rows = emit_calib()
|
||||
with open(os.path.join(a.out_dir, "calib.jsonl"), "w") as f:
|
||||
for r in rows:
|
||||
f.write(json.dumps(r) + "\n")
|
||||
nk, krows = emit_kld()
|
||||
nk, krows, scored = emit_kld()
|
||||
with open(os.path.join(a.out_dir, "kld.jsonl"), "w") as f:
|
||||
for r in krows:
|
||||
f.write(json.dumps(r) + "\n")
|
||||
scored = sum(len(tokenizer.encode(r["prompt"], add_special_tokens=False).ids)
|
||||
- r["score_from"] for r in krows)
|
||||
manifest = {
|
||||
"seed": a.seed, "template": tname,
|
||||
"tokenizer": os.path.basename(a.tokenizer_json),
|
||||
|
||||
Reference in new issue
Block a user