fix(quant): prove window token identity, gate the tap on progress, kill the runner's orphan race
Adversarial review of the campaign pipeline. Correctness: rechunk-kld-corpus.py wrote each window as text and asserted only lengths, so a KLD reference could score a different token sequence than the corpus holds — every window is now re-encoded and the tool refuses on mismatch; make-calib-corpus.py no longer crashes on a .json that is not a content-keyed JSONL, and its KLD cursor advanced by 2 per fragment (half the pool unused). Silent waste: calib-push.py now aborts when the tap sealed nothing or no prompt was accepted, instead of re-pushing 10M tokens per round; div-by-zero gone, served model id parameterised. Races: run-qwen38-bf16.sh restore() removes the rig container before starting production (today's orphan held the GPUs while the service reported active), the calib tap dir is proven writable, the score ro-guard resolves paths; convert-qwen38-w5nl64.sh refuses a missing --arm value, an existing qwen38-convert-<arm> container, and a zero-exit convert that wrote no artifact. stack-qwen38-official-fp8.py: bare asserts in verify() (skipped under -O) and a non-128-lane PLE HD that only warned are hard failures. Speed: fragment token counts computed once, KLD scored positions accumulated instead of a second full re-encode. Tests: 576 ok / 0 fail.
This commit is contained in:
1 parent
e6fba5d97e
commit
1e4fa14772
6 files changed
+127
-32
No files matched your search
@@ -337,8 +337,12 @@ def main():
|
||||
if any(meta[shard_ids[i]][4][1] != hd for i in range(n_shards)):
|
||||
log("ERROR: PLE shards disagree on HD")
|
||||
return 2
|
||||
if hd % 128 != 0 and hd != 160:
|
||||
log(f"WARNING: PLE HD={hd} is neither 160 nor a multiple of 128")
|
||||
if hd != 160 and hd % 128 != 0:
|
||||
# The pack below writes lanes of 128 codes; any other HD would produce a
|
||||
# table that decodes to the wrong rows. The old warning let it continue.
|
||||
log(f"ERROR: PLE HD={hd} is neither 160 nor a multiple of 128, and the "
|
||||
"128-lane pack cannot express it")
|
||||
return 2
|
||||
|
||||
# ---- idempotence gate -------------------------------------------------
|
||||
stacked_names = [f"{p}.mlp.experts.gate_up_proj" for p in prefixes]
|
||||
@@ -521,12 +525,16 @@ def verify(stage, prefixes, n_e, shapes, meta, shard_ids, rows, hd,
|
||||
}
|
||||
for name, (dt, shp) in want.items():
|
||||
e = hdr[name]
|
||||
assert e["dtype"] == dt and list(e["shape"]) == shp, name
|
||||
assert e["data_offsets"][1] - e["data_offsets"][0] == \
|
||||
int(np.prod(shp)) * (1 if dt == F8 else 2), name
|
||||
if e["dtype"] != dt or list(e["shape"]) != shp:
|
||||
raise RuntimeError(f"{name}: header is {e['dtype']}{e['shape']}, expected {dt}{shp}")
|
||||
if e["data_offsets"][1] - e["data_offsets"][0] != int(np.prod(shp)) * (1 if dt == F8 else 2):
|
||||
raise RuntimeError(f"{name}: data span is {e['data_offsets']}, shape {shp} needs "
|
||||
f"{int(np.prod(shp)) * (1 if dt == F8 else 2)} B")
|
||||
_, phdr = parse_header(out_path)
|
||||
e = phdr[next(iter(phdr))]
|
||||
assert e["dtype"] == BF16 and list(e["shape"]) == [sum(rows), hd]
|
||||
if e["dtype"] != BF16 or list(e["shape"]) != [sum(rows), hd]:
|
||||
raise RuntimeError(f"PLE table header is {e['dtype']}{e['shape']}, "
|
||||
f"expected {BF16}[{sum(rows)}, {hd}]")
|
||||
|
||||
# sampled PLE rows: recompute and compare bytes exactly
|
||||
lut = e4m3_lut()
|
||||
@@ -546,7 +554,8 @@ def verify(stage, prefixes, n_e, shapes, meta, shard_ids, rows, hd,
|
||||
with open(out_path, "rb") as ofh:
|
||||
ofh.seek(out_off + got_row * hd * 2)
|
||||
got = ofh.read(hd * 2)
|
||||
assert got == want, f"PLE row {got_row} mismatch"
|
||||
if got != want:
|
||||
raise RuntimeError(f"PLE row {got_row} does not match the decoded source")
|
||||
mm.close()
|
||||
log("verify: new-file headers, sizes and sampled PLE rows all exact; "
|
||||
"expert slabs were byte-checked in-stack")
|
||||
|
||||
Reference in new issue
Block a user