fix(quant): prove window token identity, gate the tap on progress, kill the runner's orphan race
Adversarial review of the campaign pipeline. Correctness: rechunk-kld-corpus.py wrote each window as text and asserted only lengths, so a KLD reference could score a different token sequence than the corpus holds — every window is now re-encoded and the tool refuses on mismatch; make-calib-corpus.py no longer crashes on a .json that is not a content-keyed JSONL, and its KLD cursor advanced by 2 per fragment (half the pool unused). Silent waste: calib-push.py now aborts when the tap sealed nothing or no prompt was accepted, instead of re-pushing 10M tokens per round; div-by-zero gone, served model id parameterised. Races: run-qwen38-bf16.sh restore() removes the rig container before starting production (today's orphan held the GPUs while the service reported active), the calib tap dir is proven writable, the score ro-guard resolves paths; convert-qwen38-w5nl64.sh refuses a missing --arm value, an existing qwen38-convert-<arm> container, and a zero-exit convert that wrote no artifact. stack-qwen38-official-fp8.py: bare asserts in verify() (skipped under -O) and a non-128-lane PLE HD that only warned are hard failures. Speed: fragment token counts computed once, KLD scored positions accumulated instead of a second full re-encode. Tests: 576 ok / 0 fail.
This commit is contained in:
1 parent
e6fba5d97e
commit
1e4fa14772
6 files changed
+127
-32
No files matched your search
+45
-7
@@ -29,6 +29,7 @@ Usage: calib-push.py [--port 7770] [--parallel 64] [--container arbench]
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
@@ -49,9 +50,9 @@ def tap_done(container):
|
||||
return DONE_MARK in (out.stdout + out.stderr)
|
||||
|
||||
|
||||
def push_one(url, prompt):
|
||||
def push_one(url, prompt, model):
|
||||
body = json.dumps({
|
||||
"model": "frank-archon",
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"max_tokens": 1,
|
||||
"temperature": 0,
|
||||
@@ -68,11 +69,33 @@ def push_one(url, prompt):
|
||||
return False
|
||||
|
||||
|
||||
def tap_bytes(path):
|
||||
"""Bytes the tap has sealed so far; 0 when the directory is absent.
|
||||
|
||||
A tap whose directory is wrong, full or read-only stops writing while the
|
||||
engine keeps answering 200 — the driver then re-pushes the whole corpus for
|
||||
every round it is allowed and the loss is invisible until the hours are gone.
|
||||
"""
|
||||
total = 0
|
||||
try:
|
||||
for root, _, files in os.walk(path):
|
||||
for f in files:
|
||||
try:
|
||||
total += os.path.getsize(os.path.join(root, f))
|
||||
except OSError:
|
||||
pass
|
||||
except OSError:
|
||||
return 0
|
||||
return total
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--port", type=int, default=7770)
|
||||
ap.add_argument("--parallel", type=int, default=64)
|
||||
ap.add_argument("--container", default="arbench")
|
||||
ap.add_argument("--model", default="frank-archon")
|
||||
ap.add_argument("--tap-dir", default="/home/dank/campaign/qwen38-w5nl64/calib")
|
||||
ap.add_argument("--corpus", default="/home/dank/corpora/out/calib.jsonl")
|
||||
ap.add_argument("--max-rounds", type=int, default=4)
|
||||
a = ap.parse_args()
|
||||
@@ -84,22 +107,37 @@ def main():
|
||||
line = line.strip()
|
||||
if line:
|
||||
prompts.append(json.loads(line)["prompt"])
|
||||
print(f"corpus: {len(prompts)} prompts", flush=True)
|
||||
chars = sum(len(p) for p in prompts)
|
||||
print(f"corpus: {len(prompts)} prompts, ~{chars // 4} tokens (chars/4)",
|
||||
flush=True)
|
||||
if not prompts:
|
||||
print("REFUSED: the corpus holds no prompts", flush=True)
|
||||
sys.exit(2)
|
||||
|
||||
for rnd in range(1, a.max_rounds + 1):
|
||||
if tap_done(a.container):
|
||||
break
|
||||
before = tap_bytes(a.tap_dir)
|
||||
t0 = time.time()
|
||||
ok = 0
|
||||
with concurrent.futures.ThreadPoolExecutor(a.parallel) as ex:
|
||||
for good in ex.map(lambda p: push_one(url, p), prompts):
|
||||
for good in ex.map(lambda p: push_one(url, p, a.model), prompts):
|
||||
ok += 1 if good else 0
|
||||
dt = time.time() - t0
|
||||
dt = max(time.time() - t0, 1e-9)
|
||||
print(f"round {rnd}: {ok}/{len(prompts)} ok in {dt:.0f}s "
|
||||
f"({sum(len(p) for p in prompts)/4/dt:.0f} chars/s est)",
|
||||
flush=True)
|
||||
f"({chars / 4 / dt:.0f} tokens/s est)", flush=True)
|
||||
if ok == 0:
|
||||
print("FAIL: not one prompt was accepted; fix the engine or the argv "
|
||||
"before spending another round", flush=True)
|
||||
sys.exit(1)
|
||||
if tap_done(a.container):
|
||||
break
|
||||
after = tap_bytes(a.tap_dir)
|
||||
if after <= before:
|
||||
print(f"FAIL: the tap at {a.tap_dir} sealed nothing this round "
|
||||
f"({before} B before, {after} B after) — the engine is serving "
|
||||
"without recording (wrong dir, full disk, read-only mount)", flush=True)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("max rounds exhausted, tap not done", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
Reference in new issue
Block a user