Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f6298e3501 | ||
|
|
a6fb0209e8 |
No files matched your search
+35
-7
@@ -22,9 +22,12 @@ Mechanics notes:
|
|||||||
prompts do not bias the Hessians (the corpus is drawn round-robin over
|
prompts do not bias the Hessians (the corpus is drawn round-robin over
|
||||||
source classes, so loss is class-neutral as long as failures are rare;
|
source classes, so loss is class-neutral as long as failures are rare;
|
||||||
a systematic 4xx is loud in the log and must be fixed, not ignored).
|
a systematic 4xx is loud in the log and must be fixed, not ignored).
|
||||||
- a round that seals no tap bytes is a FAILURE, not a retry: the engine
|
- a round that seals no tap bytes is NORMAL and continues: RADIANCE_CALIB_TOKENS
|
||||||
answering 200 while the tap directory is wrong, full or read-only is
|
is a pooled floor and this campaign's corpus is 10.03M tokens against a 12M
|
||||||
how a corpus gets pushed four times for nothing.
|
floor, so round 1 seals nothing by construction. A tap that LOSES bytes fails
|
||||||
|
the run, and a run that never sees the engine's completion line exits non-zero.
|
||||||
|
The first version failed on a flat round, which aborted exactly one round too
|
||||||
|
early (2026-10-29).
|
||||||
- before any prompt, the tap must carry the rig's own marker
|
- before any prompt, the tap must carry the rig's own marker
|
||||||
(`--tap-marker`, written by run-qwen38-bf16.sh calib) naming THIS
|
(`--tap-marker`, written by run-qwen38-bf16.sh calib) naming THIS
|
||||||
container, and that container must be running. Measured 2026-10-29:
|
container, and that container must be running. Measured 2026-10-29:
|
||||||
@@ -112,6 +115,27 @@ def tap_bytes(path):
|
|||||||
return total
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
def round_verdict(before, after, done):
|
||||||
|
"""What one round's tap bytes mean.
|
||||||
|
|
||||||
|
'done' the engine logged that it sealed the tap — the run is over
|
||||||
|
'shrink' the tap LOST bytes: reset, pruned or replaced under us
|
||||||
|
'grew' sealed bytes this round
|
||||||
|
'flat' sealed nothing — which is legitimate: RADIANCE_CALIB_TOKENS is a
|
||||||
|
pooled floor, and a round that does not reach it has nothing to
|
||||||
|
write yet (the campaign's corpus is 10.03M tokens against a 12M
|
||||||
|
floor, so round 1 is always flat). Failing on it aborted the run
|
||||||
|
exactly one round too early.
|
||||||
|
"""
|
||||||
|
if done:
|
||||||
|
return "done"
|
||||||
|
if after < before:
|
||||||
|
return "shrink"
|
||||||
|
if after == before:
|
||||||
|
return "flat"
|
||||||
|
return "grew"
|
||||||
|
|
||||||
|
|
||||||
def require_rig(a):
|
def require_rig(a):
|
||||||
"""Refuse unless the rig that opened this tap is up and serving right now.
|
"""Refuse unless the rig that opened this tap is up and serving right now.
|
||||||
|
|
||||||
@@ -198,11 +222,15 @@ def main():
|
|||||||
if tap_done(a.container):
|
if tap_done(a.container):
|
||||||
break
|
break
|
||||||
after = tap_bytes(a.tap_dir)
|
after = tap_bytes(a.tap_dir)
|
||||||
if after <= before:
|
verdict = round_verdict(before, after, False)
|
||||||
print(f"FAIL: the tap at {a.tap_dir} sealed nothing this round "
|
if verdict == "shrink":
|
||||||
f"({before} B before, {after} B after) — the engine is serving "
|
print(f"FAIL: the tap at {a.tap_dir} LOST bytes this round "
|
||||||
"without recording (wrong dir, full disk, read-only mount)", flush=True)
|
f"({before} B before, {after} B after) — it was reset or replaced "
|
||||||
|
"under us", flush=True)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
if verdict == "flat":
|
||||||
|
print(f"round {rnd}: the tap sealed nothing yet (byte floor not reached; "
|
||||||
|
f"{before} B) — continuing", flush=True)
|
||||||
else:
|
else:
|
||||||
print("max rounds exhausted, tap not done", flush=True)
|
print("max rounds exhausted, tap not done", flush=True)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|||||||
@@ -92,7 +92,8 @@
|
|||||||
# the container reads through /models), FRANK_QWEN38_LOG_DIR, and the step
|
# the container reads through /models), FRANK_QWEN38_LOG_DIR, and the step
|
||||||
# and slot counts inherited as FRANK_ARCHON_MAX_BATCHED_TOKENS /
|
# and slot counts inherited as FRANK_ARCHON_MAX_BATCHED_TOKENS /
|
||||||
# FRANK_ARCHON_MAX_SEQS — no per-mode literals (the audit law: a mode that
|
# FRANK_ARCHON_MAX_SEQS — no per-mode literals (the audit law: a mode that
|
||||||
# carries its own step measures a shape nothing serves) —
|
# carries its own step measures a shape nothing serves; calib is exempt, see
|
||||||
|
# Q38_CALIB_MNBT / Q38_CALIB_SEQS below — its output is never compared) —
|
||||||
# and FRANK_QWEN38_MAX_MODEL_LEN — the window every mode boots at (32,768;
|
# and FRANK_QWEN38_MAX_MODEL_LEN — the window every mode boots at (32,768;
|
||||||
# see the corpus census above, and raise it together with the corpus).
|
# see the corpus census above, and raise it together with the corpus).
|
||||||
# kldref refuses (exit 2) when the longest record of the corpus it is about to
|
# kldref refuses (exit 2) when the longest record of the corpus it is about to
|
||||||
@@ -185,6 +186,24 @@ if ! [[ "$MNBT" =~ ^[0-9]+$ ]]; then
|
|||||||
exit 2
|
exit 2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Calib's shape is NOT a measuring shape. kldref and score produce KL numbers that are only
|
||||||
|
# comparable across boots at an identical shape, which is why they inherit production's step and
|
||||||
|
# slots to the letter. The tap's output is layer Hessians, never a number any report compares, so
|
||||||
|
# its step and slot count are its own — and they are the tap's dominant cost: a 7,168-token
|
||||||
|
# document cut into four 2,048-token steps re-streams each layer's routed expert union four
|
||||||
|
# times from the container, so one wide step is worth up to ~4x the read volume. Measured
|
||||||
|
# 2026-10-29 at the inherited 2,048/4: prefill 71 tok/s and a 47-hour tap; the probe exists to
|
||||||
|
# find out what the wide step is worth before the campaign pays for two days of drained serving.
|
||||||
|
# Both default to the inherited value, so nothing moves unless a probe asks it to.
|
||||||
|
CALIB_MNBT="${Q38_CALIB_MNBT:-$MNBT}"
|
||||||
|
CALIB_SEQS="${Q38_CALIB_SEQS:-$MAX_SEQS}"
|
||||||
|
for pair in "Q38_CALIB_MNBT=$CALIB_MNBT" "Q38_CALIB_SEQS=$CALIB_SEQS"; do
|
||||||
|
if ! [[ "${pair#*=}" =~ ^[0-9]+$ ]]; then
|
||||||
|
echo "REFUSED: ${pair%%=*} must be a number (got: ${pair#*=})" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
# The window every mode boots at. Over-length is a REFUSAL, not a truncation (the
|
# The window every mode boots at. Over-length is a REFUSAL, not a truncation (the
|
||||||
# env file says so for production), so this is the number the corpus must fit in.
|
# env file says so for production), so this is the number the corpus must fit in.
|
||||||
MAX_MODEL_LEN="${FRANK_QWEN38_MAX_MODEL_LEN:-32768}"
|
MAX_MODEL_LEN="${FRANK_QWEN38_MAX_MODEL_LEN:-32768}"
|
||||||
@@ -296,7 +315,7 @@ case "$MODE" in
|
|||||||
# VRAM knob. The slots follow frank-archon.env's FRANK_ARCHON_MAX_SEQS
|
# VRAM knob. The slots follow frank-archon.env's FRANK_ARCHON_MAX_SEQS
|
||||||
# through MAX_SEQS, exactly as kldref and score do.
|
# through MAX_SEQS, exactly as kldref and score do.
|
||||||
ARGS+=(--model "$MODEL_IN" --port 7770
|
ARGS+=(--model "$MODEL_IN" --port 7770
|
||||||
--max-num-seqs "$MAX_SEQS" --max-num-batched-tokens "$MNBT")
|
--max-num-seqs "$CALIB_SEQS" --max-num-batched-tokens "$CALIB_MNBT")
|
||||||
# The tap is only real if this directory exists and the engine can write it:
|
# The tap is only real if this directory exists and the engine can write it:
|
||||||
# without the check a typo'd CALIB_DIR auto-creates a root-owned empty dir in
|
# without the check a typo'd CALIB_DIR auto-creates a root-owned empty dir in
|
||||||
# whatever path was given, and the run burns hours serving prompts into a tap
|
# whatever path was given, and the run burns hours serving prompts into a tap
|
||||||
|
|||||||
@@ -699,3 +699,35 @@ window_fixture() {
|
|||||||
[[ "$output" == *"calib-push.py"* ]]
|
[[ "$output" == *"calib-push.py"* ]]
|
||||||
[[ "$output" == *"AGENTTRASH"* ]]
|
[[ "$output" == *"AGENTTRASH"* ]]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@test "a flat calibration round is not a failure: the tap's floor is pooled" {
|
||||||
|
# round 1 of a 10.03M-token corpus against a 12M floor seals nothing, and the
|
||||||
|
# first version of this gate aborted the run there — one round too early.
|
||||||
|
run python3 - "$PUSHER" <<'PY'
|
||||||
|
import importlib.util, sys
|
||||||
|
spec = importlib.util.spec_from_file_location("cp", sys.argv[1])
|
||||||
|
m = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(m)
|
||||||
|
print(m.round_verdict(8192, 8192, False), # flat: continue
|
||||||
|
m.round_verdict(8192, 4096, False), # shrink: fail
|
||||||
|
m.round_verdict(8192, 20000, False), # grew: continue
|
||||||
|
m.round_verdict(0, 0, True)) # done: stop
|
||||||
|
PY
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[ "$output" = "flat shrink grew done" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the calibration boot's step is its own knob; the KL modes keep production's" {
|
||||||
|
# Calib is the only phase whose cost is read volume per document: four 2,048-token steps
|
||||||
|
# re-stream each layer's routed experts four times. Measured 2026-10-29: 71 tok/s, 47 h tap.
|
||||||
|
run env Q38_CALIB_MNBT=8192 bash "$RUNNER" calib --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
grep -A1 -x -- '--max-num-batched-tokens' <<<"$output" | grep -qx '8192'
|
||||||
|
# and the same knob must NOT reach the measuring shape those numbers are compared in
|
||||||
|
run env Q38_CALIB_MNBT=8192 bash "$RUNNER" kldref --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
! grep -qx '8192' <<<"$output"
|
||||||
|
run env Q38_CALIB_MNBT=wide bash "$RUNNER" calib --dry-run
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"Q38_CALIB_MNBT must be a number"* ]]
|
||||||
|
}
|
||||||
Reference in new issue
Block a user