From f6298e35013c171592090873860cdced29a2c3ed Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 29 Oct 2026 16:50:50 -0700 Subject: [PATCH] perf(quant): the calibration boot's step and slots are its own knobs (probe: 71 tok/s, 47 h tap) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured from the engine's own counters on the 2026-10-29 drain: prefill 71 tok/s steady state, a 180 s window showing 7 prefill steps of the capped 2,048 tokens and 3 preemptions. One 7,168-token document is therefore ~100 s, round 1 (2,367 docs, 10.03M tokens) ~39 h, and the 12M-position floor ~47 h of drained production — not the ~18 h the brief assumed. The tap is the only phase whose cost is read volume per document: at the inherited 2,048-token step a document is four steps, and each step re-streams every layer's routed expert union from the container (43.4 GiB a rank live on disk). One wide step reads each layer's union once. The 'one step, one slot count' audit law binds kldref and score because their KL numbers are only comparable at an identical shape. Calib produces layer Hessians and no number any report compares, so Q38_CALIB_MNBT / Q38_CALIB_SEQS are its own, defaulting to the inherited value — nothing moves unless a probe asks. The KL modes ignore both. Tests: 589/0 — the knob reaches calib's argv, does not reach kldref's, and a non-numeric value is refused by name. --- scripts/run-qwen38-bf16.sh | 23 +++++++++++++++++++++-- tests/qwen38_campaign.bats | 15 +++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/scripts/run-qwen38-bf16.sh b/scripts/run-qwen38-bf16.sh index cf7e730..e1db5a9 100755 --- a/scripts/run-qwen38-bf16.sh +++ b/scripts/run-qwen38-bf16.sh @@ -92,7 +92,8 @@ # the container reads through /models), FRANK_QWEN38_LOG_DIR, and the step # and slot counts inherited as FRANK_ARCHON_MAX_BATCHED_TOKENS / # FRANK_ARCHON_MAX_SEQS — no per-mode literals (the audit law: a mode that -# carries its own step measures a shape nothing serves) — +# carries its own step measures a shape nothing serves; calib is exempt, see +# Q38_CALIB_MNBT / Q38_CALIB_SEQS below — its output is never compared) — # and FRANK_QWEN38_MAX_MODEL_LEN — the window every mode boots at (32,768; # see the corpus census above, and raise it together with the corpus). # kldref refuses (exit 2) when the longest record of the corpus it is about to @@ -185,6 +186,24 @@ if ! [[ "$MNBT" =~ ^[0-9]+$ ]]; then exit 2 fi +# Calib's shape is NOT a measuring shape. kldref and score produce KL numbers that are only +# comparable across boots at an identical shape, which is why they inherit production's step and +# slots to the letter. The tap's output is layer Hessians, never a number any report compares, so +# its step and slot count are its own — and they are the tap's dominant cost: a 7,168-token +# document cut into four 2,048-token steps re-streams each layer's routed expert union four +# times from the container, so one wide step is worth up to ~4x the read volume. Measured +# 2026-10-29 at the inherited 2,048/4: prefill 71 tok/s and a 47-hour tap; the probe exists to +# find out what the wide step is worth before the campaign pays for two days of drained serving. +# Both default to the inherited value, so nothing moves unless a probe asks it to. +CALIB_MNBT="${Q38_CALIB_MNBT:-$MNBT}" +CALIB_SEQS="${Q38_CALIB_SEQS:-$MAX_SEQS}" +for pair in "Q38_CALIB_MNBT=$CALIB_MNBT" "Q38_CALIB_SEQS=$CALIB_SEQS"; do + if ! [[ "${pair#*=}" =~ ^[0-9]+$ ]]; then + echo "REFUSED: ${pair%%=*} must be a number (got: ${pair#*=})" >&2 + exit 2 + fi +done + # The window every mode boots at. Over-length is a REFUSAL, not a truncation (the # env file says so for production), so this is the number the corpus must fit in. MAX_MODEL_LEN="${FRANK_QWEN38_MAX_MODEL_LEN:-32768}" @@ -296,7 +315,7 @@ case "$MODE" in # VRAM knob. The slots follow frank-archon.env's FRANK_ARCHON_MAX_SEQS # through MAX_SEQS, exactly as kldref and score do. ARGS+=(--model "$MODEL_IN" --port 7770 - --max-num-seqs "$MAX_SEQS" --max-num-batched-tokens "$MNBT") + --max-num-seqs "$CALIB_SEQS" --max-num-batched-tokens "$CALIB_MNBT") # The tap is only real if this directory exists and the engine can write it: # without the check a typo'd CALIB_DIR auto-creates a root-owned empty dir in # whatever path was given, and the run burns hours serving prompts into a tap diff --git a/tests/qwen38_campaign.bats b/tests/qwen38_campaign.bats index f9ce0fe..fdaece6 100755 --- a/tests/qwen38_campaign.bats +++ b/tests/qwen38_campaign.bats @@ -716,3 +716,18 @@ PY [ "$status" -eq 0 ] [ "$output" = "flat shrink grew done" ] } + +@test "the calibration boot's step is its own knob; the KL modes keep production's" { + # Calib is the only phase whose cost is read volume per document: four 2,048-token steps + # re-stream each layer's routed experts four times. Measured 2026-10-29: 71 tok/s, 47 h tap. + run env Q38_CALIB_MNBT=8192 bash "$RUNNER" calib --dry-run + [ "$status" -eq 0 ] + grep -A1 -x -- '--max-num-batched-tokens' <<<"$output" | grep -qx '8192' + # and the same knob must NOT reach the measuring shape those numbers are compared in + run env Q38_CALIB_MNBT=8192 bash "$RUNNER" kldref --dry-run + [ "$status" -eq 0 ] + ! grep -qx '8192' <<<"$output" + run env Q38_CALIB_MNBT=wide bash "$RUNNER" calib --dry-run + [ "$status" -eq 2 ] + [[ "$output" == *"Q38_CALIB_MNBT must be a number"* ]] +}