From b4a1e6069e788f0a52082091196ba5a1d964d946 Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 30 Oct 2026 14:18:03 -0700 Subject: [PATCH 1/2] fix(qwen38): pin calib to 1.2.0, gate the tap's disk, kill orphaned engines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 17-hour calibration tap on 2026-10-29/30 wrote ZERO bytes: the runner booted FRANK_ARCHON_IMAGE (1.8.1) for every mode, and RADIANCE_CALIB_DIR/TOKENS only arms the expert tap on the 1.2.x line. 1.8.1 still carries the plugin strings and the "calibration wrote layer Hessians" message, /health answered, /stats showed 1,700 documents and 118 tok/s, and nothing anywhere said the tap was dead. Four gates, each with the measurement that motivated it: - CALIB_IMAGE pins the 1.2.0 digest for calib mode (stilldeadcode/radiance @sha256:665775ed…), overridable only by naming --image deliberately. Proven by the engagement test: `calibration wrote layer Hessians after 10207 tokens`. - The tap dir's filesystem must have >= 60 GiB free (Q38_CALIB_MIN_FREE_MIB). One seal wrote 39.4 GiB in seconds; pointed at a 37 GiB root it filled the filesystem to zero and every later write came back short. - restore() kills orphaned `radiance` processes still holding /dev/kfd before it starts the service. `podman rm -f` does not kill the engine: an orphan made production's ar_oneshot_2rank_wht6 init fail `device error`, and the unit restarted 100 times with /health never answering. - FRANK_QWEN38_CALIB_DIR / Q38_WINDOW_TRASH_ROOT make the tap dir and the bank target configurable, so the tap lives on /mnt/entropy instead of the root fs. --- scripts/qwen38-calib-window.sh | 14 ++++++++++-- scripts/run-qwen38-bf16.sh | 39 ++++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/scripts/qwen38-calib-window.sh b/scripts/qwen38-calib-window.sh index 15e3c51..5399fc8 100755 --- a/scripts/qwen38-calib-window.sh +++ b/scripts/qwen38-calib-window.sh @@ -36,7 +36,7 @@ CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}" RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}" CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}" KLDREF="$CAMPAIGN/kld-ref/kld.json" -CALIB="$CAMPAIGN/calib" +CALIB="${FRANK_QWEN38_CALIB_DIR:-$CAMPAIGN/calib}" HEALTH_TIMEOUT="${Q38_WINDOW_HEALTH_TIMEOUT:-3600}" ROUNDS="${Q38_WINDOW_ROUNDS:-6}" RIG=qwen38-bf16 @@ -49,6 +49,16 @@ PY="${PY:-python3}" [ -f "$CORPUS" ] || { echo "REFUSED: no calibration corpus at $CORPUS" >&2; exit 2; } [ -d "$CALIB" ] || { echo "REFUSED: no tap dir at $CALIB" >&2; exit 2; } [ -w "$CALIB" ] || { echo "REFUSED: tap dir $CALIB is not writable by $(id -un)" >&2; exit 2; } +# One tap seal writes ~40 GiB (measured 2026-10-30). The runner refuses too; this +# names the same number before a card is touched, so a full volume never becomes +# a 17-hour dead run. +avail_mib=$(df --output=avail -B 1M "$CALIB" 2>/dev/null | tail -1 | tr -d ' ') +[[ "$avail_mib" =~ ^[0-9]+$ ]] || avail_mib=0 +need_mib="${Q38_CALIB_MIN_FREE_MIB:-61440}" +[ "$avail_mib" -ge "$need_mib" ] || { + echo "REFUSED: tap dir $CALIB has ${avail_mib} MiB free; one seal writes ~40 GiB and this window needs ${need_mib} MiB (Q38_CALIB_MIN_FREE_MIB overrides)" >&2 + exit 2 +} ! [[ "$HEALTH_TIMEOUT" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_HEALTH_TIMEOUT must be seconds (got: $HEALTH_TIMEOUT)" >&2; exit 2; } ! [[ "$ROUNDS" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_ROUNDS must be a round count (got: $ROUNDS)" >&2; exit 2; } @@ -115,7 +125,7 @@ say "TAP BANKED: $(ls -1 "$CALIB" | wc -l) file(s) in $CALIB" if [ "${Q38_WINDOW_KEEP_RAD:-0}" = "1" ]; then say "reference .rad kept (Q38_WINDOW_KEEP_RAD=1)" else - T="$HOME/.AGENTTRASH/$(date +%Y%m%d)-qwen38-bf16-rad" + T="${Q38_WINDOW_TRASH_ROOT:-$HOME/.AGENTTRASH}/$(date +%Y%m%d)-qwen38-bf16-rad" mkdir -p "$T" || { echo "FAIL: cannot create trash dir $T" >&2; exit 1; } mv "$RAD" "$T/" && say "reference .rad banked in $T (the checkpoint that rebuilds it stays)" fi diff --git a/scripts/run-qwen38-bf16.sh b/scripts/run-qwen38-bf16.sh index e1db5a9..5527b60 100755 --- a/scripts/run-qwen38-bf16.sh +++ b/scripts/run-qwen38-bf16.sh @@ -113,6 +113,13 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1091 source "$SCRIPT_DIR/frank-archon.env" IMAGE="${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}" +# THE CALIBRATION PIN. RADIANCE_CALIB_DIR/TOKENS arms the expert tap on the 1.2.x +# line only. 1.8.1 still carries every plugin string and the seal message, and a +# 17-hour run on it wrote ZERO bytes into the tap dir while /health, /stats and +# the push all looked healthy (measured 2026-10-30) — so calib boots THIS digest +# and any other image must be named deliberately with --image. +CALIB_IMAGE="${Q38_CALIB_IMAGE:-stilldeadcode/radiance@sha256:665775ed1d90de6dbb25558e07fc60d20328f1346f96f3c6e00845eb763f01fa}" +IMAGE_EXPLICIT=0 MODE="" POS=() @@ -122,7 +129,7 @@ while [ $# -gt 0 ]; do kldref|calib|score) MODE="$1" ;; --dry-run) DRY=1 ;; --image) [ "$#" -ge 2 ] || { echo "--image needs a reference" >&2; exit 2; } - IMAGE="$2"; shift ;; + IMAGE="$2"; IMAGE_EXPLICIT=1; shift ;; *) # score's two positional arguments, and nothing else may be bare. if [ "$MODE" = score ] && [ "${#POS[@]}" -lt 2 ]; then @@ -152,7 +159,7 @@ TEMPLATE="$SCRIPT_DIR/qwen38-froggeric-v22.5.jinja" LOG_DIR="${FRANK_QWEN38_LOG_DIR:-/home/dank/logs}" LOG="$LOG_DIR/qwen38_bf16_${MODE}.log" KLD_REF_DIR="$CAMPAIGN/kld-ref" -CALIB_DIR="$CAMPAIGN/calib" +CALIB_DIR="${FRANK_QWEN38_CALIB_DIR:-$CAMPAIGN/calib}" C=qwen38-bf16 SERVICE=frank-archon # The card split, taken from the file that owns it instead of a private pair: @@ -314,6 +321,21 @@ case "$MODE" in # buys throughput with the expert slab, so price it like any other # VRAM knob. The slots follow frank-archon.env's FRANK_ARCHON_MAX_SEQS # through MAX_SEQS, exactly as kldref and score do. + [ "$IMAGE_EXPLICIT" = 1 ] || IMAGE="$CALIB_IMAGE" + # ONE TAP SEAL WRITES ~40 GiB. Measured 2026-10-30: a 8,192-token floor wrote + # 39.4 GiB into the tap dir within seconds; pointed at a 37 GiB root it filled + # the filesystem to zero, after which every write came back short ("calibration + # cannot write", "write was short") and the system itself began refusing + # writes. The room is demanded up front, on the tap's own filesystem. + if [ "$DRY" != 1 ]; then + avail_mib=$(df --output=avail -B 1M "$CALIB_DIR" 2>/dev/null | tail -1 | tr -d ' ') + [[ "$avail_mib" =~ ^[0-9]+$ ]] || avail_mib=0 + need_mib="${Q38_CALIB_MIN_FREE_MIB:-61440}" + if [ "$avail_mib" -lt "$need_mib" ]; then + echo "REFUSED: tap dir $CALIB_DIR has ${avail_mib} MiB free; one seal writes ~40 GiB and this run needs ${need_mib} MiB (Q38_CALIB_MIN_FREE_MIB overrides)" >&2 + exit 2 + fi + fi ARGS+=(--model "$MODEL_IN" --port 7770 --max-num-seqs "$CALIB_SEQS" --max-num-batched-tokens "$CALIB_MNBT") # The tap is only real if this directory exists and the engine can write it: @@ -417,6 +439,19 @@ cp -f "$TEMPLATE" "$TPL" || { # where the service reported `active` while its engine starved on the rig's VRAM. restore() { podman rm -f "$C" >/dev/null 2>&1 || true + # Removing the container does NOT kill its engine. Measured 2026-10-30: a rig + # radiance kept /dev/kfd after `podman rm -f`, production's + # ar_oneshot_2rank_wht6 init then failed `device error`, the engine exited 0, + # and Restart=always looped it 100 times with /health never answering. The + # service is stopped for the whole run, so any radiance still holding the + # device here is an orphan — kill it BY PID (pkill -f matches the caller's + # own command line, which is how a session gets killed instead of the orphan). + for p in $(fuser /dev/kfd 2>/dev/null); do + if [ "$(cat "/proc/$p/comm" 2>/dev/null)" = radiance ]; then + echo "restore: killing orphaned radiance pid $p still holding /dev/kfd" >&2 + kill -9 "$p" 2>/dev/null || true + fi + done sudo systemctl start "$SERVICE" >/dev/null 2>&1 || true } trap restore EXIT From 3e32cb9cd1a2a67d2e16ceecabffadd5c2e76de6 Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 30 Oct 2026 15:02:58 -0700 Subject: [PATCH 2/2] campaign(qwen38): the tap producer is latest radiance, at a vendor-backed floor The tap did not arm on the production pin: a 17-hour calib run on 1.8.1 wrote zero bytes while /health, /stats and the push all read healthy. On :latest (v1.14.0, revision 4a61d7296bc5) the same boot prints 'recorded passes: off, the program accumulates calibration Grams (gram_accum)' and a 9,129-token push sealed 24,768 gram files (42.9 GB) in the fitting shape - 4 slots at a 2,048-token step, production's own. The vendor's calibration shape (--max-num-batched-tokens 65536, --max-num-seqs 64, ~930 tok/s) cannot boot two 32 GiB R9700s: the activation arena alone is 12.83 GiB of the 27.95 GiB claimable, and every variant stops in the VRAM resolver (short 3.19 GiB .. 86 MiB, then a starved cache). Floor moves 12M -> 1.5M tokens: the vendor's own measurement is 'a pooled Gram from 10M tokens beat one from 1M by under 1%', and 1.5M is hours at this rig's rate instead of the 29 h a 6M floor cost. The converter moves to the same digest: rad-convert's gptq:calib= input is version-stable, so a latest-written tap converts on latest. --- scripts/convert-qwen38-w5nl64.sh | 7 ++++++- scripts/run-qwen38-bf16.sh | 31 ++++++++++++++++++++----------- tests/qwen38_reference.bats | 7 +++++-- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/scripts/convert-qwen38-w5nl64.sh b/scripts/convert-qwen38-w5nl64.sh index 58cb384..d2ac331 100755 --- a/scripts/convert-qwen38-w5nl64.sh +++ b/scripts/convert-qwen38-w5nl64.sh @@ -47,7 +47,12 @@ set -u -o pipefail SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -IMAGE=${IMAGE:-stilldeadcode/radiance@sha256:d590e300234bf6d2f942a0b8d456d183153fd808f28820e6c2fa69c9389178ed} +# THE TOOLCHAIN PIN. `:latest` on 2026-10-30 is v1.14.0 (revision 4a61d7296bc5), +# the same image the tap producer boots: a tap must be converted by the line that +# is current, and rad-convert's `gptq:calib=` input contract is version-stable +# (RADGRAM2 + gram.r..bin), so nothing else about the recipe moves with +# it. Production stays on its own pin — this is the campaign's toolchain. +IMAGE=${IMAGE:-stilldeadcode/radiance@sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093} CKPT=${CKPT:-/home/dank/models/Qwen3.8-Flash-Next} CAMPAIGN=${CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64} RECIPE=${RECIPE:-$SELF/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe} diff --git a/scripts/run-qwen38-bf16.sh b/scripts/run-qwen38-bf16.sh index 5527b60..2782938 100755 --- a/scripts/run-qwen38-bf16.sh +++ b/scripts/run-qwen38-bf16.sh @@ -25,13 +25,17 @@ # through the ro /models mount); OUT lands on a writable # mount because --kld-out F also writes F.rows beside it. # calib serve on :7770 with the calibration tap open -# (RADIANCE_CALIB_DIR + RADIANCE_CALIB_TOKENS=12000000). A +# (RADIANCE_CALIB_DIR + RADIANCE_CALIB_TOKENS=1500000). A # separate driver — scripts/calib-push.py --port 7770 # --container qwen38-bf16 --corpus # /home/dank/corpora/out/calib.jsonl — pushes the corpus; at -# 12M pooled positions the engine writes the Grams and the -# tap switches itself off (calib-push prints TAP_DONE). Stop -# this script after that; the trap brings frank-archon back. +# the floor the engine writes the Grams and the tap switches +# itself off (calib-push prints TAP_DONE). Stop this script +# after that; the trap brings frank-archon back. The vendor's +# own measurement is the floor's warrant — "a pooled Gram +# from 10 M tokens beat one from 1 M by under 1 %" — and at +# this rig's measured push rate a 1.5 M floor is hours, not +# the 29 h a 6 M floor cost. One seal writes ~40 GiB. # # Shape: the vendor's bf16 calibration lines — expert_tiered placement with the # weights disk tier, deterministic (the resident set pinned to the loader's plan, @@ -113,12 +117,17 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1091 source "$SCRIPT_DIR/frank-archon.env" IMAGE="${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}" -# THE CALIBRATION PIN. RADIANCE_CALIB_DIR/TOKENS arms the expert tap on the 1.2.x -# line only. 1.8.1 still carries every plugin string and the seal message, and a -# 17-hour run on it wrote ZERO bytes into the tap dir while /health, /stats and -# the push all looked healthy (measured 2026-10-30) — so calib boots THIS digest -# and any other image must be named deliberately with --image. -CALIB_IMAGE="${Q38_CALIB_IMAGE:-stilldeadcode/radiance@sha256:665775ed1d90de6dbb25558e07fc60d20328f1346f96f3c6e00845eb763f01fa}" +# THE CALIBRATION PIN: the LATEST radiance, pinned by digest because a tag is a +# moving target and a tap must name the producer that wrote it. `:latest` on +# 2026-10-30 resolves to v1.14.0 (revision 4a61d7296bc5), and on that image a +# calib boot prints the arming line — "recorded passes: off, the program +# accumulates calibration Grams (gram_accum)" — where the production pin +# (1.8.1) prints nothing of the kind: a 17-hour run on 1.8.1 wrote ZERO bytes +# into the tap dir while /health, /stats and the push all looked healthy +# (measured 2026-10-30). The tap code is identical in both trees' sources, so +# the shipped 1.8.1 build is what drops it; do not spend another window on it. +# Any other image must be named deliberately with --image. +CALIB_IMAGE="${Q38_CALIB_IMAGE:-stilldeadcode/radiance@sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093}" IMAGE_EXPLICIT=0 MODE="" @@ -348,7 +357,7 @@ case "$MODE" in } MOUNTS+=(-v "$CALIB_DIR:/data/calib") TAPENV=(-e RADIANCE_CALIB_DIR=/data/calib - -e "RADIANCE_CALIB_TOKENS=${FRANK_QWEN38_CALIB_TOKENS:-12000000}") + -e "RADIANCE_CALIB_TOKENS=${FRANK_QWEN38_CALIB_TOKENS:-1500000}") # The tap's own authorization. A pusher must see this marker before it sends # a prompt (calib-push.py --tap-marker), it names the container so a push # aimed anywhere else refuses, and a boot that fails removes it again below. diff --git a/tests/qwen38_reference.bats b/tests/qwen38_reference.bats index a45415d..3e1ee86 100755 --- a/tests/qwen38_reference.bats +++ b/tests/qwen38_reference.bats @@ -157,12 +157,15 @@ flag_value() { [[ "$output" != *"--vram-kv-mib"* ]] } -@test "the calib dry run opens the tap at 12M tokens and serves on 7770" { +@test "the calib dry run opens the tap at the 1.5M floor, on the latest pin, and serves on 7770" { run bash "$RUNNER" calib --dry-run [ "$status" -eq 0 ] flat="$(oneline)" - [[ "$flat" == *"-e RADIANCE_CALIB_TOKENS=12000000"* ]] + [[ "$flat" == *"-e RADIANCE_CALIB_TOKENS=1500000"* ]] [[ "$flat" == *"-e RADIANCE_CALIB_DIR=/data/calib"* ]] + # the tap producer is the campaign's own pin (latest radiance), not production's + [[ "$flat" == *"radiance@sha256:69add58f"* ]] + [[ "$flat" != *"radiance@sha256:d590e300"* ]] [[ "$flat" == *"--port 7770"* ]] [[ "$flat" == *":/data/calib "* ]] # calib serves; it does not record