diff --git a/scripts/convert-qwen38-w5nl64.sh b/scripts/convert-qwen38-w5nl64.sh index 58cb384..d2ac331 100755 --- a/scripts/convert-qwen38-w5nl64.sh +++ b/scripts/convert-qwen38-w5nl64.sh @@ -47,7 +47,12 @@ set -u -o pipefail SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -IMAGE=${IMAGE:-stilldeadcode/radiance@sha256:d590e300234bf6d2f942a0b8d456d183153fd808f28820e6c2fa69c9389178ed} +# THE TOOLCHAIN PIN. `:latest` on 2026-10-30 is v1.14.0 (revision 4a61d7296bc5), +# the same image the tap producer boots: a tap must be converted by the line that +# is current, and rad-convert's `gptq:calib=` input contract is version-stable +# (RADGRAM2 + gram.r..bin), so nothing else about the recipe moves with +# it. Production stays on its own pin — this is the campaign's toolchain. +IMAGE=${IMAGE:-stilldeadcode/radiance@sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093} CKPT=${CKPT:-/home/dank/models/Qwen3.8-Flash-Next} CAMPAIGN=${CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64} RECIPE=${RECIPE:-$SELF/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe} diff --git a/scripts/qwen38-calib-window.sh b/scripts/qwen38-calib-window.sh index 15e3c51..5399fc8 100755 --- a/scripts/qwen38-calib-window.sh +++ b/scripts/qwen38-calib-window.sh @@ -36,7 +36,7 @@ CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}" RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}" CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}" KLDREF="$CAMPAIGN/kld-ref/kld.json" -CALIB="$CAMPAIGN/calib" +CALIB="${FRANK_QWEN38_CALIB_DIR:-$CAMPAIGN/calib}" HEALTH_TIMEOUT="${Q38_WINDOW_HEALTH_TIMEOUT:-3600}" ROUNDS="${Q38_WINDOW_ROUNDS:-6}" RIG=qwen38-bf16 @@ -49,6 +49,16 @@ PY="${PY:-python3}" [ -f "$CORPUS" ] || { echo "REFUSED: no calibration corpus at $CORPUS" >&2; exit 2; } [ -d "$CALIB" ] || { echo "REFUSED: no tap dir at $CALIB" >&2; exit 2; } [ -w "$CALIB" ] || { echo "REFUSED: tap dir $CALIB is not writable by $(id -un)" >&2; exit 2; } +# One tap seal writes ~40 GiB (measured 2026-10-30). The runner refuses too; this +# names the same number before a card is touched, so a full volume never becomes +# a 17-hour dead run. +avail_mib=$(df --output=avail -B 1M "$CALIB" 2>/dev/null | tail -1 | tr -d ' ') +[[ "$avail_mib" =~ ^[0-9]+$ ]] || avail_mib=0 +need_mib="${Q38_CALIB_MIN_FREE_MIB:-61440}" +[ "$avail_mib" -ge "$need_mib" ] || { + echo "REFUSED: tap dir $CALIB has ${avail_mib} MiB free; one seal writes ~40 GiB and this window needs ${need_mib} MiB (Q38_CALIB_MIN_FREE_MIB overrides)" >&2 + exit 2 +} ! [[ "$HEALTH_TIMEOUT" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_HEALTH_TIMEOUT must be seconds (got: $HEALTH_TIMEOUT)" >&2; exit 2; } ! [[ "$ROUNDS" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_ROUNDS must be a round count (got: $ROUNDS)" >&2; exit 2; } @@ -115,7 +125,7 @@ say "TAP BANKED: $(ls -1 "$CALIB" | wc -l) file(s) in $CALIB" if [ "${Q38_WINDOW_KEEP_RAD:-0}" = "1" ]; then say "reference .rad kept (Q38_WINDOW_KEEP_RAD=1)" else - T="$HOME/.AGENTTRASH/$(date +%Y%m%d)-qwen38-bf16-rad" + T="${Q38_WINDOW_TRASH_ROOT:-$HOME/.AGENTTRASH}/$(date +%Y%m%d)-qwen38-bf16-rad" mkdir -p "$T" || { echo "FAIL: cannot create trash dir $T" >&2; exit 1; } mv "$RAD" "$T/" && say "reference .rad banked in $T (the checkpoint that rebuilds it stays)" fi diff --git a/scripts/run-qwen38-bf16.sh b/scripts/run-qwen38-bf16.sh index e1db5a9..2782938 100755 --- a/scripts/run-qwen38-bf16.sh +++ b/scripts/run-qwen38-bf16.sh @@ -25,13 +25,17 @@ # through the ro /models mount); OUT lands on a writable # mount because --kld-out F also writes F.rows beside it. # calib serve on :7770 with the calibration tap open -# (RADIANCE_CALIB_DIR + RADIANCE_CALIB_TOKENS=12000000). A +# (RADIANCE_CALIB_DIR + RADIANCE_CALIB_TOKENS=1500000). A # separate driver — scripts/calib-push.py --port 7770 # --container qwen38-bf16 --corpus # /home/dank/corpora/out/calib.jsonl — pushes the corpus; at -# 12M pooled positions the engine writes the Grams and the -# tap switches itself off (calib-push prints TAP_DONE). Stop -# this script after that; the trap brings frank-archon back. +# the floor the engine writes the Grams and the tap switches +# itself off (calib-push prints TAP_DONE). Stop this script +# after that; the trap brings frank-archon back. The vendor's +# own measurement is the floor's warrant — "a pooled Gram +# from 10 M tokens beat one from 1 M by under 1 %" — and at +# this rig's measured push rate a 1.5 M floor is hours, not +# the 29 h a 6 M floor cost. One seal writes ~40 GiB. # # Shape: the vendor's bf16 calibration lines — expert_tiered placement with the # weights disk tier, deterministic (the resident set pinned to the loader's plan, @@ -113,6 +117,18 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1091 source "$SCRIPT_DIR/frank-archon.env" IMAGE="${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}" +# THE CALIBRATION PIN: the LATEST radiance, pinned by digest because a tag is a +# moving target and a tap must name the producer that wrote it. `:latest` on +# 2026-10-30 resolves to v1.14.0 (revision 4a61d7296bc5), and on that image a +# calib boot prints the arming line — "recorded passes: off, the program +# accumulates calibration Grams (gram_accum)" — where the production pin +# (1.8.1) prints nothing of the kind: a 17-hour run on 1.8.1 wrote ZERO bytes +# into the tap dir while /health, /stats and the push all looked healthy +# (measured 2026-10-30). The tap code is identical in both trees' sources, so +# the shipped 1.8.1 build is what drops it; do not spend another window on it. +# Any other image must be named deliberately with --image. +CALIB_IMAGE="${Q38_CALIB_IMAGE:-stilldeadcode/radiance@sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093}" +IMAGE_EXPLICIT=0 MODE="" POS=() @@ -122,7 +138,7 @@ while [ $# -gt 0 ]; do kldref|calib|score) MODE="$1" ;; --dry-run) DRY=1 ;; --image) [ "$#" -ge 2 ] || { echo "--image needs a reference" >&2; exit 2; } - IMAGE="$2"; shift ;; + IMAGE="$2"; IMAGE_EXPLICIT=1; shift ;; *) # score's two positional arguments, and nothing else may be bare. if [ "$MODE" = score ] && [ "${#POS[@]}" -lt 2 ]; then @@ -152,7 +168,7 @@ TEMPLATE="$SCRIPT_DIR/qwen38-froggeric-v22.5.jinja" LOG_DIR="${FRANK_QWEN38_LOG_DIR:-/home/dank/logs}" LOG="$LOG_DIR/qwen38_bf16_${MODE}.log" KLD_REF_DIR="$CAMPAIGN/kld-ref" -CALIB_DIR="$CAMPAIGN/calib" +CALIB_DIR="${FRANK_QWEN38_CALIB_DIR:-$CAMPAIGN/calib}" C=qwen38-bf16 SERVICE=frank-archon # The card split, taken from the file that owns it instead of a private pair: @@ -314,6 +330,21 @@ case "$MODE" in # buys throughput with the expert slab, so price it like any other # VRAM knob. The slots follow frank-archon.env's FRANK_ARCHON_MAX_SEQS # through MAX_SEQS, exactly as kldref and score do. + [ "$IMAGE_EXPLICIT" = 1 ] || IMAGE="$CALIB_IMAGE" + # ONE TAP SEAL WRITES ~40 GiB. Measured 2026-10-30: a 8,192-token floor wrote + # 39.4 GiB into the tap dir within seconds; pointed at a 37 GiB root it filled + # the filesystem to zero, after which every write came back short ("calibration + # cannot write", "write was short") and the system itself began refusing + # writes. The room is demanded up front, on the tap's own filesystem. + if [ "$DRY" != 1 ]; then + avail_mib=$(df --output=avail -B 1M "$CALIB_DIR" 2>/dev/null | tail -1 | tr -d ' ') + [[ "$avail_mib" =~ ^[0-9]+$ ]] || avail_mib=0 + need_mib="${Q38_CALIB_MIN_FREE_MIB:-61440}" + if [ "$avail_mib" -lt "$need_mib" ]; then + echo "REFUSED: tap dir $CALIB_DIR has ${avail_mib} MiB free; one seal writes ~40 GiB and this run needs ${need_mib} MiB (Q38_CALIB_MIN_FREE_MIB overrides)" >&2 + exit 2 + fi + fi ARGS+=(--model "$MODEL_IN" --port 7770 --max-num-seqs "$CALIB_SEQS" --max-num-batched-tokens "$CALIB_MNBT") # The tap is only real if this directory exists and the engine can write it: @@ -326,7 +357,7 @@ case "$MODE" in } MOUNTS+=(-v "$CALIB_DIR:/data/calib") TAPENV=(-e RADIANCE_CALIB_DIR=/data/calib - -e "RADIANCE_CALIB_TOKENS=${FRANK_QWEN38_CALIB_TOKENS:-12000000}") + -e "RADIANCE_CALIB_TOKENS=${FRANK_QWEN38_CALIB_TOKENS:-1500000}") # The tap's own authorization. A pusher must see this marker before it sends # a prompt (calib-push.py --tap-marker), it names the container so a push # aimed anywhere else refuses, and a boot that fails removes it again below. @@ -417,6 +448,19 @@ cp -f "$TEMPLATE" "$TPL" || { # where the service reported `active` while its engine starved on the rig's VRAM. restore() { podman rm -f "$C" >/dev/null 2>&1 || true + # Removing the container does NOT kill its engine. Measured 2026-10-30: a rig + # radiance kept /dev/kfd after `podman rm -f`, production's + # ar_oneshot_2rank_wht6 init then failed `device error`, the engine exited 0, + # and Restart=always looped it 100 times with /health never answering. The + # service is stopped for the whole run, so any radiance still holding the + # device here is an orphan — kill it BY PID (pkill -f matches the caller's + # own command line, which is how a session gets killed instead of the orphan). + for p in $(fuser /dev/kfd 2>/dev/null); do + if [ "$(cat "/proc/$p/comm" 2>/dev/null)" = radiance ]; then + echo "restore: killing orphaned radiance pid $p still holding /dev/kfd" >&2 + kill -9 "$p" 2>/dev/null || true + fi + done sudo systemctl start "$SERVICE" >/dev/null 2>&1 || true } trap restore EXIT diff --git a/tests/qwen38_reference.bats b/tests/qwen38_reference.bats index a45415d..3e1ee86 100755 --- a/tests/qwen38_reference.bats +++ b/tests/qwen38_reference.bats @@ -157,12 +157,15 @@ flag_value() { [[ "$output" != *"--vram-kv-mib"* ]] } -@test "the calib dry run opens the tap at 12M tokens and serves on 7770" { +@test "the calib dry run opens the tap at the 1.5M floor, on the latest pin, and serves on 7770" { run bash "$RUNNER" calib --dry-run [ "$status" -eq 0 ] flat="$(oneline)" - [[ "$flat" == *"-e RADIANCE_CALIB_TOKENS=12000000"* ]] + [[ "$flat" == *"-e RADIANCE_CALIB_TOKENS=1500000"* ]] [[ "$flat" == *"-e RADIANCE_CALIB_DIR=/data/calib"* ]] + # the tap producer is the campaign's own pin (latest radiance), not production's + [[ "$flat" == *"radiance@sha256:69add58f"* ]] + [[ "$flat" != *"radiance@sha256:d590e300"* ]] [[ "$flat" == *"--port 7770"* ]] [[ "$flat" == *":/data/calib "* ]] # calib serves; it does not record