From b4a1e6069e788f0a52082091196ba5a1d964d946 Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 30 Oct 2026 14:18:03 -0700 Subject: [PATCH] fix(qwen38): pin calib to 1.2.0, gate the tap's disk, kill orphaned engines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 17-hour calibration tap on 2026-10-29/30 wrote ZERO bytes: the runner booted FRANK_ARCHON_IMAGE (1.8.1) for every mode, and RADIANCE_CALIB_DIR/TOKENS only arms the expert tap on the 1.2.x line. 1.8.1 still carries the plugin strings and the "calibration wrote layer Hessians" message, /health answered, /stats showed 1,700 documents and 118 tok/s, and nothing anywhere said the tap was dead. Four gates, each with the measurement that motivated it: - CALIB_IMAGE pins the 1.2.0 digest for calib mode (stilldeadcode/radiance @sha256:665775ed…), overridable only by naming --image deliberately. Proven by the engagement test: `calibration wrote layer Hessians after 10207 tokens`. - The tap dir's filesystem must have >= 60 GiB free (Q38_CALIB_MIN_FREE_MIB). One seal wrote 39.4 GiB in seconds; pointed at a 37 GiB root it filled the filesystem to zero and every later write came back short. - restore() kills orphaned `radiance` processes still holding /dev/kfd before it starts the service. `podman rm -f` does not kill the engine: an orphan made production's ar_oneshot_2rank_wht6 init fail `device error`, and the unit restarted 100 times with /health never answering. - FRANK_QWEN38_CALIB_DIR / Q38_WINDOW_TRASH_ROOT make the tap dir and the bank target configurable, so the tap lives on /mnt/entropy instead of the root fs. --- scripts/qwen38-calib-window.sh | 14 ++++++++++-- scripts/run-qwen38-bf16.sh | 39 ++++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/scripts/qwen38-calib-window.sh b/scripts/qwen38-calib-window.sh index 15e3c51..5399fc8 100755 --- a/scripts/qwen38-calib-window.sh +++ b/scripts/qwen38-calib-window.sh @@ -36,7 +36,7 @@ CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}" RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}" CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}" KLDREF="$CAMPAIGN/kld-ref/kld.json" -CALIB="$CAMPAIGN/calib" +CALIB="${FRANK_QWEN38_CALIB_DIR:-$CAMPAIGN/calib}" HEALTH_TIMEOUT="${Q38_WINDOW_HEALTH_TIMEOUT:-3600}" ROUNDS="${Q38_WINDOW_ROUNDS:-6}" RIG=qwen38-bf16 @@ -49,6 +49,16 @@ PY="${PY:-python3}" [ -f "$CORPUS" ] || { echo "REFUSED: no calibration corpus at $CORPUS" >&2; exit 2; } [ -d "$CALIB" ] || { echo "REFUSED: no tap dir at $CALIB" >&2; exit 2; } [ -w "$CALIB" ] || { echo "REFUSED: tap dir $CALIB is not writable by $(id -un)" >&2; exit 2; } +# One tap seal writes ~40 GiB (measured 2026-10-30). The runner refuses too; this +# names the same number before a card is touched, so a full volume never becomes +# a 17-hour dead run. +avail_mib=$(df --output=avail -B 1M "$CALIB" 2>/dev/null | tail -1 | tr -d ' ') +[[ "$avail_mib" =~ ^[0-9]+$ ]] || avail_mib=0 +need_mib="${Q38_CALIB_MIN_FREE_MIB:-61440}" +[ "$avail_mib" -ge "$need_mib" ] || { + echo "REFUSED: tap dir $CALIB has ${avail_mib} MiB free; one seal writes ~40 GiB and this window needs ${need_mib} MiB (Q38_CALIB_MIN_FREE_MIB overrides)" >&2 + exit 2 +} ! [[ "$HEALTH_TIMEOUT" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_HEALTH_TIMEOUT must be seconds (got: $HEALTH_TIMEOUT)" >&2; exit 2; } ! [[ "$ROUNDS" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_ROUNDS must be a round count (got: $ROUNDS)" >&2; exit 2; } @@ -115,7 +125,7 @@ say "TAP BANKED: $(ls -1 "$CALIB" | wc -l) file(s) in $CALIB" if [ "${Q38_WINDOW_KEEP_RAD:-0}" = "1" ]; then say "reference .rad kept (Q38_WINDOW_KEEP_RAD=1)" else - T="$HOME/.AGENTTRASH/$(date +%Y%m%d)-qwen38-bf16-rad" + T="${Q38_WINDOW_TRASH_ROOT:-$HOME/.AGENTTRASH}/$(date +%Y%m%d)-qwen38-bf16-rad" mkdir -p "$T" || { echo "FAIL: cannot create trash dir $T" >&2; exit 1; } mv "$RAD" "$T/" && say "reference .rad banked in $T (the checkpoint that rebuilds it stays)" fi diff --git a/scripts/run-qwen38-bf16.sh b/scripts/run-qwen38-bf16.sh index e1db5a9..5527b60 100755 --- a/scripts/run-qwen38-bf16.sh +++ b/scripts/run-qwen38-bf16.sh @@ -113,6 +113,13 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1091 source "$SCRIPT_DIR/frank-archon.env" IMAGE="${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}" +# THE CALIBRATION PIN. RADIANCE_CALIB_DIR/TOKENS arms the expert tap on the 1.2.x +# line only. 1.8.1 still carries every plugin string and the seal message, and a +# 17-hour run on it wrote ZERO bytes into the tap dir while /health, /stats and +# the push all looked healthy (measured 2026-10-30) — so calib boots THIS digest +# and any other image must be named deliberately with --image. +CALIB_IMAGE="${Q38_CALIB_IMAGE:-stilldeadcode/radiance@sha256:665775ed1d90de6dbb25558e07fc60d20328f1346f96f3c6e00845eb763f01fa}" +IMAGE_EXPLICIT=0 MODE="" POS=() @@ -122,7 +129,7 @@ while [ $# -gt 0 ]; do kldref|calib|score) MODE="$1" ;; --dry-run) DRY=1 ;; --image) [ "$#" -ge 2 ] || { echo "--image needs a reference" >&2; exit 2; } - IMAGE="$2"; shift ;; + IMAGE="$2"; IMAGE_EXPLICIT=1; shift ;; *) # score's two positional arguments, and nothing else may be bare. if [ "$MODE" = score ] && [ "${#POS[@]}" -lt 2 ]; then @@ -152,7 +159,7 @@ TEMPLATE="$SCRIPT_DIR/qwen38-froggeric-v22.5.jinja" LOG_DIR="${FRANK_QWEN38_LOG_DIR:-/home/dank/logs}" LOG="$LOG_DIR/qwen38_bf16_${MODE}.log" KLD_REF_DIR="$CAMPAIGN/kld-ref" -CALIB_DIR="$CAMPAIGN/calib" +CALIB_DIR="${FRANK_QWEN38_CALIB_DIR:-$CAMPAIGN/calib}" C=qwen38-bf16 SERVICE=frank-archon # The card split, taken from the file that owns it instead of a private pair: @@ -314,6 +321,21 @@ case "$MODE" in # buys throughput with the expert slab, so price it like any other # VRAM knob. The slots follow frank-archon.env's FRANK_ARCHON_MAX_SEQS # through MAX_SEQS, exactly as kldref and score do. + [ "$IMAGE_EXPLICIT" = 1 ] || IMAGE="$CALIB_IMAGE" + # ONE TAP SEAL WRITES ~40 GiB. Measured 2026-10-30: a 8,192-token floor wrote + # 39.4 GiB into the tap dir within seconds; pointed at a 37 GiB root it filled + # the filesystem to zero, after which every write came back short ("calibration + # cannot write", "write was short") and the system itself began refusing + # writes. The room is demanded up front, on the tap's own filesystem. + if [ "$DRY" != 1 ]; then + avail_mib=$(df --output=avail -B 1M "$CALIB_DIR" 2>/dev/null | tail -1 | tr -d ' ') + [[ "$avail_mib" =~ ^[0-9]+$ ]] || avail_mib=0 + need_mib="${Q38_CALIB_MIN_FREE_MIB:-61440}" + if [ "$avail_mib" -lt "$need_mib" ]; then + echo "REFUSED: tap dir $CALIB_DIR has ${avail_mib} MiB free; one seal writes ~40 GiB and this run needs ${need_mib} MiB (Q38_CALIB_MIN_FREE_MIB overrides)" >&2 + exit 2 + fi + fi ARGS+=(--model "$MODEL_IN" --port 7770 --max-num-seqs "$CALIB_SEQS" --max-num-batched-tokens "$CALIB_MNBT") # The tap is only real if this directory exists and the engine can write it: @@ -417,6 +439,19 @@ cp -f "$TEMPLATE" "$TPL" || { # where the service reported `active` while its engine starved on the rig's VRAM. restore() { podman rm -f "$C" >/dev/null 2>&1 || true + # Removing the container does NOT kill its engine. Measured 2026-10-30: a rig + # radiance kept /dev/kfd after `podman rm -f`, production's + # ar_oneshot_2rank_wht6 init then failed `device error`, the engine exited 0, + # and Restart=always looped it 100 times with /health never answering. The + # service is stopped for the whole run, so any radiance still holding the + # device here is an orphan — kill it BY PID (pkill -f matches the caller's + # own command line, which is how a session gets killed instead of the orphan). + for p in $(fuser /dev/kfd 2>/dev/null); do + if [ "$(cat "/proc/$p/comm" 2>/dev/null)" = radiance ]; then + echo "restore: killing orphaned radiance pid $p still holding /dev/kfd" >&2 + kill -9 "$p" 2>/dev/null || true + fi + done sudo systemctl start "$SERVICE" >/dev/null 2>&1 || true } trap restore EXIT