Merge pull request 'fix(qwen38): the calibration tap runs on latest radiance, not the production pin' (#87) from fix/quant-pipeline-hardening into main
This commit was merged in pull request #87.
This commit is contained in:
dank committed
commit
7b2b5b66fe
4 files changed
+74
-12
No files matched your search
@@ -47,7 +47,12 @@
|
||||
set -u -o pipefail
|
||||
|
||||
SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
IMAGE=${IMAGE:-stilldeadcode/radiance@sha256:d590e300234bf6d2f942a0b8d456d183153fd808f28820e6c2fa69c9389178ed}
|
||||
# THE TOOLCHAIN PIN. `:latest` on 2026-10-30 is v1.14.0 (revision 4a61d7296bc5),
|
||||
# the same image the tap producer boots: a tap must be converted by the line that
|
||||
# is current, and rad-convert's `gptq:calib=<dir>` input contract is version-stable
|
||||
# (RADGRAM2 + gram.r<rank>.<stem>.bin), so nothing else about the recipe moves with
|
||||
# it. Production stays on its own pin — this is the campaign's toolchain.
|
||||
IMAGE=${IMAGE:-stilldeadcode/radiance@sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093}
|
||||
CKPT=${CKPT:-/home/dank/models/Qwen3.8-Flash-Next}
|
||||
CAMPAIGN=${CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}
|
||||
RECIPE=${RECIPE:-$SELF/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe}
|
||||
|
||||
@@ -36,7 +36,7 @@ CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}"
|
||||
RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}"
|
||||
CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}"
|
||||
KLDREF="$CAMPAIGN/kld-ref/kld.json"
|
||||
CALIB="$CAMPAIGN/calib"
|
||||
CALIB="${FRANK_QWEN38_CALIB_DIR:-$CAMPAIGN/calib}"
|
||||
HEALTH_TIMEOUT="${Q38_WINDOW_HEALTH_TIMEOUT:-3600}"
|
||||
ROUNDS="${Q38_WINDOW_ROUNDS:-6}"
|
||||
RIG=qwen38-bf16
|
||||
@@ -49,6 +49,16 @@ PY="${PY:-python3}"
|
||||
[ -f "$CORPUS" ] || { echo "REFUSED: no calibration corpus at $CORPUS" >&2; exit 2; }
|
||||
[ -d "$CALIB" ] || { echo "REFUSED: no tap dir at $CALIB" >&2; exit 2; }
|
||||
[ -w "$CALIB" ] || { echo "REFUSED: tap dir $CALIB is not writable by $(id -un)" >&2; exit 2; }
|
||||
# One tap seal writes ~40 GiB (measured 2026-10-30). The runner refuses too; this
|
||||
# names the same number before a card is touched, so a full volume never becomes
|
||||
# a 17-hour dead run.
|
||||
avail_mib=$(df --output=avail -B 1M "$CALIB" 2>/dev/null | tail -1 | tr -d ' ')
|
||||
[[ "$avail_mib" =~ ^[0-9]+$ ]] || avail_mib=0
|
||||
need_mib="${Q38_CALIB_MIN_FREE_MIB:-61440}"
|
||||
[ "$avail_mib" -ge "$need_mib" ] || {
|
||||
echo "REFUSED: tap dir $CALIB has ${avail_mib} MiB free; one seal writes ~40 GiB and this window needs ${need_mib} MiB (Q38_CALIB_MIN_FREE_MIB overrides)" >&2
|
||||
exit 2
|
||||
}
|
||||
! [[ "$HEALTH_TIMEOUT" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_HEALTH_TIMEOUT must be seconds (got: $HEALTH_TIMEOUT)" >&2; exit 2; }
|
||||
! [[ "$ROUNDS" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_ROUNDS must be a round count (got: $ROUNDS)" >&2; exit 2; }
|
||||
|
||||
@@ -115,7 +125,7 @@ say "TAP BANKED: $(ls -1 "$CALIB" | wc -l) file(s) in $CALIB"
|
||||
if [ "${Q38_WINDOW_KEEP_RAD:-0}" = "1" ]; then
|
||||
say "reference .rad kept (Q38_WINDOW_KEEP_RAD=1)"
|
||||
else
|
||||
T="$HOME/.AGENTTRASH/$(date +%Y%m%d)-qwen38-bf16-rad"
|
||||
T="${Q38_WINDOW_TRASH_ROOT:-$HOME/.AGENTTRASH}/$(date +%Y%m%d)-qwen38-bf16-rad"
|
||||
mkdir -p "$T" || { echo "FAIL: cannot create trash dir $T" >&2; exit 1; }
|
||||
mv "$RAD" "$T/" && say "reference .rad banked in $T (the checkpoint that rebuilds it stays)"
|
||||
fi
|
||||
|
||||
@@ -25,13 +25,17 @@
|
||||
# through the ro /models mount); OUT lands on a writable
|
||||
# mount because --kld-out F also writes F.rows beside it.
|
||||
# calib serve on :7770 with the calibration tap open
|
||||
# (RADIANCE_CALIB_DIR + RADIANCE_CALIB_TOKENS=12000000). A
|
||||
# (RADIANCE_CALIB_DIR + RADIANCE_CALIB_TOKENS=1500000). A
|
||||
# separate driver — scripts/calib-push.py --port 7770
|
||||
# --container qwen38-bf16 --corpus
|
||||
# /home/dank/corpora/out/calib.jsonl — pushes the corpus; at
|
||||
# 12M pooled positions the engine writes the Grams and the
|
||||
# tap switches itself off (calib-push prints TAP_DONE). Stop
|
||||
# this script after that; the trap brings frank-archon back.
|
||||
# the floor the engine writes the Grams and the tap switches
|
||||
# itself off (calib-push prints TAP_DONE). Stop this script
|
||||
# after that; the trap brings frank-archon back. The vendor's
|
||||
# own measurement is the floor's warrant — "a pooled Gram
|
||||
# from 10 M tokens beat one from 1 M by under 1 %" — and at
|
||||
# this rig's measured push rate a 1.5 M floor is hours, not
|
||||
# the 29 h a 6 M floor cost. One seal writes ~40 GiB.
|
||||
#
|
||||
# Shape: the vendor's bf16 calibration lines — expert_tiered placement with the
|
||||
# weights disk tier, deterministic (the resident set pinned to the loader's plan,
|
||||
@@ -113,6 +117,18 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/frank-archon.env"
|
||||
IMAGE="${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}"
|
||||
# THE CALIBRATION PIN: the LATEST radiance, pinned by digest because a tag is a
|
||||
# moving target and a tap must name the producer that wrote it. `:latest` on
|
||||
# 2026-10-30 resolves to v1.14.0 (revision 4a61d7296bc5), and on that image a
|
||||
# calib boot prints the arming line — "recorded passes: off, the program
|
||||
# accumulates calibration Grams (gram_accum)" — where the production pin
|
||||
# (1.8.1) prints nothing of the kind: a 17-hour run on 1.8.1 wrote ZERO bytes
|
||||
# into the tap dir while /health, /stats and the push all looked healthy
|
||||
# (measured 2026-10-30). The tap code is identical in both trees' sources, so
|
||||
# the shipped 1.8.1 build is what drops it; do not spend another window on it.
|
||||
# Any other image must be named deliberately with --image.
|
||||
CALIB_IMAGE="${Q38_CALIB_IMAGE:-stilldeadcode/radiance@sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093}"
|
||||
IMAGE_EXPLICIT=0
|
||||
|
||||
MODE=""
|
||||
POS=()
|
||||
@@ -122,7 +138,7 @@ while [ $# -gt 0 ]; do
|
||||
kldref|calib|score) MODE="$1" ;;
|
||||
--dry-run) DRY=1 ;;
|
||||
--image) [ "$#" -ge 2 ] || { echo "--image needs a reference" >&2; exit 2; }
|
||||
IMAGE="$2"; shift ;;
|
||||
IMAGE="$2"; IMAGE_EXPLICIT=1; shift ;;
|
||||
*)
|
||||
# score's two positional arguments, and nothing else may be bare.
|
||||
if [ "$MODE" = score ] && [ "${#POS[@]}" -lt 2 ]; then
|
||||
@@ -152,7 +168,7 @@ TEMPLATE="$SCRIPT_DIR/qwen38-froggeric-v22.5.jinja"
|
||||
LOG_DIR="${FRANK_QWEN38_LOG_DIR:-/home/dank/logs}"
|
||||
LOG="$LOG_DIR/qwen38_bf16_${MODE}.log"
|
||||
KLD_REF_DIR="$CAMPAIGN/kld-ref"
|
||||
CALIB_DIR="$CAMPAIGN/calib"
|
||||
CALIB_DIR="${FRANK_QWEN38_CALIB_DIR:-$CAMPAIGN/calib}"
|
||||
C=qwen38-bf16
|
||||
SERVICE=frank-archon
|
||||
# The card split, taken from the file that owns it instead of a private pair:
|
||||
@@ -314,6 +330,21 @@ case "$MODE" in
|
||||
# buys throughput with the expert slab, so price it like any other
|
||||
# VRAM knob. The slots follow frank-archon.env's FRANK_ARCHON_MAX_SEQS
|
||||
# through MAX_SEQS, exactly as kldref and score do.
|
||||
[ "$IMAGE_EXPLICIT" = 1 ] || IMAGE="$CALIB_IMAGE"
|
||||
# ONE TAP SEAL WRITES ~40 GiB. Measured 2026-10-30: a 8,192-token floor wrote
|
||||
# 39.4 GiB into the tap dir within seconds; pointed at a 37 GiB root it filled
|
||||
# the filesystem to zero, after which every write came back short ("calibration
|
||||
# cannot write", "write was short") and the system itself began refusing
|
||||
# writes. The room is demanded up front, on the tap's own filesystem.
|
||||
if [ "$DRY" != 1 ]; then
|
||||
avail_mib=$(df --output=avail -B 1M "$CALIB_DIR" 2>/dev/null | tail -1 | tr -d ' ')
|
||||
[[ "$avail_mib" =~ ^[0-9]+$ ]] || avail_mib=0
|
||||
need_mib="${Q38_CALIB_MIN_FREE_MIB:-61440}"
|
||||
if [ "$avail_mib" -lt "$need_mib" ]; then
|
||||
echo "REFUSED: tap dir $CALIB_DIR has ${avail_mib} MiB free; one seal writes ~40 GiB and this run needs ${need_mib} MiB (Q38_CALIB_MIN_FREE_MIB overrides)" >&2
|
||||
exit 2
|
||||
fi
|
||||
fi
|
||||
ARGS+=(--model "$MODEL_IN" --port 7770
|
||||
--max-num-seqs "$CALIB_SEQS" --max-num-batched-tokens "$CALIB_MNBT")
|
||||
# The tap is only real if this directory exists and the engine can write it:
|
||||
@@ -326,7 +357,7 @@ case "$MODE" in
|
||||
}
|
||||
MOUNTS+=(-v "$CALIB_DIR:/data/calib")
|
||||
TAPENV=(-e RADIANCE_CALIB_DIR=/data/calib
|
||||
-e "RADIANCE_CALIB_TOKENS=${FRANK_QWEN38_CALIB_TOKENS:-12000000}")
|
||||
-e "RADIANCE_CALIB_TOKENS=${FRANK_QWEN38_CALIB_TOKENS:-1500000}")
|
||||
# The tap's own authorization. A pusher must see this marker before it sends
|
||||
# a prompt (calib-push.py --tap-marker), it names the container so a push
|
||||
# aimed anywhere else refuses, and a boot that fails removes it again below.
|
||||
@@ -417,6 +448,19 @@ cp -f "$TEMPLATE" "$TPL" || {
|
||||
# where the service reported `active` while its engine starved on the rig's VRAM.
|
||||
restore() {
|
||||
podman rm -f "$C" >/dev/null 2>&1 || true
|
||||
# Removing the container does NOT kill its engine. Measured 2026-10-30: a rig
|
||||
# radiance kept /dev/kfd after `podman rm -f`, production's
|
||||
# ar_oneshot_2rank_wht6 init then failed `device error`, the engine exited 0,
|
||||
# and Restart=always looped it 100 times with /health never answering. The
|
||||
# service is stopped for the whole run, so any radiance still holding the
|
||||
# device here is an orphan — kill it BY PID (pkill -f matches the caller's
|
||||
# own command line, which is how a session gets killed instead of the orphan).
|
||||
for p in $(fuser /dev/kfd 2>/dev/null); do
|
||||
if [ "$(cat "/proc/$p/comm" 2>/dev/null)" = radiance ]; then
|
||||
echo "restore: killing orphaned radiance pid $p still holding /dev/kfd" >&2
|
||||
kill -9 "$p" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
sudo systemctl start "$SERVICE" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap restore EXIT
|
||||
|
||||
@@ -157,12 +157,15 @@ flag_value() {
|
||||
[[ "$output" != *"--vram-kv-mib"* ]]
|
||||
}
|
||||
|
||||
@test "the calib dry run opens the tap at 12M tokens and serves on 7770" {
|
||||
@test "the calib dry run opens the tap at the 1.5M floor, on the latest pin, and serves on 7770" {
|
||||
run bash "$RUNNER" calib --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
flat="$(oneline)"
|
||||
[[ "$flat" == *"-e RADIANCE_CALIB_TOKENS=12000000"* ]]
|
||||
[[ "$flat" == *"-e RADIANCE_CALIB_TOKENS=1500000"* ]]
|
||||
[[ "$flat" == *"-e RADIANCE_CALIB_DIR=/data/calib"* ]]
|
||||
# the tap producer is the campaign's own pin (latest radiance), not production's
|
||||
[[ "$flat" == *"radiance@sha256:69add58f"* ]]
|
||||
[[ "$flat" != *"radiance@sha256:d590e300"* ]]
|
||||
[[ "$flat" == *"--port 7770"* ]]
|
||||
[[ "$flat" == *":/data/calib "* ]]
|
||||
# calib serves; it does not record
|
||||
|
||||
Reference in new issue
Block a user