diff --git a/scripts/frank-dragoon.env b/scripts/frank-dragoon.env index c662000..89548dd 100644 --- a/scripts/frank-dragoon.env +++ b/scripts/frank-dragoon.env @@ -114,10 +114,13 @@ HALOGEN_NPU_MODELS=qwen3-embedding-0.6b FRANK_DRAGOON_NPU_DIR=/home/dank/models/halogen-v2-abliterated/npu # Node paths and the XRT directory are variables so the bats seam can drive # these gates on a box with no NPU. Ubuntu's distro XRT (libxrt2 + libxrt-npu2) -# is mounted FILE BY FILE and TWICE EACH — once where the image looks for XRT, -# once at its own path, because those libraries find each other at their install -# path; the vendor forbids mounting an /opt/xilinx/xrt of symlinks into the -# system dir, which dangle inside the container. +# is mounted FILE BY FILE and THREE TIMES EACH — where the image looks for XRT +# (/opt/xilinx/xrt/lib), at its own path (those libraries find each other at +# their install path), and at /opt/xilinx/lib// (the NPU child +# searches that third prefix; without it the lane dies naming it and the +# supervisor takes the container down — 2026-10-21 boot loop). The basename of +# this directory IS the multiarch name. The vendor forbids mounting an +# /opt/xilinx/xrt of symlinks into the system dir, which dangle in the container. FRANK_DRAGOON_DEV_ACCEL=/dev/accel/accel0 FRANK_DRAGOON_XRT_DIR=/usr/lib/x86_64-linux-gnu # sysfs root the fabric-clock preflight reads (same knob the vendor's diff --git a/scripts/serve-frank-dragoon.sh b/scripts/serve-frank-dragoon.sh index cb5e6c9..e2fea17 100755 --- a/scripts/serve-frank-dragoon.sh +++ b/scripts/serve-frank-dragoon.sh @@ -136,10 +136,17 @@ fi # The engine re-checks size+sha256 against that catalog at boot; it cannot # fetch anything here because /models is read-only and HALOGEN_DOWNLOAD # stays unset — no outbound connection at boot), -# 2. the distro XRT's three libraries, each mounted twice (where the image -# looks for XRT, and at its own path — the libraries find each other there; -# an /opt/xilinx/xrt of symlinks into the system dir dangles inside the -# container, which the vendor calls out explicitly), +# 2. the distro XRT's three libraries, each mounted THREE times: where the +# image looks for XRT (/opt/xilinx/xrt/lib), at their own path (the +# libraries find each other there), and at /opt/xilinx/lib/ +# — the NPU child searches that third prefix too, and since the host's +# libxrt2 was reconfigured 2026-10-20 it is the ONLY prefix the child +# names: omitting it crashes the lane with `No such library +# '/opt/xilinx/lib/x86_64-linux-gnu/libxrt_core.so.2'` and the engine +# supervisor takes the dead lane down on the whole container (measured +# 2026-10-21: 88-restart boot loop). Real files only — an /opt/xilinx/xrt +# of symlinks into the system dir dangles inside the container, which the +# vendor calls out explicitly), # 3. the GPU fabric clock HELD: on Strix Halo the iGPU and the NPU co-running # hang the machine while FCLK changes speed, and the engine refuses to # start when it moves — it cannot read host sysfs from inside the @@ -190,7 +197,9 @@ if [ -n "$NPU_MODELS" ]; then exit 1 fi REAL="$(readlink -f "$XRT_DIR/$L")" - NPU_ARGS+=(-v "$REAL:/opt/xilinx/xrt/lib/$L:ro" -v "$REAL:$XRT_DIR/$L:ro") + MULTIARCH="${XRT_DIR##*/}" + NPU_ARGS+=(-v "$REAL:/opt/xilinx/xrt/lib/$L:ro" -v "$REAL:$XRT_DIR/$L:ro" + -v "$REAL:/opt/xilinx/lib/$MULTIARCH/$L:ro") done HELD=0 for D in "$SYSFS"/class/drm/card*/device; do diff --git a/serverAppConfigs-docs/LLM/frank-dragoon.md b/serverAppConfigs-docs/LLM/frank-dragoon.md index a0077fc..04c8424 100644 --- a/serverAppConfigs-docs/LLM/frank-dragoon.md +++ b/serverAppConfigs-docs/LLM/frank-dragoon.md @@ -256,9 +256,14 @@ lane is **armed and serving** since 2026-10-19 (`HALOGEN_NPU_MODELS=qwen3-embedd (`dankclaw/udev/98-halogen-npu-access.rules`, installed and re-triggered by the deployer, `0666` like the GPU compute nodes beside it); the serve script refuses any node that is not other-accessible, so the fault cannot return as a crash-loop. -2. The host's XRT reachable inside: there is **no `/opt/xilinx/xrt`** here — distro packages put the three files the - container needs (`libxrt_coreutil.so.2`, `libxrt_core.so.2`, `libxrt_driver_xdna.so.2`) in - `/usr/lib/x86_64-linux-gnu`, where all three exist. +2. The host's XRT reachable inside, at **THREE prefixes per library**: there is **no `/opt/xilinx/xrt`** here — + distro packages put the three files the container needs (`libxrt_coreutil.so.2`, `libxrt_core.so.2`, + `libxrt_driver_xdna.so.2`) in `/usr/lib/x86_64-linux-gnu`. The serve script mounts each REAL file (never a + symlink) at `/opt/xilinx/xrt/lib/`, at its own path, AND at `/opt/xilinx/lib//` — the NPU child + searches that third prefix and a missing one is fatal: since the host's `libxrt2` was reconfigured + 2026-10-20 the child died with `No such library '/opt/xilinx/lib/x86_64-linux-gnu/libxrt_core.so.2'` and the + engine supervisor took the WHOLE container down on it (88-restart loop, chat dead beside a healthy GPU line). + A dead lane is not survivable here — arm the lane only with all three prefixes mounted. 3. **The GPU fabric clock held at its top level, or the engine refuses to start.** GPU and NPU work share the die: while `pp_dpm_fclk` is changing speed, concurrent GPU work can hang the machine and corrupt NPU results. The engine holds the clock itself only when it runs as root with `/sys` mounted; `frank-dragoon` is a **rootless** diff --git a/tests/frank_dragoon_unit.bats b/tests/frank_dragoon_unit.bats index 715fd63..e8fcfb2 100755 --- a/tests/frank_dragoon_unit.bats +++ b/tests/frank_dragoon_unit.bats @@ -430,9 +430,12 @@ npu_fixture() { # the stock root:render 0660 mode, and the serve gate checks exactly that. touch "$TEST_TMP/accel0" chmod 666 "$TEST_TMP/accel0" - mkdir -p "$TEST_TMP/xrt" + # basename carries a multiarch shape: the third mount prefix is derived + # from the XRT dir's basename (the NPU child searches + # /opt/xilinx/lib//, measured 2026-10-21). + mkdir -p "$TEST_TMP/x86_64-linux-gnu" for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do - touch "$TEST_TMP/xrt/$l" + touch "$TEST_TMP/x86_64-linux-gnu/$l" done mkdir -p "$TEST_TMP/npu/qwen3-embedding-0.6b/devices" mkdir -p "$TEST_TMP/sys/class/drm/card1/device" @@ -442,7 +445,7 @@ npu_fixture() { "HALOGEN_NPU_MODELS=qwen3-embedding-0.6b" "FRANK_DRAGOON_NPU_DIR=$TEST_TMP/npu" "FRANK_DRAGOON_DEV_ACCEL=$TEST_TMP/accel0" - "FRANK_DRAGOON_XRT_DIR=$TEST_TMP/xrt" + "FRANK_DRAGOON_XRT_DIR=$TEST_TMP/x86_64-linux-gnu" "FRANK_DRAGOON_SYSFS=$TEST_TMP/sys" ) } @@ -472,11 +475,15 @@ $(cat "$TEST_TMP/serve.log")" ! grep -q 'accel0' "$TEST_TMP/serve.log" } -@test "the lane arms the accel node, the staged model dir and each XRT library twice" { - # Each library mounts at the image's XRT path AND at its own path: the - # distro libraries find each other where they were installed, and an - # /opt/xilinx/xrt of symlinks into the system dir dangles inside the - # container (the vendor calls that out by name). +@test "the lane arms the accel node, the staged model dir and each XRT library three times" { + # Each library mounts at the image's XRT path, at its own path (the distro + # libraries find each other where they were installed), and at + # /opt/xilinx/lib// — the prefix the NPU child searches and + # names when it is missing (`No such library + # '/opt/xilinx/lib/x86_64-linux-gnu/libxrt_core.so.2'`, measured + # 2026-10-21: the lane died and the supervisor took the whole container + # with it, 88-restart loop). An /opt/xilinx/xrt of symlinks into the + # system dir dangles inside the container (the vendor calls that out). podman_stub good_env npu_fixture @@ -491,8 +498,9 @@ $(cat "$TEST_TMP/serve.log")" [[ "$output" != *":/models/npu"* ]] [[ "$output" == *"-e HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"* ]] for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do - [[ "$output" == *"-v $TEST_TMP/xrt/$l:/opt/xilinx/xrt/lib/$l:ro"* ]] - [[ "$output" == *"-v $TEST_TMP/xrt/$l:$TEST_TMP/xrt/$l:ro"* ]] + [[ "$output" == *"-v $TEST_TMP/x86_64-linux-gnu/$l:/opt/xilinx/xrt/lib/$l:ro"* ]] + [[ "$output" == *"-v $TEST_TMP/x86_64-linux-gnu/$l:$TEST_TMP/x86_64-linux-gnu/$l:ro"* ]] + [[ "$output" == *"-v $TEST_TMP/x86_64-linux-gnu/$l:/opt/xilinx/lib/x86_64-linux-gnu/$l:ro"* ]] done # The Flash payload still rides the same container: the lane is additive. [[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]] @@ -567,7 +575,7 @@ $(cat "$TEST_TMP/serve.log")" podman_stub good_env npu_fixture - rm "$TEST_TMP/xrt/libxrt_driver_xdna.so.2" + rm "$TEST_TMP/x86_64-linux-gnu/libxrt_driver_xdna.so.2" run_with_log [ "$status" -ne 0 ] [[ "$output" == *"libxrt_driver_xdna.so.2 missing"* ]]