feat(frank-dragoon): open the NPU node the way rootless can actually reach it #46
@@ -0,0 +1,26 @@
|
||||
# /etc/udev/rules.d/98-halogen-npu-access.rules
|
||||
#
|
||||
# Why the NPU node is world-writable: `frank-dragoon` runs rootless podman under a
|
||||
# unit with `User=dank`, and the container's user namespace maps exactly one uid and
|
||||
# one gid (dank). `/dev/accel/accel0` is `root:render 0660`, so inside the container
|
||||
# the node reads `nobody:nogroup` and the engine dies with its own diagnosis:
|
||||
#
|
||||
# halogen npu: /dev/accel/accel0 is here, but this process cannot open it
|
||||
# (crw-rw---- nobody:nogroup).
|
||||
#
|
||||
# `--group-add keep-groups` is the vendor's advice and it does NOT work in this
|
||||
# shape: measured 2026-10-19 from the host as a `dank` shell that was already in
|
||||
# `render`, with and without the flag, the node still read `nobody:nogroup` and the
|
||||
# open was refused — an unmapped gid cannot be granted to a process inside a
|
||||
# namespace that does not map it. Same conclusion `99-comfyui-gpu-access.rules`
|
||||
# reached for `/dev/kfd` from the systemd-user-spawn direction: the mode is the only
|
||||
# spawn-context-independent door short of running the stack rootful.
|
||||
#
|
||||
# Blast radius: any process in this host's single uid domain (every rootless
|
||||
# container here shares it) can submit to the XDNA2 NPU. The NPU holds no user data
|
||||
# and no privileged context — the same trade already accepted for the GPU compute
|
||||
# nodes beside it. Installed by `deploy-remote.sh` when the NPU lane is armed and left
|
||||
# in place afterwards — the mode costs nothing while the lane is idle. `serve-frank-dragoon.sh`
|
||||
# refuses to arm the lane on a node that is not other-accessible, so a host that lost this
|
||||
# file fails loudly at boot instead of crash-looping.
|
||||
KERNEL=="accel[0-9]*", SUBSYSTEM=="accel", MODE="0666"
|
||||
@@ -1452,6 +1452,20 @@ deploy_llm_dragoon() {
|
||||
echo "ERROR: NPU node $LLM_DRAGOON_DEV_ACCEL absent — amdxdna is not bound (is iommu=pt still on the boot line?)" >&2
|
||||
return 1
|
||||
fi
|
||||
# Access, not existence: a rootless container cannot open a root:render
|
||||
# 0660 node (its namespace maps one uid/gid, and keep-groups cannot grant
|
||||
# an unmapped gid — measured 2026-10-19). Install the mode rule, re-trigger
|
||||
# the node, then PROVE the node is other-accessible before the restart.
|
||||
run_privileged install -m 0644 dankclaw/udev/98-halogen-npu-access.rules \
|
||||
/etc/udev/rules.d/98-halogen-npu-access.rules
|
||||
run_privileged udevadm control --reload-rules
|
||||
run_privileged udevadm trigger --sysname-match="$(basename "$LLM_DRAGOON_DEV_ACCEL")"
|
||||
# (`-o FILE` is not "other-writable" — it means "exists", and `-o` is also
|
||||
# the OR operator inside test), so the bit is read straight off the mode.
|
||||
if [[ $(( 8#$(stat -c '%a' "$LLM_DRAGOON_DEV_ACCEL") & 8#2 )) -eq 0 ]]; then
|
||||
echo "ERROR: $LLM_DRAGOON_DEV_ACCEL is still $(stat -c '%a %U:%G' "$LLM_DRAGOON_DEV_ACCEL") after installing the access rule — the lane cannot open it rootless" >&2
|
||||
return 1
|
||||
fi
|
||||
for xl in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||||
if [[ ! -e "$LLM_DRAGOON_XRT_DIR/$xl" ]]; then
|
||||
echo "ERROR: XRT NPU library $LLM_DRAGOON_XRT_DIR/$xl absent (Ubuntu libxrt2 + libxrt-npu2)" >&2
|
||||
|
||||
@@ -165,6 +165,18 @@ if [ -n "$NPU_MODELS" ]; then
|
||||
echo "frank-dragoon: $DEV_ACCEL missing — the NPU lane needs the amdxdna node (is iommu=pt still on the boot line?)" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Existence is not access. This endpoint runs rootless, so the container's
|
||||
# namespace maps exactly one uid/gid (dank) and a root:render 0660 node reads
|
||||
# `nobody:nogroup` inside it — the engine then dies naming the node itself
|
||||
# (measured 2026-10-19). --group-add keep-groups cannot help: an unmapped gid
|
||||
# cannot be granted into a namespace that does not map it, verified with and
|
||||
# without the flag from a shell already in `render`. The door is the udev mode.
|
||||
# NB: `[ -o FILE ]` is NOT "other-writable" — it means "exists", and `-o` is
|
||||
# also the OR operator inside test — so the permission bit is read directly.
|
||||
if [ $(( 8#$(stat -c '%a' "$DEV_ACCEL") & 8#2 )) -eq 0 ]; then
|
||||
echo "frank-dragoon: $DEV_ACCEL is $(stat -c '%a %U:%G' "$DEV_ACCEL") — a rootless container cannot open it; install dankclaw/udev/98-halogen-npu-access.rules and re-trigger udev (sudo udevadm control --reload-rules && sudo udevadm trigger --sysname-match=accel0)" >&2
|
||||
exit 1
|
||||
fi
|
||||
for M in ${NPU_MODELS//,/ }; do
|
||||
if [ ! -d "$NPU_DIR/$M" ]; then
|
||||
echo "frank-dragoon: NPU model $M not staged in $NPU_DIR;" >&2
|
||||
|
||||
@@ -240,8 +240,15 @@ precondition that surfaces as an engine crash-loop is a gate that was never writ
|
||||
box as of 2026-10-19; the endpoint still runs with the lane **off** until that day's mount fix merges, and the first
|
||||
armed boot's evidence lands here with it:
|
||||
|
||||
1. `--device /dev/accel/accel0` plus `--group-add keep-groups` (the pattern ComfyUI uses for `/dev/kfd`; `dank` is
|
||||
already in `render`).
|
||||
1. `--device /dev/accel/accel0` **and** a node the container's namespace can open. `/dev/accel/accel0` is
|
||||
`root:render 0660` and a rootless namespace maps exactly one uid/gid, so the engine's own first words are
|
||||
*"this process cannot open it (crw-rw---- nobody:nogroup)"*. `--group-add keep-groups` — the vendor's advice and
|
||||
the ComfyUI pattern for `/dev/kfd` — is **dead in this shape**: measured 2026-10-19 from a `dank` shell already
|
||||
in `render`, with and without the flag the node stayed `nobody:nogroup` and the open was refused; an unmapped
|
||||
gid cannot be granted across a namespace that does not map it. The door is the udev mode
|
||||
(`dankclaw/udev/98-halogen-npu-access.rules`, installed and re-triggered by the deployer, `0666` like the GPU
|
||||
compute nodes beside it); the serve script refuses any node that is not other-accessible, so the fault cannot
|
||||
return as a crash-loop.
|
||||
2. The host's XRT reachable inside: there is **no `/opt/xilinx/xrt`** here — distro packages put the three files the
|
||||
container needs (`libxrt_coreutil.so.2`, `libxrt_core.so.2`, `libxrt_driver_xdna.so.2`) in
|
||||
`/usr/lib/x86_64-linux-gnu`, where all three exist.
|
||||
|
||||
@@ -30,6 +30,7 @@ setup() {
|
||||
HOST_UNIT="$ROOT/dankclaw/host/halogen-fabric-clock"
|
||||
HOST_SERVICE="$ROOT/dankclaw/host/halogen-fabric-clock.service"
|
||||
NPU_FETCH="$ROOT/scripts/download-frank-dragoon-npu.sh"
|
||||
NPU_RULE="$ROOT/dankclaw/udev/98-halogen-npu-access.rules"
|
||||
BUILD="$ROOT/scripts/build-rocmfpx.sh"
|
||||
TEST_TMP="$(mktemp -d)"
|
||||
touch "$TEST_TMP/empty.env"
|
||||
@@ -418,7 +419,10 @@ $(cat "$TEST_TMP/serve.log")"
|
||||
# refused BY NAME instead of surfacing as an engine crash-loop.
|
||||
|
||||
npu_fixture() {
|
||||
# 0666 like the installed node: a rootless container's namespace cannot open
|
||||
# the stock root:render 0660 mode, and the serve gate checks exactly that.
|
||||
touch "$TEST_TMP/accel0"
|
||||
chmod 666 "$TEST_TMP/accel0"
|
||||
mkdir -p "$TEST_TMP/xrt"
|
||||
for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||||
touch "$TEST_TMP/xrt/$l"
|
||||
@@ -526,6 +530,32 @@ $(cat "$TEST_TMP/serve.log")"
|
||||
[[ "$output" == *"iommu=pt"* ]]
|
||||
}
|
||||
|
||||
@test "the lane refuses an NPU node the container namespace could not open, and names the rule" {
|
||||
# Stock Debian mode is root:render 0660. The engine's own first words on such
|
||||
# a node are "this process cannot open it (crw-rw---- nobody:nogroup)" and no
|
||||
# flag fixes a mode an unmapped gid cannot cross (keep-groups measured dead
|
||||
# 2026-10-19), so the unit refuses instead of crash-looping 189 times.
|
||||
podman_stub
|
||||
good_env
|
||||
npu_fixture
|
||||
chmod 660 "$TEST_TMP/accel0"
|
||||
run_with_log
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"$TEST_TMP/accel0 is 660"* ]]
|
||||
[[ "$output" == *"98-halogen-npu-access.rules"* ]]
|
||||
[[ "$output" != *"--device"* ]]
|
||||
}
|
||||
|
||||
@test "the NPU access rule is the accel counterpart of the GPU compute rule" {
|
||||
[ -f "$NPU_RULE" ]
|
||||
grep -qE '^KERNEL=="accel\[0-9\]\*", SUBSYSTEM=="accel", MODE="0666"$' "$NPU_RULE"
|
||||
# The deployer installs the rule and re-triggers the node BEFORE it trusts
|
||||
# the mode, so a machine that never had it self-heals instead of refusing.
|
||||
grep -qE '98-halogen-npu-access\.rules' "$DEPLOYER"
|
||||
grep -qE 'udevadm control --reload-rules' "$DEPLOYER"
|
||||
grep -qE 'udevadm trigger --sysname-match=' "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the lane refuses when an XRT NPU library is missing" {
|
||||
podman_stub
|
||||
good_env
|
||||
|
||||
Reference in New Issue
Block a user