Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fcf94562d1 | ||
|
|
3ef711f146 |
No files matched your search
@@ -0,0 +1,26 @@
|
||||
# /etc/udev/rules.d/98-halogen-npu-access.rules
|
||||
#
|
||||
# Why the NPU node is world-writable: `frank-dragoon` runs rootless podman under a
|
||||
# unit with `User=dank`, and the container's user namespace maps exactly one uid and
|
||||
# one gid (dank). `/dev/accel/accel0` is `root:render 0660`, so inside the container
|
||||
# the node reads `nobody:nogroup` and the engine dies with its own diagnosis:
|
||||
#
|
||||
# halogen npu: /dev/accel/accel0 is here, but this process cannot open it
|
||||
# (crw-rw---- nobody:nogroup).
|
||||
#
|
||||
# `--group-add keep-groups` is the vendor's advice and it does NOT work in this
|
||||
# shape: measured 2026-10-19 from the host as a `dank` shell that was already in
|
||||
# `render`, with and without the flag, the node still read `nobody:nogroup` and the
|
||||
# open was refused — an unmapped gid cannot be granted to a process inside a
|
||||
# namespace that does not map it. Same conclusion `99-comfyui-gpu-access.rules`
|
||||
# reached for `/dev/kfd` from the systemd-user-spawn direction: the mode is the only
|
||||
# spawn-context-independent door short of running the stack rootful.
|
||||
#
|
||||
# Blast radius: any process in this host's single uid domain (every rootless
|
||||
# container here shares it) can submit to the XDNA2 NPU. The NPU holds no user data
|
||||
# and no privileged context — the same trade already accepted for the GPU compute
|
||||
# nodes beside it. Installed by `deploy-remote.sh` when the NPU lane is armed and left
|
||||
# in place afterwards — the mode costs nothing while the lane is idle. `serve-frank-dragoon.sh`
|
||||
# refuses to arm the lane on a node that is not other-accessible, so a host that lost this
|
||||
# file fails loudly at boot instead of crash-looping.
|
||||
KERNEL=="accel[0-9]*", SUBSYSTEM=="accel", MODE="0666"
|
||||
@@ -1329,6 +1329,9 @@ deploy_anubis() {
|
||||
# port gate is the same as every cutover since 2026-09-16: nothing
|
||||
# starts until 7770 is released.
|
||||
LLM_DRAGOON_IMAGE="${LLM_DRAGOON_IMAGE:-ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a}"
|
||||
# The payload dir is its own twin because the NPU lane's containment gate below
|
||||
# reads it; every artifact path under it stays a full literal as usual.
|
||||
LLM_DRAGOON_MODEL_DIR="${LLM_DRAGOON_MODEL_DIR:-/home/dank/models/halogen-v2-abliterated}"
|
||||
LLM_DRAGOON_CKPT="${LLM_DRAGOON_CKPT:-/home/dank/models/halogen-v2-abliterated/qwen38-flash-next-v2-abliterated.hgn}"
|
||||
LLM_DRAGOON_CKPT_BYTES="${LLM_DRAGOON_CKPT_BYTES:-66687678432}"
|
||||
LLM_DRAGOON_NGRAM="${LLM_DRAGOON_NGRAM:-/home/dank/models/halogen-v2-abliterated/qwen38-flash-next-ngram.hgn}"
|
||||
@@ -1441,16 +1444,28 @@ deploy_llm_dragoon() {
|
||||
# payload dir. A staging dir outside it needs a nested bind whose
|
||||
# mountpoint runc cannot create through a read-only parent, so the
|
||||
# parity twin of the serve gate refuses the shape before the restart.
|
||||
npu_model_dir="$(sed -nE 's/^FRANK_DRAGOON_MODEL_DIR=(.*)$/\1/p' \
|
||||
"$CHECKOUT/scripts/frank-dragoon.env" | tail -1)"
|
||||
if [[ -n "$npu_model_dir" && "$LLM_DRAGOON_NPU_DIR/" != "$npu_model_dir"/* ]]; then
|
||||
echo "ERROR: NPU staging dir $LLM_DRAGOON_NPU_DIR is outside the payload dir $npu_model_dir — the lane's files reach the container only through the read-only /models mount" >&2
|
||||
if [[ "$LLM_DRAGOON_NPU_DIR/" != "$LLM_DRAGOON_MODEL_DIR"/* ]]; then
|
||||
echo "ERROR: NPU staging dir $LLM_DRAGOON_NPU_DIR is outside the payload dir $LLM_DRAGOON_MODEL_DIR — the lane's files reach the container only through the read-only /models mount" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -e "$LLM_DRAGOON_DEV_ACCEL" ]]; then
|
||||
echo "ERROR: NPU node $LLM_DRAGOON_DEV_ACCEL absent — amdxdna is not bound (is iommu=pt still on the boot line?)" >&2
|
||||
return 1
|
||||
fi
|
||||
# Access, not existence: a rootless container cannot open a root:render
|
||||
# 0660 node (its namespace maps one uid/gid, and keep-groups cannot grant
|
||||
# an unmapped gid — measured 2026-10-19). Install the mode rule, re-trigger
|
||||
# the node, then PROVE the node is other-accessible before the restart.
|
||||
run_privileged install -m 0644 dankclaw/udev/98-halogen-npu-access.rules \
|
||||
/etc/udev/rules.d/98-halogen-npu-access.rules
|
||||
run_privileged udevadm control --reload-rules
|
||||
run_privileged udevadm trigger --sysname-match="$(basename "$LLM_DRAGOON_DEV_ACCEL")"
|
||||
# (`-o FILE` is not "other-writable" — it means "exists", and `-o` is also
|
||||
# the OR operator inside test), so the bit is read straight off the mode.
|
||||
if [[ $(( 8#$(stat -c '%a' "$LLM_DRAGOON_DEV_ACCEL") & 8#2 )) -eq 0 ]]; then
|
||||
echo "ERROR: $LLM_DRAGOON_DEV_ACCEL is still $(stat -c '%a %U:%G' "$LLM_DRAGOON_DEV_ACCEL") after installing the access rule — the lane cannot open it rootless" >&2
|
||||
return 1
|
||||
fi
|
||||
for xl in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||||
if [[ ! -e "$LLM_DRAGOON_XRT_DIR/$xl" ]]; then
|
||||
echo "ERROR: XRT NPU library $LLM_DRAGOON_XRT_DIR/$xl absent (Ubuntu libxrt2 + libxrt-npu2)" >&2
|
||||
|
||||
@@ -165,6 +165,18 @@ if [ -n "$NPU_MODELS" ]; then
|
||||
echo "frank-dragoon: $DEV_ACCEL missing — the NPU lane needs the amdxdna node (is iommu=pt still on the boot line?)" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Existence is not access. This endpoint runs rootless, so the container's
|
||||
# namespace maps exactly one uid/gid (dank) and a root:render 0660 node reads
|
||||
# `nobody:nogroup` inside it — the engine then dies naming the node itself
|
||||
# (measured 2026-10-19). --group-add keep-groups cannot help: an unmapped gid
|
||||
# cannot be granted into a namespace that does not map it, verified with and
|
||||
# without the flag from a shell already in `render`. The door is the udev mode.
|
||||
# NB: `[ -o FILE ]` is NOT "other-writable" — it means "exists", and `-o` is
|
||||
# also the OR operator inside test — so the permission bit is read directly.
|
||||
if [ $(( 8#$(stat -c '%a' "$DEV_ACCEL") & 8#2 )) -eq 0 ]; then
|
||||
echo "frank-dragoon: $DEV_ACCEL is $(stat -c '%a %U:%G' "$DEV_ACCEL") — a rootless container cannot open it; install dankclaw/udev/98-halogen-npu-access.rules and re-trigger udev (sudo udevadm control --reload-rules && sudo udevadm trigger --sysname-match=accel0)" >&2
|
||||
exit 1
|
||||
fi
|
||||
for M in ${NPU_MODELS//,/ }; do
|
||||
if [ ! -d "$NPU_DIR/$M" ]; then
|
||||
echo "frank-dragoon: NPU model $M not staged in $NPU_DIR;" >&2
|
||||
|
||||
@@ -240,8 +240,15 @@ precondition that surfaces as an engine crash-loop is a gate that was never writ
|
||||
box as of 2026-10-19; the endpoint still runs with the lane **off** until that day's mount fix merges, and the first
|
||||
armed boot's evidence lands here with it:
|
||||
|
||||
1. `--device /dev/accel/accel0` plus `--group-add keep-groups` (the pattern ComfyUI uses for `/dev/kfd`; `dank` is
|
||||
already in `render`).
|
||||
1. `--device /dev/accel/accel0` **and** a node the container's namespace can open. `/dev/accel/accel0` is
|
||||
`root:render 0660` and a rootless namespace maps exactly one uid/gid, so the engine's own first words are
|
||||
*"this process cannot open it (crw-rw---- nobody:nogroup)"*. `--group-add keep-groups` — the vendor's advice and
|
||||
the ComfyUI pattern for `/dev/kfd` — is **dead in this shape**: measured 2026-10-19 from a `dank` shell already
|
||||
in `render`, with and without the flag the node stayed `nobody:nogroup` and the open was refused; an unmapped
|
||||
gid cannot be granted across a namespace that does not map it. The door is the udev mode
|
||||
(`dankclaw/udev/98-halogen-npu-access.rules`, installed and re-triggered by the deployer, `0666` like the GPU
|
||||
compute nodes beside it); the serve script refuses any node that is not other-accessible, so the fault cannot
|
||||
return as a crash-loop.
|
||||
2. The host's XRT reachable inside: there is **no `/opt/xilinx/xrt`** here — distro packages put the three files the
|
||||
container needs (`libxrt_coreutil.so.2`, `libxrt_core.so.2`, `libxrt_driver_xdna.so.2`) in
|
||||
`/usr/lib/x86_64-linux-gnu`, where all three exist.
|
||||
|
||||
@@ -30,6 +30,7 @@ setup() {
|
||||
HOST_UNIT="$ROOT/dankclaw/host/halogen-fabric-clock"
|
||||
HOST_SERVICE="$ROOT/dankclaw/host/halogen-fabric-clock.service"
|
||||
NPU_FETCH="$ROOT/scripts/download-frank-dragoon-npu.sh"
|
||||
NPU_RULE="$ROOT/dankclaw/udev/98-halogen-npu-access.rules"
|
||||
BUILD="$ROOT/scripts/build-rocmfpx.sh"
|
||||
TEST_TMP="$(mktemp -d)"
|
||||
touch "$TEST_TMP/empty.env"
|
||||
@@ -357,6 +358,9 @@ $(cat "$TEST_TMP/serve.log")"
|
||||
npu_models="$(sed -nE 's/^HALOGEN_NPU_MODELS=(.*)$/\1/p' "$ENV_FILE")"
|
||||
grep -qF "LLM_DRAGOON_NPU_DIR=\"\${LLM_DRAGOON_NPU_DIR:-$npu_dir}\"" "$DEPLOYER"
|
||||
grep -qF "LLM_DRAGOON_NPU_MODELS=\"\${LLM_DRAGOON_NPU_MODELS:-$npu_models}\"" "$DEPLOYER"
|
||||
# The deployer's own copy of the payload dir is what the lane's containment
|
||||
# gate compares against — a stale twin would wave a bad staging path through.
|
||||
grep -qF "LLM_DRAGOON_MODEL_DIR=\"\${LLM_DRAGOON_MODEL_DIR:-$dir}\"" "$DEPLOYER"
|
||||
net="$(sed -nE 's/^FRANK_DRAGOON_NETWORK=(.*)$/\1/p' "$ENV_FILE")"
|
||||
grep -qF "LLM_DRAGOON_NETWORK=\"\${LLM_DRAGOON_NETWORK:-$net}\"" "$DEPLOYER"
|
||||
}
|
||||
@@ -415,7 +419,10 @@ $(cat "$TEST_TMP/serve.log")"
|
||||
# refused BY NAME instead of surfacing as an engine crash-loop.
|
||||
|
||||
npu_fixture() {
|
||||
# 0666 like the installed node: a rootless container's namespace cannot open
|
||||
# the stock root:render 0660 mode, and the serve gate checks exactly that.
|
||||
touch "$TEST_TMP/accel0"
|
||||
chmod 666 "$TEST_TMP/accel0"
|
||||
mkdir -p "$TEST_TMP/xrt"
|
||||
for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||||
touch "$TEST_TMP/xrt/$l"
|
||||
@@ -523,6 +530,32 @@ $(cat "$TEST_TMP/serve.log")"
|
||||
[[ "$output" == *"iommu=pt"* ]]
|
||||
}
|
||||
|
||||
@test "the lane refuses an NPU node the container namespace could not open, and names the rule" {
|
||||
# Stock Debian mode is root:render 0660. The engine's own first words on such
|
||||
# a node are "this process cannot open it (crw-rw---- nobody:nogroup)" and no
|
||||
# flag fixes a mode an unmapped gid cannot cross (keep-groups measured dead
|
||||
# 2026-10-19), so the unit refuses instead of crash-looping 189 times.
|
||||
podman_stub
|
||||
good_env
|
||||
npu_fixture
|
||||
chmod 660 "$TEST_TMP/accel0"
|
||||
run_with_log
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"$TEST_TMP/accel0 is 660"* ]]
|
||||
[[ "$output" == *"98-halogen-npu-access.rules"* ]]
|
||||
[[ "$output" != *"--device"* ]]
|
||||
}
|
||||
|
||||
@test "the NPU access rule is the accel counterpart of the GPU compute rule" {
|
||||
[ -f "$NPU_RULE" ]
|
||||
grep -qE '^KERNEL=="accel\[0-9\]\*", SUBSYSTEM=="accel", MODE="0666"$' "$NPU_RULE"
|
||||
# The deployer installs the rule and re-triggers the node BEFORE it trusts
|
||||
# the mode, so a machine that never had it self-heals instead of refusing.
|
||||
grep -qE '98-halogen-npu-access\.rules' "$DEPLOYER"
|
||||
grep -qE 'udevadm control --reload-rules' "$DEPLOYER"
|
||||
grep -qE 'udevadm trigger --sysname-match=' "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the lane refuses when an XRT NPU library is missing" {
|
||||
podman_stub
|
||||
good_env
|
||||
|
||||
Reference in new issue
Block a user