fix(frank-dragoon): stage the NPU lane inside the payload mount

The lane's first boot died at container init, not at the engine: runc create failed: ... create mountpoint for /models/npu mount: mkdirat .../merged/models/npu: read-only file system The image ships no /models, and the serve script binds the payload dir there read-only, so a nested `-v $NPU_DIR:/models/npu:ro` had to create its own mountpoint underneath a read-only parent — impossible. The engine's own entrypoint confirms the container path is fixed: it rejects HALOGEN_NPU_DIR outright ("our NPU models live under /models/npu/<id>/"). Fix is the mount shape, not the engine: FRANK_DRAGOON_NPU_DIR now lives inside FRANK_DRAGOON_MODEL_DIR, so the payload's single read-only mount puts the files at /models/npu/<id>/ by itself and the nested bind is gone. Verified on the running container: after the move, `podman exec frank-dragoon-halogen ls /models/npu/qwen3-embedding-0.6b` answers with the staged tree while the lane is still armed off. Both gates refuse the old shape by name (serve + deployer parity) so a future payload swap that moves the model dir cannot reintroduce a crash that lands after the deployer has already stopped the running unit.
2026-10-19 21:42:31 -07:00
parent b1ca4e02fa
commit b19f1c49d6
5 changed files with 74 additions and 13 deletions
@@ -1345,7 +1345,7 @@ LLM_DRAGOON_TOK_FILES="${LLM_DRAGOON_TOK_FILES:-tokenizer.json chat_template.jin
 # hang the machine while the fabric clock moves, so halogen refuses to boot with
 # the lane armed unless that clock is held.
 LLM_DRAGOON_NPU_MODELS="${LLM_DRAGOON_NPU_MODELS:-qwen3-embedding-0.6b}"
-LLM_DRAGOON_NPU_DIR="${LLM_DRAGOON_NPU_DIR:-/home/dank/models/halogen-npu}"
+LLM_DRAGOON_NPU_DIR="${LLM_DRAGOON_NPU_DIR:-/home/dank/models/halogen-v2-abliterated/npu}"
 LLM_DRAGOON_DEV_ACCEL="${LLM_DRAGOON_DEV_ACCEL:-/dev/accel/accel0}"
 LLM_DRAGOON_XRT_DIR="${LLM_DRAGOON_XRT_DIR:-/usr/lib/x86_64-linux-gnu}"
 LLM_DRAGOON_NETWORK="${LLM_DRAGOON_NETWORK:-dankclaw_default}"
@@ -1435,8 +1435,18 @@ deploy_llm_dragoon() {
        return 1
    fi

-    # --- NPU lane: the same three predicates the serve script checks ---------
+    # --- NPU lane: the same predicates the serve script checks --------------
    if [[ -n "$LLM_DRAGOON_NPU_MODELS" ]]; then
+        # One door into the container: the read-only /models mount of the
+        # payload dir. A staging dir outside it needs a nested bind whose
+        # mountpoint runc cannot create through a read-only parent, so the
+        # parity twin of the serve gate refuses the shape before the restart.
+        npu_model_dir="$(sed -nE 's/^FRANK_DRAGOON_MODEL_DIR=(.*)$/\1/p' \
+            "$CHECKOUT/scripts/frank-dragoon.env" | tail -1)"
+        if [[ -n "$npu_model_dir" && "$LLM_DRAGOON_NPU_DIR/" != "$npu_model_dir"/* ]]; then
+            echo "ERROR: NPU staging dir $LLM_DRAGOON_NPU_DIR is outside the payload dir $npu_model_dir — the lane's files reach the container only through the read-only /models mount" >&2
+            return 1
+        fi
        if [[ ! -e "$LLM_DRAGOON_DEV_ACCEL" ]]; then
            echo "ERROR: NPU node $LLM_DRAGOON_DEV_ACCEL absent — amdxdna is not bound (is iommu=pt still on the boot line?)" >&2
            return 1
@@ -29,7 +29,14 @@ set -euo pipefail

 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}"
-DEST="${FRANK_DRAGOON_NPU_DIR:-$HOME/models/halogen-npu}"
+# The env file owns the staging path (it must sit inside the payload dir, which
+# is the container's only /models mount); an exported FRANK_DRAGOON_NPU_DIR wins
+# so the bats seam can stage somewhere disposable.
+DEST="${FRANK_DRAGOON_NPU_DIR:-}"
+if [ -z "$DEST" ] && [ -f "$ENV_FILE" ]; then
+    DEST="$(sed -nE 's/^FRANK_DRAGOON_NPU_DIR=(.*)$/\1/p' "$ENV_FILE" | tail -1)"
+fi
+: "${DEST:?FRANK_DRAGOON_NPU_DIR is not set and $ENV_FILE names no FRANK_DRAGOON_NPU_DIR}"
 HF="https://huggingface.co"

 MODELS="${HALOGEN_NPU_MODELS:-}"
@@ -87,18 +87,24 @@ HALOGEN_WEIGHTS_LOCK=1
 # fabric-clock dependency, no /v1/embeddings). Rolling the lane back is blanking
 # this one line and re-deploying — nothing else moves.
 # Names are the vendor catalog's folder names (`/opt/halogen/npu/models.txt`
-# inside the image). The engine cannot fetch them here: /models is read-only and
+# inside the image) and the engine reads a model ONLY at /models/npu/<name>/ —
+# HALOGEN_NPU_DIR is gone from the image and setting it is a boot refusal.
+# The engine cannot fetch anything here: /models is read-only and
 # HALOGEN_DOWNLOAD stays unset so the container opens no outbound connection at
-# boot, so scripts/download-frank-dragoon-npu.sh must stage them into
-# $FRANK_DRAGOON_NPU_DIR first (the serve gate and the deployer both refuse by
-# name when a named model is not staged).
+# boot, so scripts/download-frank-dragoon-npu.sh must stage the files first
+# (the serve gate and the deployer both refuse by name when a named model is
+# missing). FRANK_DRAGOON_NPU_DIR therefore lives INSIDE the payload dir: the
+# payload mount is the container's only handle on /models, and runc cannot
+# mkdir a nested /models/npu mountpoint through a read-only bind — the lane's
+# first boot died on precisely that (2026-10-19). Swap the payload dir and this
+# path moves with it.
 # Cost: ~1.5 GiB extra pinned host RAM per named model, and a hard dependency on
 # the host holding the GPU fabric clock — on Strix Halo the iGPU and the NPU
 # together hang the machine (and corrupt NPU results) while FCLK changes speed,
 # so halogen refuses to start with this variable set unless the clock is held
 # (dankclaw/host/halogen-fabric-clock; the deployer installs and gates it).
 HALOGEN_NPU_MODELS=qwen3-embedding-0.6b
-FRANK_DRAGOON_NPU_DIR=/home/dank/models/halogen-npu
+FRANK_DRAGOON_NPU_DIR=/home/dank/models/halogen-v2-abliterated/npu
 # Node paths and the XRT directory are variables so the bats seam can drive
 # these gates on a box with no NPU. Ubuntu's distro XRT (libxrt2 + libxrt-npu2)
 # is mounted FILE BY FILE and TWICE EACH — once where the image looks for XRT,
@@ -125,9 +125,17 @@ fi
 # Empty = the endpoint exactly as it ran before the lane existed: no device, no
 # extra mounts, no gate below. Set = three host preconditions, each refused BY
 # NAME (the 2026-10-01 law: the serve gate and the deployer gate must agree):
-#   1. the staged model files (the engine checks size+sha256 against its own
-#      catalog at boot; it cannot fetch them here because /models is read-only
-#      and HALOGEN_DOWNLOAD stays unset — no outbound connection at boot),
+#   1. the staged model files under $FRANK_DRAGOON_NPU_DIR — a path that MUST
+#      live INSIDE $FRANK_DRAGOON_MODEL_DIR, because the container's only
+#      handle on them is the read-only /models mount. A nested bind at
+#      /models/npu cannot work: runc has to mkdir its mountpoint and the parent
+#      bind is ro (`mkdirat .../merged/models/npu: read-only file system`,
+#      measured 2026-10-19), and the image ships no /models at all for it to
+#      land on. `$MODEL_DIR/npu/<id>` is visible at /models/npu/<id> through
+#      the existing mount, which is also where the vendor's catalog expects it.
+#      The engine re-checks size+sha256 against that catalog at boot; it cannot
+#      fetch anything here because /models is read-only and HALOGEN_DOWNLOAD
+#      stays unset — no outbound connection at boot),
 #   2. the distro XRT's three libraries, each mounted twice (where the image
 #      looks for XRT, and at its own path — the libraries find each other there;
 #      an /opt/xilinx/xrt of symlinks into the system dir dangles inside the
@@ -144,6 +152,15 @@ if [ -n "$NPU_MODELS" ]; then
    NPU_DIR="${FRANK_DRAGOON_NPU_DIR:?HALOGEN_NPU_MODELS is set, so FRANK_DRAGOON_NPU_DIR must name the staged model dir}"
    XRT_DIR="${FRANK_DRAGOON_XRT_DIR:-/usr/lib/x86_64-linux-gnu}"
    SYSFS="${FRANK_DRAGOON_SYSFS:-/sys}"
+    # The lane's files have exactly one door into the container: the read-only
+    # payload mount. A staging dir OUTSIDE it would need a nested bind whose
+    # mountpoint runc cannot create through that read-only parent, so refuse the
+    # shape by name here rather than let runc die at container init (2026-10-19).
+    case "$NPU_DIR/" in
+        "$FRANK_DRAGOON_MODEL_DIR"/*) ;;
+        *) echo "frank-dragoon: FRANK_DRAGOON_NPU_DIR=$NPU_DIR is outside the payload dir $FRANK_DRAGOON_MODEL_DIR — the lane's files reach the container only through the read-only /models mount; move the staging dir inside it" >&2
+           exit 1 ;;
+    esac
    if [ ! -e "$DEV_ACCEL" ]; then
        echo "frank-dragoon: $DEV_ACCEL missing — the NPU lane needs the amdxdna node (is iommu=pt still on the boot line?)" >&2
        exit 1
@@ -183,7 +200,6 @@ if [ -n "$NPU_MODELS" ]; then
        exit 1
    fi
    NPU_ARGS+=(--device "$DEV_ACCEL" \
-               -v "$NPU_DIR:/models/npu:ro" \
               -e HALOGEN_NPU_MODELS="$NPU_MODELS")
    echo "frank-dragoon: NPU lane armed: $NPU_MODELS on $DEV_ACCEL (fabric clock held)" >&2
 fi
@@ -469,7 +469,12 @@ $(cat "$TEST_TMP/serve.log")"
    run_with_log
    [ "$status" -eq 0 ]
    [[ "$output" == *"--device $TEST_TMP/accel0"* ]]
-    [[ "$output" == *"-v $TEST_TMP/npu:/models/npu:ro"* ]]
+    # The lane's files ride the payload mount: /models is a read-only bind and
+    # runc cannot mkdir a nested /models/npu mountpoint through it (measured
+    # 2026-10-19), so the staging dir lives INSIDE the payload dir and gets no
+    # -v of its own.
+    [[ "$output" == *"-v $TEST_TMP:/models:ro"* ]]
+    [[ "$output" != *":/models/npu"* ]]
    [[ "$output" == *"-e HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"* ]]
    for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
        [[ "$output" == *"-v $TEST_TMP/xrt/$l:/opt/xilinx/xrt/lib/$l:ro"* ]]
@@ -479,6 +484,20 @@ $(cat "$TEST_TMP/serve.log")"
    [[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]]
 }

+@test "the lane refuses a staging dir outside the payload dir and says why" {
+    # A staging dir outside the payload dir needs a nested bind the read-only
+    # /models mount cannot host: runc dies at container init, which is AFTER the
+    # deployer has stopped the old unit — so the shape is refused here, before
+    # anything is rendered.
+    podman_stub
+    good_env
+    npu_fixture
+    run_with_log FRANK_DRAGOON_NPU_DIR=/elsewhere/npu
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"/elsewhere/npu"*"outside the payload dir"* ]]
+    [[ "$output" != *"--device"* ]]
+}
+
@test "the lane refuses a model that is not staged and names the fetch script" {
    # The engine cannot fetch these at boot here (/models is read-only and
    # HALOGEN_DOWNLOAD stays unset), so an un-staged name must stop the unit
@@ -560,6 +579,9 @@ $(cat "$TEST_TMP/serve.log")"
    grep -qE 'download-frank-dragoon-npu\.sh' "$DEPLOYER"
    # The lane's own health gate is a real embedding, not a 200 on /health.
    grep -qE '/v1/embeddings' "$DEPLOYER"
+    # Parity with the serve gate: a staging dir outside the payload dir is
+    # refused before the restart, not left to runc.
+    grep -qE 'outside the payload dir' "$DEPLOYER"
 }

@test "the fabric-clock host unit is the vendor shape and boots on its own" {