Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e914ecbca6 | ||
|
|
1eebf3b129 | ||
|
|
af5a1f69a2 | ||
|
|
0930510f9b | ||
|
|
af327bcc5e | ||
|
|
fcf94562d1 | ||
|
|
861920797d | ||
|
|
3ef711f146 | ||
|
|
823fd52d0e | ||
|
|
bda011d26f | ||
|
|
b19f1c49d6 | ||
|
|
b1ca4e02fa | ||
|
|
7f65cf0252 | ||
|
|
346e1d22f4 | ||
|
|
53ef121180 | ||
|
|
fa093e617d | ||
|
|
8bac97ec87 | ||
|
|
ee6d8610b3 |
Executable
+65
@@ -0,0 +1,65 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# halogen-fabric-clock hold | release | status: the AMD GPU's fabric clock (FCLK), held at its top level.
|
||||||
|
#
|
||||||
|
# On Strix Halo the GPU and the NPU working at the same time can hang the machine, and corrupt the NPU's results,
|
||||||
|
# while the GPU's fabric clock changes speed; held at its top level, the two run together cleanly. halogen-flash-server
|
||||||
|
# with HALOGEN_NPU_MODELS refuses to start until the clock is held. This holds it: the performance level `manual` and
|
||||||
|
# only the top pp_dpm_fclk level, on every amdgpu device that has the control (the GPU and SoC clocks stay automatic).
|
||||||
|
# Cost: about 1 W at idle. Run as root; halogen-fabric-clock.service runs `hold` at every boot and after a resume.
|
||||||
|
#
|
||||||
|
# hold hold it (waits up to 60 s for the GPU driver to come up)
|
||||||
|
# release give it back to the driver (the performance level `auto`)
|
||||||
|
# status print each device's state; exit 0 when every device is held
|
||||||
|
set -u
|
||||||
|
SYS="${HALOGEN_SYSFS:-/sys}"
|
||||||
|
|
||||||
|
devices() {
|
||||||
|
for d in "$SYS"/class/drm/card*/device; do
|
||||||
|
[ -w "$d/pp_dpm_fclk" ] || [ -r "$d/pp_dpm_fclk" ] || continue
|
||||||
|
[ -e "$d/power_dpm_force_performance_level" ] || continue
|
||||||
|
echo "$d"
|
||||||
|
done | sort -u
|
||||||
|
}
|
||||||
|
|
||||||
|
held() { # held DEVICE_DIR: the performance level high, or manual with only the top fclk level selected
|
||||||
|
perf=$(cat "$1/power_dpm_force_performance_level" 2>/dev/null)
|
||||||
|
[ "$perf" = high ] && return 0
|
||||||
|
[ "$perf" = manual ] || return 1
|
||||||
|
top=$(grep . "$1/pp_dpm_fclk" | tail -n 1)
|
||||||
|
[ "$(grep -c '\*' "$1/pp_dpm_fclk")" = 1 ] && [ "${top%\*}" != "$top" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd="${1:-status}"
|
||||||
|
case "$cmd" in
|
||||||
|
hold)
|
||||||
|
n=0
|
||||||
|
while [ -z "$(devices)" ] && [ "$n" -lt 60 ]; do sleep 1; n=$((n + 1)); done
|
||||||
|
[ -n "$(devices)" ] || { echo "halogen-fabric-clock: no amdgpu device with a fabric clock control (pp_dpm_fclk) under $SYS/class/drm" >&2; exit 1; }
|
||||||
|
rc=0
|
||||||
|
for d in $(devices); do
|
||||||
|
lvl=$(grep . "$d/pp_dpm_fclk" | tail -n 1 | cut -d: -f1)
|
||||||
|
# the driver applies the level several seconds after the write, so the read-back waits up to 30 s
|
||||||
|
ok=0
|
||||||
|
if echo manual > "$d/power_dpm_force_performance_level" && echo "$lvl" > "$d/pp_dpm_fclk"; then
|
||||||
|
t=0; until held "$d" || [ "$t" -ge 300 ]; do sleep 0.1; t=$((t + 1)); done
|
||||||
|
held "$d" && ok=1
|
||||||
|
fi
|
||||||
|
if [ "$ok" = 1 ]; then
|
||||||
|
echo "halogen-fabric-clock: $d: held at level $lvl ($(grep . "$d/pp_dpm_fclk" | tail -n 1 | awk '{print $2}'))"
|
||||||
|
else
|
||||||
|
echo "halogen-fabric-clock: $d: could not hold it (level $lvl)" >&2; rc=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
exit "$rc" ;;
|
||||||
|
release)
|
||||||
|
for d in $(devices); do echo auto > "$d/power_dpm_force_performance_level" && echo "halogen-fabric-clock: $d: released (auto)"; done ;;
|
||||||
|
status)
|
||||||
|
[ -n "$(devices)" ] || { echo "halogen-fabric-clock: no amdgpu device with a fabric clock control"; exit 1; }
|
||||||
|
rc=0
|
||||||
|
for d in $(devices); do
|
||||||
|
if held "$d"; then echo "$d: held ($(cat "$d/power_dpm_force_performance_level"))"
|
||||||
|
else echo "$d: NOT held ($(cat "$d/power_dpm_force_performance_level"))"; rc=1; fi
|
||||||
|
done
|
||||||
|
exit "$rc" ;;
|
||||||
|
*) echo "usage: halogen-fabric-clock hold | release | status" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
# Holds the AMD GPU's fabric clock at its top level at boot and after every resume, for halogen-flash-server with
|
||||||
|
# the NPU (HALOGEN_NPU_MODELS): on Strix Halo the GPU and the NPU at once are safe only with that clock held.
|
||||||
|
# Install (as root), from this directory:
|
||||||
|
# install -m 755 halogen-fabric-clock /usr/local/sbin/
|
||||||
|
# install -m 644 halogen-fabric-clock.service /etc/systemd/system/
|
||||||
|
# systemctl daemon-reload && systemctl enable --now halogen-fabric-clock.service
|
||||||
|
# Check: halogen-fabric-clock status. Undo: systemctl disable halogen-fabric-clock.service, then
|
||||||
|
# halogen-fabric-clock release (or reboot). The unit runs once at boot and once after each resume (a resume can hand
|
||||||
|
# the clock back to the driver); it holds nothing open, so it shows inactive between runs.
|
||||||
|
[Unit]
|
||||||
|
Description=Hold the AMD GPU fabric clock at its top level (GPU and NPU together)
|
||||||
|
After=systemd-modules-load.service suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/usr/local/sbin/halogen-fabric-clock hold
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
# /etc/udev/rules.d/98-halogen-npu-access.rules
|
||||||
|
#
|
||||||
|
# Why the NPU node is world-writable: `frank-dragoon` runs rootless podman under a
|
||||||
|
# unit with `User=dank`, and the container's user namespace maps exactly one uid and
|
||||||
|
# one gid (dank). `/dev/accel/accel0` is `root:render 0660`, so inside the container
|
||||||
|
# the node reads `nobody:nogroup` and the engine dies with its own diagnosis:
|
||||||
|
#
|
||||||
|
# halogen npu: /dev/accel/accel0 is here, but this process cannot open it
|
||||||
|
# (crw-rw---- nobody:nogroup).
|
||||||
|
#
|
||||||
|
# `--group-add keep-groups` is the vendor's advice and it does NOT work in this
|
||||||
|
# shape: measured 2026-10-19 from the host as a `dank` shell that was already in
|
||||||
|
# `render`, with and without the flag, the node still read `nobody:nogroup` and the
|
||||||
|
# open was refused — an unmapped gid cannot be granted to a process inside a
|
||||||
|
# namespace that does not map it. Same conclusion `99-comfyui-gpu-access.rules`
|
||||||
|
# reached for `/dev/kfd` from the systemd-user-spawn direction: the mode is the only
|
||||||
|
# spawn-context-independent door short of running the stack rootful.
|
||||||
|
#
|
||||||
|
# Blast radius: any process in this host's single uid domain (every rootless
|
||||||
|
# container here shares it) can submit to the XDNA2 NPU. The NPU holds no user data
|
||||||
|
# and no privileged context — the same trade already accepted for the GPU compute
|
||||||
|
# nodes beside it. Installed by `deploy-remote.sh` when the NPU lane is armed and left
|
||||||
|
# in place afterwards — the mode costs nothing while the lane is idle. `serve-frank-dragoon.sh`
|
||||||
|
# refuses to arm the lane on a node that is not other-accessible, so a host that lost this
|
||||||
|
# file fails loudly at boot instead of crash-looping.
|
||||||
|
KERNEL=="accel[0-9]*", SUBSYSTEM=="accel", MODE="0666"
|
||||||
@@ -293,6 +293,17 @@ def services_for_path(path: str, target: str) -> set[str]:
|
|||||||
if len(parts) >= 2 and parts[1] in {"caddy", "comfyui", "loki", "alloy", "hermes", "anubis", "seerr"}:
|
if len(parts) >= 2 and parts[1] in {"caddy", "comfyui", "loki", "alloy", "hermes", "anubis", "seerr"}:
|
||||||
_add(services, parts[1])
|
_add(services, parts[1])
|
||||||
return services
|
return services
|
||||||
|
if len(parts) >= 3 and parts[1] == "host":
|
||||||
|
# Host state a service needs at boot lives under dankclaw/host/.
|
||||||
|
# The fabric-clock hold is frank-dragoon's NPU precondition (the
|
||||||
|
# engine refuses to start with HALOGEN_NPU_MODELS set while the
|
||||||
|
# clock moves) and the deployer installs and gates it, so changing
|
||||||
|
# it re-dispatches that endpoint. `grub` is the tracked boot line,
|
||||||
|
# which moves only through scripts/set-dankclaw-bootline.sh and is
|
||||||
|
# a reboot decision — it plans no service.
|
||||||
|
if parts[2].startswith("halogen-fabric-clock"):
|
||||||
|
_add(services, "frank-dragoon")
|
||||||
|
return services
|
||||||
if len(parts) >= 4 and parts[1] == "authentik" and parts[2] == "quadlet":
|
if len(parts) >= 4 and parts[1] == "authentik" and parts[2] == "quadlet":
|
||||||
name = parts[3].removesuffix(".container")
|
name = parts[3].removesuffix(".container")
|
||||||
if name in {"authentik-postgres", "authentik-server", "authentik-worker", "authentik-outpost", "authentik-outpost-ldap"}:
|
if name in {"authentik-postgres", "authentik-server", "authentik-worker", "authentik-outpost", "authentik-outpost-ldap"}:
|
||||||
|
|||||||
@@ -1329,6 +1329,9 @@ deploy_anubis() {
|
|||||||
# port gate is the same as every cutover since 2026-09-16: nothing
|
# port gate is the same as every cutover since 2026-09-16: nothing
|
||||||
# starts until 7770 is released.
|
# starts until 7770 is released.
|
||||||
LLM_DRAGOON_IMAGE="${LLM_DRAGOON_IMAGE:-ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a}"
|
LLM_DRAGOON_IMAGE="${LLM_DRAGOON_IMAGE:-ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a}"
|
||||||
|
# The payload dir is its own twin because the NPU lane's containment gate below
|
||||||
|
# reads it; every artifact path under it stays a full literal as usual.
|
||||||
|
LLM_DRAGOON_MODEL_DIR="${LLM_DRAGOON_MODEL_DIR:-/home/dank/models/halogen-v2-abliterated}"
|
||||||
LLM_DRAGOON_CKPT="${LLM_DRAGOON_CKPT:-/home/dank/models/halogen-v2-abliterated/qwen38-flash-next-v2-abliterated.hgn}"
|
LLM_DRAGOON_CKPT="${LLM_DRAGOON_CKPT:-/home/dank/models/halogen-v2-abliterated/qwen38-flash-next-v2-abliterated.hgn}"
|
||||||
LLM_DRAGOON_CKPT_BYTES="${LLM_DRAGOON_CKPT_BYTES:-66687678432}"
|
LLM_DRAGOON_CKPT_BYTES="${LLM_DRAGOON_CKPT_BYTES:-66687678432}"
|
||||||
LLM_DRAGOON_NGRAM="${LLM_DRAGOON_NGRAM:-/home/dank/models/halogen-v2-abliterated/qwen38-flash-next-ngram.hgn}"
|
LLM_DRAGOON_NGRAM="${LLM_DRAGOON_NGRAM:-/home/dank/models/halogen-v2-abliterated/qwen38-flash-next-ngram.hgn}"
|
||||||
@@ -1337,6 +1340,18 @@ LLM_DRAGOON_VISION="${LLM_DRAGOON_VISION:-/home/dank/models/halogen-v2-abliterat
|
|||||||
LLM_DRAGOON_VISION_BYTES="${LLM_DRAGOON_VISION_BYTES:-897916416}"
|
LLM_DRAGOON_VISION_BYTES="${LLM_DRAGOON_VISION_BYTES:-897916416}"
|
||||||
LLM_DRAGOON_TOK_DIR="${LLM_DRAGOON_TOK_DIR:-/home/dank/models/halogen-v2-abliterated/tokenizer}"
|
LLM_DRAGOON_TOK_DIR="${LLM_DRAGOON_TOK_DIR:-/home/dank/models/halogen-v2-abliterated/tokenizer}"
|
||||||
LLM_DRAGOON_TOK_FILES="${LLM_DRAGOON_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json}"
|
LLM_DRAGOON_TOK_FILES="${LLM_DRAGOON_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json}"
|
||||||
|
# NPU lane (2026-10-19). HALOGEN_NPU_MODELS is the whole switch and the parity
|
||||||
|
# twin of the env file's line: empty here AND in the env = the pre-lane
|
||||||
|
# endpoint. The gates below mirror serve-frank-dragoon.sh, and the vendor's
|
||||||
|
# fabric-clock host unit (dankclaw/host/, verbatim at tag v0.16.2) is installed
|
||||||
|
# ONLY while the lane is named — on Strix Halo the iGPU and the NPU co-running
|
||||||
|
# hang the machine while the fabric clock moves, so halogen refuses to boot with
|
||||||
|
# the lane armed unless that clock is held.
|
||||||
|
LLM_DRAGOON_NPU_MODELS="${LLM_DRAGOON_NPU_MODELS:-qwen3-embedding-0.6b}"
|
||||||
|
LLM_DRAGOON_NPU_DIR="${LLM_DRAGOON_NPU_DIR:-/home/dank/models/halogen-v2-abliterated/npu}"
|
||||||
|
LLM_DRAGOON_DEV_ACCEL="${LLM_DRAGOON_DEV_ACCEL:-/dev/accel/accel0}"
|
||||||
|
LLM_DRAGOON_XRT_DIR="${LLM_DRAGOON_XRT_DIR:-/usr/lib/x86_64-linux-gnu}"
|
||||||
|
LLM_DRAGOON_NETWORK="${LLM_DRAGOON_NETWORK:-dankclaw_default}"
|
||||||
|
|
||||||
retire_halogen_dragoon_quadlet() {
|
retire_halogen_dragoon_quadlet() {
|
||||||
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
||||||
@@ -1415,6 +1430,77 @@ deploy_llm_dragoon() {
|
|||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# The container network has to exist before the run line names it (a typo
|
||||||
|
# there is a restart-looping unit, not a config error anyone reads).
|
||||||
|
if [[ -n "$LLM_DRAGOON_NETWORK" ]] && ! podman network exists "$LLM_DRAGOON_NETWORK"; then
|
||||||
|
echo "ERROR: podman network $LLM_DRAGOON_NETWORK does not exist" >&2
|
||||||
|
echo "frank-dragoon: create it or blank FRANK_DRAGOON_NETWORK (pasta default)" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- NPU lane: the same predicates the serve script checks --------------
|
||||||
|
if [[ -n "$LLM_DRAGOON_NPU_MODELS" ]]; then
|
||||||
|
# One door into the container: the read-only /models mount of the
|
||||||
|
# payload dir. A staging dir outside it needs a nested bind whose
|
||||||
|
# mountpoint runc cannot create through a read-only parent, so the
|
||||||
|
# parity twin of the serve gate refuses the shape before the restart.
|
||||||
|
if [[ "$LLM_DRAGOON_NPU_DIR/" != "$LLM_DRAGOON_MODEL_DIR"/* ]]; then
|
||||||
|
echo "ERROR: NPU staging dir $LLM_DRAGOON_NPU_DIR is outside the payload dir $LLM_DRAGOON_MODEL_DIR — the lane's files reach the container only through the read-only /models mount" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if [[ ! -e "$LLM_DRAGOON_DEV_ACCEL" ]]; then
|
||||||
|
echo "ERROR: NPU node $LLM_DRAGOON_DEV_ACCEL absent — amdxdna is not bound (is iommu=pt still on the boot line?)" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
# Access, not existence: a rootless container cannot open a root:render
|
||||||
|
# 0660 node (its namespace maps one uid/gid, and keep-groups cannot grant
|
||||||
|
# an unmapped gid — measured 2026-10-19). Install the mode rule, re-trigger
|
||||||
|
# the node, then PROVE the node is other-accessible before the restart.
|
||||||
|
run_privileged install -m 0644 dankclaw/udev/98-halogen-npu-access.rules \
|
||||||
|
/etc/udev/rules.d/98-halogen-npu-access.rules
|
||||||
|
run_privileged udevadm control --reload-rules
|
||||||
|
run_privileged udevadm trigger --sysname-match="$(basename "$LLM_DRAGOON_DEV_ACCEL")"
|
||||||
|
# (`-o FILE` is not "other-writable" — it means "exists", and `-o` is also
|
||||||
|
# the OR operator inside test), so the bit is read straight off the mode.
|
||||||
|
if [[ $(( 8#$(stat -c '%a' "$LLM_DRAGOON_DEV_ACCEL") & 8#2 )) -eq 0 ]]; then
|
||||||
|
echo "ERROR: $LLM_DRAGOON_DEV_ACCEL is still $(stat -c '%a %U:%G' "$LLM_DRAGOON_DEV_ACCEL") after installing the access rule — the lane cannot open it rootless" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
for xl in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||||||
|
if [[ ! -e "$LLM_DRAGOON_XRT_DIR/$xl" ]]; then
|
||||||
|
echo "ERROR: XRT NPU library $LLM_DRAGOON_XRT_DIR/$xl absent (Ubuntu libxrt2 + libxrt-npu2)" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
for nm in ${LLM_DRAGOON_NPU_MODELS//,/ }; do
|
||||||
|
if [[ ! -d "$LLM_DRAGOON_NPU_DIR/$nm" ]]; then
|
||||||
|
echo "ERROR: NPU model $nm not staged in $LLM_DRAGOON_NPU_DIR" >&2
|
||||||
|
echo "frank-dragoon: stage it: bash scripts/download-frank-dragoon-npu.sh" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
# The hold must exist BEFORE the engine starts and survive every boot
|
||||||
|
# (a resume hands the clock back too — the unit's WantedBy covers it).
|
||||||
|
run_privileged install -m 0755 dankclaw/host/halogen-fabric-clock \
|
||||||
|
/usr/local/sbin/halogen-fabric-clock
|
||||||
|
run_privileged install -m 0644 dankclaw/host/halogen-fabric-clock.service \
|
||||||
|
/etc/systemd/system/halogen-fabric-clock.service
|
||||||
|
run_privileged systemctl daemon-reload
|
||||||
|
run_privileged systemctl enable --now halogen-fabric-clock.service
|
||||||
|
if ! run_privileged /usr/local/sbin/halogen-fabric-clock status; then
|
||||||
|
echo "ERROR: fabric clock is not held — halogen refuses to start with HALOGEN_NPU_MODELS set" >&2
|
||||||
|
echo "frank-dragoon: journalctl -u halogen-fabric-clock.service -n 40" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
elif [[ -x /usr/local/sbin/halogen-fabric-clock ]]; then
|
||||||
|
# Lane rolled back: hand the clock back to the driver and drop the boot
|
||||||
|
# hook. `disable` deletes the INSTALLED copy of this unit — unlike the
|
||||||
|
# tracked LLM unit symlinks the fleet's disable law protects, this file
|
||||||
|
# is repo-owned and the next lane-on deploy reinstalls it.
|
||||||
|
run_privileged systemctl disable halogen-fabric-clock.service 2>/dev/null || true
|
||||||
|
run_privileged /usr/local/sbin/halogen-fabric-clock release || true
|
||||||
|
fi
|
||||||
|
|
||||||
retire_halogen_dragoon_quadlet
|
retire_halogen_dragoon_quadlet
|
||||||
# The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops
|
# The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops
|
||||||
# the container) before waiting on the port. The stray sweep is keyed
|
# the container) before waiting on the port. The stray sweep is keyed
|
||||||
@@ -1434,6 +1520,33 @@ deploy_llm_dragoon() {
|
|||||||
run_privileged systemctl enable frank-dragoon.service
|
run_privileged systemctl enable frank-dragoon.service
|
||||||
run_privileged systemctl restart frank-dragoon.service
|
run_privileged systemctl restart frank-dragoon.service
|
||||||
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200
|
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200
|
||||||
|
# The lane's own gate. /health proves the engine is up, nothing about the
|
||||||
|
# NPU programs; one real embedding is the observable output. The first
|
||||||
|
# request after a boot is a warmup artifact (fleet law), so retry before
|
||||||
|
# calling a booted lane dead.
|
||||||
|
if [[ -n "$LLM_DRAGOON_NPU_MODELS" ]]; then
|
||||||
|
for nm in ${LLM_DRAGOON_NPU_MODELS//,/ }; do
|
||||||
|
lane_ok=0
|
||||||
|
for _try in 1 2 3 4 5; do
|
||||||
|
if curl -fsS -m 60 -H 'content-type: application/json' \
|
||||||
|
-d "{\"model\":\"$nm\",\"input\":[\"frank-dragoon npu lane gate\"]}" \
|
||||||
|
http://127.0.0.1:7770/v1/embeddings 2>/dev/null \
|
||||||
|
| python3 -c 'import json,sys
|
||||||
|
v = json.load(sys.stdin)["data"][0]["embedding"]
|
||||||
|
raise SystemExit(0 if isinstance(v, list) and len(v) and all(isinstance(x, (int, float)) for x in v) else 1)'; then
|
||||||
|
lane_ok=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 10
|
||||||
|
done
|
||||||
|
if [[ "$lane_ok" != 1 ]]; then
|
||||||
|
echo "ERROR: NPU model $nm answered no vector on /v1/embeddings" >&2
|
||||||
|
echo "frank-dragoon: read the boot ledger: tail -100 /home/dank/logs/frank_dragoon.log" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "frank-dragoon: NPU lane verified — $nm embedded"
|
||||||
|
done
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# deploy_llm_zealot — frank-zealot is llama.cpp (PrismML fork build) on the
|
# deploy_llm_zealot — frank-zealot is llama.cpp (PrismML fork build) on the
|
||||||
|
|||||||
Executable
+117
@@ -0,0 +1,117 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# download-frank-dragoon-npu.sh — stage the NPU half of frank-dragoon: the
|
||||||
|
# vendor-precompiled AIE programs halogen-flash-server runs on the XDNA2 NPU
|
||||||
|
# when HALOGEN_NPU_MODELS names them. These are NOT model weights you can
|
||||||
|
# quantize yourself: XDNA2 executes only programs compiled by AMD's AIE
|
||||||
|
# compiler, so the only source is peonist-ai's per-model NPU repo, and the
|
||||||
|
# engine checks every file's size and sha256 against its own catalog at boot
|
||||||
|
# (/opt/halogen/npu/models.txt inside the image) and refuses to load a model
|
||||||
|
# whose files do not match.
|
||||||
|
#
|
||||||
|
# The manifest below IS that catalog block (dumped from the pinned 0.16.2
|
||||||
|
# image on 2026-10-19), so the files land in the layout the entrypoint
|
||||||
|
# expects — <dir>/<model-id>/devices/*, <dir>/<model-id>/<model-id>.hnpw and
|
||||||
|
# <dir>/<model-id>/tokenizer/tokenizer.json — and the endpoint can keep
|
||||||
|
# /models read-only and open no outbound connection at boot (HALOGEN_DOWNLOAD
|
||||||
|
# stays unset; the vendor's own README: with it unset the container makes no
|
||||||
|
# outbound connections at all).
|
||||||
|
#
|
||||||
|
# The fetch law, same as every sibling script: curl -f (an access error must
|
||||||
|
# not become a "successful" 153-byte artifact), the exact byte count and the
|
||||||
|
# sha256 from the manifest, --verify-only to re-check a staged set.
|
||||||
|
#
|
||||||
|
# Adding a second NPU model: read its block out of the image's catalog and add
|
||||||
|
# a case below —
|
||||||
|
# podman exec frank-dragoon-halogen grep "^file <model-id> " /opt/halogen/npu/models.txt
|
||||||
|
# podman exec frank-dragoon-halogen grep "^model <model-id> " /opt/halogen/npu/models.txt
|
||||||
|
# and add the model id to HALOGEN_NPU_MODELS in frank-dragoon.env.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}"
|
||||||
|
# The env file owns the staging path (it must sit inside the payload dir, which
|
||||||
|
# is the container's only /models mount); an exported FRANK_DRAGOON_NPU_DIR wins
|
||||||
|
# so the bats seam can stage somewhere disposable.
|
||||||
|
DEST="${FRANK_DRAGOON_NPU_DIR:-}"
|
||||||
|
if [ -z "$DEST" ] && [ -f "$ENV_FILE" ]; then
|
||||||
|
DEST="$(sed -nE 's/^FRANK_DRAGOON_NPU_DIR=(.*)$/\1/p' "$ENV_FILE" | tail -1)"
|
||||||
|
fi
|
||||||
|
: "${DEST:?FRANK_DRAGOON_NPU_DIR is not set and $ENV_FILE names no FRANK_DRAGOON_NPU_DIR}"
|
||||||
|
HF="https://huggingface.co"
|
||||||
|
|
||||||
|
MODELS="${HALOGEN_NPU_MODELS:-}"
|
||||||
|
if [ -z "$MODELS" ] && [ -f "$ENV_FILE" ]; then
|
||||||
|
MODELS="$(sed -nE 's/^HALOGEN_NPU_MODELS=(.*)$/\1/p' "$ENV_FILE" | tail -1)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
VERIFY_ONLY=0
|
||||||
|
case "${1:-}" in
|
||||||
|
--verify-only) VERIFY_ONLY=1 ;;
|
||||||
|
"") ;;
|
||||||
|
*) echo "usage: ${0##*/} [--verify-only] (models come from HALOGEN_NPU_MODELS)" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
[ -n "$MODELS" ] || { echo "no NPU model named — set HALOGEN_NPU_MODELS or run with it in the environment" >&2; exit 1; }
|
||||||
|
|
||||||
|
# manifest <model-id> -> repo revision + one "path bytes sha256" per file.
|
||||||
|
manifest() {
|
||||||
|
case "$1" in
|
||||||
|
qwen3-embedding-0.6b)
|
||||||
|
NPU_REPO="peonist-ai/halogen-npu-qwen3-embedding-0.6b"
|
||||||
|
NPU_REV="4960fd9d4423dd15373ad21c83db48dbdb3b2798"
|
||||||
|
FILES=(
|
||||||
|
"devices/NOTICE.md 3248 0f267b3d2523ddf14997ee58660a492a9eaf10ccf6b08bf19dfdc85b9a478b91"
|
||||||
|
"devices/consts.hnpb 5086496 ae430e41545a0b1012938cda373b0c450364d3be631111cca3a8c43679f93336"
|
||||||
|
"devices/devices.hnpm 24047 4e9bca9d95d084fb89ce5b99f579a00666c8ca1efae2613c53e3a4878cc9f174"
|
||||||
|
"devices/u0.elf 4931968 b38f165886cb037b7d3951d9cb878473eaba58a432acd45416ff85fd88aa4e46"
|
||||||
|
"devices/u1.elf 5456000 b1811feee851b8af833b37721da92099d89772110815bbbe2c3fa1c276f0ff42"
|
||||||
|
"devices/u2.elf 6505552 406ea9a95b470628a7382e376f4a72cdf4d7aa4c59b4ddf0a4bdd139d63d6ff2"
|
||||||
|
"devices/u3.elf 6817216 bc205d51b523d770151a0c61968f7c70bf624412827580bb6fe0aaf4650dc2ee"
|
||||||
|
"devices/u4.elf 7444032 a5a91338a2603acdcd5c11df532033fda2d3f5479ea44ae87f98fafbdff9a62d"
|
||||||
|
"devices/u5.elf 7432416 6c2bf41835569e0eba25ded7c05c7264ae69c6700bdeb4791057d884f2a4c126"
|
||||||
|
"devices/u6.elf 4941232 919d0038ca088eafddf195f3466ff38db27d3635be1cffa5d4d4b5140902a68e"
|
||||||
|
"devices/u7.elf 5488304 d904cf8afae9428dd167960549e18cce5441745f18c62f8c50975dffc077d3c2"
|
||||||
|
"devices/u8.elf 4941232 db3aef69b17825854ee0dd2df475125a3f5c872f95c686793d6af1ee83596fa7"
|
||||||
|
"devices/u9.elf 5488304 d182727e551084db8fb9600441d8aa373c7f09a3d267ae0a5e921ef92326db1c"
|
||||||
|
"qwen3-embedding-0.6b.hnpw 813032248 b9f9901429df7b46ec17153aa3899f738adbea8e4356080da0c6e6586fb750d1"
|
||||||
|
"tokenizer/tokenizer.json 11423705 def76fb086971c7867b829c23a26261e38d9d74e02139253b38aeb9df8b4b50a"
|
||||||
|
) ;;
|
||||||
|
*)
|
||||||
|
echo "unknown NPU model '$1' — add its catalog block to manifest() in this script" >&2
|
||||||
|
return 1 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
for MODEL in ${MODELS//,/ }; do
|
||||||
|
manifest "$MODEL" || exit 1
|
||||||
|
echo "== $MODEL ($NPU_REPO@$NPU_REV) =="
|
||||||
|
for entry in "${FILES[@]}"; do
|
||||||
|
read -r f want sha <<<"$entry"
|
||||||
|
path="$DEST/$MODEL/$f"
|
||||||
|
mkdir -p "$(dirname "$path")"
|
||||||
|
if [ "$VERIFY_ONLY" -eq 0 ]; then
|
||||||
|
if [ -f "$path" ] && [ "$(stat -c %s "$path")" = "$want" ]; then
|
||||||
|
echo "HAVE $f ($want bytes) — skipping fetch"
|
||||||
|
else
|
||||||
|
echo "FETCH $f -> $path"
|
||||||
|
# HF answers /resolve/ with a 302 to the Xet CDN and curl
|
||||||
|
# DROPS the Range header across redirects, so resolve the
|
||||||
|
# effective URL first and resume against the CDN (the sibling
|
||||||
|
# 62 GiB fetch hit this on 2026-10-18).
|
||||||
|
url=$(curl -sfIL -o /dev/null -w '%{url_effective}' "$HF/$NPU_REPO/resolve/$NPU_REV/$f") \
|
||||||
|
|| { echo "cannot resolve $NPU_REPO/$f" >&2; exit 1; }
|
||||||
|
curl -fL --retry 5 -C - -o "$path" "$url"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
[ -f "$path" ] || { echo "MISSING $path (run without --verify-only)" >&2; exit 1; }
|
||||||
|
if [ -L "$path" ]; then
|
||||||
|
echo "SYMLINK $path — the serve gate refuses symlinks; replace with a real file" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
got="$(stat -c %s "$path")"
|
||||||
|
[ "$got" = "$want" ] || { echo "SIZE $path: $got != $want" >&2; exit 1; }
|
||||||
|
got="$(sha256sum "$path" | cut -d' ' -f1)"
|
||||||
|
[ "$got" = "$sha" ] || { echo "SHA256 $path: $got != $sha" >&2; exit 1; }
|
||||||
|
echo "OK $f ($want bytes, sha256)"
|
||||||
|
done
|
||||||
|
done
|
||||||
|
echo "DOWNLOAD_DONE $DEST ($MODELS)"
|
||||||
@@ -80,3 +80,48 @@ HALOGEN_TOP_K=20
|
|||||||
# Weights stay pinned: the engine pins by default and only the retired w4b build
|
# Weights stay pinned: the engine pins by default and only the retired w4b build
|
||||||
# honored HALOGEN_FLASH_PIN_TRUNK=0, which this payload rejects.
|
# honored HALOGEN_FLASH_PIN_TRUNK=0, which this payload rejects.
|
||||||
HALOGEN_WEIGHTS_LOCK=1
|
HALOGEN_WEIGHTS_LOCK=1
|
||||||
|
|
||||||
|
# --- NPU lane (XDNA2) -------------------------------------------------------
|
||||||
|
# HALOGEN_NPU_MODELS is the whole switch: EMPTY or unset = the endpoint exactly
|
||||||
|
# as it ran before this lane existed (no accel device, no XRT mount, no
|
||||||
|
# fabric-clock dependency, no /v1/embeddings). Rolling the lane back is blanking
|
||||||
|
# this one line and re-deploying — nothing else moves.
|
||||||
|
# Names are the vendor catalog's folder names (`/opt/halogen/npu/models.txt`
|
||||||
|
# inside the image) and the engine reads a model ONLY at /models/npu/<name>/ —
|
||||||
|
# HALOGEN_NPU_DIR is gone from the image and setting it is a boot refusal.
|
||||||
|
# The engine cannot fetch anything here: /models is read-only and
|
||||||
|
# HALOGEN_DOWNLOAD stays unset so the container opens no outbound connection at
|
||||||
|
# boot, so scripts/download-frank-dragoon-npu.sh must stage the files first
|
||||||
|
# (the serve gate and the deployer both refuse by name when a named model is
|
||||||
|
# missing). FRANK_DRAGOON_NPU_DIR therefore lives INSIDE the payload dir: the
|
||||||
|
# payload mount is the container's only handle on /models, and runc cannot
|
||||||
|
# mkdir a nested /models/npu mountpoint through a read-only bind — the lane's
|
||||||
|
# first boot died on precisely that (2026-10-19). Swap the payload dir and this
|
||||||
|
# path moves with it.
|
||||||
|
# Cost: ~1.5 GiB extra pinned host RAM per named model, and a hard dependency on
|
||||||
|
# the host holding the GPU fabric clock — on Strix Halo the iGPU and the NPU
|
||||||
|
# together hang the machine (and corrupt NPU results) while FCLK changes speed,
|
||||||
|
# so halogen refuses to start with this variable set unless the clock is held
|
||||||
|
# (dankclaw/host/halogen-fabric-clock; the deployer installs and gates it).
|
||||||
|
HALOGEN_NPU_MODELS=qwen3-embedding-0.6b
|
||||||
|
FRANK_DRAGOON_NPU_DIR=/home/dank/models/halogen-v2-abliterated/npu
|
||||||
|
# Node paths and the XRT directory are variables so the bats seam can drive
|
||||||
|
# these gates on a box with no NPU. Ubuntu's distro XRT (libxrt2 + libxrt-npu2)
|
||||||
|
# is mounted FILE BY FILE and TWICE EACH — once where the image looks for XRT,
|
||||||
|
# once at its own path, because those libraries find each other at their install
|
||||||
|
# path; the vendor forbids mounting an /opt/xilinx/xrt of symlinks into the
|
||||||
|
# system dir, which dangle inside the container.
|
||||||
|
FRANK_DRAGOON_DEV_ACCEL=/dev/accel/accel0
|
||||||
|
FRANK_DRAGOON_XRT_DIR=/usr/lib/x86_64-linux-gnu
|
||||||
|
# sysfs root the fabric-clock preflight reads (same knob the vendor's
|
||||||
|
# halogen-fabric-clock status reads; the seam points it at a fixture).
|
||||||
|
FRANK_DRAGOON_SYSFS=/sys
|
||||||
|
# Container network. Empty = podman's rootless default (`pasta`): a private
|
||||||
|
# netns that nothing else can join — podman refuses a second network while the
|
||||||
|
# primary is pasta, the published port is refused from dankclaw_default
|
||||||
|
# containers, and host.containers.internal resolves link-local, which
|
||||||
|
# open-notebook's URL guard refuses to store. Named = the app bridge, so
|
||||||
|
# open-notebook can dial http://frank-dragoon-halogen:8731 for the NPU
|
||||||
|
# endpoints. The engine's unauthenticated 8730 stays private either way (it
|
||||||
|
# binds the container's own loopback); only the API port becomes name-resolvable.
|
||||||
|
FRANK_DRAGOON_NETWORK=dankclaw_default
|
||||||
|
|||||||
@@ -27,6 +27,11 @@
|
|||||||
# the flash-server line reads it; it ships as a container; the root unit
|
# the flash-server line reads it; it ships as a container; the root unit
|
||||||
# keeps LimitMEMLOCK=infinity for parity and to outlive the login session,
|
# keeps LimitMEMLOCK=infinity for parity and to outlive the login session,
|
||||||
# exactly like frank-archon's.
|
# exactly like frank-archon's.
|
||||||
|
# Since 2026-10-19 the container also carries the XDNA2 NPU lane
|
||||||
|
# (HALOGEN_NPU_MODELS): the vendor's precompiled AIE programs for the models
|
||||||
|
# named there run on the NPU inside THIS container and answer /v1/embeddings,
|
||||||
|
# beside the Flash model, on the same port. Opt-in — the variable empty is the
|
||||||
|
# pre-lane endpoint, byte for byte.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
@@ -115,6 +120,119 @@ if [ ! -e "$DEV_KFD" ] || [ ! -d "$DEV_DRI" ]; then
|
|||||||
echo "frank-dragoon: $DEV_KFD or $DEV_DRI missing — no gfx1151 to serve on" >&2
|
echo "frank-dragoon: $DEV_KFD or $DEV_DRI missing — no gfx1151 to serve on" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# --- NPU lane (XDNA2), opt-in via HALOGEN_NPU_MODELS ------------------------
|
||||||
|
# Empty = the endpoint exactly as it ran before the lane existed: no device, no
|
||||||
|
# extra mounts, no gate below. Set = three host preconditions, each refused BY
|
||||||
|
# NAME (the 2026-10-01 law: the serve gate and the deployer gate must agree):
|
||||||
|
# 1. the staged model files under $FRANK_DRAGOON_NPU_DIR — a path that MUST
|
||||||
|
# live INSIDE $FRANK_DRAGOON_MODEL_DIR, because the container's only
|
||||||
|
# handle on them is the read-only /models mount. A nested bind at
|
||||||
|
# /models/npu cannot work: runc has to mkdir its mountpoint and the parent
|
||||||
|
# bind is ro (`mkdirat .../merged/models/npu: read-only file system`,
|
||||||
|
# measured 2026-10-19), and the image ships no /models at all for it to
|
||||||
|
# land on. `$MODEL_DIR/npu/<id>` is visible at /models/npu/<id> through
|
||||||
|
# the existing mount, which is also where the vendor's catalog expects it.
|
||||||
|
# The engine re-checks size+sha256 against that catalog at boot; it cannot
|
||||||
|
# fetch anything here because /models is read-only and HALOGEN_DOWNLOAD
|
||||||
|
# stays unset — no outbound connection at boot),
|
||||||
|
# 2. the distro XRT's three libraries, each mounted twice (where the image
|
||||||
|
# looks for XRT, and at its own path — the libraries find each other there;
|
||||||
|
# an /opt/xilinx/xrt of symlinks into the system dir dangles inside the
|
||||||
|
# container, which the vendor calls out explicitly),
|
||||||
|
# 3. the GPU fabric clock HELD: on Strix Halo the iGPU and the NPU co-running
|
||||||
|
# hang the machine while FCLK changes speed, and the engine refuses to
|
||||||
|
# start when it moves — it cannot read host sysfs from inside the
|
||||||
|
# container, so the proof lives here, checked exactly the way the vendor's
|
||||||
|
# `halogen-fabric-clock status` checks it.
|
||||||
|
NPU_ARGS=()
|
||||||
|
NPU_MODELS="${HALOGEN_NPU_MODELS:-}"
|
||||||
|
if [ -n "$NPU_MODELS" ]; then
|
||||||
|
DEV_ACCEL="${FRANK_DRAGOON_DEV_ACCEL:-/dev/accel/accel0}"
|
||||||
|
NPU_DIR="${FRANK_DRAGOON_NPU_DIR:?HALOGEN_NPU_MODELS is set, so FRANK_DRAGOON_NPU_DIR must name the staged model dir}"
|
||||||
|
XRT_DIR="${FRANK_DRAGOON_XRT_DIR:-/usr/lib/x86_64-linux-gnu}"
|
||||||
|
SYSFS="${FRANK_DRAGOON_SYSFS:-/sys}"
|
||||||
|
# The lane's files have exactly one door into the container: the read-only
|
||||||
|
# payload mount. A staging dir OUTSIDE it would need a nested bind whose
|
||||||
|
# mountpoint runc cannot create through that read-only parent, so refuse the
|
||||||
|
# shape by name here rather than let runc die at container init (2026-10-19).
|
||||||
|
case "$NPU_DIR/" in
|
||||||
|
"$FRANK_DRAGOON_MODEL_DIR"/*) ;;
|
||||||
|
*) echo "frank-dragoon: FRANK_DRAGOON_NPU_DIR=$NPU_DIR is outside the payload dir $FRANK_DRAGOON_MODEL_DIR — the lane's files reach the container only through the read-only /models mount; move the staging dir inside it" >&2
|
||||||
|
exit 1 ;;
|
||||||
|
esac
|
||||||
|
if [ ! -e "$DEV_ACCEL" ]; then
|
||||||
|
echo "frank-dragoon: $DEV_ACCEL missing — the NPU lane needs the amdxdna node (is iommu=pt still on the boot line?)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
# Existence is not access. This endpoint runs rootless, so the container's
|
||||||
|
# namespace maps exactly one uid/gid (dank) and a root:render 0660 node reads
|
||||||
|
# `nobody:nogroup` inside it — the engine then dies naming the node itself
|
||||||
|
# (measured 2026-10-19). --group-add keep-groups cannot help: an unmapped gid
|
||||||
|
# cannot be granted into a namespace that does not map it, verified with and
|
||||||
|
# without the flag from a shell already in `render`. The door is the udev mode.
|
||||||
|
# NB: `[ -o FILE ]` is NOT "other-writable" — it means "exists", and `-o` is
|
||||||
|
# also the OR operator inside test — so the permission bit is read directly.
|
||||||
|
if [ $(( 8#$(stat -c '%a' "$DEV_ACCEL") & 8#2 )) -eq 0 ]; then
|
||||||
|
echo "frank-dragoon: $DEV_ACCEL is $(stat -c '%a %U:%G' "$DEV_ACCEL") — a rootless container cannot open it; install dankclaw/udev/98-halogen-npu-access.rules and re-trigger udev (sudo udevadm control --reload-rules && sudo udevadm trigger --sysname-match=accel0)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
for M in ${NPU_MODELS//,/ }; do
|
||||||
|
if [ ! -d "$NPU_DIR/$M" ]; then
|
||||||
|
echo "frank-dragoon: NPU model $M not staged in $NPU_DIR;" >&2
|
||||||
|
echo " fetch it: bash scripts/download-frank-dragoon-npu.sh" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
for L in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||||||
|
if [ ! -e "$XRT_DIR/$L" ]; then
|
||||||
|
echo "frank-dragoon: XRT NPU library $XRT_DIR/$L missing — the NPU lane cannot load (Ubuntu libxrt2 + libxrt-npu2)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
REAL="$(readlink -f "$XRT_DIR/$L")"
|
||||||
|
NPU_ARGS+=(-v "$REAL:/opt/xilinx/xrt/lib/$L:ro" -v "$REAL:$XRT_DIR/$L:ro")
|
||||||
|
done
|
||||||
|
HELD=0
|
||||||
|
for D in "$SYSFS"/class/drm/card*/device; do
|
||||||
|
[ -e "$D/pp_dpm_fclk" ] || continue
|
||||||
|
[ -e "$D/power_dpm_force_performance_level" ] || continue
|
||||||
|
PERF="$(cat "$D/power_dpm_force_performance_level" 2>/dev/null || true)"
|
||||||
|
TOP="$(grep . "$D/pp_dpm_fclk" | tail -n 1)"
|
||||||
|
if [ "$PERF" = high ] || { [ "$PERF" = manual ] \
|
||||||
|
&& [ "$(grep -c '\*' "$D/pp_dpm_fclk")" = 1 ] && [ "${TOP%\*}" != "$TOP" ]; }; then
|
||||||
|
HELD=1
|
||||||
|
else
|
||||||
|
echo "frank-dragoon: fabric clock at $D is '$PERF', not held — the NPU lane refuses to start (GPU+NPU together need FCLK pinned)" >&2
|
||||||
|
echo " hold it: sudo halogen-fabric-clock hold" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ "$HELD" = 0 ]; then
|
||||||
|
echo "frank-dragoon: no amdgpu device with a fabric-clock control under $SYSFS/class/drm — cannot prove FCLK is held" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
NPU_ARGS+=(--device "$DEV_ACCEL" \
|
||||||
|
-e HALOGEN_NPU_MODELS="$NPU_MODELS")
|
||||||
|
echo "frank-dragoon: NPU lane armed: $NPU_MODELS on $DEV_ACCEL (fabric clock held)" >&2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Container network. Empty = podman's rootless default (`pasta`), which is a
|
||||||
|
# private netns and CANNOT be joined by anything else: podman refuses a second
|
||||||
|
# network while the primary is pasta ("cannot set multiple networks without
|
||||||
|
# bridge network mode"), the endpoint's published port answers 200 on the host
|
||||||
|
# and on the LAN but is REFUSED from a dankclaw_default container (measured
|
||||||
|
# 2026-10-19), and `host.containers.internal:7770` answers 200 yet resolves to
|
||||||
|
# 169.254.1.2 — link-local, which open-notebook's URL guard refuses to store.
|
||||||
|
# So an endpoint on pasta has no address an app container can use; naming the
|
||||||
|
# app bridge here is the only shape that lets open-notebook dial
|
||||||
|
# http://frank-dragoon-halogen:8731 for the NPU endpoints. It does not widen
|
||||||
|
# the surface: the engine's unauthenticated 8730 is bound to the container's
|
||||||
|
# OWN loopback (/proc/net/tcp: 127.0.0.1:8730 vs 0.0.0.0:8731, checked the same
|
||||||
|
# day), so only the API port is reachable by name.
|
||||||
|
NET_ARGS=()
|
||||||
|
if [ -n "${FRANK_DRAGOON_NETWORK:-}" ]; then
|
||||||
|
NET_ARGS=(--network "$FRANK_DRAGOON_NETWORK")
|
||||||
|
fi
|
||||||
# Host-RAM preflight — the engine's pin gate, checked BEFORE podman run.
|
# Host-RAM preflight — the engine's pin gate, checked BEFORE podman run.
|
||||||
# The engine refuses to pin when MemAvailable is below the checkpoint plus
|
# The engine refuses to pin when MemAvailable is below the checkpoint plus
|
||||||
# a 16 GiB floor: pinned pages cannot be swapped, and under the floor the
|
# a 16 GiB floor: pinned pages cannot be swapped, and under the floor the
|
||||||
@@ -189,6 +307,8 @@ podman run --name "$FRANK_DRAGOON_CONTAINER" \
|
|||||||
${HALOGEN_WEIGHTS_LOCK:+-e HALOGEN_WEIGHTS_LOCK="$HALOGEN_WEIGHTS_LOCK"} \
|
${HALOGEN_WEIGHTS_LOCK:+-e HALOGEN_WEIGHTS_LOCK="$HALOGEN_WEIGHTS_LOCK"} \
|
||||||
-v "$FRANK_DRAGOON_MODEL_DIR:/models:ro" \
|
-v "$FRANK_DRAGOON_MODEL_DIR:/models:ro" \
|
||||||
-v "$FRANK_DRAGOON_MODEL_DIR/tokenizer:/tokenizer:ro" \
|
-v "$FRANK_DRAGOON_MODEL_DIR/tokenizer:/tokenizer:ro" \
|
||||||
|
"${NET_ARGS[@]}" \
|
||||||
|
"${NPU_ARGS[@]}" \
|
||||||
"$FRANK_DRAGOON_IMAGE" \
|
"$FRANK_DRAGOON_IMAGE" \
|
||||||
> "$LOG" 2>&1
|
> "$LOG" 2>&1
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ tags:
|
|||||||
- netavark
|
- netavark
|
||||||
- rootless
|
- rootless
|
||||||
- dankclaw
|
- dankclaw
|
||||||
timestamp: 2026-09-02
|
timestamp: 2026-10-19
|
||||||
---
|
---
|
||||||
|
|
||||||
# Container Network Namespaces
|
# Container Network Namespaces
|
||||||
@@ -99,18 +99,42 @@ private `systemd-open-notebook`), each address aimed at a live listener:
|
|||||||
| Dial | Result | Mechanism |
|
| Dial | Result | Mechanism |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `192.168.0.143:3000` (Gitea) | **200** | published by a **rootless** container — rootless egress crosses the host netfilter where netavark installed that DNAT |
|
| `192.168.0.143:3000` (Gitea) | **200** | published by a **rootless** container — rootless egress crosses the host netfilter where netavark installed that DNAT |
|
||||||
| `192.168.0.143:7770` (`frank-dragoon`) | refused | published by a **root** container. Root and rootless Podman carry separate netavark rule sets, so a rootless container's traffic never crosses root's and a root-published port is invisible to it |
|
| `192.168.0.143:7770` (`frank-dragoon`) | refused | the container is **rootless on podman's default `pasta` netns** (the root unit's `User=dank` runs `podman run`, so it lands in dank's storage). Pasta forwards the published port in the host netns — the host and the LAN both answer 200 — but installs no netavark DNAT for traffic arriving from a rootless bridge, so the same address is refused from a container. This is NOT a root/rootless rule-set split: `podman ps` as `dank` lists it |
|
||||||
|
| `host.containers.internal:7770` (`169.254.1.2`) | **200** | the pasta forwarder is bound `0.0.0.0` on the host, so the gateway alias reaches it — and the alias is link-local, which is the dead end below |
|
||||||
| `192.168.0.143:7771` (`frank-zealot`) | refused | host-netns listener, unpublished — the rule above, reconfirmed |
|
| `192.168.0.143:7771` (`frank-zealot`) | refused | host-netns listener, unpublished — the rule above, reconfirmed |
|
||||||
| `10.89.1.1:7771` | refused | the bridge gateway address belongs to the rootlesskit netns, not to the host |
|
| `10.89.1.1:7771` | refused | the bridge gateway address belongs to the rootlesskit netns, not to the host |
|
||||||
| `host.containers.internal:7771` (`169.254.1.2`) | **200** | the gateway alias is the one address that carries a rootless container to a host-netns listener. Why `hermes-dashboard` timed out on this alias in 2026-09-02 while `open-notebook` answers today is **not established**; the difference to check first is the second network and where the default route lands |
|
| `host.containers.internal:7771` (`169.254.1.2`) | **200** | the gateway alias is the one address that carries a rootless container to a host-netns listener. Why `hermes-dashboard` timed out on this alias in 2026-09-02 while `open-notebook` answers today is **not established**; the difference to check first is the second network and where the default route lands |
|
||||||
| `192.168.0.235:7770` (`frank-archon`) | **200** | a different machine: plain routing, no DNAT in the path |
|
| `192.168.0.235:7770` (`frank-archon`) | **200** | a different machine: plain routing, no DNAT in the path |
|
||||||
|
|
||||||
The consequence that changes designs: **a rootless container's menu of local model
|
The consequence that changed designs: **a rootless container's menu of local model
|
||||||
endpoints is `frank-tts` (container DNS, same network) and `frank-archon` (LAN
|
endpoints was only `frank-tts` (container DNS, same network) and `frank-archon` (LAN
|
||||||
address on another host).** Both Dankclaw endpoints are out of reach —
|
address on another host).** A `pasta` endpoint has no address an app container can
|
||||||
`frank-dragoon` behind root's publish, `frank-zealot` behind the host netns — and
|
use: the LAN address is refused, and the one alias that answers is link-local,
|
||||||
the only address that does reach them, `169.254.1.2`, is link-local, which some
|
which some applications refuse to store at all ([[Open Notebook]] meets exactly
|
||||||
applications refuse to store at all ([[Open Notebook]] meets exactly that refusal).
|
that refusal). Pasta cannot be *added to* either —
|
||||||
|
|
||||||
|
```
|
||||||
|
podman run --network pasta --network dankclaw_default …
|
||||||
|
Error: cannot set multiple networks without bridge network mode, selected mode pasta
|
||||||
|
```
|
||||||
|
|
||||||
|
so the only shape that makes such an endpoint nameable is **moving it onto
|
||||||
|
`dankclaw_default`** (`FRANK_DRAGOON_NETWORK` in `scripts/frank-dragoon.env`,
|
||||||
|
2026-10-19). For frank-dragoon that is not an exposure change: the engine's
|
||||||
|
unauthenticated token port binds the container's **own loopback**
|
||||||
|
(`/proc/net/tcp` on 2026-10-19: `127.0.0.1:8730`, `0.0.0.0:8731`), so only the
|
||||||
|
API port becomes name-resolvable, which is the same trust the published 7770
|
||||||
|
already carries on the LAN. `frank-zealot` (7771) stays a host-netns binary and
|
||||||
|
remains unreachable except through the link-local alias.
|
||||||
|
|
||||||
|
The move is verified from the consumer side, so the table above reads as the
|
||||||
|
**pre-move** census and not as today's menu: inside `open-notebook`,
|
||||||
|
`getent hosts frank-dragoon-halogen` → `10.89.1.39`, `GET :8731/v1/models`
|
||||||
|
lists both served models, and the engine's log carries the app's
|
||||||
|
`POST /v1/embeddings 200` from that address (2026-10-19) — see
|
||||||
|
[[Open Notebook]] for what runs over it. What the table still owns is the
|
||||||
|
pasta rule itself: nothing about pasta's forwarding changed, the endpoint was
|
||||||
|
moved out of it.
|
||||||
|
|
||||||
## Operational rules
|
## Operational rules
|
||||||
|
|
||||||
|
|||||||
@@ -64,8 +64,10 @@ Credentials live in the database, encrypted with `OPEN_NOTEBOOK_ENCRYPTION_KEY`,
|
|||||||
|--------|-------|
|
|--------|-------|
|
||||||
| Credential `frank-tts` | modality `text_to_speech`, base URL `http://frank-tts:7772/v1`, no API key (the shim does not authenticate) |
|
| Credential `frank-tts` | modality `text_to_speech`, base URL `http://frank-tts:7772/v1`, no API key (the shim does not authenticate) |
|
||||||
| Credential `frank-archon` | modality `language`, base URL `http://192.168.0.235:7770/v1` |
|
| Credential `frank-archon` | modality `language`, base URL `http://192.168.0.235:7770/v1` |
|
||||||
|
| Credential `frank-dragoon NPU` | modality `embedding`, base URL `http://frank-dragoon-halogen:8731/v1`, dummy API key (the lane does not authenticate) — a container name on `dankclaw_default`, which is the address shape that clears the URL guard below |
|
||||||
| Model row `voxcpm2` | type `text_to_speech` on the frank-tts credential. The name is decorative: frank-tts ignores the request's `model` field and synthesizes whatever `voice` names ([[Frank TTS]]) |
|
| Model row `voxcpm2` | type `text_to_speech` on the frank-tts credential. The name is decorative: frank-tts ignores the request's `model` field and synthesizes whatever `voice` names ([[Frank TTS]]) |
|
||||||
| Model row `frank-archon` | type `language`; `PUT /api/models/defaults` points chat, transformation and large-context at it. **The embedding default stays null — nothing in the fleet serves `/v1/embeddings`** |
|
| Model row `frank-archon` | type `language`; `PUT /api/models/defaults` points chat, transformation and large-context at it |
|
||||||
|
| Model row `qwen3-embedding-0.6b` | type `embedding` on the NPU-lane credential; `default_embedding_model` points at it, so every note save embeds through the XDNA2 lane ([[frank-dragoon]]) |
|
||||||
| Speaker profile `dank-duo` | two speakers, Frank → `voice_id: narrator1`, Comrade → `narrator2`, `voice_model` = the TTS row |
|
| Speaker profile `dank-duo` | two speakers, Frank → `voice_id: narrator1`, Comrade → `narrator2`, `voice_model` = the TTS row |
|
||||||
| Episode profile `dank-daily` | `num_segments: 4`, `speaker_config` = `dank-duo`, outline and transcript LLM = the archon row, briefing that ends "Never invent numbers." |
|
| Episode profile `dank-daily` | `num_segments: 4`, `speaker_config` = `dank-duo`, outline and transcript LLM = the archon row, briefing that ends "Never invent numbers." |
|
||||||
|
|
||||||
@@ -78,11 +80,16 @@ Order that works over REST: credentials → model rows (`POST /api/models` with
|
|||||||
- `192.168.0.143:7771` (frank-zealot, a host-netns listener) and `192.168.0.143:7770` (frank-dragoon, a **root**-published port) both refuse the connection; `host.containers.internal:7771` answers. Full census: [[Container Network Namespaces]].
|
- `192.168.0.143:7771` (frank-zealot, a host-netns listener) and `192.168.0.143:7770` (frank-dragoon, a **root**-published port) both refuse the connection; `host.containers.internal:7771` answers. Full census: [[Container Network Namespaces]].
|
||||||
- The app refuses to **store** that one working address: its URL guard rejects any base URL that resolves inside link-local `169.254.0.0/16` (`open_notebook/utils/url_validation.py`, "Link-local addresses are not allowed (metadata endpoints)"), unconditionally — no env var touches it, and `AddHost=` cannot dodge it because the check runs on the resolved address.
|
- The app refuses to **store** that one working address: its URL guard rejects any base URL that resolves inside link-local `169.254.0.0/16` (`open_notebook/utils/url_validation.py`, "Link-local addresses are not allowed (metadata endpoints)"), unconditionally — no env var touches it, and `AddHost=` cannot dodge it because the check runs on the resolved address.
|
||||||
|
|
||||||
`http://192.168.0.235:7770/v1` — another machine, plain RFC1918 — is the only fleet LLM address that passes both tests. Cost: frank-archon runs `--max-num-seqs 2` and the operator session itself lives there, so a render is a slow-down for whoever else is talking to that endpoint, not a stall; `ESPERANTO_LLM_TIMEOUT=180` covers an outline or transcript call comfortably.
|
`http://192.168.0.235:7770/v1` — another machine, plain RFC1918 — was, when this was first written, the only fleet LLM address that passed both tests; a container name on `dankclaw_default` passes them as well (the embedding credential below is stored that way), but frank-archon stays the scriptwriter for the reason in the next paragraph. Cost: frank-archon runs `--max-num-seqs 2` and the operator session itself lives there, so a render is a slow-down for whoever else is talking to that endpoint, not a stall; `ESPERANTO_LLM_TIMEOUT=180` covers an outline or transcript call comfortably.
|
||||||
|
|
||||||
**frank-dragoon can write transcripts, but only greedily — and open-notebook cannot ask for greedy.** Measured 2026-10-19 against the live endpoint: `response_format` as `json_schema` or `json_object` returns schema-valid JSON at `temperature: 0` and a **400** at any sampled temperature — halogen decodes greedy under a schema, and because its sampler default is server-side, omitting `temperature` is sampled, not greedy. That 400 is the trap: esperanto's `is_json_schema_unsupported_error` matches the words *not implemented* in it, so `_SchemaFallbackModel` in `podcast_creator/nodes.py` reads it as "this endpoint has no schema support", downgrades to `json_object`, gets the same 400, and **fails the run** — esperanto's own rescue (drop `response_format` entirely) fires only on the different message `'response_format.type' must be 'json_schema'`. The full law is on [[frank-dragoon]]. And the fix is not reachable from here: a model record is `{credential, name, provider, type}` and a model response adds only timestamps (`GET /openapi.json`, `ModelCreate`/`ModelResponse`) — **there is no per-row config field**, no way to attach `temperature: 0` to one row from the UI or the API. Making `frank-dragoon` the writer would therefore need something in the path that rewrites request bodies, which is a heavier tenant than the contention it removes. **frank-archon stays the scriptwriter.** Reachability would be a separate problem anyway: `frank-dragoon` is not on `dankclaw_default` (the serve script passes no `--network`), so nothing on this network has a name for it, and the one address that does answer — `host.containers.internal:7770` — is the link-local the guard refuses to store ([[Container Network Namespaces]]).
|
**frank-dragoon can write transcripts, but only greedily — and open-notebook cannot ask for greedy.** Measured 2026-10-19 against the live endpoint: `response_format` as `json_schema` or `json_object` returns schema-valid JSON at `temperature: 0` and a **400** at any sampled temperature — halogen decodes greedy under a schema, and because its sampler default is server-side, omitting `temperature` is sampled, not greedy. That 400 is the trap: esperanto's `is_json_schema_unsupported_error` matches the words *not implemented* in it, so `_SchemaFallbackModel` in `podcast_creator/nodes.py` reads it as "this endpoint has no schema support", downgrades to `json_object`, gets the same 400, and **fails the run** — esperanto's own rescue (drop `response_format` entirely) fires only on the different message `'response_format.type' must be 'json_schema'`. The full law is on [[frank-dragoon]]. And the fix is not reachable from here: a model record is `{credential, name, provider, type}` and a model response adds only timestamps (`GET /openapi.json`, `ModelCreate`/`ModelResponse`) — **there is no per-row config field**, no way to attach `temperature: 0` to one row from the UI or the API. Making `frank-dragoon` the writer would therefore need something in the path that rewrites request bodies, which is a heavier tenant than the contention it removes. **frank-archon stays the scriptwriter.** Reachability stopped being the objection on 2026-10-19 — the endpoint joined `dankclaw_default`, which is exactly why the embedding lane below is dialable by name — but no network change reaches the no-per-row-config wall.
|
||||||
|
|
||||||
**No embedding model is deployed, and saving a note is where that shows up.** Every note save submits an `embed_note` command that fails in the worker with `No embedding model configured. Please configure one in the Models section.` The note itself is stored and readable; semantic search, source ingestion and notebook-context questions stay dead until something serves `/v1/embeddings`. Podcast generation from `content` does not need embeddings, which is why renders work today. The cheapest door is **halogen's NPU lane, not a new endpoint**: `qwen3-embedding-0.6b` is already in the catalog of the image `frank-dragoon` runs and answers `/v1/embeddings` on the same port (1024 unit-length floats; the instruct prefix goes in the text, and `check_embedding_ctx_length=False` for LangChain). What enabling it costs — the accel device, the host XRT mount, and the fabric-clock hold that GPU+NPU co-run requires — is on [[frank-dragoon]]; after that, the embedding row still has to clear the reachability rule above.
|
**Embeddings run on the XDNA2 lane — provisioned 2026-10-19.** `qwen3-embedding-0.6b` answers `/v1/embeddings` at `http://frank-dragoon-halogen:8731/v1`, the flash container's own bridge address; a container name on `dankclaw_default` resolves to `10.89.1.39`, plain RFC1918, so it clears the URL guard that ruled out `host.containers.internal` above. The lane's shape (one program, the token ceiling, what it costs the Flash model) is owned by [[frank-dragoon]]; the two shapes that bite on the app side are open-notebook's own:
|
||||||
|
|
||||||
|
- **`PUT /api/models/defaults` stores the record ID, not the name.** `"default_embedding_model": "qwen3-embedding-0.6b"` is accepted and echoed back, and then every embed dies in the worker with `Model with ID qwen3-embedding-0.6b not found`. The value has to be the row's `model:…` id — the other defaults in that same object are ids, so mirror them.
|
||||||
|
- **`POST /api/embed` answers 500 while succeeding.** Upstream's `EmbedResponse.command_id` declares `str` and the worker hands it a `RecordID`, so FastAPI raises on serialization *after* the vector is written. The truth is the worker line `Successfully embedded note … in 0.19s`, never the HTTP code. Saving a note (`POST /api/notes`) submits `embed_note` by itself, so that route is only needed to backfill old rows.
|
||||||
|
|
||||||
|
Proof from the app's own paths, 2026-10-19: `POST /api/models/{id}/test` → `Embedding dimensions: 1024`; two notes with nothing in common, then `POST /api/search` with `type: "vector"` (the enum is `text` or `vector`; the score field is `similarity`) — "a yellow waterbird beside a glowing beacon tower" put the grebe note at **0.659** and the boiler note at 0.292, and "quarterly depreciation schedule for industrial boilers" inverted it to **0.906** / 0.318. Embed cost 0.19–0.43 s per note. Semantic search, source ingestion and notebook-context questions are live; podcast generation never needed any of it.
|
||||||
|
|
||||||
**Secrets never go in a command line.** The API's bearer token *is* `OPEN_NOTEBOOK_PASSWORD`, and a `-H "Authorization: Bearer …"` argument is world-readable in `/proc/<pid>/cmdline`; put it in a mode-600 curl config and pass `-K`:
|
**Secrets never go in a command line.** The API's bearer token *is* `OPEN_NOTEBOOK_PASSWORD`, and a `-H "Authorization: Bearer …"` argument is world-readable in `/proc/<pid>/cmdline`; put it in a mode-600 curl config and pass `-K`:
|
||||||
|
|
||||||
@@ -164,6 +171,7 @@ podman pull docker.io/lfnovo/open_notebook:v1-latest@sha256:54b2025a4a4853c188f7
|
|||||||
- Local measurements (2026-10-18, workstation, pinned digests): `surreal start --help` (credential and bind options carry `[env: SURREAL_*]`); running the pinned SurrealDB digest with env-only credentials (credential init in the log, correct login authenticates, wrong password refused, `/surreal is-ready` → `OK` exit 0, named volume owned by the mapped uid); the v1-compatible Quadlet generator dry-run of all three unit files (two `--env-file` in order, two `--network` in order, named volume + host bind, both health probes, `--pull never`)
|
- Local measurements (2026-10-18, workstation, pinned digests): `surreal start --help` (credential and bind options carry `[env: SURREAL_*]`); running the pinned SurrealDB digest with env-only credentials (credential init in the log, correct login authenticates, wrong password refused, `/surreal is-ready` → `OK` exit 0, named volume owned by the mapped uid); the v1-compatible Quadlet generator dry-run of all three unit files (two `--env-file` in order, two `--network` in order, named volume + host bind, both health probes, `--pull never`)
|
||||||
- Host evidence (Dankclaw, 2026-10-18, read-only): `ss -ltn` (`127.0.0.1:5055` held by seerr, `0.0.0.0:7771` a host listener, nothing on 7772/8502/5056/8000), `podman ps -a` (no open-notebook container, `dankclaw_default` present), subnet registry from `dankclaw/open-webui/quadlet/open-webui.network`
|
- Host evidence (Dankclaw, 2026-10-18, read-only): `ss -ltn` (`127.0.0.1:5055` held by seerr, `0.0.0.0:7771` a host listener, nothing on 7772/8502/5056/8000), `podman ps -a` (no open-notebook container, `dankclaw_default` present), subnet registry from `dankclaw/open-webui/quadlet/open-webui.network`
|
||||||
- Deploy + provisioning (Dankclaw, 2026-10-19): first-boot log — migrations 1→25, `Credentials were provided, and no root users were found. The root user 'open_notebook' will be created`, `api`/`worker`/`frontend` all RUNNING under supervisord; `GET :8502/config` → the `API_URL` value; `GET /api/providers` (the `openai_compatible` modality list); `GET /openapi.json` schemas for `CreateCredentialRequest` / `ModelCreate` / `SpeakerProfileCreate` / `PodcastGenerateRequest`; the POSTs listed above; the note-save worker error `No embedding model configured.`
|
- Deploy + provisioning (Dankclaw, 2026-10-19): first-boot log — migrations 1→25, `Credentials were provided, and no root users were found. The root user 'open_notebook' will be created`, `api`/`worker`/`frontend` all RUNNING under supervisord; `GET :8502/config` → the `API_URL` value; `GET /api/providers` (the `openai_compatible` modality list); `GET /openapi.json` schemas for `CreateCredentialRequest` / `ModelCreate` / `SpeakerProfileCreate` / `PodcastGenerateRequest`; the POSTs listed above; the note-save worker error `No embedding model configured.`
|
||||||
|
- Embedding lane (Dankclaw, 2026-10-19): `POST /api/credentials` → `credential:7fdqz09tt0c4xp68x7cm`, `POST /api/models` → `model:p3ki598dydfinu93tbny`, `POST /api/models/{id}/test` → `Embedding dimensions: 1024`; worker lines `Successfully embedded note … in 0.43s` / `0.19s`; the name-not-id failure `Model with ID qwen3-embedding-0.6b not found`; the `/api/embed` 500 body (`EmbedResponse.command_id`, `input_value=RecordID(table_name=command…)`); `POST /api/search` `similarity` 0.6590/0.2921 and 0.9059/0.3176 for the two-note pair; `podman logs frank-dragoon-halogen` showing `POST /v1/embeddings 200` from `10.89.1.21`. Both probe notes deleted afterwards.
|
||||||
- First render (2026-10-19): job `command:9rdojq5ne7dj9caicq20`, `execution_time: 301.21 s`, `episode:7ft03thj98sn0ek9xf29`; `episodes/d5fed29b-33e9-40ac-b46a-e0537c56c256/clips/0001.mp3 … 0023.mp3`; `…/audio/d5fed29b-….mp3` = 11,957,420 bytes, mp3 24 kHz mono, `ffprobe` duration 747.31 s; `GET /api/podcasts/episodes/{id}/audio` → 200 `audio/mpeg` with the bearer, 401 without.
|
- First render (2026-10-19): job `command:9rdojq5ne7dj9caicq20`, `execution_time: 301.21 s`, `episode:7ft03thj98sn0ek9xf29`; `episodes/d5fed29b-33e9-40ac-b46a-e0537c56c256/clips/0001.mp3 … 0023.mp3`; `…/audio/d5fed29b-….mp3` = 11,957,420 bytes, mp3 24 kHz mono, `ffprobe` duration 747.31 s; `GET /api/podcasts/episodes/{id}/audio` → 200 `audio/mpeg` with the bearer, 401 without.
|
||||||
- Upstream code read inside the pinned image (2026-10-19): `frontend/next.config.ts:26-43` (the `/api/*` rewrite and the one-port claim), `frontend/src/app/config/route.ts:122-173` (`/config` priority order; autodetect appends `:5055`), `frontend/src/app/api/_sse-proxy.ts` (SSE proxied with `no-transform`), `open_notebook/utils/url_validation.py` (link-local refusal, `is_blocked_ip`, no environment override anywhere in the file).
|
- Upstream code read inside the pinned image (2026-10-19): `frontend/next.config.ts:26-43` (the `/api/*` rewrite and the one-port claim), `frontend/src/app/config/route.ts:122-173` (`/config` priority order; autodetect appends `:5055`), `frontend/src/app/api/_sse-proxy.ts` (SSE proxied with `no-transform`), `open_notebook/utils/url_validation.py` (link-local refusal, `is_blocked_ip`, no environment override anywhere in the file).
|
||||||
- Guard chain (2026-10-19): provider `notebook-proxy` created through `podman exec -i authentik-server ak shell` with the flows copied from `embedded-outpost` (pk 1), Application `Open Notebook`/`notebook-guard`, embedded-outpost provider set 9 → 10; afterwards the outpost logged zero `Failed to fetch outpost from API` and `seerr`/`grafana` still answered 200. Before the route existed, `notebook.franksdank.cloud` resolved to nothing (`000`).
|
- Guard chain (2026-10-19): provider `notebook-proxy` created through `podman exec -i authentik-server ak shell` with the flows copied from `embedded-outpost` (pk 1), Application `Open Notebook`/`notebook-guard`, embedded-outpost provider set 9 → 10; afterwards the outpost logged zero `Failed to fetch outpost from API` and `seerr`/`grafana` still answered 200. Before the route existed, `notebook.franksdank.cloud` resolved to nothing (`000`).
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ description: "Pi-hole is the LAN DNS server for the homelab."
|
|||||||
tags:
|
tags:
|
||||||
- infrastructure
|
- infrastructure
|
||||||
- pi-hole
|
- pi-hole
|
||||||
timestamp: 2026-08-28
|
timestamp: 2026-10-19
|
||||||
---
|
---
|
||||||
|
|
||||||
# Pi-hole
|
# Pi-hole
|
||||||
@@ -46,6 +46,8 @@ The file format is hosts-style (`<IP> <name> [<name> ...]`), one A record per Ca
|
|||||||
|
|
||||||
**Pitfall (2026-08-24):** the first quadlet mounted the file into `/etc/dnsmasq.d/`. FTL never read it; names only resolved from the query cache. A fresh name returned NXDOMAIN. Mount into `/etc/pihole/hosts/`, not `/etc/dnsmasq.d/`.
|
**Pitfall (2026-08-24):** the first quadlet mounted the file into `/etc/dnsmasq.d/`. FTL never read it; names only resolved from the query cache. A fresh name returned NXDOMAIN. Mount into `/etc/pihole/hosts/`, not `/etc/dnsmasq.d/`.
|
||||||
|
|
||||||
|
**Pitfall (2026-10-19): FTL can come up with the record file mounted and still serve none of it.** A plain `systemctl --user restart pihole` (16:25, inside the open-notebook deploy window) left every `franksdank.cloud` name NXDOMAIN for LAN clients while **external names kept answering** — from FTL's cache, which is exactly what makes the failure read like a client or an upstream problem. **The trigger is a container restart, not a boot**, and that matters for where the check belongs: `uptime -s` reads 2026-10-18 01:24 (two days of uptime) while `journalctl --user -u pihole.service` shows `Stopping pihole.service` at 16:25:23 and the container's `StartedAt` at 16:25:27 — so verify after **any** pihole restart, deploy-triggered or manual, not only after a reboot. The instrument is on the box: `getent hosts grafana.franksdank.cloud` (or `dig +short @192.168.0.143 <name>`) answering empty means FTL is holding no local records at all; here the file was present and readable inside the container (`-rw-rw-r-- … /etc/pihole/hosts/addn-hosts`, 44 records) and `dns.hosts` in `pihole.toml` was intact. `podman exec pihole pihole restartdns reload` had every name answering within seconds. The mechanism is **not established** — the same mounts and the same config read fine before and after the reload — so treat it as a load failure of the record sources at FTL start, not as a lost or unmounted file. What it cost: the whole Playwright gate, 18 specs across every vhost failing `ERR_NAME_NOT_RESOLVED` while the fleet itself was healthy ([[E2E Suite]]).
|
||||||
|
|
||||||
## Upstream
|
## Upstream
|
||||||
|
|
||||||
The slirp netns has no useful `resolv.conf`. The env file pins the upstreams: `DNS1=1.1.1.1`, `DNS2=1.0.0.1` (mirrors the retired `dnsmasq.conf` `server=` lines). Every query that is not a local record goes to those resolvers. The zone apex and unknown subdomains fall through to Porkbun.
|
The slirp netns has no useful `resolv.conf`. The env file pins the upstreams: `DNS1=1.1.1.1`, `DNS2=1.0.0.1` (mirrors the retired `dnsmasq.conf` `server=` lines). Every query that is not a local record goes to those resolvers. The zone apex and unknown subdomains fall through to Porkbun.
|
||||||
@@ -74,6 +76,8 @@ Both prerequisites survive the dnsmasq → Pi-hole swap. Do not remove them.
|
|||||||
| Upstream queries fail (local names still answer) | `DNS1`/`DNS2` in the env file are missing or wrong |
|
| Upstream queries fail (local names still answer) | `DNS1`/`DNS2` in the env file are missing or wrong |
|
||||||
| Port 53 `bind: permission denied` crash loop after a reboot | sysctl conflict: the value must be 53 at boot, and remember `/etc/sysctl.conf` is applied **after** all of `sysctl.d/`. Redeploy pihole — `deploy_pihole` reinstalls the drop-in, neutralizes conflicting drop-ins, and comments out a stale `/etc/sysctl.conf` line |
|
| Port 53 `bind: permission denied` crash loop after a reboot | sysctl conflict: the value must be 53 at boot, and remember `/etc/sysctl.conf` is applied **after** all of `sysctl.d/`. Redeploy pihole — `deploy_pihole` reinstalls the drop-in, neutralizes conflicting drop-ins, and comments out a stale `/etc/sysctl.conf` line |
|
||||||
| Single-label name (for example `dankclaw`) answers `127.0.1.1` | The rootless container `/etc/hosts` carries the host entry (`127.0.1.1 <host>`), and FTL serves it. Use the FQDN, or add an `/etc/hosts` entry on the client |
|
| Single-label name (for example `dankclaw`) answers `127.0.1.1` | The rootless container `/etc/hosts` carries the host entry (`127.0.1.1 <host>`), and FTL serves it. Use the FQDN, or add an `/etc/hosts` entry on the client |
|
||||||
|
| Every `franksdank.cloud` name is NXDOMAIN for LAN clients while external names still answer | FTL holds no local records: `getent hosts <name>` on Dankclaw comes back empty → `podman exec pihole pihole restartdns reload` (the 2026-10-19 pitfall under **Records**) |
|
||||||
|
| One client gets nothing while every other client is fine | FTL rate-limiting that client: `podman exec pihole grep Rate-limiting /var/log/pihole/FTL.log`. A `10.0.2.x` client address is a pasta container's DNS forwarder, not a machine |
|
||||||
|
|
||||||
## See Also
|
## See Also
|
||||||
|
|
||||||
|
|||||||
@@ -215,9 +215,10 @@ is in `render`, so a quadlet needs `AddDevice=/dev/accel/accel0` +
|
|||||||
`GroupAdd=keep-groups`. **No NPU endpoint is deployed** — stays the
|
`GroupAdd=keep-groups`. **No NPU endpoint is deployed** — stays the
|
||||||
history of the retired endpoint; what changed is only its precondition.
|
history of the retired endpoint; what changed is only its precondition.
|
||||||
|
|
||||||
**Deliberately deferred:** the `pp_dpm_fclk` pin (level 7, 2000 MHz) from the issue's
|
**The `pp_dpm_fclk` pin (level 7, 2000 MHz) went in on 2026-10-19**, with the first thing that needed it: the
|
||||||
co-run step. Pinning the fabric clock changes iGPU timing for every workload, and
|
vendor's root host unit, installed and enabled by `frank-dragoon`'s deployer while its NPU lane is named. Pinning
|
||||||
nothing co-runs on the NPU yet; it goes in when an NPU endpoint actually exists.
|
the fabric clock changes iGPU timing for every workload, which is exactly why it waited for a co-run to exist; the
|
||||||
|
law, the hold and the serve-side re-check live in [[frank-dragoon]].
|
||||||
PerfOpt still needs Linux 7.4 — `7.0.0-38-generic` has zero `perfopt` symbols and the
|
PerfOpt still needs Linux 7.4 — `7.0.0-38-generic` has zero `perfopt` symbols and the
|
||||||
26.04 archive caps at `7.0.0-38.38`.
|
26.04 archive caps at `7.0.0-38.38`.
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ tags:
|
|||||||
- gfx1151
|
- gfx1151
|
||||||
- halogen
|
- halogen
|
||||||
- endpoint
|
- endpoint
|
||||||
timestamp: 2026-10-18
|
timestamp: 2026-10-19
|
||||||
---
|
---
|
||||||
|
|
||||||
# frank-dragoon — halogen-flash-server 0.16.2, native v2 abliterated (Dankclaw iGPU)
|
# frank-dragoon — halogen-flash-server 0.16.2, native v2 abliterated (Dankclaw iGPU)
|
||||||
@@ -219,7 +219,7 @@ created it — earlier topologies in .
|
|||||||
- Deploy waits for :7770 to be free: a rootless container binds it at CREATE and exits 126, and a
|
- Deploy waits for :7770 to be free: a rootless container binds it at CREATE and exits 126, and a
|
||||||
leftover Quadlet beside the root unit guarantees the race.
|
leftover Quadlet beside the root unit guarantees the race.
|
||||||
|
|
||||||
## The NPU lane (present in the image, not enabled)
|
## The NPU lane (XDNA2, armed by one env var)
|
||||||
|
|
||||||
XDNA2 executes only vendor-precompiled AIE partition programs — that is the wall the TTS attempt died at
|
XDNA2 executes only vendor-precompiled AIE partition programs — that is the wall the TTS attempt died at
|
||||||
([[Frank TTS]]) — and flash-server ships those programs. `HALOGEN_NPU_MODELS=<names>` starts a second engine **in the
|
([[Frank TTS]]) — and flash-server ships those programs. `HALOGEN_NPU_MODELS=<names>` starts a second engine **in the
|
||||||
@@ -235,11 +235,19 @@ Embedding contract on that lane: `POST /v1/embeddings`, `model: "qwen3-embedding
|
|||||||
(`Instruct: …\nQuery:…`), `base64` accepted, and LangChain's `OpenAIEmbeddings` needs
|
(`Instruct: …\nQuery:…`), `base64` accepted, and LangChain's `OpenAIEmbeddings` needs
|
||||||
`check_embedding_ctx_length=False`.
|
`check_embedding_ctx_length=False`.
|
||||||
|
|
||||||
Four prerequisites the trio does not satisfy today, from the vendor's own host list — and this box is one of the two
|
Four host prerequisites, each refused **by name** in the serve script and mirrored in `deploy_llm_dragoon` — a
|
||||||
configurations they test (Ubuntu 26.04's XRT 2.21 `libxrt2`+`libxrt-npu2`, the kernel's `amdxdna`, `iommu=pt`):
|
precondition that surfaces as an engine crash-loop is a gate that was never written. All four are satisfied and the
|
||||||
|
lane is **armed and serving** since 2026-10-19 (`HALOGEN_NPU_MODELS=qwen3-embedding-0.6b`):
|
||||||
|
|
||||||
1. `--device /dev/accel/accel0` plus `--group-add keep-groups` (the pattern ComfyUI uses for `/dev/kfd`; `dank` is
|
1. `--device /dev/accel/accel0` **and** a node the container's namespace can open. `/dev/accel/accel0` is
|
||||||
already in `render`).
|
`root:render 0660` and a rootless namespace maps exactly one uid/gid, so the engine's own first words are
|
||||||
|
*"this process cannot open it (crw-rw---- nobody:nogroup)"*. `--group-add keep-groups` — the vendor's advice and
|
||||||
|
the ComfyUI pattern for `/dev/kfd` — is **dead in this shape**: measured 2026-10-19 from a `dank` shell already
|
||||||
|
in `render`, with and without the flag the node stayed `nobody:nogroup` and the open was refused; an unmapped
|
||||||
|
gid cannot be granted across a namespace that does not map it. The door is the udev mode
|
||||||
|
(`dankclaw/udev/98-halogen-npu-access.rules`, installed and re-triggered by the deployer, `0666` like the GPU
|
||||||
|
compute nodes beside it); the serve script refuses any node that is not other-accessible, so the fault cannot
|
||||||
|
return as a crash-loop.
|
||||||
2. The host's XRT reachable inside: there is **no `/opt/xilinx/xrt`** here — distro packages put the three files the
|
2. The host's XRT reachable inside: there is **no `/opt/xilinx/xrt`** here — distro packages put the three files the
|
||||||
container needs (`libxrt_coreutil.so.2`, `libxrt_core.so.2`, `libxrt_driver_xdna.so.2`) in
|
container needs (`libxrt_coreutil.so.2`, `libxrt_core.so.2`, `libxrt_driver_xdna.so.2`) in
|
||||||
`/usr/lib/x86_64-linux-gnu`, where all three exist.
|
`/usr/lib/x86_64-linux-gnu`, where all three exist.
|
||||||
@@ -248,17 +256,27 @@ configurations they test (Ubuntu 26.04's XRT 2.21 `libxrt2`+`libxrt-npu2`, the k
|
|||||||
engine holds the clock itself only when it runs as root with `/sys` mounted; `frank-dragoon` is a **rootless**
|
engine holds the clock itself only when it runs as root with `/sys` mounted; `frank-dragoon` is a **rootless**
|
||||||
`podman run` under `User=dank`, so the path is the vendor's root host unit (`deploy/host/halogen-fabric-clock`) —
|
`podman run` under `User=dank`, so the path is the vendor's root host unit (`deploy/host/halogen-fabric-clock`) —
|
||||||
which is exactly the `pp_dpm_fclk` level-7 pin deferred in [[dankclaw-iommu-translation-off]] with the words "it
|
which is exactly the `pp_dpm_fclk` level-7 pin deferred in [[dankclaw-iommu-translation-off]] with the words "it
|
||||||
goes in when an NPU endpoint actually exists". Held: about a watt. Unheld: the hang. Measured 2026-10-19 nothing
|
goes in when an NPU endpoint actually exists". Held: about a watt. Unheld: the hang. The deployer installs and
|
||||||
is held (fabric DPM lists 400–2000 MHz with no forced level, graphics clock sitting at 857 of 2900 MHz).
|
enables that unit whenever the lane is named, and `halogen-fabric-clock status` reports `held (manual)` here as of
|
||||||
4. `HALOGEN_DOWNLOAD=peonist-ai/halogen-qwen3.8-flash-next` on a boot with a **writable** `/models/npu` — the payload
|
2026-10-19; the serve script re-reads the same two sysfs files at every start and refuses by name if the top level
|
||||||
mount is `:ro` by law, so the programs land via a throwaway writable run or a dedicated read-write bind for that
|
is not the starred one, because a hold done by hand is a countdown to the next reboot.
|
||||||
one subdirectory; the vendor notes a read-only volume works but re-runs the conversion at every start.
|
4. The programs are staged under `$FRANK_DRAGOON_NPU_DIR`, which **must sit inside the payload dir**
|
||||||
|
(`/home/dank/models/halogen-v2-abliterated/npu`): the payload's read-only mount is the container's only handle on
|
||||||
|
`/models`. A nested bind at `/models/npu` cannot work — the image ships no `/models`, so runc has to `mkdir` the
|
||||||
|
mountpoint through a read-only parent and dies at container init (measured 2026-10-19: 52 crash-loop restarts with
|
||||||
|
chat down before the mount shape was fixed). `HALOGEN_NPU_DIR` is no escape; the shipped entrypoint rejects it by
|
||||||
|
name ("our NPU models live under `/models/npu/<id>/`"). Nothing downloads at boot — `HALOGEN_DOWNLOAD` stays
|
||||||
|
unset, so the container opens no outbound connection; `scripts/download-frank-dragoon-npu.sh` pins every file by
|
||||||
|
size and sha256 against the catalog and the payload mount carries the tree in.
|
||||||
|
|
||||||
Price: the programs and their weights sit in host RAM beside the 62.9 GiB of locked weights ("a few GB in all" for
|
Armed, the boot ledger reads (`/home/dank/logs/frank_dragoon.log`, 2026-10-19): the GPU fabric clock is held at its top level (2000 MHz) **before** the NPU starts; `/dev/accel/accel0` is `crw-rw-rw- nobody:nogroup` under driver `amdxdna` with the distro XRT and its NPU plugin; the program arrives as `15 files checked against this image's record`; then `npu_api: qwen3-embedding-0.6b (embeddings) on the NPU engine at 127.0.0.1:8740 (buckets u0…u9; inputs up to 4096 tokens)` and `up to 64 passes queued … up to 7200 s`. Callers never see `:8740` — the lane answers on the same `:8731` as the Flash model and a request picks the engine by its `model` field, so `/v1/models` lists `frank-dragoon` **and** `qwen3-embedding-0.6b`. The fleet's consumer is open-notebook's embedding default ([[Open Notebook]]); that page owns the app-side shapes (the URL guard, defaults-by-id, the `/api/embed` 500 that still writes the vector).
|
||||||
the default set; ~1.1 GiB for `embeddinggemma-2`), and **the Flash model decodes slower while the NPU works** — same
|
|
||||||
memory bus, same power budget. Under this page's RAM law the question is pool versus n-gram table, so read the boot
|
Price: the programs and their weights sit in host RAM beside the locked weights ("a few GB in all" for the default
|
||||||
ledger after the first NPU boot instead of modelling it. If the lane is opened, prefer **0.18.8 or newer**: that
|
set; ~1.1 GiB for `embeddinggemma-2`), and **the Flash model decodes slower while the NPU works** — same memory bus,
|
||||||
release budgets NPU memory before asking for it, fixing kernel errors from several NPU models under a concurrent mix
|
same power budget. Measured 2026-10-19 with one program armed and the endpoint idle, the whole container holds
|
||||||
|
**1.55 GiB** of host RSS (`podman stats`), so this lane is not what the RAM law above is negotiating against; a
|
||||||
|
multi-model set is where that question returns. If the lane is widened, prefer **0.18.8 or newer**: that release
|
||||||
|
budgets NPU memory before asking for it, fixing kernel errors from several NPU models under a concurrent mix
|
||||||
**on the stock kernel driver** — the driver this box runs.
|
**on the stock kernel driver** — the driver this box runs.
|
||||||
|
|
||||||
## Files
|
## Files
|
||||||
|
|||||||
@@ -247,3 +247,21 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|||||||
}
|
}
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@test "the fabric-clock host unit re-dispatches frank-dragoon and stages nothing else" {
|
||||||
|
# The hold is the NPU lane's boot precondition and the deployer installs and
|
||||||
|
# gates it, so a change to it has to reach the endpoint. The tracked boot
|
||||||
|
# line under the same directory is a hand-taken reboot decision and must
|
||||||
|
# never plan a restart, and staging the NPU model files is host-side state
|
||||||
|
# like every other download script.
|
||||||
|
run python3 "$PLANNER" --target dankclaw --paths \
|
||||||
|
dankclaw/host/halogen-fabric-clock dankclaw/host/halogen-fabric-clock.service
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[ "$output" = "frank-dragoon" ]
|
||||||
|
run python3 "$PLANNER" --target dankclaw --paths dankclaw/host/grub
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[ -z "$output" ]
|
||||||
|
run python3 "$PLANNER" --target dankclaw --paths scripts/download-frank-dragoon-npu.sh
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[ -z "$output" ]
|
||||||
|
}
|
||||||
|
|||||||
@@ -27,6 +27,10 @@ setup() {
|
|||||||
ENV_FILE="$ROOT/scripts/frank-dragoon.env"
|
ENV_FILE="$ROOT/scripts/frank-dragoon.env"
|
||||||
UNIT="$ROOT/scripts/frank-dragoon.service"
|
UNIT="$ROOT/scripts/frank-dragoon.service"
|
||||||
DEPLOYER="$ROOT/scripts/deploy-remote.sh"
|
DEPLOYER="$ROOT/scripts/deploy-remote.sh"
|
||||||
|
HOST_UNIT="$ROOT/dankclaw/host/halogen-fabric-clock"
|
||||||
|
HOST_SERVICE="$ROOT/dankclaw/host/halogen-fabric-clock.service"
|
||||||
|
NPU_FETCH="$ROOT/scripts/download-frank-dragoon-npu.sh"
|
||||||
|
NPU_RULE="$ROOT/dankclaw/udev/98-halogen-npu-access.rules"
|
||||||
BUILD="$ROOT/scripts/build-rocmfpx.sh"
|
BUILD="$ROOT/scripts/build-rocmfpx.sh"
|
||||||
TEST_TMP="$(mktemp -d)"
|
TEST_TMP="$(mktemp -d)"
|
||||||
touch "$TEST_TMP/empty.env"
|
touch "$TEST_TMP/empty.env"
|
||||||
@@ -167,6 +171,14 @@ good_env() {
|
|||||||
kv_gib=$(( pool * 28 * 1024 / 1073741824 ))
|
kv_gib=$(( pool * 28 * 1024 / 1073741824 ))
|
||||||
arena_tenths=$(( 38 + 84 * mt / 16384 ))
|
arena_tenths=$(( 38 + 84 * mt / 16384 ))
|
||||||
total_gib=$(( 62 + kv_gib + arena_tenths / 10 + 2 + reserve + 15 ))
|
total_gib=$(( 62 + kv_gib + arena_tenths / 10 + 2 + reserve + 15 ))
|
||||||
|
# The NPU lane bills extra pinned host RAM per named model (the vendor's
|
||||||
|
# budget note: ~2 GiB for an embedding/decider/rerank-sized program, more
|
||||||
|
# for FLUX-class ones), and the lane is named in the shipped env, so the
|
||||||
|
# ledger must still close on a 121 GiB unified-memory host.
|
||||||
|
npu_models=0
|
||||||
|
npu="$(sed -nE 's/^HALOGEN_NPU_MODELS=(.*)$/\1/p' "$ENV_FILE")"
|
||||||
|
[ -n "$npu" ] && npu_models="$(echo "${npu//,/ }" | wc -w)"
|
||||||
|
total_gib=$(( total_gib + 2 * npu_models ))
|
||||||
[ "$total_gib" -le 121 ]
|
[ "$total_gib" -le 121 ]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -339,6 +351,18 @@ $(cat "$TEST_TMP/serve.log")"
|
|||||||
grep -qF "LLM_DRAGOON_TOK_FILES=\"\${LLM_DRAGOON_TOK_FILES:-$tokf}\"" "$DEPLOYER"
|
grep -qF "LLM_DRAGOON_TOK_FILES=\"\${LLM_DRAGOON_TOK_FILES:-$tokf}\"" "$DEPLOYER"
|
||||||
bytes="$(sed -nE 's/^FRANK_DRAGOON_CHECKPOINT_BYTES=([0-9]+)$/\1/p' "$ENV_FILE")"
|
bytes="$(sed -nE 's/^FRANK_DRAGOON_CHECKPOINT_BYTES=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||||
grep -qF "LLM_DRAGOON_CKPT_BYTES=\"\${LLM_DRAGOON_CKPT_BYTES:-$bytes}\"" "$DEPLOYER"
|
grep -qF "LLM_DRAGOON_CKPT_BYTES=\"\${LLM_DRAGOON_CKPT_BYTES:-$bytes}\"" "$DEPLOYER"
|
||||||
|
# The NPU twins: the lane switches on the same value on both sides. A blank
|
||||||
|
# env line with a named deployer default (or the reverse) is the 2026-10-01
|
||||||
|
# disagreement again — one side gates a lane the other does not build.
|
||||||
|
npu_dir="$(sed -nE 's/^FRANK_DRAGOON_NPU_DIR=(.*)$/\1/p' "$ENV_FILE")"
|
||||||
|
npu_models="$(sed -nE 's/^HALOGEN_NPU_MODELS=(.*)$/\1/p' "$ENV_FILE")"
|
||||||
|
grep -qF "LLM_DRAGOON_NPU_DIR=\"\${LLM_DRAGOON_NPU_DIR:-$npu_dir}\"" "$DEPLOYER"
|
||||||
|
grep -qF "LLM_DRAGOON_NPU_MODELS=\"\${LLM_DRAGOON_NPU_MODELS:-$npu_models}\"" "$DEPLOYER"
|
||||||
|
# The deployer's own copy of the payload dir is what the lane's containment
|
||||||
|
# gate compares against — a stale twin would wave a bad staging path through.
|
||||||
|
grep -qF "LLM_DRAGOON_MODEL_DIR=\"\${LLM_DRAGOON_MODEL_DIR:-$dir}\"" "$DEPLOYER"
|
||||||
|
net="$(sed -nE 's/^FRANK_DRAGOON_NETWORK=(.*)$/\1/p' "$ENV_FILE")"
|
||||||
|
grep -qF "LLM_DRAGOON_NETWORK=\"\${LLM_DRAGOON_NETWORK:-$net}\"" "$DEPLOYER"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "the deployer points a missing artifact at the v2abl fetch script" {
|
@test "the deployer points a missing artifact at the v2abl fetch script" {
|
||||||
@@ -383,3 +407,257 @@ $(cat "$TEST_TMP/serve.log")"
|
|||||||
grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD"
|
grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD"
|
||||||
grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD"
|
grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# --- the NPU lane (XDNA2) ----------------------------------------------------
|
||||||
|
# Opt-in lane added 2026-10-19: halogen runs the vendor's precompiled AIE
|
||||||
|
# programs for the models HALOGEN_NPU_MODELS names on the NPU inside the SAME
|
||||||
|
# container (they answer /v1/embeddings on 7770). Two things make this seam
|
||||||
|
# worth pinning: the lane must be genuinely OFF when the variable is blank (the
|
||||||
|
# rollback is one line, so the render must show nothing lane-specific), and
|
||||||
|
# every host precondition the engine needs — the accel node, the distro XRT's
|
||||||
|
# three libraries, the staged model files, the held GPU fabric clock — must be
|
||||||
|
# refused BY NAME instead of surfacing as an engine crash-loop.
|
||||||
|
|
||||||
|
npu_fixture() {
|
||||||
|
# 0666 like the installed node: a rootless container's namespace cannot open
|
||||||
|
# the stock root:render 0660 mode, and the serve gate checks exactly that.
|
||||||
|
touch "$TEST_TMP/accel0"
|
||||||
|
chmod 666 "$TEST_TMP/accel0"
|
||||||
|
mkdir -p "$TEST_TMP/xrt"
|
||||||
|
for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||||||
|
touch "$TEST_TMP/xrt/$l"
|
||||||
|
done
|
||||||
|
mkdir -p "$TEST_TMP/npu/qwen3-embedding-0.6b/devices"
|
||||||
|
mkdir -p "$TEST_TMP/sys/class/drm/card1/device"
|
||||||
|
printf '0: 400Mhz \n7: 2000Mhz *\n' > "$TEST_TMP/sys/class/drm/card1/device/pp_dpm_fclk"
|
||||||
|
printf 'manual\n' > "$TEST_TMP/sys/class/drm/card1/device/power_dpm_force_performance_level"
|
||||||
|
NPU_ENV=(
|
||||||
|
"HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"
|
||||||
|
"FRANK_DRAGOON_NPU_DIR=$TEST_TMP/npu"
|
||||||
|
"FRANK_DRAGOON_DEV_ACCEL=$TEST_TMP/accel0"
|
||||||
|
"FRANK_DRAGOON_XRT_DIR=$TEST_TMP/xrt"
|
||||||
|
"FRANK_DRAGOON_SYSFS=$TEST_TMP/sys"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
# run_with_log <args...> — run the serve script and fold its own stderr plus
|
||||||
|
# the podman stub's argv (which the script redirects into the log) into
|
||||||
|
# $output, so both refusal text and render assertions read one variable.
|
||||||
|
run_with_log() {
|
||||||
|
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" "${NPU_ENV[@]}" "$@" "$SERVE"
|
||||||
|
if [ -f "$TEST_TMP/serve.log" ]; then
|
||||||
|
output="$output
|
||||||
|
$(cat "$TEST_TMP/serve.log")"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the lane is off when HALOGEN_NPU_MODELS is blank: no device, no mounts, no engine env" {
|
||||||
|
# The rollback is blanking one env line, so "off" has to be observable in
|
||||||
|
# the command podman would run — not just implied by an absent log line.
|
||||||
|
podman_stub
|
||||||
|
good_env
|
||||||
|
run_with_log HALOGEN_NPU_MODELS=
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"--publish 7770:8731"* ]]
|
||||||
|
! grep -q '/models/npu' "$TEST_TMP/serve.log"
|
||||||
|
! grep -q 'HALOGEN_NPU_MODELS' "$TEST_TMP/serve.log"
|
||||||
|
! grep -q 'xilinx' "$TEST_TMP/serve.log"
|
||||||
|
! grep -q 'accel0' "$TEST_TMP/serve.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the lane arms the accel node, the staged model dir and each XRT library twice" {
|
||||||
|
# Each library mounts at the image's XRT path AND at its own path: the
|
||||||
|
# distro libraries find each other where they were installed, and an
|
||||||
|
# /opt/xilinx/xrt of symlinks into the system dir dangles inside the
|
||||||
|
# container (the vendor calls that out by name).
|
||||||
|
podman_stub
|
||||||
|
good_env
|
||||||
|
npu_fixture
|
||||||
|
run_with_log
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"--device $TEST_TMP/accel0"* ]]
|
||||||
|
# The lane's files ride the payload mount: /models is a read-only bind and
|
||||||
|
# runc cannot mkdir a nested /models/npu mountpoint through it (measured
|
||||||
|
# 2026-10-19), so the staging dir lives INSIDE the payload dir and gets no
|
||||||
|
# -v of its own.
|
||||||
|
[[ "$output" == *"-v $TEST_TMP:/models:ro"* ]]
|
||||||
|
[[ "$output" != *":/models/npu"* ]]
|
||||||
|
[[ "$output" == *"-e HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"* ]]
|
||||||
|
for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||||||
|
[[ "$output" == *"-v $TEST_TMP/xrt/$l:/opt/xilinx/xrt/lib/$l:ro"* ]]
|
||||||
|
[[ "$output" == *"-v $TEST_TMP/xrt/$l:$TEST_TMP/xrt/$l:ro"* ]]
|
||||||
|
done
|
||||||
|
# The Flash payload still rides the same container: the lane is additive.
|
||||||
|
[[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the lane refuses a staging dir outside the payload dir and says why" {
|
||||||
|
# A staging dir outside the payload dir needs a nested bind the read-only
|
||||||
|
# /models mount cannot host: runc dies at container init, which is AFTER the
|
||||||
|
# deployer has stopped the old unit — so the shape is refused here, before
|
||||||
|
# anything is rendered.
|
||||||
|
podman_stub
|
||||||
|
good_env
|
||||||
|
npu_fixture
|
||||||
|
run_with_log FRANK_DRAGOON_NPU_DIR=/elsewhere/npu
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"/elsewhere/npu"*"outside the payload dir"* ]]
|
||||||
|
[[ "$output" != *"--device"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the lane refuses a model that is not staged and names the fetch script" {
|
||||||
|
# The engine cannot fetch these at boot here (/models is read-only and
|
||||||
|
# HALOGEN_DOWNLOAD stays unset), so an un-staged name must stop the unit
|
||||||
|
# before podman runs, with the command that fixes it.
|
||||||
|
podman_stub
|
||||||
|
good_env
|
||||||
|
npu_fixture
|
||||||
|
rm -rf "$TEST_TMP/npu/qwen3-embedding-0.6b"
|
||||||
|
run_with_log
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"qwen3-embedding-0.6b"*"not staged"* ]]
|
||||||
|
[[ "$output" == *"download-frank-dragoon-npu.sh"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the lane refuses without the accel node and says what to check" {
|
||||||
|
podman_stub
|
||||||
|
good_env
|
||||||
|
npu_fixture
|
||||||
|
rm "$TEST_TMP/accel0"
|
||||||
|
run_with_log
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"$TEST_TMP/accel0 missing"* ]]
|
||||||
|
[[ "$output" == *"iommu=pt"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the lane refuses an NPU node the container namespace could not open, and names the rule" {
|
||||||
|
# Stock Debian mode is root:render 0660. The engine's own first words on such
|
||||||
|
# a node are "this process cannot open it (crw-rw---- nobody:nogroup)" and no
|
||||||
|
# flag fixes a mode an unmapped gid cannot cross (keep-groups measured dead
|
||||||
|
# 2026-10-19), so the unit refuses instead of crash-looping 189 times.
|
||||||
|
podman_stub
|
||||||
|
good_env
|
||||||
|
npu_fixture
|
||||||
|
chmod 660 "$TEST_TMP/accel0"
|
||||||
|
run_with_log
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"$TEST_TMP/accel0 is 660"* ]]
|
||||||
|
[[ "$output" == *"98-halogen-npu-access.rules"* ]]
|
||||||
|
[[ "$output" != *"--device"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the NPU access rule is the accel counterpart of the GPU compute rule" {
|
||||||
|
[ -f "$NPU_RULE" ]
|
||||||
|
grep -qE '^KERNEL=="accel\[0-9\]\*", SUBSYSTEM=="accel", MODE="0666"$' "$NPU_RULE"
|
||||||
|
# The deployer installs the rule and re-triggers the node BEFORE it trusts
|
||||||
|
# the mode, so a machine that never had it self-heals instead of refusing.
|
||||||
|
grep -qE '98-halogen-npu-access\.rules' "$DEPLOYER"
|
||||||
|
grep -qE 'udevadm control --reload-rules' "$DEPLOYER"
|
||||||
|
grep -qE 'udevadm trigger --sysname-match=' "$DEPLOYER"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the lane refuses when an XRT NPU library is missing" {
|
||||||
|
podman_stub
|
||||||
|
good_env
|
||||||
|
npu_fixture
|
||||||
|
rm "$TEST_TMP/xrt/libxrt_driver_xdna.so.2"
|
||||||
|
run_with_log
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"libxrt_driver_xdna.so.2 missing"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the lane refuses while the fabric clock is moving" {
|
||||||
|
# The engine itself refuses to start in this state (GPU+NPU co-run hangs
|
||||||
|
# Strix Halo while FCLK changes speed) but cannot see host sysfs from
|
||||||
|
# inside the container — the proof has to happen on the host side.
|
||||||
|
podman_stub
|
||||||
|
good_env
|
||||||
|
npu_fixture
|
||||||
|
printf 'auto\n' > "$TEST_TMP/sys/class/drm/card1/device/power_dpm_force_performance_level"
|
||||||
|
run_with_log
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"not held"* ]]
|
||||||
|
[[ "$output" == *"sudo halogen-fabric-clock hold"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the lane refuses when no fabric-clock control can be read" {
|
||||||
|
# A host whose sysfs exposes nothing cannot PROVE the hold, and a silent
|
||||||
|
# pass here is a boot that dies in the engine instead.
|
||||||
|
podman_stub
|
||||||
|
good_env
|
||||||
|
npu_fixture
|
||||||
|
rm -rf "$TEST_TMP/sys/class/drm"
|
||||||
|
run_with_log
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"cannot prove"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the lane also passes when the clock is held at the high level" {
|
||||||
|
# The vendor's status accepts `high` as well as `manual` + the top fclk
|
||||||
|
# level; refusing a correctly-held host would block a deploy for nothing.
|
||||||
|
podman_stub
|
||||||
|
good_env
|
||||||
|
npu_fixture
|
||||||
|
printf 'high\n' > "$TEST_TMP/sys/class/drm/card1/device/power_dpm_force_performance_level"
|
||||||
|
run_with_log
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"-e HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the deployer gates the same lane artifacts and installs the fabric-clock unit" {
|
||||||
|
grep -qE 'LLM_DRAGOON_NPU_MODELS' "$DEPLOYER"
|
||||||
|
grep -qE 'install -m 0755 dankclaw/host/halogen-fabric-clock' "$DEPLOYER"
|
||||||
|
grep -qE 'install -m 0644 dankclaw/host/halogen-fabric-clock\.service' "$DEPLOYER"
|
||||||
|
grep -qE 'halogen-fabric-clock status' "$DEPLOYER"
|
||||||
|
grep -qE 'download-frank-dragoon-npu\.sh' "$DEPLOYER"
|
||||||
|
# The lane's own health gate is a real embedding, not a 200 on /health.
|
||||||
|
grep -qE '/v1/embeddings' "$DEPLOYER"
|
||||||
|
# Parity with the serve gate: a staging dir outside the payload dir is
|
||||||
|
# refused before the restart, not left to runc.
|
||||||
|
grep -qE 'outside the payload dir' "$DEPLOYER"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the fabric-clock host unit is the vendor shape and boots on its own" {
|
||||||
|
[ -x "$HOST_UNIT" ]
|
||||||
|
grep -qE '^ (hold\)|release\)|status\))' "$HOST_UNIT"
|
||||||
|
grep -qE 'pp_dpm_fclk' "$HOST_UNIT"
|
||||||
|
grep -qE '^Type=oneshot$' "$HOST_SERVICE"
|
||||||
|
grep -qE '^WantedBy=.*multi-user\.target' "$HOST_SERVICE"
|
||||||
|
# A resume hands the clock back to the driver, so the unit must re-run then.
|
||||||
|
grep -qE '^WantedBy=.*suspend\.target' "$HOST_SERVICE"
|
||||||
|
grep -qE '^ExecStart=.*/usr/local/sbin/halogen-fabric-clock hold$' "$HOST_SERVICE"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the NPU fetch script pins the vendor catalog by size and sha256" {
|
||||||
|
# XDNA2 runs only AIE-compiled programs, so the vendor's per-model repo is
|
||||||
|
# the only source; the engine re-checks size+sha256 against its own catalog
|
||||||
|
# at boot, so this manifest has to agree with it or the boot refuses.
|
||||||
|
[ -x "$NPU_FETCH" ]
|
||||||
|
grep -qE 'curl -fL' "$NPU_FETCH"
|
||||||
|
grep -qE 'qwen3-embedding-0\.6b\.hnpw 813032248 [0-9a-f]{64}' "$NPU_FETCH"
|
||||||
|
grep -qE 'sha256sum' "$NPU_FETCH"
|
||||||
|
# curl -f alone is not enough on Hugging Face: the resolve URL redirects
|
||||||
|
# and curl drops Range across it (the sibling 62 GiB fetch, 2026-10-18).
|
||||||
|
grep -qE 'url_effective' "$NPU_FETCH"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the container network is podman's default when unnamed and the app bridge when named" {
|
||||||
|
# The app-to-endpoint leg hangs on this: pasta is a private netns nothing
|
||||||
|
# else can join (podman refuses a second network, the published port is
|
||||||
|
# refused from app containers, and host.containers.internal is link-local,
|
||||||
|
# which open-notebook refuses to store), so reaching the NPU endpoints by
|
||||||
|
# name means the container lives on dankclaw_default. Both arms are
|
||||||
|
# observable in the command podman would run.
|
||||||
|
podman_stub
|
||||||
|
good_env
|
||||||
|
run_with_log FRANK_DRAGOON_NETWORK=
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
! grep -q -- '--network' "$TEST_TMP/serve.log"
|
||||||
|
run_with_log FRANK_DRAGOON_NETWORK=dankclaw_default
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"--network dankclaw_default"* ]]
|
||||||
|
# The deployer refuses a network that does not exist rather than starting a
|
||||||
|
# unit that restart-loops on it.
|
||||||
|
grep -qE 'podman network exists' "$DEPLOYER"
|
||||||
|
grep -qE '^FRANK_DRAGOON_NETWORK=dankclaw_default$' "$ENV_FILE"
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user