diff --git a/dankclaw/host/halogen-fabric-clock b/dankclaw/host/halogen-fabric-clock new file mode 100755 index 0000000..92e8d07 --- /dev/null +++ b/dankclaw/host/halogen-fabric-clock @@ -0,0 +1,65 @@ +#!/bin/sh +# halogen-fabric-clock hold | release | status: the AMD GPU's fabric clock (FCLK), held at its top level. +# +# On Strix Halo the GPU and the NPU working at the same time can hang the machine, and corrupt the NPU's results, +# while the GPU's fabric clock changes speed; held at its top level, the two run together cleanly. halogen-flash-server +# with HALOGEN_NPU_MODELS refuses to start until the clock is held. This holds it: the performance level `manual` and +# only the top pp_dpm_fclk level, on every amdgpu device that has the control (the GPU and SoC clocks stay automatic). +# Cost: about 1 W at idle. Run as root; halogen-fabric-clock.service runs `hold` at every boot and after a resume. +# +# hold hold it (waits up to 60 s for the GPU driver to come up) +# release give it back to the driver (the performance level `auto`) +# status print each device's state; exit 0 when every device is held +set -u +SYS="${HALOGEN_SYSFS:-/sys}" + +devices() { + for d in "$SYS"/class/drm/card*/device; do + [ -w "$d/pp_dpm_fclk" ] || [ -r "$d/pp_dpm_fclk" ] || continue + [ -e "$d/power_dpm_force_performance_level" ] || continue + echo "$d" + done | sort -u +} + +held() { # held DEVICE_DIR: the performance level high, or manual with only the top fclk level selected + perf=$(cat "$1/power_dpm_force_performance_level" 2>/dev/null) + [ "$perf" = high ] && return 0 + [ "$perf" = manual ] || return 1 + top=$(grep . "$1/pp_dpm_fclk" | tail -n 1) + [ "$(grep -c '\*' "$1/pp_dpm_fclk")" = 1 ] && [ "${top%\*}" != "$top" ] +} + +cmd="${1:-status}" +case "$cmd" in + hold) + n=0 + while [ -z "$(devices)" ] && [ "$n" -lt 60 ]; do sleep 1; n=$((n + 1)); done + [ -n "$(devices)" ] || { echo "halogen-fabric-clock: no amdgpu device with a fabric clock control (pp_dpm_fclk) under $SYS/class/drm" >&2; exit 1; } + rc=0 + for d in $(devices); do + lvl=$(grep . "$d/pp_dpm_fclk" | tail -n 1 | cut -d: -f1) + # the driver applies the level several seconds after the write, so the read-back waits up to 30 s + ok=0 + if echo manual > "$d/power_dpm_force_performance_level" && echo "$lvl" > "$d/pp_dpm_fclk"; then + t=0; until held "$d" || [ "$t" -ge 300 ]; do sleep 0.1; t=$((t + 1)); done + held "$d" && ok=1 + fi + if [ "$ok" = 1 ]; then + echo "halogen-fabric-clock: $d: held at level $lvl ($(grep . "$d/pp_dpm_fclk" | tail -n 1 | awk '{print $2}'))" + else + echo "halogen-fabric-clock: $d: could not hold it (level $lvl)" >&2; rc=1 + fi + done + exit "$rc" ;; + release) + for d in $(devices); do echo auto > "$d/power_dpm_force_performance_level" && echo "halogen-fabric-clock: $d: released (auto)"; done ;; + status) + [ -n "$(devices)" ] || { echo "halogen-fabric-clock: no amdgpu device with a fabric clock control"; exit 1; } + rc=0 + for d in $(devices); do + if held "$d"; then echo "$d: held ($(cat "$d/power_dpm_force_performance_level"))" + else echo "$d: NOT held ($(cat "$d/power_dpm_force_performance_level"))"; rc=1; fi + done + exit "$rc" ;; + *) echo "usage: halogen-fabric-clock hold | release | status" >&2; exit 2 ;; +esac diff --git a/dankclaw/host/halogen-fabric-clock.service b/dankclaw/host/halogen-fabric-clock.service new file mode 100644 index 0000000..bba4fe0 --- /dev/null +++ b/dankclaw/host/halogen-fabric-clock.service @@ -0,0 +1,19 @@ +# Holds the AMD GPU's fabric clock at its top level at boot and after every resume, for halogen-flash-server with +# the NPU (HALOGEN_NPU_MODELS): on Strix Halo the GPU and the NPU at once are safe only with that clock held. +# Install (as root), from this directory: +# install -m 755 halogen-fabric-clock /usr/local/sbin/ +# install -m 644 halogen-fabric-clock.service /etc/systemd/system/ +# systemctl daemon-reload && systemctl enable --now halogen-fabric-clock.service +# Check: halogen-fabric-clock status. Undo: systemctl disable halogen-fabric-clock.service, then +# halogen-fabric-clock release (or reboot). The unit runs once at boot and once after each resume (a resume can hand +# the clock back to the driver); it holds nothing open, so it shows inactive between runs. +[Unit] +Description=Hold the AMD GPU fabric clock at its top level (GPU and NPU together) +After=systemd-modules-load.service suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/halogen-fabric-clock hold + +[Install] +WantedBy=multi-user.target suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py index 8684c14..7388e1d 100755 --- a/scripts/deploy-plan.py +++ b/scripts/deploy-plan.py @@ -293,6 +293,17 @@ def services_for_path(path: str, target: str) -> set[str]: if len(parts) >= 2 and parts[1] in {"caddy", "comfyui", "loki", "alloy", "hermes", "anubis", "seerr"}: _add(services, parts[1]) return services + if len(parts) >= 3 and parts[1] == "host": + # Host state a service needs at boot lives under dankclaw/host/. + # The fabric-clock hold is frank-dragoon's NPU precondition (the + # engine refuses to start with HALOGEN_NPU_MODELS set while the + # clock moves) and the deployer installs and gates it, so changing + # it re-dispatches that endpoint. `grub` is the tracked boot line, + # which moves only through scripts/set-dankclaw-bootline.sh and is + # a reboot decision — it plans no service. + if parts[2].startswith("halogen-fabric-clock"): + _add(services, "frank-dragoon") + return services if len(parts) >= 4 and parts[1] == "authentik" and parts[2] == "quadlet": name = parts[3].removesuffix(".container") if name in {"authentik-postgres", "authentik-server", "authentik-worker", "authentik-outpost", "authentik-outpost-ldap"}: diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 9767627..717605d 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1337,6 +1337,18 @@ LLM_DRAGOON_VISION="${LLM_DRAGOON_VISION:-/home/dank/models/halogen-v2-abliterat LLM_DRAGOON_VISION_BYTES="${LLM_DRAGOON_VISION_BYTES:-897916416}" LLM_DRAGOON_TOK_DIR="${LLM_DRAGOON_TOK_DIR:-/home/dank/models/halogen-v2-abliterated/tokenizer}" LLM_DRAGOON_TOK_FILES="${LLM_DRAGOON_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json}" +# NPU lane (2026-10-19). HALOGEN_NPU_MODELS is the whole switch and the parity +# twin of the env file's line: empty here AND in the env = the pre-lane +# endpoint. The gates below mirror serve-frank-dragoon.sh, and the vendor's +# fabric-clock host unit (dankclaw/host/, verbatim at tag v0.16.2) is installed +# ONLY while the lane is named — on Strix Halo the iGPU and the NPU co-running +# hang the machine while the fabric clock moves, so halogen refuses to boot with +# the lane armed unless that clock is held. +LLM_DRAGOON_NPU_MODELS="${LLM_DRAGOON_NPU_MODELS:-qwen3-embedding-0.6b}" +LLM_DRAGOON_NPU_DIR="${LLM_DRAGOON_NPU_DIR:-/home/dank/models/halogen-npu}" +LLM_DRAGOON_DEV_ACCEL="${LLM_DRAGOON_DEV_ACCEL:-/dev/accel/accel0}" +LLM_DRAGOON_XRT_DIR="${LLM_DRAGOON_XRT_DIR:-/usr/lib/x86_64-linux-gnu}" +LLM_DRAGOON_NETWORK="${LLM_DRAGOON_NETWORK:-dankclaw_default}" retire_halogen_dragoon_quadlet() { local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" @@ -1415,6 +1427,55 @@ deploy_llm_dragoon() { fi done + # The container network has to exist before the run line names it (a typo + # there is a restart-looping unit, not a config error anyone reads). + if [[ -n "$LLM_DRAGOON_NETWORK" ]] && ! podman network exists "$LLM_DRAGOON_NETWORK"; then + echo "ERROR: podman network $LLM_DRAGOON_NETWORK does not exist" >&2 + echo "frank-dragoon: create it or blank FRANK_DRAGOON_NETWORK (pasta default)" >&2 + return 1 + fi + + # --- NPU lane: the same three predicates the serve script checks --------- + if [[ -n "$LLM_DRAGOON_NPU_MODELS" ]]; then + if [[ ! -e "$LLM_DRAGOON_DEV_ACCEL" ]]; then + echo "ERROR: NPU node $LLM_DRAGOON_DEV_ACCEL absent — amdxdna is not bound (is iommu=pt still on the boot line?)" >&2 + return 1 + fi + for xl in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do + if [[ ! -e "$LLM_DRAGOON_XRT_DIR/$xl" ]]; then + echo "ERROR: XRT NPU library $LLM_DRAGOON_XRT_DIR/$xl absent (Ubuntu libxrt2 + libxrt-npu2)" >&2 + return 1 + fi + done + for nm in ${LLM_DRAGOON_NPU_MODELS//,/ }; do + if [[ ! -d "$LLM_DRAGOON_NPU_DIR/$nm" ]]; then + echo "ERROR: NPU model $nm not staged in $LLM_DRAGOON_NPU_DIR" >&2 + echo "frank-dragoon: stage it: bash scripts/download-frank-dragoon-npu.sh" >&2 + return 1 + fi + done + # The hold must exist BEFORE the engine starts and survive every boot + # (a resume hands the clock back too — the unit's WantedBy covers it). + run_privileged install -m 0755 dankclaw/host/halogen-fabric-clock \ + /usr/local/sbin/halogen-fabric-clock + run_privileged install -m 0644 dankclaw/host/halogen-fabric-clock.service \ + /etc/systemd/system/halogen-fabric-clock.service + run_privileged systemctl daemon-reload + run_privileged systemctl enable --now halogen-fabric-clock.service + if ! run_privileged /usr/local/sbin/halogen-fabric-clock status; then + echo "ERROR: fabric clock is not held — halogen refuses to start with HALOGEN_NPU_MODELS set" >&2 + echo "frank-dragoon: journalctl -u halogen-fabric-clock.service -n 40" >&2 + return 1 + fi + elif [[ -x /usr/local/sbin/halogen-fabric-clock ]]; then + # Lane rolled back: hand the clock back to the driver and drop the boot + # hook. `disable` deletes the INSTALLED copy of this unit — unlike the + # tracked LLM unit symlinks the fleet's disable law protects, this file + # is repo-owned and the next lane-on deploy reinstalls it. + run_privileged systemctl disable halogen-fabric-clock.service 2>/dev/null || true + run_privileged /usr/local/sbin/halogen-fabric-clock release || true + fi + retire_halogen_dragoon_quadlet # The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops # the container) before waiting on the port. The stray sweep is keyed @@ -1434,6 +1495,33 @@ deploy_llm_dragoon() { run_privileged systemctl enable frank-dragoon.service run_privileged systemctl restart frank-dragoon.service wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200 + # The lane's own gate. /health proves the engine is up, nothing about the + # NPU programs; one real embedding is the observable output. The first + # request after a boot is a warmup artifact (fleet law), so retry before + # calling a booted lane dead. + if [[ -n "$LLM_DRAGOON_NPU_MODELS" ]]; then + for nm in ${LLM_DRAGOON_NPU_MODELS//,/ }; do + lane_ok=0 + for _try in 1 2 3 4 5; do + if curl -fsS -m 60 -H 'content-type: application/json' \ + -d "{\"model\":\"$nm\",\"input\":[\"frank-dragoon npu lane gate\"]}" \ + http://127.0.0.1:7770/v1/embeddings 2>/dev/null \ + | python3 -c 'import json,sys +v = json.load(sys.stdin)["data"][0]["embedding"] +raise SystemExit(0 if isinstance(v, list) and len(v) and all(isinstance(x, (int, float)) for x in v) else 1)'; then + lane_ok=1 + break + fi + sleep 10 + done + if [[ "$lane_ok" != 1 ]]; then + echo "ERROR: NPU model $nm answered no vector on /v1/embeddings" >&2 + echo "frank-dragoon: read the boot ledger: tail -100 /home/dank/logs/frank_dragoon.log" >&2 + return 1 + fi + echo "frank-dragoon: NPU lane verified — $nm embedded" + done + fi } # deploy_llm_zealot — frank-zealot is llama.cpp (PrismML fork build) on the diff --git a/scripts/download-frank-dragoon-npu.sh b/scripts/download-frank-dragoon-npu.sh new file mode 100755 index 0000000..b397ac2 --- /dev/null +++ b/scripts/download-frank-dragoon-npu.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# download-frank-dragoon-npu.sh — stage the NPU half of frank-dragoon: the +# vendor-precompiled AIE programs halogen-flash-server runs on the XDNA2 NPU +# when HALOGEN_NPU_MODELS names them. These are NOT model weights you can +# quantize yourself: XDNA2 executes only programs compiled by AMD's AIE +# compiler, so the only source is peonist-ai's per-model NPU repo, and the +# engine checks every file's size and sha256 against its own catalog at boot +# (/opt/halogen/npu/models.txt inside the image) and refuses to load a model +# whose files do not match. +# +# The manifest below IS that catalog block (dumped from the pinned 0.16.2 +# image on 2026-10-19), so the files land in the layout the entrypoint +# expects — //devices/*, //.hnpw and +# //tokenizer/tokenizer.json — and the endpoint can keep +# /models read-only and open no outbound connection at boot (HALOGEN_DOWNLOAD +# stays unset; the vendor's own README: with it unset the container makes no +# outbound connections at all). +# +# The fetch law, same as every sibling script: curl -f (an access error must +# not become a "successful" 153-byte artifact), the exact byte count and the +# sha256 from the manifest, --verify-only to re-check a staged set. +# +# Adding a second NPU model: read its block out of the image's catalog and add +# a case below — +# podman exec frank-dragoon-halogen grep "^file " /opt/halogen/npu/models.txt +# podman exec frank-dragoon-halogen grep "^model " /opt/halogen/npu/models.txt +# and add the model id to HALOGEN_NPU_MODELS in frank-dragoon.env. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}" +DEST="${FRANK_DRAGOON_NPU_DIR:-$HOME/models/halogen-npu}" +HF="https://huggingface.co" + +MODELS="${HALOGEN_NPU_MODELS:-}" +if [ -z "$MODELS" ] && [ -f "$ENV_FILE" ]; then + MODELS="$(sed -nE 's/^HALOGEN_NPU_MODELS=(.*)$/\1/p' "$ENV_FILE" | tail -1)" +fi + +VERIFY_ONLY=0 +case "${1:-}" in + --verify-only) VERIFY_ONLY=1 ;; + "") ;; + *) echo "usage: ${0##*/} [--verify-only] (models come from HALOGEN_NPU_MODELS)" >&2; exit 2 ;; +esac +[ -n "$MODELS" ] || { echo "no NPU model named — set HALOGEN_NPU_MODELS or run with it in the environment" >&2; exit 1; } + +# manifest -> repo revision + one "path bytes sha256" per file. +manifest() { + case "$1" in + qwen3-embedding-0.6b) + NPU_REPO="peonist-ai/halogen-npu-qwen3-embedding-0.6b" + NPU_REV="4960fd9d4423dd15373ad21c83db48dbdb3b2798" + FILES=( + "devices/NOTICE.md 3248 0f267b3d2523ddf14997ee58660a492a9eaf10ccf6b08bf19dfdc85b9a478b91" + "devices/consts.hnpb 5086496 ae430e41545a0b1012938cda373b0c450364d3be631111cca3a8c43679f93336" + "devices/devices.hnpm 24047 4e9bca9d95d084fb89ce5b99f579a00666c8ca1efae2613c53e3a4878cc9f174" + "devices/u0.elf 4931968 b38f165886cb037b7d3951d9cb878473eaba58a432acd45416ff85fd88aa4e46" + "devices/u1.elf 5456000 b1811feee851b8af833b37721da92099d89772110815bbbe2c3fa1c276f0ff42" + "devices/u2.elf 6505552 406ea9a95b470628a7382e376f4a72cdf4d7aa4c59b4ddf0a4bdd139d63d6ff2" + "devices/u3.elf 6817216 bc205d51b523d770151a0c61968f7c70bf624412827580bb6fe0aaf4650dc2ee" + "devices/u4.elf 7444032 a5a91338a2603acdcd5c11df532033fda2d3f5479ea44ae87f98fafbdff9a62d" + "devices/u5.elf 7432416 6c2bf41835569e0eba25ded7c05c7264ae69c6700bdeb4791057d884f2a4c126" + "devices/u6.elf 4941232 919d0038ca088eafddf195f3466ff38db27d3635be1cffa5d4d4b5140902a68e" + "devices/u7.elf 5488304 d904cf8afae9428dd167960549e18cce5441745f18c62f8c50975dffc077d3c2" + "devices/u8.elf 4941232 db3aef69b17825854ee0dd2df475125a3f5c872f95c686793d6af1ee83596fa7" + "devices/u9.elf 5488304 d182727e551084db8fb9600441d8aa373c7f09a3d267ae0a5e921ef92326db1c" + "qwen3-embedding-0.6b.hnpw 813032248 b9f9901429df7b46ec17153aa3899f738adbea8e4356080da0c6e6586fb750d1" + "tokenizer/tokenizer.json 11423705 def76fb086971c7867b829c23a26261e38d9d74e02139253b38aeb9df8b4b50a" + ) ;; + *) + echo "unknown NPU model '$1' — add its catalog block to manifest() in this script" >&2 + return 1 ;; + esac +} + +for MODEL in ${MODELS//,/ }; do + manifest "$MODEL" || exit 1 + echo "== $MODEL ($NPU_REPO@$NPU_REV) ==" + for entry in "${FILES[@]}"; do + read -r f want sha <<<"$entry" + path="$DEST/$MODEL/$f" + mkdir -p "$(dirname "$path")" + if [ "$VERIFY_ONLY" -eq 0 ]; then + if [ -f "$path" ] && [ "$(stat -c %s "$path")" = "$want" ]; then + echo "HAVE $f ($want bytes) — skipping fetch" + else + echo "FETCH $f -> $path" + # HF answers /resolve/ with a 302 to the Xet CDN and curl + # DROPS the Range header across redirects, so resolve the + # effective URL first and resume against the CDN (the sibling + # 62 GiB fetch hit this on 2026-10-18). + url=$(curl -sfIL -o /dev/null -w '%{url_effective}' "$HF/$NPU_REPO/resolve/$NPU_REV/$f") \ + || { echo "cannot resolve $NPU_REPO/$f" >&2; exit 1; } + curl -fL --retry 5 -C - -o "$path" "$url" + fi + fi + [ -f "$path" ] || { echo "MISSING $path (run without --verify-only)" >&2; exit 1; } + if [ -L "$path" ]; then + echo "SYMLINK $path — the serve gate refuses symlinks; replace with a real file" >&2 + exit 1 + fi + got="$(stat -c %s "$path")" + [ "$got" = "$want" ] || { echo "SIZE $path: $got != $want" >&2; exit 1; } + got="$(sha256sum "$path" | cut -d' ' -f1)" + [ "$got" = "$sha" ] || { echo "SHA256 $path: $got != $sha" >&2; exit 1; } + echo "OK $f ($want bytes, sha256)" + done +done +echo "DOWNLOAD_DONE $DEST ($MODELS)" diff --git a/scripts/frank-dragoon.env b/scripts/frank-dragoon.env index 045a770..ee67a94 100644 --- a/scripts/frank-dragoon.env +++ b/scripts/frank-dragoon.env @@ -80,3 +80,42 @@ HALOGEN_TOP_K=20 # Weights stay pinned: the engine pins by default and only the retired w4b build # honored HALOGEN_FLASH_PIN_TRUNK=0, which this payload rejects. HALOGEN_WEIGHTS_LOCK=1 + +# --- NPU lane (XDNA2) ------------------------------------------------------- +# HALOGEN_NPU_MODELS is the whole switch: EMPTY or unset = the endpoint exactly +# as it ran before this lane existed (no accel device, no XRT mount, no +# fabric-clock dependency, no /v1/embeddings). Rolling the lane back is blanking +# this one line and re-deploying — nothing else moves. +# Names are the vendor catalog's folder names (`/opt/halogen/npu/models.txt` +# inside the image). The engine cannot fetch them here: /models is read-only and +# HALOGEN_DOWNLOAD stays unset so the container opens no outbound connection at +# boot, so scripts/download-frank-dragoon-npu.sh must stage them into +# $FRANK_DRAGOON_NPU_DIR first (the serve gate and the deployer both refuse by +# name when a named model is not staged). +# Cost: ~1.5 GiB extra pinned host RAM per named model, and a hard dependency on +# the host holding the GPU fabric clock — on Strix Halo the iGPU and the NPU +# together hang the machine (and corrupt NPU results) while FCLK changes speed, +# so halogen refuses to start with this variable set unless the clock is held +# (dankclaw/host/halogen-fabric-clock; the deployer installs and gates it). +HALOGEN_NPU_MODELS=qwen3-embedding-0.6b +FRANK_DRAGOON_NPU_DIR=/home/dank/models/halogen-npu +# Node paths and the XRT directory are variables so the bats seam can drive +# these gates on a box with no NPU. Ubuntu's distro XRT (libxrt2 + libxrt-npu2) +# is mounted FILE BY FILE and TWICE EACH — once where the image looks for XRT, +# once at its own path, because those libraries find each other at their install +# path; the vendor forbids mounting an /opt/xilinx/xrt of symlinks into the +# system dir, which dangle inside the container. +FRANK_DRAGOON_DEV_ACCEL=/dev/accel/accel0 +FRANK_DRAGOON_XRT_DIR=/usr/lib/x86_64-linux-gnu +# sysfs root the fabric-clock preflight reads (same knob the vendor's +# halogen-fabric-clock status reads; the seam points it at a fixture). +FRANK_DRAGOON_SYSFS=/sys +# Container network. Empty = podman's rootless default (`pasta`): a private +# netns that nothing else can join — podman refuses a second network while the +# primary is pasta, the published port is refused from dankclaw_default +# containers, and host.containers.internal resolves link-local, which +# open-notebook's URL guard refuses to store. Named = the app bridge, so +# open-notebook can dial http://frank-dragoon-halogen:8731 for the NPU +# endpoints. The engine's unauthenticated 8730 stays private either way (it +# binds the container's own loopback); only the API port becomes name-resolvable. +FRANK_DRAGOON_NETWORK=dankclaw_default diff --git a/scripts/serve-frank-dragoon.sh b/scripts/serve-frank-dragoon.sh index b51fe42..9f90f27 100755 --- a/scripts/serve-frank-dragoon.sh +++ b/scripts/serve-frank-dragoon.sh @@ -27,6 +27,11 @@ # the flash-server line reads it; it ships as a container; the root unit # keeps LimitMEMLOCK=infinity for parity and to outlive the login session, # exactly like frank-archon's. +# Since 2026-10-19 the container also carries the XDNA2 NPU lane +# (HALOGEN_NPU_MODELS): the vendor's precompiled AIE programs for the models +# named there run on the NPU inside THIS container and answer /v1/embeddings, +# beside the Flash model, on the same port. Opt-in — the variable empty is the +# pre-lane endpoint, byte for byte. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -115,6 +120,91 @@ if [ ! -e "$DEV_KFD" ] || [ ! -d "$DEV_DRI" ]; then echo "frank-dragoon: $DEV_KFD or $DEV_DRI missing — no gfx1151 to serve on" >&2 exit 1 fi + +# --- NPU lane (XDNA2), opt-in via HALOGEN_NPU_MODELS ------------------------ +# Empty = the endpoint exactly as it ran before the lane existed: no device, no +# extra mounts, no gate below. Set = three host preconditions, each refused BY +# NAME (the 2026-10-01 law: the serve gate and the deployer gate must agree): +# 1. the staged model files (the engine checks size+sha256 against its own +# catalog at boot; it cannot fetch them here because /models is read-only +# and HALOGEN_DOWNLOAD stays unset — no outbound connection at boot), +# 2. the distro XRT's three libraries, each mounted twice (where the image +# looks for XRT, and at its own path — the libraries find each other there; +# an /opt/xilinx/xrt of symlinks into the system dir dangles inside the +# container, which the vendor calls out explicitly), +# 3. the GPU fabric clock HELD: on Strix Halo the iGPU and the NPU co-running +# hang the machine while FCLK changes speed, and the engine refuses to +# start when it moves — it cannot read host sysfs from inside the +# container, so the proof lives here, checked exactly the way the vendor's +# `halogen-fabric-clock status` checks it. +NPU_ARGS=() +NPU_MODELS="${HALOGEN_NPU_MODELS:-}" +if [ -n "$NPU_MODELS" ]; then + DEV_ACCEL="${FRANK_DRAGOON_DEV_ACCEL:-/dev/accel/accel0}" + NPU_DIR="${FRANK_DRAGOON_NPU_DIR:?HALOGEN_NPU_MODELS is set, so FRANK_DRAGOON_NPU_DIR must name the staged model dir}" + XRT_DIR="${FRANK_DRAGOON_XRT_DIR:-/usr/lib/x86_64-linux-gnu}" + SYSFS="${FRANK_DRAGOON_SYSFS:-/sys}" + if [ ! -e "$DEV_ACCEL" ]; then + echo "frank-dragoon: $DEV_ACCEL missing — the NPU lane needs the amdxdna node (is iommu=pt still on the boot line?)" >&2 + exit 1 + fi + for M in ${NPU_MODELS//,/ }; do + if [ ! -d "$NPU_DIR/$M" ]; then + echo "frank-dragoon: NPU model $M not staged in $NPU_DIR;" >&2 + echo " fetch it: bash scripts/download-frank-dragoon-npu.sh" >&2 + exit 1 + fi + done + for L in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do + if [ ! -e "$XRT_DIR/$L" ]; then + echo "frank-dragoon: XRT NPU library $XRT_DIR/$L missing — the NPU lane cannot load (Ubuntu libxrt2 + libxrt-npu2)" >&2 + exit 1 + fi + REAL="$(readlink -f "$XRT_DIR/$L")" + NPU_ARGS+=(-v "$REAL:/opt/xilinx/xrt/lib/$L:ro" -v "$REAL:$XRT_DIR/$L:ro") + done + HELD=0 + for D in "$SYSFS"/class/drm/card*/device; do + [ -e "$D/pp_dpm_fclk" ] || continue + [ -e "$D/power_dpm_force_performance_level" ] || continue + PERF="$(cat "$D/power_dpm_force_performance_level" 2>/dev/null || true)" + TOP="$(grep . "$D/pp_dpm_fclk" | tail -n 1)" + if [ "$PERF" = high ] || { [ "$PERF" = manual ] \ + && [ "$(grep -c '\*' "$D/pp_dpm_fclk")" = 1 ] && [ "${TOP%\*}" != "$TOP" ]; }; then + HELD=1 + else + echo "frank-dragoon: fabric clock at $D is '$PERF', not held — the NPU lane refuses to start (GPU+NPU together need FCLK pinned)" >&2 + echo " hold it: sudo halogen-fabric-clock hold" >&2 + exit 1 + fi + done + if [ "$HELD" = 0 ]; then + echo "frank-dragoon: no amdgpu device with a fabric-clock control under $SYSFS/class/drm — cannot prove FCLK is held" >&2 + exit 1 + fi + NPU_ARGS+=(--device "$DEV_ACCEL" \ + -v "$NPU_DIR:/models/npu:ro" \ + -e HALOGEN_NPU_MODELS="$NPU_MODELS") + echo "frank-dragoon: NPU lane armed: $NPU_MODELS on $DEV_ACCEL (fabric clock held)" >&2 +fi + +# Container network. Empty = podman's rootless default (`pasta`), which is a +# private netns and CANNOT be joined by anything else: podman refuses a second +# network while the primary is pasta ("cannot set multiple networks without +# bridge network mode"), the endpoint's published port answers 200 on the host +# and on the LAN but is REFUSED from a dankclaw_default container (measured +# 2026-10-19), and `host.containers.internal:7770` answers 200 yet resolves to +# 169.254.1.2 — link-local, which open-notebook's URL guard refuses to store. +# So an endpoint on pasta has no address an app container can use; naming the +# app bridge here is the only shape that lets open-notebook dial +# http://frank-dragoon-halogen:8731 for the NPU endpoints. It does not widen +# the surface: the engine's unauthenticated 8730 is bound to the container's +# OWN loopback (/proc/net/tcp: 127.0.0.1:8730 vs 0.0.0.0:8731, checked the same +# day), so only the API port is reachable by name. +NET_ARGS=() +if [ -n "${FRANK_DRAGOON_NETWORK:-}" ]; then + NET_ARGS=(--network "$FRANK_DRAGOON_NETWORK") +fi # Host-RAM preflight — the engine's pin gate, checked BEFORE podman run. # The engine refuses to pin when MemAvailable is below the checkpoint plus # a 16 GiB floor: pinned pages cannot be swapped, and under the floor the @@ -189,6 +279,8 @@ podman run --name "$FRANK_DRAGOON_CONTAINER" \ ${HALOGEN_WEIGHTS_LOCK:+-e HALOGEN_WEIGHTS_LOCK="$HALOGEN_WEIGHTS_LOCK"} \ -v "$FRANK_DRAGOON_MODEL_DIR:/models:ro" \ -v "$FRANK_DRAGOON_MODEL_DIR/tokenizer:/tokenizer:ro" \ + "${NET_ARGS[@]}" \ + "${NPU_ARGS[@]}" \ "$FRANK_DRAGOON_IMAGE" \ > "$LOG" 2>&1 diff --git a/serverAppConfigs-docs/Architecture/Container Network Namespaces.md b/serverAppConfigs-docs/Architecture/Container Network Namespaces.md index 954686d..eaa574e 100644 --- a/serverAppConfigs-docs/Architecture/Container Network Namespaces.md +++ b/serverAppConfigs-docs/Architecture/Container Network Namespaces.md @@ -99,7 +99,8 @@ private `systemd-open-notebook`), each address aimed at a live listener: | Dial | Result | Mechanism | |---|---|---| | `192.168.0.143:3000` (Gitea) | **200** | published by a **rootless** container — rootless egress crosses the host netfilter where netavark installed that DNAT | -| `192.168.0.143:7770` (`frank-dragoon`) | refused | published by a **root** container. Root and rootless Podman carry separate netavark rule sets, so a rootless container's traffic never crosses root's and a root-published port is invisible to it | +| `192.168.0.143:7770` (`frank-dragoon`) | refused | the container is **rootless on podman's default `pasta` netns** (the root unit's `User=dank` runs `podman run`, so it lands in dank's storage). Pasta forwards the published port in the host netns — the host and the LAN both answer 200 — but installs no netavark DNAT for traffic arriving from a rootless bridge, so the same address is refused from a container. This is NOT a root/rootless rule-set split: `podman ps` as `dank` lists it | +| `host.containers.internal:7770` (`169.254.1.2`) | **200** | the pasta forwarder is bound `0.0.0.0` on the host, so the gateway alias reaches it — and the alias is link-local, which is the dead end below | | `192.168.0.143:7771` (`frank-zealot`) | refused | host-netns listener, unpublished — the rule above, reconfirmed | | `10.89.1.1:7771` | refused | the bridge gateway address belongs to the rootlesskit netns, not to the host | | `host.containers.internal:7771` (`169.254.1.2`) | **200** | the gateway alias is the one address that carries a rootless container to a host-netns listener. Why `hermes-dashboard` timed out on this alias in 2026-09-02 while `open-notebook` answers today is **not established**; the difference to check first is the second network and where the default route lands | @@ -107,10 +108,24 @@ private `systemd-open-notebook`), each address aimed at a live listener: The consequence that changes designs: **a rootless container's menu of local model endpoints is `frank-tts` (container DNS, same network) and `frank-archon` (LAN -address on another host).** Both Dankclaw endpoints are out of reach — -`frank-dragoon` behind root's publish, `frank-zealot` behind the host netns — and -the only address that does reach them, `169.254.1.2`, is link-local, which some -applications refuse to store at all ([[Open Notebook]] meets exactly that refusal). +address on another host).** A `pasta` endpoint has no address an app container can +use: the LAN address is refused, and the one alias that answers is link-local, +which some applications refuse to store at all ([[Open Notebook]] meets exactly +that refusal). Pasta cannot be *added to* either — + +``` +podman run --network pasta --network dankclaw_default … +Error: cannot set multiple networks without bridge network mode, selected mode pasta +``` + +so the only shape that makes such an endpoint nameable is **moving it onto +`dankclaw_default`** (`FRANK_DRAGOON_NETWORK` in `scripts/frank-dragoon.env`, +2026-10-19). For frank-dragoon that is not an exposure change: the engine's +unauthenticated token port binds the container's **own loopback** +(`/proc/net/tcp` on 2026-10-19: `127.0.0.1:8730`, `0.0.0.0:8731`), so only the +API port becomes name-resolvable, which is the same trust the published 7770 +already carries on the LAN. `frank-zealot` (7771) stays a host-netns binary and +remains unreachable except through the link-local alias. ## Operational rules diff --git a/tests/deployment_scoping.bats b/tests/deployment_scoping.bats index 5c2e24c..3b957fa 100644 --- a/tests/deployment_scoping.bats +++ b/tests/deployment_scoping.bats @@ -247,3 +247,21 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" } done } + +@test "the fabric-clock host unit re-dispatches frank-dragoon and stages nothing else" { + # The hold is the NPU lane's boot precondition and the deployer installs and + # gates it, so a change to it has to reach the endpoint. The tracked boot + # line under the same directory is a hand-taken reboot decision and must + # never plan a restart, and staging the NPU model files is host-side state + # like every other download script. + run python3 "$PLANNER" --target dankclaw --paths \ + dankclaw/host/halogen-fabric-clock dankclaw/host/halogen-fabric-clock.service + [ "$status" -eq 0 ] + [ "$output" = "frank-dragoon" ] + run python3 "$PLANNER" --target dankclaw --paths dankclaw/host/grub + [ "$status" -eq 0 ] + [ -z "$output" ] + run python3 "$PLANNER" --target dankclaw --paths scripts/download-frank-dragoon-npu.sh + [ "$status" -eq 0 ] + [ -z "$output" ] +} diff --git a/tests/frank_dragoon_unit.bats b/tests/frank_dragoon_unit.bats index 8fab8e5..f0e35e3 100755 --- a/tests/frank_dragoon_unit.bats +++ b/tests/frank_dragoon_unit.bats @@ -27,6 +27,9 @@ setup() { ENV_FILE="$ROOT/scripts/frank-dragoon.env" UNIT="$ROOT/scripts/frank-dragoon.service" DEPLOYER="$ROOT/scripts/deploy-remote.sh" + HOST_UNIT="$ROOT/dankclaw/host/halogen-fabric-clock" + HOST_SERVICE="$ROOT/dankclaw/host/halogen-fabric-clock.service" + NPU_FETCH="$ROOT/scripts/download-frank-dragoon-npu.sh" BUILD="$ROOT/scripts/build-rocmfpx.sh" TEST_TMP="$(mktemp -d)" touch "$TEST_TMP/empty.env" @@ -167,6 +170,14 @@ good_env() { kv_gib=$(( pool * 28 * 1024 / 1073741824 )) arena_tenths=$(( 38 + 84 * mt / 16384 )) total_gib=$(( 62 + kv_gib + arena_tenths / 10 + 2 + reserve + 15 )) + # The NPU lane bills extra pinned host RAM per named model (the vendor's + # budget note: ~2 GiB for an embedding/decider/rerank-sized program, more + # for FLUX-class ones), and the lane is named in the shipped env, so the + # ledger must still close on a 121 GiB unified-memory host. + npu_models=0 + npu="$(sed -nE 's/^HALOGEN_NPU_MODELS=(.*)$/\1/p' "$ENV_FILE")" + [ -n "$npu" ] && npu_models="$(echo "${npu//,/ }" | wc -w)" + total_gib=$(( total_gib + 2 * npu_models )) [ "$total_gib" -le 121 ] } @@ -339,6 +350,15 @@ $(cat "$TEST_TMP/serve.log")" grep -qF "LLM_DRAGOON_TOK_FILES=\"\${LLM_DRAGOON_TOK_FILES:-$tokf}\"" "$DEPLOYER" bytes="$(sed -nE 's/^FRANK_DRAGOON_CHECKPOINT_BYTES=([0-9]+)$/\1/p' "$ENV_FILE")" grep -qF "LLM_DRAGOON_CKPT_BYTES=\"\${LLM_DRAGOON_CKPT_BYTES:-$bytes}\"" "$DEPLOYER" + # The NPU twins: the lane switches on the same value on both sides. A blank + # env line with a named deployer default (or the reverse) is the 2026-10-01 + # disagreement again — one side gates a lane the other does not build. + npu_dir="$(sed -nE 's/^FRANK_DRAGOON_NPU_DIR=(.*)$/\1/p' "$ENV_FILE")" + npu_models="$(sed -nE 's/^HALOGEN_NPU_MODELS=(.*)$/\1/p' "$ENV_FILE")" + grep -qF "LLM_DRAGOON_NPU_DIR=\"\${LLM_DRAGOON_NPU_DIR:-$npu_dir}\"" "$DEPLOYER" + grep -qF "LLM_DRAGOON_NPU_MODELS=\"\${LLM_DRAGOON_NPU_MODELS:-$npu_models}\"" "$DEPLOYER" + net="$(sed -nE 's/^FRANK_DRAGOON_NETWORK=(.*)$/\1/p' "$ENV_FILE")" + grep -qF "LLM_DRAGOON_NETWORK=\"\${LLM_DRAGOON_NETWORK:-$net}\"" "$DEPLOYER" } @test "the deployer points a missing artifact at the v2abl fetch script" { @@ -383,3 +403,206 @@ $(cat "$TEST_TMP/serve.log")" grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD" grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD" } + +# --- the NPU lane (XDNA2) ---------------------------------------------------- +# Opt-in lane added 2026-10-19: halogen runs the vendor's precompiled AIE +# programs for the models HALOGEN_NPU_MODELS names on the NPU inside the SAME +# container (they answer /v1/embeddings on 7770). Two things make this seam +# worth pinning: the lane must be genuinely OFF when the variable is blank (the +# rollback is one line, so the render must show nothing lane-specific), and +# every host precondition the engine needs — the accel node, the distro XRT's +# three libraries, the staged model files, the held GPU fabric clock — must be +# refused BY NAME instead of surfacing as an engine crash-loop. + +npu_fixture() { + touch "$TEST_TMP/accel0" + mkdir -p "$TEST_TMP/xrt" + for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do + touch "$TEST_TMP/xrt/$l" + done + mkdir -p "$TEST_TMP/npu/qwen3-embedding-0.6b/devices" + mkdir -p "$TEST_TMP/sys/class/drm/card1/device" + printf '0: 400Mhz \n7: 2000Mhz *\n' > "$TEST_TMP/sys/class/drm/card1/device/pp_dpm_fclk" + printf 'manual\n' > "$TEST_TMP/sys/class/drm/card1/device/power_dpm_force_performance_level" + NPU_ENV=( + "HALOGEN_NPU_MODELS=qwen3-embedding-0.6b" + "FRANK_DRAGOON_NPU_DIR=$TEST_TMP/npu" + "FRANK_DRAGOON_DEV_ACCEL=$TEST_TMP/accel0" + "FRANK_DRAGOON_XRT_DIR=$TEST_TMP/xrt" + "FRANK_DRAGOON_SYSFS=$TEST_TMP/sys" + ) +} + +# run_with_log — run the serve script and fold its own stderr plus +# the podman stub's argv (which the script redirects into the log) into +# $output, so both refusal text and render assertions read one variable. +run_with_log() { + run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" "${NPU_ENV[@]}" "$@" "$SERVE" + if [ -f "$TEST_TMP/serve.log" ]; then + output="$output +$(cat "$TEST_TMP/serve.log")" + fi +} + +@test "the lane is off when HALOGEN_NPU_MODELS is blank: no device, no mounts, no engine env" { + # The rollback is blanking one env line, so "off" has to be observable in + # the command podman would run — not just implied by an absent log line. + podman_stub + good_env + run_with_log HALOGEN_NPU_MODELS= + [ "$status" -eq 0 ] + [[ "$output" == *"--publish 7770:8731"* ]] + ! grep -q '/models/npu' "$TEST_TMP/serve.log" + ! grep -q 'HALOGEN_NPU_MODELS' "$TEST_TMP/serve.log" + ! grep -q 'xilinx' "$TEST_TMP/serve.log" + ! grep -q 'accel0' "$TEST_TMP/serve.log" +} + +@test "the lane arms the accel node, the staged model dir and each XRT library twice" { + # Each library mounts at the image's XRT path AND at its own path: the + # distro libraries find each other where they were installed, and an + # /opt/xilinx/xrt of symlinks into the system dir dangles inside the + # container (the vendor calls that out by name). + podman_stub + good_env + npu_fixture + run_with_log + [ "$status" -eq 0 ] + [[ "$output" == *"--device $TEST_TMP/accel0"* ]] + [[ "$output" == *"-v $TEST_TMP/npu:/models/npu:ro"* ]] + [[ "$output" == *"-e HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"* ]] + for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do + [[ "$output" == *"-v $TEST_TMP/xrt/$l:/opt/xilinx/xrt/lib/$l:ro"* ]] + [[ "$output" == *"-v $TEST_TMP/xrt/$l:$TEST_TMP/xrt/$l:ro"* ]] + done + # The Flash payload still rides the same container: the lane is additive. + [[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]] +} + +@test "the lane refuses a model that is not staged and names the fetch script" { + # The engine cannot fetch these at boot here (/models is read-only and + # HALOGEN_DOWNLOAD stays unset), so an un-staged name must stop the unit + # before podman runs, with the command that fixes it. + podman_stub + good_env + npu_fixture + rm -rf "$TEST_TMP/npu/qwen3-embedding-0.6b" + run_with_log + [ "$status" -ne 0 ] + [[ "$output" == *"qwen3-embedding-0.6b"*"not staged"* ]] + [[ "$output" == *"download-frank-dragoon-npu.sh"* ]] +} + +@test "the lane refuses without the accel node and says what to check" { + podman_stub + good_env + npu_fixture + rm "$TEST_TMP/accel0" + run_with_log + [ "$status" -ne 0 ] + [[ "$output" == *"$TEST_TMP/accel0 missing"* ]] + [[ "$output" == *"iommu=pt"* ]] +} + +@test "the lane refuses when an XRT NPU library is missing" { + podman_stub + good_env + npu_fixture + rm "$TEST_TMP/xrt/libxrt_driver_xdna.so.2" + run_with_log + [ "$status" -ne 0 ] + [[ "$output" == *"libxrt_driver_xdna.so.2 missing"* ]] +} + +@test "the lane refuses while the fabric clock is moving" { + # The engine itself refuses to start in this state (GPU+NPU co-run hangs + # Strix Halo while FCLK changes speed) but cannot see host sysfs from + # inside the container — the proof has to happen on the host side. + podman_stub + good_env + npu_fixture + printf 'auto\n' > "$TEST_TMP/sys/class/drm/card1/device/power_dpm_force_performance_level" + run_with_log + [ "$status" -ne 0 ] + [[ "$output" == *"not held"* ]] + [[ "$output" == *"sudo halogen-fabric-clock hold"* ]] +} + +@test "the lane refuses when no fabric-clock control can be read" { + # A host whose sysfs exposes nothing cannot PROVE the hold, and a silent + # pass here is a boot that dies in the engine instead. + podman_stub + good_env + npu_fixture + rm -rf "$TEST_TMP/sys/class/drm" + run_with_log + [ "$status" -ne 0 ] + [[ "$output" == *"cannot prove"* ]] +} + +@test "the lane also passes when the clock is held at the high level" { + # The vendor's status accepts `high` as well as `manual` + the top fclk + # level; refusing a correctly-held host would block a deploy for nothing. + podman_stub + good_env + npu_fixture + printf 'high\n' > "$TEST_TMP/sys/class/drm/card1/device/power_dpm_force_performance_level" + run_with_log + [ "$status" -eq 0 ] + [[ "$output" == *"-e HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"* ]] +} + +@test "the deployer gates the same lane artifacts and installs the fabric-clock unit" { + grep -qE 'LLM_DRAGOON_NPU_MODELS' "$DEPLOYER" + grep -qE 'install -m 0755 dankclaw/host/halogen-fabric-clock' "$DEPLOYER" + grep -qE 'install -m 0644 dankclaw/host/halogen-fabric-clock\.service' "$DEPLOYER" + grep -qE 'halogen-fabric-clock status' "$DEPLOYER" + grep -qE 'download-frank-dragoon-npu\.sh' "$DEPLOYER" + # The lane's own health gate is a real embedding, not a 200 on /health. + grep -qE '/v1/embeddings' "$DEPLOYER" +} + +@test "the fabric-clock host unit is the vendor shape and boots on its own" { + [ -x "$HOST_UNIT" ] + grep -qE '^ (hold\)|release\)|status\))' "$HOST_UNIT" + grep -qE 'pp_dpm_fclk' "$HOST_UNIT" + grep -qE '^Type=oneshot$' "$HOST_SERVICE" + grep -qE '^WantedBy=.*multi-user\.target' "$HOST_SERVICE" + # A resume hands the clock back to the driver, so the unit must re-run then. + grep -qE '^WantedBy=.*suspend\.target' "$HOST_SERVICE" + grep -qE '^ExecStart=.*/usr/local/sbin/halogen-fabric-clock hold$' "$HOST_SERVICE" +} + +@test "the NPU fetch script pins the vendor catalog by size and sha256" { + # XDNA2 runs only AIE-compiled programs, so the vendor's per-model repo is + # the only source; the engine re-checks size+sha256 against its own catalog + # at boot, so this manifest has to agree with it or the boot refuses. + [ -x "$NPU_FETCH" ] + grep -qE 'curl -fL' "$NPU_FETCH" + grep -qE 'qwen3-embedding-0\.6b\.hnpw 813032248 [0-9a-f]{64}' "$NPU_FETCH" + grep -qE 'sha256sum' "$NPU_FETCH" + # curl -f alone is not enough on Hugging Face: the resolve URL redirects + # and curl drops Range across it (the sibling 62 GiB fetch, 2026-10-18). + grep -qE 'url_effective' "$NPU_FETCH" +} + +@test "the container network is podman's default when unnamed and the app bridge when named" { + # The app-to-endpoint leg hangs on this: pasta is a private netns nothing + # else can join (podman refuses a second network, the published port is + # refused from app containers, and host.containers.internal is link-local, + # which open-notebook refuses to store), so reaching the NPU endpoints by + # name means the container lives on dankclaw_default. Both arms are + # observable in the command podman would run. + podman_stub + good_env + run_with_log FRANK_DRAGOON_NETWORK= + [ "$status" -eq 0 ] + ! grep -q -- '--network' "$TEST_TMP/serve.log" + run_with_log FRANK_DRAGOON_NETWORK=dankclaw_default + [ "$status" -eq 0 ] + [[ "$output" == *"--network dankclaw_default"* ]] + # The deployer refuses a network that does not exist rather than starting a + # unit that restart-loops on it. + grep -qE 'podman network exists' "$DEPLOYER" + grep -qE '^FRANK_DRAGOON_NETWORK=dankclaw_default$' "$ENV_FILE" +}