Frank
f68bda56c7
The CIRU launcher gates on hip/lib/libamdhip64.so + rocr/lib/libhsa-runtime64.so under its runtime root and exits 2 without them; the build links the private venv through its RUNPATH, so point those entries at the venv's unversioned devel symlinks (idempotent ln -sfn). Without this the unit crash-looped for the full 2400s health gate on first deploy.
95 lines
4.4 KiB
Bash
Executable File
95 lines
4.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca as frank-large on Dankclaw's
|
|
# Strix Halo iGPU (gfx1151), using the CIRU v4.4.1 runtime with the QSA
|
|
# window hotfix.
|
|
#
|
|
# Runtime: the v4.4.1 source with the QSA window hotfix pre-applied
|
|
# (ciru-runtime-v4.4.1-qsa-hotfix-source.tar.gz, shipped in the model repo
|
|
# under runtime/v4.4.1/qsa-window-hotfix/), built ON the host against the
|
|
# private ROCm 10.0.0 SDK that scripts/ciru/setup-linux-amd.sh installs into
|
|
# the runtime checkout's own venv. The launcher below re-verifies on every
|
|
# start that the built libraries carry the v4.4.1 corrections (it greps the
|
|
# common and HIP libraries for the fix markers) before exec.
|
|
#
|
|
# The launcher owns the flag list: -sm none (one iGPU, no split), -ngl all,
|
|
# f16 KV, flash attention, cont-batching, the PLE sidecar, the MTP drafter
|
|
# (depth 4 for the Orca variant) and the production kernel-bundle profile.
|
|
# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides it;
|
|
# anything not set there falls through to the launcher's own defaults.
|
|
#
|
|
# It replaces the halogen-flash-server quadlet, which served
|
|
# peonist-ai/halogen-qwen3.8-flash-next on this same iGPU since 2026-09-16.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) ------------------
|
|
: "${RUNTIME_DIR:=/home/dank/GitPackages/ciru-runtime-v4.4.1-qsa-hotfix}"
|
|
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca}"
|
|
: "${HOST:=0.0.0.0}"
|
|
: "${PORT:=7770}"
|
|
: "${CONTEXT_SIZE:=262144}"
|
|
: "${PARALLEL_SLOTS:=1}"
|
|
|
|
LAUNCHER="$RUNTIME_DIR/scripts/ciru/run-orca-server.sh"
|
|
SERVER_BIN="$RUNTIME_DIR/build-gfx1151-sdk/bin/llama-server"
|
|
|
|
if [[ ! -x "$LAUNCHER" ]]; then
|
|
echo "missing runtime launcher: $LAUNCHER" >&2
|
|
echo "fetch the hotfix source tarball from the model repo" >&2
|
|
echo "(runtime/v4.4.1/qsa-window-hotfix/) and build it:" >&2
|
|
echo " bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2
|
|
exit 1
|
|
fi
|
|
if [[ ! -x "$SERVER_BIN" ]]; then
|
|
echo "missing built runtime: $SERVER_BIN" >&2
|
|
echo "build it: bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The launcher gates on a runtime root holding hip/lib/libamdhip64.so and
|
|
# rocr/lib/libhsa-runtime64.so (it prepends both to LD_LIBRARY_PATH). Our
|
|
# build links the private SDK venv through its RUNPATH, so point those
|
|
# entries at the venv's unversioned devel symlinks. Idempotent: ln -sfn.
|
|
VENV_LIB="$(ls -d "$RUNTIME_DIR"/.venv-rocm/lib/python*/site-packages/_rocm_sdk_devel/lib 2>/dev/null | head -1 || true)"
|
|
if [[ -z "$VENV_LIB" || ! -e "$VENV_LIB/libamdhip64.so" || ! -e "$VENV_LIB/libhsa-runtime64.so" ]]; then
|
|
echo "missing SDK venv runtime under $RUNTIME_DIR/.venv-rocm (libamdhip64.so, libhsa-runtime64.so)" >&2
|
|
echo "build it: bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2
|
|
exit 1
|
|
fi
|
|
mkdir -p "$RUNTIME_DIR/runtime/hip/lib" "$RUNTIME_DIR/runtime/rocr/lib"
|
|
ln -sfn "$VENV_LIB/libamdhip64.so" "$RUNTIME_DIR/runtime/hip/lib/libamdhip64.so"
|
|
ln -sfn "$VENV_LIB/libhsa-runtime64.so" "$RUNTIME_DIR/runtime/rocr/lib/libhsa-runtime64.so"
|
|
if [[ ! -r "$MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" ]]; then
|
|
echo "missing target model: $MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" >&2
|
|
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The launcher checks the MTP head and the PLE sidecar itself, but a missing
|
|
# payload should be a deploy error with a pointer, not a runtime exit 2.
|
|
for required in \
|
|
"$MODEL_DIR/mtp/Qwen3.8-Flash-CIRU-STRIX-Orca-MTP-Q8_0.gguf" \
|
|
"$MODEL_DIR/ple/ple.payload.bin" \
|
|
"$MODEL_DIR/ple/ple.manifest.json" \
|
|
"$MODEL_DIR/ple/ple.scale.bf16"; do
|
|
if [[ ! -r "$required" ]]; then
|
|
echo "missing payload: $required" >&2
|
|
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# The Qwen3.8 family template keeps prior thinking blocks across turns when
|
|
# preserve_thinking is set; the launcher passes --jinja but no template
|
|
# kwargs, so name them here. A request-level chat_template_kwargs still
|
|
# wins over this.
|
|
exec "$LAUNCHER" --chat-template-kwargs '{"preserve_thinking":true}'
|