147 lines
7.0 KiB
Bash
Executable File
147 lines
7.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# serve-frank-large.sh — frank-large endpoint: halogen (closed-source
|
|
# C++/HIP engine, image ghcr.io/peonist-ai/halogen, digest-pinned in
|
|
# frank-large.env) serving peonist-ai/halogen-qwen3.8-27b on Dankclaw's
|
|
# Strix Halo iGPU, host :7770. 2026-10-05 cutover from the ROCmFPX
|
|
# llama.cpp shape — revert the swap commit's trio to roll back (payload
|
|
# and fork build stay on disk).
|
|
#
|
|
# The engine is driven by `podman run` DIRECTLY (the fleet's frank-primary
|
|
# pattern): the image entrypoint `halogen-docker` in default "all" mode
|
|
# starts engine + OpenAI frontend in one container. The frontend listens on
|
|
# 8731 inside (HALOGEN_API_PORT default) and is published as host 7770; the
|
|
# engine's own token port (8730, no auth) stays unpublished on the
|
|
# container's loopback. The image ships NO default checkpoint:
|
|
# HALOGEN_CHECKPOINT selects the file inside the /models mount.
|
|
#
|
|
# Gates mirror the deployer EXACTLY (the 2026-10-01 crash-loop lesson —
|
|
# serve gate and deploy gate disagreeing = green deploy, red unit):
|
|
# tokenizer directory present with every named file a FLAT REGULAR FILE —
|
|
# the model card ships `tokenizer/` as a flat directory and the engine
|
|
# defaults HALOGEN_TOKENIZER to /tokenizer, so a HF cache snapshot's
|
|
# symlinks into blobs/ dangle inside the mount; GPU device nodes present.
|
|
#
|
|
# Why containers instead of the bare binary (the old frank-large shape):
|
|
# the payload is .hgn and only halogen reads it; halogen ships as a
|
|
# container; the root unit keeps LimitMEMLOCK=infinity for parity and to
|
|
# outlive the login session, exactly like frank-primary's.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
|
if [ ! -f "$ENV_FILE" ]; then
|
|
echo "frank-large: env file not found: $ENV_FILE" >&2
|
|
exit 1
|
|
fi
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
|
|
: "${FRANK_LARGE_IMAGE:?FRANK_LARGE_IMAGE must name the digest-pinned image}"
|
|
: "${FRANK_LARGE_CONTAINER:?FRANK_LARGE_CONTAINER must name the container}"
|
|
: "${FRANK_LARGE_PUBLISH:?FRANK_LARGE_PUBLISH must be host:container}"
|
|
: "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR must name the payload dir}"
|
|
: "${FRANK_LARGE_CHECKPOINT:?FRANK_LARGE_CHECKPOINT must name the .hgn inside it}"
|
|
: "${FRANK_LARGE_CHECKPOINT_BYTES:?FRANK_LARGE_CHECKPOINT_BYTES must pin the size}"
|
|
: "${FRANK_LARGE_TOKENIZER_FILES:?FRANK_LARGE_TOKENIZER_FILES must list the tokenizer dir contents}"
|
|
: "${HALOGEN_MODEL_ID:?HALOGEN_MODEL_ID must be the served alias (frank-large)}"
|
|
: "${HALOGEN_SLOT_CTX:?HALOGEN_SLOT_CTX must be set}"
|
|
: "${HALOGEN_KV_SLOTS:?HALOGEN_KV_SLOTS must be set (1 keeps the drafter on)}"
|
|
: "${HALOGEN_DRAFTER:?HALOGEN_DRAFTER must be set (2 = DFlash2)}"
|
|
|
|
# Capture podman's output BEFORE grepping: under `set -o pipefail` the
|
|
# `podman images | grep -q` pattern races (grep exits on first match,
|
|
# podman dies SIGPIPE 141, pipefail turns the 141 into "no match" — the
|
|
# 2026-09-30 deploy refusal).
|
|
DIGESTS="$(podman images --format '{{.RepoDigests}}')"
|
|
if ! grep -qF "${FRANK_LARGE_IMAGE#*@}" <<<"$DIGESTS"; then
|
|
echo "frank-large: image $FRANK_LARGE_IMAGE not in podman storage;" >&2
|
|
echo " pull it: podman pull ${FRANK_LARGE_IMAGE%@*}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The checkpoint gate is size-exact: a truncated curl exits 0 on some
|
|
# failures and a partial .hgn loads far enough to look alive, so the byte
|
|
# count from the model card is the contract (the fleet fetch law).
|
|
CKPT="$FRANK_LARGE_MODEL_DIR/$FRANK_LARGE_CHECKPOINT"
|
|
if [ ! -f "$CKPT" ]; then
|
|
echo "frank-large: checkpoint $CKPT missing;" >&2
|
|
echo " fetch it: bash scripts/download-frank-large-halogen.sh" >&2
|
|
exit 1
|
|
fi
|
|
CKPT_SIZE="$(stat -c %s "$CKPT")"
|
|
if [ "$CKPT_SIZE" != "$FRANK_LARGE_CHECKPOINT_BYTES" ]; then
|
|
echo "frank-large: checkpoint is $CKPT_SIZE bytes, expected $FRANK_LARGE_CHECKPOINT_BYTES (partial download?)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The payload ships the tokenizer as a FLAT DIRECTORY (model card:
|
|
# "tokenizer/ flat tokenizer directory"). A HuggingFace cache snapshot
|
|
# symlinks each file into a sibling blobs/, and those symlinks dangle
|
|
# inside the container mount — so the directory must be real and every
|
|
# named file must be a regular file, not a link.
|
|
TOK_DIR="$FRANK_LARGE_MODEL_DIR/tokenizer"
|
|
if [ -L "$TOK_DIR" ] || [ ! -d "$TOK_DIR" ]; then
|
|
echo "frank-large: tokenizer dir $TOK_DIR missing or a symlink (HF cache snapshot?);" >&2
|
|
echo " fetch it: bash scripts/download-frank-large-halogen.sh" >&2
|
|
exit 1
|
|
fi
|
|
for TF in $FRANK_LARGE_TOKENIZER_FILES; do
|
|
if [ ! -f "$TOK_DIR/$TF" ] || [ -L "$TOK_DIR/$TF" ]; then
|
|
echo "frank-large: tokenizer file $TOK_DIR/$TF missing or a symlink;" >&2
|
|
echo " fetch it: bash scripts/download-frank-large-halogen.sh" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# GPU access is mandatory but the node paths are variables so the seam can
|
|
# drive the gate on a box without an AMD GPU.
|
|
DEV_KFD="${FRANK_LARGE_DEV_KFD:-/dev/kfd}"
|
|
DEV_DRI="${FRANK_LARGE_DEV_DRI:-/dev/dri}"
|
|
if [ ! -e "$DEV_KFD" ] || [ ! -d "$DEV_DRI" ]; then
|
|
echo "frank-large: $DEV_KFD or $DEV_DRI missing — no gfx1151 to serve on" >&2
|
|
exit 1
|
|
fi
|
|
for CTR in ${FRANK_LARGE_LEGACY_CONTAINERS:-} "$FRANK_LARGE_CONTAINER"; do
|
|
podman rm -f "$CTR" >/dev/null 2>&1 || true
|
|
done
|
|
|
|
LOG="${FRANK_LARGE_LOG:-/home/dank/logs/frank_large.log}"
|
|
mkdir -p "$(dirname "$LOG")"
|
|
# for the udev-granted supplementary group, seccomp unconfined for the ROCm
|
|
# ioctl surface, and unbounded memlock (the unit's LimitMEMLOCK=infinity is
|
|
# inherited, so runc never has to RAISE it). NO --ulimit nofile: Dankclaw's
|
|
# hard limit is 524288, rootless runc cannot raise a hard limit and dies
|
|
# with "error setting rlimit type 7: operation not permitted" (measured
|
|
# 2026-10-05 cutover), and 524288 fds is already ~100x what the engine
|
|
# wants; the model card and both sibling serve scripts pass no nofile
|
|
# either. No comment lines inside the backslash
|
|
# chain — a mid-chain # terminates the continuation (the 2026-10-05
|
|
# exit-127 crash loop on the sibling serve script).
|
|
podman run -d --name "$FRANK_LARGE_CONTAINER" \
|
|
--publish "$FRANK_LARGE_PUBLISH" \
|
|
--ipc=host \
|
|
--ulimit memlock=-1:-1 \
|
|
--device "$DEV_KFD" --device "$DEV_DRI" \
|
|
--group-add keep-groups \
|
|
--security-opt seccomp=unconfined \
|
|
-e HALOGEN_MODEL_ID="$HALOGEN_MODEL_ID" \
|
|
-e HALOGEN_CHECKPOINT="/models/$FRANK_LARGE_CHECKPOINT" \
|
|
-e HALOGEN_SLOT_CTX="$HALOGEN_SLOT_CTX" \
|
|
-e HALOGEN_KV_SLOTS="${HALOGEN_KV_SLOTS:-1}" \
|
|
-e HALOGEN_DRAFTER="$HALOGEN_DRAFTER" \
|
|
-e HALOGEN_MAX_TOKENS_CAP="${HALOGEN_MAX_TOKENS_CAP:-65536}" \
|
|
-e HALOGEN_QUEUE_TIMEOUT="${HALOGEN_QUEUE_TIMEOUT:-7200}" \
|
|
${HALOGEN_CACHE_MB:+-e HALOGEN_CACHE_MB="$HALOGEN_CACHE_MB"} \
|
|
-v "$FRANK_LARGE_MODEL_DIR:/models:ro" \
|
|
-v "$FRANK_LARGE_MODEL_DIR/tokenizer:/tokenizer:ro" \
|
|
"$FRANK_LARGE_IMAGE" \
|
|
> "$LOG" 2>&1
|
|
|
|
# This script's lifetime IS the container's: podman wait returns the
|
|
# container's exit code, so a crash is a nonzero ExecStart and the unit's
|
|
# Restart=always is real. systemd stop goes through the unit's ExecStop
|
|
# (podman wait alone cannot stop the child).
|
|
exec podman wait "$FRANK_LARGE_CONTAINER"
|