serverAppConfigs/scripts/serve-frank-primary.sh

#!/usr/bin/env bash
# serve-frank-primary.sh — frank-primary endpoint (tcclaviger RDNA4 vLLM fork,
# Intel/GLM-5.3-Flash-W4A16-AutoRound, 2x R9700, dankserver :7770).
#
# 2026-09-30 cutover from orcarouter Qwen3.8-Flash-Next-Uncensored-FP8 + MTP4.
# Rollback: commit 2339fae carries the whole pre-cutover scripts/ tree; restore
# it and deploy. The qwen payload and the R9V state dirs stay on disk.
#
# The engine is driven by `podman run` DIRECTLY: the image's entrypoint execs
# `vllm serve "$@"`, so the run line below IS the engine argv.
#
# Why this payload runs the offload path at all: 170 GiB of int4 experts cannot
# fit in 64 GiB of VRAM, so the engine keeps 12,096 experts per rank in pinned
# host memory and serves them through a 2,383-slot VRAM LRU arena. The MoE
# kernel is libr4d's grouped W4A16 GEMM (R4dW4a16MoEMethod), reached because the
# checkpoint's quant_method "auto-round" is overridden to "inc" by the fork.
#
# Shape: MAXLEN 131,072 x 2 seqs, bf16 KV, util 0.98, MTP OFF, prefetch OFF.
# Measured 2026-09-30: decode 9.9-10.0 tok/s, prefill 918 tok/s warm. The
# 35 tok/s target is NOT met — the bound is the fork's expert-gather read path
# (8.2 GB/s effective vs the link's 22.74 GB/s); see frank-primary.env and the
# runbook's arithmetic section.
#
# The root unit (scripts/frank-primary.service, User=dank,
# LimitMEMLOCK=infinity) grants the pin budget for the 73.64 GiB/rank of pinned
# host expert rows. A plain shell session also works, but the unit is the
# fleet-parity boot-persistent shape.
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_PRIMARY_ENV_FILE:-$SCRIPT_DIR/frank-primary.env}"

if [ ! -f "$ENV_FILE" ]; then
    echo "serve-frank-primary: env file missing: $ENV_FILE" >&2
    exit 1
fi
set -a
# shellcheck disable=SC1090
source "$ENV_FILE"
set +a

: "${FRANK_PRIMARY_IMAGE:?FRANK_PRIMARY_IMAGE must name the digest-pinned image}"
: "${FRANK_PRIMARY_MODEL_DIR:?FRANK_PRIMARY_MODEL_DIR must name the FP8 payload}"
: "${FRANK_PRIMARY_MODEL_SHARDS:?FRANK_PRIMARY_MODEL_SHARDS must give the shard count}"
: "${FRANK_PRIMARY_CONTAINER:?FRANK_PRIMARY_CONTAINER must name the container}"
: "${FRANK_PRIMARY_SERVED_NAME:?FRANK_PRIMARY_SERVED_NAME must name the alias}"
: "${FRANK_PRIMARY_PUBLISH:?FRANK_PRIMARY_PUBLISH must be host:container}"
: "${FRANK_PRIMARY_MAXLEN:?FRANK_PRIMARY_MAXLEN must be set}"
: "${FRANK_PRIMARY_SEQS:?FRANK_PRIMARY_SEQS must be set}"
: "${FRANK_PRIMARY_NBATCH:?FRANK_PRIMARY_NBATCH must be set}"
: "${FRANK_PRIMARY_TOOLPARSER:?FRANK_PRIMARY_TOOLPARSER must be set}"
: "${FRANK_PRIMARY_REASONINGPARSER:?FRANK_PRIMARY_REASONINGPARSER must be set}"
# Capture podman's output BEFORE grepping: under `set -o pipefail`, the
# classic `podman images | grep -q` pattern races — grep exits on its first
# match, podman dies SIGPIPE (141) writing the remaining lines, and pipefail
# turns the 141 into "no match" (the 2026-09-30 deploy refusal).
DIGESTS="$(podman images --format '{{.RepoDigests}}')"
if ! grep -qF "${FRANK_PRIMARY_IMAGE#*@}" <<<"$DIGESTS"; then
    echo "serve-frank-primary: pinned image absent from rootless storage: $FRANK_PRIMARY_IMAGE" >&2
    echo "  fetch it: podman pull $FRANK_PRIMARY_IMAGE" >&2
    exit 1
fi

if [ ! -f "$FRANK_PRIMARY_MODEL_DIR/config.json" ]; then
    echo "serve-frank-primary: payload config missing: $FRANK_PRIMARY_MODEL_DIR/config.json" >&2
    exit 1
fi
SHARDS="$(find "$FRANK_PRIMARY_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)"
if [ "$SHARDS" != "$FRANK_PRIMARY_MODEL_SHARDS" ]; then
    echo "serve-frank-primary: payload shard count wrong: $FRANK_PRIMARY_MODEL_DIR" >&2
    echo "  found $SHARDS safetensors, want $FRANK_PRIMARY_MODEL_SHARDS" >&2
    echo "  re-fetch: scripts/download-frank-primary.sh" >&2
    exit 1
fi
if [ ! -f "$FRANK_PRIMARY_MODEL_DIR/model.safetensors.index.json" ]; then
    echo "serve-frank-primary: weight-map index missing: $FRANK_PRIMARY_MODEL_DIR/model.safetensors.index.json" >&2
    exit 1
fi

if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then
    echo "serve-frank-primary: GPU device nodes absent (/dev/kfd, /dev/dri)" >&2
    exit 1
fi

# A crashed/killed unit can leave the container name registered (systemd
# SIGKILLs conmon before cleanup runs). Clear OUR name plus the legacy
# identities (the retired R9V container and the ad hoc experiment container
# both hold these GPUs); all idempotent.
for CTR in $FRANK_PRIMARY_LEGACY_CONTAINERS "$FRANK_PRIMARY_CONTAINER"; do
    podman rm -f "$CTR" >/dev/null 2>&1 || true
done

LOG=/home/dank/logs/frank_primary.log
mkdir -p "$(dirname "$LOG")"

# --device /dev/dri is the WHOLE DIRECTORY (individual renderD nodes enumerate
# without the card nodes and HIP refuses contexts — "No CUDA GPUs are
# available" while device_count() says 2). --ipc=host for the engine's shared
# memory; memlock unbounded for the pinned host slabs.
# The entrypoint already execs `vllm serve "$@"`, so no `serve` here.
podman run -d --name "$FRANK_PRIMARY_CONTAINER" \
  --publish "$FRANK_PRIMARY_PUBLISH" \
  --ipc=host --ulimit memlock=-1:-1 \
  --cap-add SYS_PTRACE \
  --device /dev/kfd --device /dev/dri --device /dev/mem \
  -e HIP_VISIBLE_DEVICES=0,1 -e VLLM_ROCM_USE_AITER=0 \
  -v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \
  "$FRANK_PRIMARY_IMAGE" /models \
  --served-model-name "$FRANK_PRIMARY_SERVED_NAME" --port 7770 \
  --tensor-parallel-size 2 \
  --max-model-len "$FRANK_PRIMARY_MAXLEN" \
  --max-num-seqs "$FRANK_PRIMARY_SEQS" \
  --max-num-batched-tokens "$FRANK_PRIMARY_NBATCH" \
  --gpu-memory-utilization "$FRANK_PRIMARY_UTIL" \
  --enable-expert-offload \
  --expert-precise-memory \
  --enable-prefix-caching \
  --enable-auto-tool-choice \
  --tool-call-parser "$FRANK_PRIMARY_TOOLPARSER" \
  --reasoning-parser "$FRANK_PRIMARY_REASONINGPARSER" \
  --trust-remote-code \
  > "$LOG" 2>&1

# NOT passed, deliberately (each measured on this payload — see the env file):
#   --kv-cache-dtype             fp8 disables the sparse MLA backend's rope-free
#                                Triton path, which is the only MLA decode on
#                                gfx12 (aiter's MLA op is gated on CDNA>2).
#   --speculative-config         MTP head loads and accepts well, but buys zero
#                                decode and collapses prefill on an
#                                offload-bound shape.
#   --expert-prefetch*           36% precision; deleting it bought +40-51%.
#   --kv-cache-memory-bytes      the plan's own reserve covers this shape.

# From here on, this script's lifetime IS the container's: wait returns the
# container's exit code, so a crash is a nonzero ExecStart and the unit's
# Restart=on-failure is real. (systemd stop goes through ExecStop in the
# unit — podman wait alone cannot stop the container child.)
exec podman wait "$FRANK_PRIMARY_CONTAINER"