Frank
ecaad36654
The llama.cpp b11030 trio becomes the vLLM 0.30 trio ahead of the FP8 cutover on :7770 — same port, same alias, no client moves. Serve shape: locally built ROCm gfx1201 image (localhost/frank-primary:vllm030) execs via podman as dank from the root unit (LimitMEMLOCK=infinity legalizes the pin). UVA offload: --offload-backend uva pins the PLE family (47.75 GiB, 138 tensors: 131 n-gram table shards + 7 per-layer conv/proj) + routed experts (114.86 GiB, 150,528 tensors) in host RAM — 162.60 GiB combined, ~81.3 GiB per rank under TP=2, hence --cpu-offload-gb 82 (80/card strands 2.6 GiB of experts on the cards). Parsers per the host's own Ornith-era precedent: qwen3_xml tools + qwen3 reasoning (vLLM 0.30 emits the channel as `reasoning` — reasoning_content is renamed server-side) + enable_thinking/preserve_thinking template kwargs. Spec decode stays env-wired and OFF at bring-up. Deployer deploy_llm_primary rewritten: image gate, index-driven shard-count gate (131), stale-container cleanup, port wait, and the false-green hardening — health is only green when the unit's own MainPID is live behind the answer (2026-09-26 lesson). Seam tests rewritten to the vLLM contract (15 gates): offload arithmetic, parser wiring, no-llama.cpp-ism, deployer gates. Suite 349/349 green.
124 lines
4.8 KiB
Bash
Executable File
124 lines
4.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# serve-frank-primary.sh — frank-primary endpoint (vLLM 0.30,
|
|
# Qwen3.8-Flash-Next-FP8 on the 2x R9700 pair, dankserver :7770).
|
|
#
|
|
# Validates the image, the model payload and the env tunables, then execs the
|
|
# container. The unit (scripts/frank-primary.service, root systemd, User=dank,
|
|
# LimitMEMLOCK=infinity) is what makes the UVA pin budget legal — a plain
|
|
# shell here carries the 8 MiB session limit and the pinned offload fails.
|
|
#
|
|
# OFFLOAD SHAPE (the only legal fit for a ~125B-param FP8 model on 2x 32 GiB
|
|
# cards): --offload-backend uva pins the PLE n-gram table (51.2B params) and
|
|
# the routed experts in host RAM and reads them in place over PCIe. The
|
|
# first start MUST be watched for the UVA engagement log (not just /health):
|
|
# a silent fallback to the copy path turns the endpoint into a slideshow.
|
|
#
|
|
# NO --model-draft, no draft GGUF: vLLM's MTP drafting comes from the model's
|
|
# own mtp.* weights via --speculative-config, and only when SPEC_CONFIG is
|
|
# non-empty. Bring-up runs with it empty on purpose.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_PRIMARY_ENV_FILE:-$SCRIPT_DIR/frank-primary.env}"
|
|
|
|
if [ ! -f "$ENV_FILE" ]; then
|
|
echo "serve-frank-primary: env file missing: $ENV_FILE" >&2
|
|
exit 1
|
|
fi
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
|
|
: "${SERVED_NAME:?SERVED_NAME must be set}"
|
|
: "${BIND_HOST:?BIND_HOST must be set}"
|
|
: "${PORT:?PORT must be set}"
|
|
: "${MODEL_DIR:?MODEL_DIR must be set}"
|
|
: "${TP:?TP must be set}"
|
|
: "${GPU_UTIL:?GPU_UTIL must be set}"
|
|
: "${CTX_SIZE:?CTX_SIZE must be set}"
|
|
: "${MAX_SEQS:?MAX_SEQS must be set}"
|
|
: "${OFFLOAD_BACKEND:?OFFLOAD_BACKEND must be set}"
|
|
: "${OFFLOAD_GB:?OFFLOAD_GB must be set}"
|
|
: "${OFFLOAD_PARAMS:?OFFLOAD_PARAMS must be set}"
|
|
: "${VLLM_ROCM_USE_AITER:?VLLM_ROCM_USE_AITER must be set}"
|
|
: "${IMAGE:?IMAGE must be set}"
|
|
: "${CONTAINER_NAME:?CONTAINER_NAME must be set}"
|
|
: "${ROCR_VISIBLE_DEVICES:?ROCR_VISIBLE_DEVICES must be set}"
|
|
: "${TOOL_PARSER:?TOOL_PARSER must be set}"
|
|
: "${REASONING_PARSER:?REASONING_PARSER must be set}"
|
|
: "${CHAT_TEMPLATE_KWARGS:?CHAT_TEMPLATE_KWARGS must be set}"
|
|
: "${CACHE_ROOT:?CACHE_ROOT must be set}"
|
|
|
|
if [ "$OFFLOAD_BACKEND" != "uva" ]; then
|
|
echo "serve-frank-primary: OFFLOAD_BACKEND=$OFFLOAD_BACKEND — this deployment is only qualified for uva" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! podman image inspect "$IMAGE" > /dev/null 2>&1; then
|
|
echo "serve-frank-primary: image missing: $IMAGE — build it before serving" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The model payload is verified by the fetch/verify step (131 shards, index
|
|
# cross-checked). The serve-time check is the load-bearing minimum: the
|
|
# wrapper config and the shard index must exist, or the container start is a
|
|
# guaranteed 20-minute timeout instead of a fast failure.
|
|
HOST_MODEL_DIR="/home/dank/models/${MODEL_DIR#/models/}"
|
|
if [ ! -f "$HOST_MODEL_DIR/config.json" ] || [ ! -f "$HOST_MODEL_DIR/model.safetensors.index.json" ]; then
|
|
echo "serve-frank-primary: model payload incomplete at $HOST_MODEL_DIR (config.json or index missing)" >&2
|
|
echo "serve-frank-primary: run scripts/download-frank-primary-fp8.sh on this host" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then
|
|
echo "serve-frank-primary: /dev/kfd or /dev/dri missing — GPU stack not up" >&2
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p /home/dank/.cache/frank-primary-vllm
|
|
|
|
SERVE_ARGS=(serve "$MODEL_DIR"
|
|
--served-model-name "$SERVED_NAME"
|
|
--host "$BIND_HOST"
|
|
--port "$PORT"
|
|
--tensor-parallel-size "$TP"
|
|
--gpu-memory-utilization "$GPU_UTIL"
|
|
--max-model-len "$CTX_SIZE"
|
|
--max-num-seqs "$MAX_SEQS"
|
|
--offload-backend "$OFFLOAD_BACKEND"
|
|
--cpu-offload-gb "$OFFLOAD_GB"
|
|
--cpu-offload-params "$OFFLOAD_PARAMS"
|
|
--enable-auto-tool-choice --tool-call-parser "$TOOL_PARSER"
|
|
--reasoning-parser "$REASONING_PARSER"
|
|
--default-chat-template-kwargs "$CHAT_TEMPLATE_KWARGS"
|
|
)
|
|
|
|
# SPEC_CONFIG (optional, JSON) is how MTP drafting is enabled after the
|
|
# bring-up measurement: --speculative-config '{"method":"mtp",...}'.
|
|
if [ -n "${SPEC_CONFIG:-}" ]; then
|
|
SERVE_ARGS+=(--speculative-config "$SPEC_CONFIG")
|
|
fi
|
|
|
|
exec podman run --rm \
|
|
--name "$CONTAINER_NAME" \
|
|
--userns=keep-id \
|
|
--network=host \
|
|
--ipc=host \
|
|
--security-opt seccomp=unconfined \
|
|
--cap-add SYS_PTRACE \
|
|
--device /dev/kfd \
|
|
--device /dev/dri \
|
|
--group-add keep-groups \
|
|
-e ROCR_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" \
|
|
-e HIP_VISIBLE_DEVICES="$HIP_VISIBLE_DEVICES" \
|
|
-e VLLM_ROCM_USE_AITER="$VLLM_ROCM_USE_AITER" \
|
|
-e TRITON_CACHE_DIR="$CACHE_ROOT/triton" \
|
|
-e TORCHINDUCTOR_CACHE_DIR="$CACHE_ROOT/inductor" \
|
|
-e VLLM_CACHE_DIR="$CACHE_ROOT/vllm" \
|
|
-e TRITON_ALLOW_NON_CONST_GLOBALS=1 \
|
|
--volume /home/dank/models:/models \
|
|
--volume /home/dank/.cache/frank-primary-vllm:"$CACHE_ROOT" \
|
|
"$IMAGE" \
|
|
vllm "${SERVE_ARGS[@]}"
|