Frank
918fe4eaec
The vllmScripts repository (LLM inference fleet: serve scripts, systemd units, env tunables, deployment runbook, research docs) is merged into this repository. Its full history is grafted under llm/ with a two-parent merge commit so no commit is lost. A follow-up commit reorganizes llm/ into the repo layout: llm/docs -> docs/LLM, llm/scripts -> scripts, llm/old -> old.
39 lines
1.5 KiB
Bash
Executable File
39 lines
1.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# serve-ling3-vllm.sh — Launch Ling-3.0-flash CIRU INT4 vLLM (gfx1151) on Dankclaw
|
|
#
|
|
# This wrapper sources the jcbtc ling3-runtime.env (sets VENV and MODEL_PATH),
|
|
# applies our production settings (host, port, served-model-name), and hands
|
|
# off to the validated run-256k.sh launcher (TRITON_MLA, MTP K1, 256k, 6 slots,
|
|
# ROCr idle-fix LD_PRELOAD). Do not hand-roll the vllm serve command; the
|
|
# launcher carries required env vars and the ROCr override.
|
|
set -Eeuo pipefail
|
|
|
|
PKG="/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native"
|
|
|
|
if [ ! -f "$PKG/ling3-runtime.env" ]; then
|
|
echo "CIRU package runtime environment not found: $PKG/ling3-runtime.env" >&2
|
|
exit 1
|
|
fi
|
|
|
|
set -a
|
|
# shellcheck disable=SC1091
|
|
source "$PKG/ling3-runtime.env"
|
|
set +a
|
|
|
|
export HOST="${HOST:-0.0.0.0}"
|
|
export PORT="${PORT:-7770}"
|
|
export SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-frank-default-large}"
|
|
# Raise the vLLM frame-buffer utilization so weights + 256k KV fit in VRAM
|
|
# (the default 0.72 from native-256k.env OOMs; native-256k uses
|
|
# ${GPU_MEMORY_UTILIZATION:-0.72}, so this exported value wins).
|
|
export GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.95}"
|
|
|
|
# Run from a directory with no top-level 'vllm/' dir.
|
|
# Neither launcher changes directory, so the process cwd becomes sys.path[0]
|
|
# for the 'python -m ...registry' inspection subprocess. From the package root,
|
|
# the top-level vllm/ repo dir shadows the editable install as a namespace
|
|
# package and breaks lazy imports (for example, SamplingParams).
|
|
cd /home/dank/ling3-ciru
|
|
|
|
exec bash "$PKG/scripts/run-256k.sh"
|