Frank
d9af7f1d19
- Move all previous scripts to old/scripts/ (retired PP/NPU-era) - frank-default-large: llama.cpp FastMTP on dankserver (HauhauCS Qwen3.8-27B IQ4_XS + FastMTP 32K draft + ngram-mod, K q8_0/V f16, 128K ctx, batch 2048/ubatch 1024, port 7770) - frank-default-small: vLLM ROCm on dankclaw (Avesed Qwen3.6-35B-A3B INT8 W8A8, full 262144 ctx, no KV quant, 4 seqs, batch 16384, ngram spec, port 52625) - retarget deploy.sh (dankserver/dankclaw, inline sudo, no helpers) - docs: deployment + index + AGENTS.md + README for new topology, superseded banners on retired docs, add frank-redo-2026-08-17 runbook
39 lines
1.5 KiB
Bash
Executable File
39 lines
1.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# serve-ling3-vllm.sh — Launch Ling-3.0-flash CIRU INT4 vLLM (gfx1151) on Dankclaw
|
|
#
|
|
# This wrapper sources the jcbtc ling3-runtime.env (sets VENV and MODEL_PATH),
|
|
# applies our production settings (host, port, served-model-name), and hands
|
|
# off to the validated run-256k.sh launcher (TRITON_MLA, MTP K1, 256k, 6 slots,
|
|
# ROCr idle-fix LD_PRELOAD). Do not hand-roll the vllm serve command; the
|
|
# launcher carries required env vars and the ROCr override.
|
|
set -Eeuo pipefail
|
|
|
|
PKG="/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native"
|
|
|
|
if [ ! -f "$PKG/ling3-runtime.env" ]; then
|
|
echo "CIRU package runtime environment not found: $PKG/ling3-runtime.env" >&2
|
|
exit 1
|
|
fi
|
|
|
|
set -a
|
|
# shellcheck disable=SC1091
|
|
source "$PKG/ling3-runtime.env"
|
|
set +a
|
|
|
|
export HOST="${HOST:-0.0.0.0}"
|
|
export PORT="${PORT:-7770}"
|
|
export SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-frank-default-large}"
|
|
# Raise the vLLM frame-buffer utilization so weights + 256k KV fit in VRAM
|
|
# (the default 0.72 from native-256k.env OOMs; native-256k uses
|
|
# ${GPU_MEMORY_UTILIZATION:-0.72}, so this exported value wins).
|
|
export GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.95}"
|
|
|
|
# Run from a directory with no top-level 'vllm/' dir.
|
|
# Neither launcher changes directory, so the process cwd becomes sys.path[0]
|
|
# for the 'python -m ...registry' inspection subprocess. From the package root,
|
|
# the top-level vllm/ repo dir shadows the editable install as a namespace
|
|
# package and breaks lazy imports (for example, SamplingParams).
|
|
cd /home/dank/ling3-ciru
|
|
|
|
exec bash "$PKG/scripts/run-256k.sh"
|