diff --git a/scripts/serve-ling3-vllm.sh b/scripts/serve-ling3-vllm.sh index d0d6084..09643d3 100755 --- a/scripts/serve-ling3-vllm.sh +++ b/scripts/serve-ling3-vllm.sh @@ -23,6 +23,10 @@ set +a export HOST="${HOST:-0.0.0.0}" export PORT="${PORT:-7770}" export SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-frank-default-large}" +# Raise the vLLM frame-buffer utilization so weights + 256k KV fit in VRAM +# (the default 0.72 from native-256k.env OOMs; native-256k uses +# ${GPU_MEMORY_UTILIZATION:-0.72}, so this exported value wins). +export GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.95}" # Run from a directory with no top-level 'vllm/' dir. # Neither launcher changes directory, so the process cwd becomes sys.path[0]