From 52ad985978b0e71fff514a396cb8fe64e402d79e Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 10 Aug 2026 23:25:02 -0700 Subject: [PATCH] feat: raise Ling-3 GPU memory utilization to 0.95 The 0.72 default from native-256k.env OOMs. Export 0.95 in the wrapper; native-256k uses ${GPU_MEMORY_UTILIZATION:-0.72} so this value wins. --- scripts/serve-ling3-vllm.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/serve-ling3-vllm.sh b/scripts/serve-ling3-vllm.sh index d0d6084..09643d3 100755 --- a/scripts/serve-ling3-vllm.sh +++ b/scripts/serve-ling3-vllm.sh @@ -23,6 +23,10 @@ set +a export HOST="${HOST:-0.0.0.0}" export PORT="${PORT:-7770}" export SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-frank-default-large}" +# Raise the vLLM frame-buffer utilization so weights + 256k KV fit in VRAM +# (the default 0.72 from native-256k.env OOMs; native-256k uses +# ${GPU_MEMORY_UTILIZATION:-0.72}, so this exported value wins). +export GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.95}" # Run from a directory with no top-level 'vllm/' dir. # Neither launcher changes directory, so the process cwd becomes sys.path[0]