feat: raise Ling-3 GPU memory utilization to 0.95
The 0.72 default from native-256k.env OOMs. Export 0.95 in the wrapper;
native-256k uses ${GPU_MEMORY_UTILIZATION:-0.72} so this value wins.
This commit is contained in:
1 parent
bf14ac7a7b
commit
52ad985978
1 file changed
+4
@@ -23,6 +23,10 @@ set +a
|
|||||||
export HOST="${HOST:-0.0.0.0}"
|
export HOST="${HOST:-0.0.0.0}"
|
||||||
export PORT="${PORT:-7770}"
|
export PORT="${PORT:-7770}"
|
||||||
export SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-frank-default-large}"
|
export SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-frank-default-large}"
|
||||||
|
# Raise the vLLM frame-buffer utilization so weights + 256k KV fit in VRAM
|
||||||
|
# (the default 0.72 from native-256k.env OOMs; native-256k uses
|
||||||
|
# ${GPU_MEMORY_UTILIZATION:-0.72}, so this exported value wins).
|
||||||
|
export GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.95}"
|
||||||
|
|
||||||
# Run from a directory with no top-level 'vllm/' dir.
|
# Run from a directory with no top-level 'vllm/' dir.
|
||||||
# Neither launcher changes directory, so the process cwd becomes sys.path[0]
|
# Neither launcher changes directory, so the process cwd becomes sys.path[0]
|
||||||
|
|||||||
Reference in new issue
Block a user