dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Reduce GPU memory to 75%, add expandable_segments to fix OOM

Browse Source
This commit is contained in:
Frank committed 2026-08-09 12:11:48 -07:00
1 parent 9dfa16fae9
commit 88aa18ef34
1 file changed
+2 -1
Download Patch File Download Diff File Expand all files Collapse all files
scripts/vllm-awq-qwen36-dflash-ampere.sh
+2 -1
View File
@@ -19,6 +19,7 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
-p "${PORT}:8000" \ -p "${PORT}:8000" \
-e OMP_NUM_THREADS=8 \ -e OMP_NUM_THREADS=8 \
-e VLLM_CPU_OMP_THREADS_BIND=8 \ -e VLLM_CPU_OMP_THREADS_BIND=8 \
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
-v ~/.cache/huggingface:/root/.cache/huggingface \ -v ~/.cache/huggingface:/root/.cache/huggingface \
vllm/vllm-openai:nightly "${MODEL}" \ vllm/vllm-openai:nightly "${MODEL}" \
--served-model-name "frank-default-small" \ --served-model-name "frank-default-small" \
@@ -32,7 +33,7 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
--enable-auto-tool-choice --tool-call-parser qwen3_xml \ --enable-auto-tool-choice --tool-call-parser qwen3_xml \
--reasoning-parser qwen3 \ --reasoning-parser qwen3 \
--trust-remote-code \ --trust-remote-code \
--gpu-memory-utilization 0.95 \ --gpu-memory-utilization 0.75 \
--cpu-offload-gb 20 \ --cpu-offload-gb 20 \
--cpu-offload-params "experts" \ --cpu-offload-params "experts" \
--default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}' --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'