From 88aa18ef34206a2e74a07985f613debe9b34e0f0 Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Sun, 9 Aug 2026 12:11:48 -0700
Subject: [PATCH] Reduce GPU memory to 75%, add expandable_segments to fix OOM

---
 scripts/vllm-awq-qwen36-dflash-ampere.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/vllm-awq-qwen36-dflash-ampere.sh b/scripts/vllm-awq-qwen36-dflash-ampere.sh
index 9d00d39..dc464af 100644
--- a/scripts/vllm-awq-qwen36-dflash-ampere.sh
+++ b/scripts/vllm-awq-qwen36-dflash-ampere.sh
@@ -19,6 +19,7 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
     -p "${PORT}:8000" \
     -e OMP_NUM_THREADS=8 \
     -e VLLM_CPU_OMP_THREADS_BIND=8 \
+    -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
     -v ~/.cache/huggingface:/root/.cache/huggingface \
     vllm/vllm-openai:nightly "${MODEL}" \
     --served-model-name "frank-default-small" \
@@ -32,7 +33,7 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
     --enable-auto-tool-choice --tool-call-parser qwen3_xml \
     --reasoning-parser qwen3 \
     --trust-remote-code \
-    --gpu-memory-utilization 0.95 \
+    --gpu-memory-utilization 0.75 \
     --cpu-offload-gb 20 \
     --cpu-offload-params "experts" \
     --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'