diff --git a/scripts/vllm-awq-qwen36-dflash-ampere.sh b/scripts/vllm-awq-qwen36-dflash-ampere.sh index 9d00d39..dc464af 100644 --- a/scripts/vllm-awq-qwen36-dflash-ampere.sh +++ b/scripts/vllm-awq-qwen36-dflash-ampere.sh @@ -19,6 +19,7 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \ -p "${PORT}:8000" \ -e OMP_NUM_THREADS=8 \ -e VLLM_CPU_OMP_THREADS_BIND=8 \ + -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ -v ~/.cache/huggingface:/root/.cache/huggingface \ vllm/vllm-openai:nightly "${MODEL}" \ --served-model-name "frank-default-small" \ @@ -32,7 +33,7 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \ --enable-auto-tool-choice --tool-call-parser qwen3_xml \ --reasoning-parser qwen3 \ --trust-remote-code \ - --gpu-memory-utilization 0.95 \ + --gpu-memory-utilization 0.75 \ --cpu-offload-gb 20 \ --cpu-offload-params "experts" \ --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'