diff --git a/vllm-qwen36.sh b/vllm-qwen36.sh index d8a947b..1e0216a 100755 --- a/vllm-qwen36.sh +++ b/vllm-qwen36.sh @@ -20,7 +20,7 @@ exec docker run --rm --name vllm-qwen36 \ --gpu-memory-utilization 0.95 \ --max-num-seqs 4 \ --max-model-len 32768 \ - --max-num-batched-tokens 8192 \ + --max-num-batched-tokens 1536 \ --trust-remote-code \ --cpu-offload-gb 40 \ --cpu-offload-params "experts" \