Add --enable-auto-tool-choice and --tool-call-parser qwen3_xml for tool choice support
This commit is contained in:
1 parent
dd19627635
commit
69bb41057c
1 file changed
+5
-6
+5
-6
@@ -1,9 +1,6 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
# Qwen3.6-35B-A3B-FP8 vLLM serving script for dankserver
|
|
||||||
# CPU expert offloading to fit MoE model on 24GB RTX 3090
|
|
||||||
|
|
||||||
MODEL="Qwen/Qwen3.6-35B-A3B-FP8"
|
MODEL="Qwen/Qwen3.6-35B-A3B-FP8"
|
||||||
PORT=8001
|
PORT=8001
|
||||||
HOST="0.0.0.0"
|
HOST="0.0.0.0"
|
||||||
@@ -15,13 +12,13 @@ exec docker run --rm --name vllm-qwen36 \
|
|||||||
-p "${PORT}:${PORT}" \
|
-p "${PORT}:${PORT}" \
|
||||||
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
|
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
|
||||||
-v /home/dank/.cache/huggingface:/root/.cache/huggingface \
|
-v /home/dank/.cache/huggingface:/root/.cache/huggingface \
|
||||||
docker.io/vllm/vllm-openai:nightly \
|
docker.io/vllm/vllm-openai:latest \
|
||||||
"${MODEL}" \
|
"${MODEL}" \
|
||||||
--port "${PORT}" \
|
--port "${PORT}" \
|
||||||
--host "${HOST}" \
|
--host "${HOST}" \
|
||||||
--kv-cache-dtype fp8 \
|
--kv-cache-dtype fp8 \
|
||||||
--gpu-memory-utilization 0.95 \
|
--gpu-memory-utilization 0.95 \
|
||||||
--max-num-seqs 3 \
|
--max-num-seqs 4 \
|
||||||
--max-model-len 16384 \
|
--max-model-len 16384 \
|
||||||
--max-num-batched-tokens 4096 \
|
--max-num-batched-tokens 4096 \
|
||||||
--trust-remote-code \
|
--trust-remote-code \
|
||||||
@@ -29,4 +26,6 @@ exec docker run --rm --name vllm-qwen36 \
|
|||||||
--cpu-offload-params "experts" \
|
--cpu-offload-params "experts" \
|
||||||
--enable-chunked-prefill \
|
--enable-chunked-prefill \
|
||||||
--enable-prefix-caching \
|
--enable-prefix-caching \
|
||||||
--served-model-name frank-default-small
|
--served-model-name frank-default-small \
|
||||||
|
--enable-auto-tool-choice \
|
||||||
|
--tool-call-parser qwen3_xml
|
||||||
Reference in new issue
Block a user