Add --enable-auto-tool-choice and --tool-call-parser qwen3_xml for tool choice support

This commit is contained in:
Frank committed 2026-08-08 11:23:57 -07:00
1 parent dd19627635
commit 69bb41057c
1 file changed
+5 -6
+5 -6
View File
@@ -1,9 +1,6 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -euo pipefail
# Qwen3.6-35B-A3B-FP8 vLLM serving script for dankserver
# CPU expert offloading to fit MoE model on 24GB RTX 3090
MODEL="Qwen/Qwen3.6-35B-A3B-FP8" MODEL="Qwen/Qwen3.6-35B-A3B-FP8"
PORT=8001 PORT=8001
HOST="0.0.0.0" HOST="0.0.0.0"
@@ -15,13 +12,13 @@ exec docker run --rm --name vllm-qwen36 \
-p "${PORT}:${PORT}" \ -p "${PORT}:${PORT}" \
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
-v /home/dank/.cache/huggingface:/root/.cache/huggingface \ -v /home/dank/.cache/huggingface:/root/.cache/huggingface \
docker.io/vllm/vllm-openai:nightly \ docker.io/vllm/vllm-openai:latest \
"${MODEL}" \ "${MODEL}" \
--port "${PORT}" \ --port "${PORT}" \
--host "${HOST}" \ --host "${HOST}" \
--kv-cache-dtype fp8 \ --kv-cache-dtype fp8 \
--gpu-memory-utilization 0.95 \ --gpu-memory-utilization 0.95 \
--max-num-seqs 3 \ --max-num-seqs 4 \
--max-model-len 16384 \ --max-model-len 16384 \
--max-num-batched-tokens 4096 \ --max-num-batched-tokens 4096 \
--trust-remote-code \ --trust-remote-code \
@@ -29,4 +26,6 @@ exec docker run --rm --name vllm-qwen36 \
--cpu-offload-params "experts" \ --cpu-offload-params "experts" \
--enable-chunked-prefill \ --enable-chunked-prefill \
--enable-prefix-caching \ --enable-prefix-caching \
--served-model-name frank-default-small --served-model-name frank-default-small \
--enable-auto-tool-choice \
--tool-call-parser qwen3_xml