Add --enable-auto-tool-choice and --tool-call-parser qwen3_xml for tool choice support

This commit is contained in:
Frank
2026-08-08 11:23:57 -07:00
parent dd19627635
commit 69bb41057c
+5 -6
View File
@@ -1,9 +1,6 @@
#!/usr/bin/env bash
set -euo pipefail
# Qwen3.6-35B-A3B-FP8 vLLM serving script for dankserver
# CPU expert offloading to fit MoE model on 24GB RTX 3090
MODEL="Qwen/Qwen3.6-35B-A3B-FP8"
PORT=8001
HOST="0.0.0.0"
@@ -15,13 +12,13 @@ exec docker run --rm --name vllm-qwen36 \
-p "${PORT}:${PORT}" \
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
-v /home/dank/.cache/huggingface:/root/.cache/huggingface \
docker.io/vllm/vllm-openai:nightly \
docker.io/vllm/vllm-openai:latest \
"${MODEL}" \
--port "${PORT}" \
--host "${HOST}" \
--kv-cache-dtype fp8 \
--gpu-memory-utilization 0.95 \
--max-num-seqs 3 \
--max-num-seqs 4 \
--max-model-len 16384 \
--max-num-batched-tokens 4096 \
--trust-remote-code \
@@ -29,4 +26,6 @@ exec docker run --rm --name vllm-qwen36 \
--cpu-offload-params "experts" \
--enable-chunked-prefill \
--enable-prefix-caching \
--served-model-name frank-default-small
--served-model-name frank-default-small \
--enable-auto-tool-choice \
--tool-call-parser qwen3_xml