Add --enable-auto-tool-choice and --tool-call-parser qwen3_xml for tool choice support
This commit is contained in:
+5
-6
@@ -1,9 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Qwen3.6-35B-A3B-FP8 vLLM serving script for dankserver
|
||||
# CPU expert offloading to fit MoE model on 24GB RTX 3090
|
||||
|
||||
MODEL="Qwen/Qwen3.6-35B-A3B-FP8"
|
||||
PORT=8001
|
||||
HOST="0.0.0.0"
|
||||
@@ -15,13 +12,13 @@ exec docker run --rm --name vllm-qwen36 \
|
||||
-p "${PORT}:${PORT}" \
|
||||
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
|
||||
-v /home/dank/.cache/huggingface:/root/.cache/huggingface \
|
||||
docker.io/vllm/vllm-openai:nightly \
|
||||
docker.io/vllm/vllm-openai:latest \
|
||||
"${MODEL}" \
|
||||
--port "${PORT}" \
|
||||
--host "${HOST}" \
|
||||
--kv-cache-dtype fp8 \
|
||||
--gpu-memory-utilization 0.95 \
|
||||
--max-num-seqs 3 \
|
||||
--max-num-seqs 4 \
|
||||
--max-model-len 16384 \
|
||||
--max-num-batched-tokens 4096 \
|
||||
--trust-remote-code \
|
||||
@@ -29,4 +26,6 @@ exec docker run --rm --name vllm-qwen36 \
|
||||
--cpu-offload-params "experts" \
|
||||
--enable-chunked-prefill \
|
||||
--enable-prefix-caching \
|
||||
--served-model-name frank-default-small
|
||||
--served-model-name frank-default-small \
|
||||
--enable-auto-tool-choice \
|
||||
--tool-call-parser qwen3_xml
|
||||
|
||||
Reference in New Issue
Block a user