diff --git a/vllm-qwen36.sh b/vllm-qwen36.sh index d29afa9..5426845 100755 --- a/vllm-qwen36.sh +++ b/vllm-qwen36.sh @@ -1,9 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -# Qwen3.6-35B-A3B-FP8 vLLM serving script for dankserver -# CPU expert offloading to fit MoE model on 24GB RTX 3090 - MODEL="Qwen/Qwen3.6-35B-A3B-FP8" PORT=8001 HOST="0.0.0.0" @@ -15,13 +12,13 @@ exec docker run --rm --name vllm-qwen36 \ -p "${PORT}:${PORT}" \ -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ -v /home/dank/.cache/huggingface:/root/.cache/huggingface \ - docker.io/vllm/vllm-openai:nightly \ + docker.io/vllm/vllm-openai:latest \ "${MODEL}" \ --port "${PORT}" \ --host "${HOST}" \ --kv-cache-dtype fp8 \ --gpu-memory-utilization 0.95 \ - --max-num-seqs 3 \ + --max-num-seqs 4 \ --max-model-len 16384 \ --max-num-batched-tokens 4096 \ --trust-remote-code \ @@ -29,4 +26,6 @@ exec docker run --rm --name vllm-qwen36 \ --cpu-offload-params "experts" \ --enable-chunked-prefill \ --enable-prefix-caching \ - --served-model-name frank-default-small + --served-model-name frank-default-small \ + --enable-auto-tool-choice \ + --tool-call-parser qwen3_xml