diff --git a/scripts/vllm-awq-qwen36-dflash-ampere.sh b/scripts/vllm-awq-qwen36-dflash-ampere.sh old mode 100644 new mode 100755 index 9835a7a..0d77df0 --- a/scripts/vllm-awq-qwen36-dflash-ampere.sh +++ b/scripts/vllm-awq-qwen36-dflash-ampere.sh @@ -10,7 +10,7 @@ # set -euo pipefail -MODEL="cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit" +MODEL="Avesed/Qwen3.6-35B-A3B-INT8-W8A8" PORT=34401 exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \ @@ -22,20 +22,23 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \ -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ -e VLLM_USE_V2_MODEL_RUNNER=0 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ - vllm/vllm-openai:latest "${MODEL}" \ + vllm/vllm-openai:nightly "${MODEL}" \ --served-model-name "frank-default-small" \ - --max-num-seqs 1 \ - --max-model-len 32768 \ + --max-num-seqs 3 \ + --max-model-len 200700 \ --tensor-parallel-size 1 \ --enable-chunked-prefill \ - --max-num-batched-tokens 4096 \ + --enable-prefix-caching \ + --max-num-batched-tokens 3072 \ --performance-mode throughput \ --enable-auto-tool-choice --tool-call-parser qwen3_xml \ --reasoning-parser qwen3 \ --trust-remote-code \ - --gpu-memory-utilization 0.96 \ - --cpu-offload-gb 40 \ + --gpu-memory-utilization 0.97 \ + --kv-cache-dtype auto \ + --cpu-offload-gb 30 \ --cpu-offload-params "experts" \ --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}' \ -O3 \ - --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 12}' + --speculative-config '{"method": "mtp", "num_speculative_tokens": 2}' +