32 lines
860 B
Bash
Executable File
32 lines
860 B
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Qwen3.6-35B-A3B-FP8 vLLM serving script for dankserver
|
|
# CPU expert offloading to fit MoE model on 24GB RTX 3090
|
|
|
|
MODEL="Qwen/Qwen3.6-35B-A3B-FP8"
|
|
PORT=8001
|
|
HOST="0.0.0.0"
|
|
|
|
exec docker run --rm --name vllm-qwen36 \
|
|
--gpus all \
|
|
--privileged \
|
|
--ipc=host \
|
|
-p "${PORT}:${PORT}" \
|
|
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
|
|
-v /home/dank/.cache/huggingface:/root/.cache/huggingface \
|
|
docker.io/vllm/vllm-openai:nightly \
|
|
"${MODEL}" \
|
|
--port "${PORT}" \
|
|
--host "${HOST}" \
|
|
--kv-cache-dtype fp8 \
|
|
--gpu-memory-utilization 0.95 \
|
|
--max-num-seqs 3 \
|
|
--max-model-len 16384 \
|
|
--max-num-batched-tokens 4096 \
|
|
--trust-remote-code \
|
|
--cpu-offload-gb 30 \
|
|
--cpu-offload-params "experts" \
|
|
--enable-chunked-prefill \
|
|
--enable-prefix-caching
|