31 lines
782 B
Bash
31 lines
782 B
Bash
#!/bin/bash
|
|
# vLLM serve script for ThinkingCap-Qwen3.6-27B-FP8 on dankclaw (ROCm)
|
|
# Run via systemd service: vllm-thinkingcap.service
|
|
|
|
set -euo pipefail
|
|
|
|
# Environment
|
|
export HSA_OVERRIDE_GFX_VERSION=11.5.1
|
|
export HF_HOME="/home/dank/.cache/huggingface"
|
|
|
|
# vLLM virtual environment — use direct paths, no activate script
|
|
VENV="/home/dank/vllm-env"
|
|
export PATH="${VENV}/bin:${PATH}"
|
|
|
|
# Model
|
|
MODEL="bottlecapai/ThinkingCap-Qwen3.6-27B-FP8"
|
|
|
|
# Serve
|
|
exec "${VENV}/bin/vllm" serve "${MODEL}" \
|
|
--port 8000 \
|
|
--host 0.0.0.0 \
|
|
--kv-cache-dtype fp8 \
|
|
--gpu-memory-utilization 0.80 \
|
|
--enforce-eager \
|
|
--max-model-len 32768 \
|
|
--max-num-batched-tokens 2048 \
|
|
--max-num-seqs 32 \
|
|
--reasoning-parser qwen3 \
|
|
--language-model-only \
|
|
--dtype auto
|