From 5452bb63f141ff0807e9bbc1c2e1be6d621452b8 Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 7 Aug 2026 18:19:40 -0700 Subject: [PATCH] Add vLLM serve script, systemd service, and validation test for Qwen3.6-35B-A3B-FP8 on dankserver --- scripts/serve-vllm-qwen36-dflash.sh | 56 +++++++++++++++++ scripts/validate-vllm.sh | 93 +++++++++++++++++++++++++++++ scripts/vllm-qwen36.service | 32 ++++++++++ 3 files changed, 181 insertions(+) create mode 100755 scripts/serve-vllm-qwen36-dflash.sh create mode 100755 scripts/validate-vllm.sh create mode 100644 scripts/vllm-qwen36.service diff --git a/scripts/serve-vllm-qwen36-dflash.sh b/scripts/serve-vllm-qwen36-dflash.sh new file mode 100755 index 0000000..01f8bfa --- /dev/null +++ b/scripts/serve-vllm-qwen36-dflash.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Serve Qwen3.6-35B-A3B-FP8 with DFlash speculative decoding via vLLM on NVIDIA RTX 3090 +# Uses Podman container with CUDA support +# 4 parallel streams, full context (262K) each +set -euo pipefail + +# Configuration +MODEL="${MODEL:-Qwen/Qwen3.6-35B-A3B-FP8}" +DRAFT_MODEL="${DRAFT_MODEL:-z-lab/Qwen3.6-35B-A3B-DFlash}" +PORT="${PORT:-8000}" +GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.95}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-4}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}" +KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" +NUM_SPEC_TOKENS="${NUM_SPEC_TOKENS:-8}" +CONTAINER_NAME="${CONTAINER_NAME:-vllm-qwen36-dflash}" +HF_CACHE="${HF_CACHE:-/home/dank/.cache/huggingface}" +GPU_LIBS="${GPU_LIBS:-/usr/lib/x86_64-linux-gnu}" + +echo "Starting vLLM server in Podman..." +echo " Target model: ${MODEL}" +echo " Draft model: ${DRAFT_MODEL}" +echo " Port: ${PORT}" +echo " GPU mem util: ${GPU_MEM_UTIL}" +echo " Max seqs: ${MAX_NUM_SEQS}" +echo " Context len: ${MAX_MODEL_LEN}" +echo " KV cache: ${KV_CACHE_DTYPE}" +echo " Spec tokens: ${NUM_SPEC_TOKENS}" +echo "" + +# Remove existing container if present +podman rm -f "${CONTAINER_NAME}" 2>/dev/null || true + +# Run vLLM in Podman container +exec podman run -d --name "${CONTAINER_NAME}" \ + --device /dev/nvidia0 \ + --device /dev/nvidiactl \ + --device /dev/nvidia-uvm \ + --device /dev/nvidia-uvm-tools \ + --device /dev/nvidia-modeset \ + -v "${GPU_LIBS}:/usr/lib/x86_64-linux-gnu:ro" \ + -v "${HF_CACHE}:/root/.cache/huggingface:ro" \ + -p "${PORT}:${PORT}" \ + --cap-add SYS_ADMIN \ + docker.io/vllm/vllm-openai:latest \ + vllm serve "${MODEL}" \ + --port "${PORT}" \ + --host "0.0.0.0" \ + --kv-cache-dtype "${KV_CACHE_DTYPE}" \ + --gpu-memory-utilization "${GPU_MEM_UTIL}" \ + --max-num-seqs "${MAX_NUM_SEQS}" \ + --max-model-len "${MAX_MODEL_LEN}" \ + --speculative-method dflash \ + --speculative-model "${DRAFT_MODEL}" \ + --num-speculative-tokens "${NUM_SPEC_TOKENS}" \ + --trust-remote-code diff --git a/scripts/validate-vllm.sh b/scripts/validate-vllm.sh new file mode 100755 index 0000000..50920c7 --- /dev/null +++ b/scripts/validate-vllm.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# End-to-end validation test for vLLM Qwen3.6-35B-A3B-FP8 on dankserver +# Sends a chat completion request and measures prompt processing speed +set -euo pipefail + +PORT="${PORT:-8000}" +HOST="${HOST:-127.0.0.1}" +URL="http://${HOST}:${PORT}" +MIN_SPEED="${MIN_SPEED:-5.0}" +MAX_TOKENS="${MAX_TOKENS:-256}" + +echo "=== vLLM Validation Test ===" +echo "Target: ${URL}" +echo "Min prompt speed: ${MIN_SPEED} tok/s" +echo "" + +# Wait for server to be ready +echo "Waiting for server..." +for i in $(seq 1 300); do + if curl -sf "${URL}/health" > /dev/null 2>&1; then + echo "Server ready after $((i * 2))s" + break + fi + if [ "$i" -eq 300 ]; then + echo "FAIL: Server did not become ready in 600s" + exit 1 + fi + sleep 2 +done + +# Send chat completion and measure prompt processing speed +echo "" +echo "--- Sending validation prompt ---" +RESPONSE=$(curl -s -X POST "${URL}/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d "{ + \"model\": \"frank-default-large\", + \"messages\": [ + {\"role\": \"user\", \"content\": \"Explain the difference between supervised learning and unsupervised learning in two sentences.\"} + ], + \"max_tokens\": ${MAX_TOKENS} + }") + +echo "Response received." +echo "" + +# Parse timing from response +echo "$RESPONSE" | python3 -c " +import sys, json + +data = json.load(sys.stdin) + +# Check for errors +if 'error' in data: + print(f'ERROR: {data[\"error\"]}') + sys.exit(1) + +# Extract usage info +usage = data.get('usage', {}) +prompt_tokens = usage.get('prompt_tokens', 0) +completion_tokens = usage.get('completion_tokens', 0) + +print(f'Prompt tokens: {prompt_tokens}') +print(f'Completion tokens: {completion_tokens}') + +# Extract timing details +prompt_details = usage.get('prompt_details', {}) +if prompt_details: + prompt_time = prompt_details.get('total_time', 0) + speed = prompt_tokens / prompt_time if prompt_time > 0 else 0 + print(f'Prompt time: {prompt_time:.4f}s') + print(f'Prompt speed: {speed:.1f} tok/s') + + if speed < float('${MIN_SPEED}'): + print(f'FAIL: Speed {speed:.1f} tok/s < {\"${MIN_SPEED}\"} tok/s') + sys.exit(1) + else: + print(f'PASS: Speed {speed:.1f} tok/s >= {\"${MIN_SPEED}\"} tok/s') +else: + # Fallback: use total_time from usage + total_time = usage.get('total_time', 0) + if total_time > 0 and prompt_tokens > 0: + speed = prompt_tokens / total_time + print(f'Total time: {total_time:.4f}s') + print(f'Est. prompt speed: {speed:.1f} tok/s') + print('PASS (response received)') + else: + print('No timing details available') + print('PASS (response received)') +" + +echo "" +echo "=== Validation complete ===" diff --git a/scripts/vllm-qwen36.service b/scripts/vllm-qwen36.service new file mode 100644 index 0000000..fe30dec --- /dev/null +++ b/scripts/vllm-qwen36.service @@ -0,0 +1,32 @@ +[Unit] +Description=vLLM Qwen3.6-35B-A3B-FP8 with DFlash Drafting (Podman) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=dank +Group=dank +Environment="PORT=8000" +Environment="GPU_MEM_UTIL=0.95" +Environment="MAX_NUM_SEQS=4" +Environment="MAX_MODEL_LEN=262144" +Environment="KV_CACHE_DTYPE=fp8" +Environment="NUM_SPEC_TOKENS=8" +ExecStart=/home/dank/Projects/vllmScripts/scripts/serve-vllm-qwen36-dflash.sh +ExecStop=/usr/bin/podman stop vllm-qwen36-dflash +ExecStopPost=/usr/bin/podman rm -f vllm-qwen36-dflash +Restart=on-failure +RestartSec=10 +TimeoutStopSec=60 + +# Ensure GPU devices are available +DevicePolicy=closed +DeviceAllow=/dev/nvidia0 rwm +DeviceAllow=/dev/nvidiactl rwm +DeviceAllow=/dev/nvidia-uvm rwm +DeviceAllow=/dev/nvidia-uvm-tools rwm +DeviceAllow=/dev/nvidia-modeset rwm + +[Install] +WantedBy=multi-user.target