Add vLLM serve script, systemd service, and validation test for Qwen3.6-35B-A3B-FP8 on dankserver

2026-08-07 18:19:40 -07:00
parent 663e47866b
commit 5452bb63f1
3 changed files with 181 additions and 0 deletions
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Serve Qwen3.6-35B-A3B-FP8 with DFlash speculative decoding via vLLM on NVIDIA RTX 3090
+# Uses Podman container with CUDA support
+# 4 parallel streams, full context (262K) each
+set -euo pipefail
+
+# Configuration
+MODEL="${MODEL:-Qwen/Qwen3.6-35B-A3B-FP8}"
+DRAFT_MODEL="${DRAFT_MODEL:-z-lab/Qwen3.6-35B-A3B-DFlash}"
+PORT="${PORT:-8000}"
+GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.95}"
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-4}"
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}"
+KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
+NUM_SPEC_TOKENS="${NUM_SPEC_TOKENS:-8}"
+CONTAINER_NAME="${CONTAINER_NAME:-vllm-qwen36-dflash}"
+HF_CACHE="${HF_CACHE:-/home/dank/.cache/huggingface}"
+GPU_LIBS="${GPU_LIBS:-/usr/lib/x86_64-linux-gnu}"
+
+echo "Starting vLLM server in Podman..."
+echo "  Target model: ${MODEL}"
+echo "  Draft model:  ${DRAFT_MODEL}"
+echo "  Port:         ${PORT}"
+echo "  GPU mem util: ${GPU_MEM_UTIL}"
+echo "  Max seqs:     ${MAX_NUM_SEQS}"
+echo "  Context len:  ${MAX_MODEL_LEN}"
+echo "  KV cache:     ${KV_CACHE_DTYPE}"
+echo "  Spec tokens:  ${NUM_SPEC_TOKENS}"
+echo ""
+
+# Remove existing container if present
+podman rm -f "${CONTAINER_NAME}" 2>/dev/null || true
+
+# Run vLLM in Podman container
+exec podman run -d --name "${CONTAINER_NAME}" \
+  --device /dev/nvidia0 \
+  --device /dev/nvidiactl \
+  --device /dev/nvidia-uvm \
+  --device /dev/nvidia-uvm-tools \
+  --device /dev/nvidia-modeset \
+  -v "${GPU_LIBS}:/usr/lib/x86_64-linux-gnu:ro" \
+  -v "${HF_CACHE}:/root/.cache/huggingface:ro" \
+  -p "${PORT}:${PORT}" \
+  --cap-add SYS_ADMIN \
+  docker.io/vllm/vllm-openai:latest \
+  vllm serve "${MODEL}" \
+    --port "${PORT}" \
+    --host "0.0.0.0" \
+    --kv-cache-dtype "${KV_CACHE_DTYPE}" \
+    --gpu-memory-utilization "${GPU_MEM_UTIL}" \
+    --max-num-seqs "${MAX_NUM_SEQS}" \
+    --max-model-len "${MAX_MODEL_LEN}" \
+    --speculative-method dflash \
+    --speculative-model "${DRAFT_MODEL}" \
+    --num-speculative-tokens "${NUM_SPEC_TOKENS}" \
+    --trust-remote-code
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+# End-to-end validation test for vLLM Qwen3.6-35B-A3B-FP8 on dankserver
+# Sends a chat completion request and measures prompt processing speed
+set -euo pipefail
+
+PORT="${PORT:-8000}"
+HOST="${HOST:-127.0.0.1}"
+URL="http://${HOST}:${PORT}"
+MIN_SPEED="${MIN_SPEED:-5.0}"
+MAX_TOKENS="${MAX_TOKENS:-256}"
+
+echo "=== vLLM Validation Test ==="
+echo "Target: ${URL}"
+echo "Min prompt speed: ${MIN_SPEED} tok/s"
+echo ""
+
+# Wait for server to be ready
+echo "Waiting for server..."
+for i in $(seq 1 300); do
+    if curl -sf "${URL}/health" > /dev/null 2>&1; then
+        echo "Server ready after $((i * 2))s"
+        break
+    fi
+    if [ "$i" -eq 300 ]; then
+        echo "FAIL: Server did not become ready in 600s"
+        exit 1
+    fi
+    sleep 2
+done
+
+# Send chat completion and measure prompt processing speed
+echo ""
+echo "--- Sending validation prompt ---"
+RESPONSE=$(curl -s -X POST "${URL}/v1/chat/completions" \
+    -H "Content-Type: application/json" \
+    -d "{
+        \"model\": \"frank-default-large\",
+        \"messages\": [
+            {\"role\": \"user\", \"content\": \"Explain the difference between supervised learning and unsupervised learning in two sentences.\"}
+        ],
+        \"max_tokens\": ${MAX_TOKENS}
+    }")
+
+echo "Response received."
+echo ""
+
+# Parse timing from response
+echo "$RESPONSE" | python3 -c "
+import sys, json
+
+data = json.load(sys.stdin)
+
+# Check for errors
+if 'error' in data:
+    print(f'ERROR: {data[\"error\"]}')
+    sys.exit(1)
+
+# Extract usage info
+usage = data.get('usage', {})
+prompt_tokens = usage.get('prompt_tokens', 0)
+completion_tokens = usage.get('completion_tokens', 0)
+
+print(f'Prompt tokens:   {prompt_tokens}')
+print(f'Completion tokens: {completion_tokens}')
+
+# Extract timing details
+prompt_details = usage.get('prompt_details', {})
+if prompt_details:
+    prompt_time = prompt_details.get('total_time', 0)
+    speed = prompt_tokens / prompt_time if prompt_time > 0 else 0
+    print(f'Prompt time:   {prompt_time:.4f}s')
+    print(f'Prompt speed:  {speed:.1f} tok/s')
+    
+    if speed < float('${MIN_SPEED}'):
+        print(f'FAIL: Speed {speed:.1f} tok/s < {\"${MIN_SPEED}\"} tok/s')
+        sys.exit(1)
+    else:
+        print(f'PASS: Speed {speed:.1f} tok/s >= {\"${MIN_SPEED}\"} tok/s')
+else:
+    # Fallback: use total_time from usage
+    total_time = usage.get('total_time', 0)
+    if total_time > 0 and prompt_tokens > 0:
+        speed = prompt_tokens / total_time
+        print(f'Total time:    {total_time:.4f}s')
+        print(f'Est. prompt speed: {speed:.1f} tok/s')
+        print('PASS (response received)')
+    else:
+        print('No timing details available')
+        print('PASS (response received)')
+"
+
+echo ""
+echo "=== Validation complete ==="
@@ -0,0 +1,32 @@
+[Unit]
+Description=vLLM Qwen3.6-35B-A3B-FP8 with DFlash Drafting (Podman)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=dank
+Group=dank
+Environment="PORT=8000"
+Environment="GPU_MEM_UTIL=0.95"
+Environment="MAX_NUM_SEQS=4"
+Environment="MAX_MODEL_LEN=262144"
+Environment="KV_CACHE_DTYPE=fp8"
+Environment="NUM_SPEC_TOKENS=8"
+ExecStart=/home/dank/Projects/vllmScripts/scripts/serve-vllm-qwen36-dflash.sh
+ExecStop=/usr/bin/podman stop vllm-qwen36-dflash
+ExecStopPost=/usr/bin/podman rm -f vllm-qwen36-dflash
+Restart=on-failure
+RestartSec=10
+TimeoutStopSec=60
+
+# Ensure GPU devices are available
+DevicePolicy=closed
+DeviceAllow=/dev/nvidia0 rwm
+DeviceAllow=/dev/nvidiactl rwm
+DeviceAllow=/dev/nvidia-uvm rwm
+DeviceAllow=/dev/nvidia-uvm-tools rwm
+DeviceAllow=/dev/nvidia-modeset rwm
+
+[Install]
+WantedBy=multi-user.target