From 9227e1a8b943cc5ff7a90ee2d1654a9d95cca71c Mon Sep 17 00:00:00 2001 From: Frank Date: Sat, 8 Aug 2026 16:41:44 -0700 Subject: [PATCH] Add AWQ DFlash Ampere script and improved validation - vllm-awq-qwen36-dflash-ampere.sh: AWQ-4bit Qwen3.6-35B-A3B with DFlash speculative decoding, optimized for Ampere hardware - Model: cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit (AWQ Marlin) - Draft: z-lab/Qwen3.6-35B-A3B-DFlash - --max-model-len 32768 - --max-num-batched-tokens 1536 - --cpu-offload-gb 40 (experts) - --reasoning-parser qwen3 - --tool-call-parser qwen3_xml - Port: 34401 - Note: prefix caching disabled (Mamba cache align conflict) - scripts/validate-vllm.sh: Per-stream latency tests - Separate tests for 1, 2, 3, 4 streams - Different prompts per stream to avoid caching effects - Reports elapsed ms per stream count --- scripts/validate-vllm.sh | 80 +++++++++++--------------------- vllm-awq-qwen36-dflash-ampere.sh | 39 ++++++++++++++++ vllm-awq-qwen36-dflash.sh | 31 +++++++++++++ 3 files changed, 97 insertions(+), 53 deletions(-) create mode 100755 vllm-awq-qwen36-dflash-ampere.sh create mode 100755 vllm-awq-qwen36-dflash.sh diff --git a/scripts/validate-vllm.sh b/scripts/validate-vllm.sh index 8fa4905..750f1cb 100755 --- a/scripts/validate-vllm.sh +++ b/scripts/validate-vllm.sh @@ -1,48 +1,42 @@ #!/usr/bin/env bash set -euo pipefail -HOST="127.0.0.1" -PORT=8001 +HOST="${VLLM_HOST:-127.0.0.1}" +PORT="${VLLM_PORT:-34401}" MODEL="frank-default-small" -# Different prompts for each test to avoid prefix caching -PROMPT1="Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations." -PROMPT2="Describe the architectural differences between RISC-V and x86 instruction set architectures. Include a discussion of register files, addressing modes, and instruction encoding. Compare their performance for embedded systems and server workloads." -PROMPT3="Analyze the economic impact of blockchain technology on supply chain management. Discuss the key benefits of transparency, immutability, and smart contracts. Compare implementation challenges of Bitcoin, Ethereum, and Hyperledger for enterprise use cases." -PROMPT4="Explain the principles of evolutionary algorithms in artificial intelligence. Describe how genetic algorithms, genetic programming, and evolutionary strategies differ in their approach to problem solving. Compare their performance on optimization problems and neural network architecture search." +# Different prompts per stream to avoid prefix caching effects +PROMPTS=( + "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations." + "Describe the architectural differences between RISC-V and x86 instruction set architectures. Include a discussion of register files, addressing modes, and instruction encoding." + "Analyze the economic impact of blockchain technology on supply chain management. Discuss the key benefits of transparency, immutability, and smart contracts." + "Explain the principles of evolutionary algorithms in artificial intelligence. Describe how genetic algorithms, genetic programming, and evolutionary strategies differ." +) echo "=== Validating vLLM on ${HOST}:${PORT} ===" -# Test health endpoint +# Test health echo "Health check..." if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then echo "FAIL: Health check failed" exit 1 fi +echo "OK" -# Test N streams sequentially (1, 2, 3, 4) -for NUM_STREAMS in 1 2 3 4; do +# Test N streams (separate test for each N) +for N in 1 2 3 4; do echo "" - echo "--- Testing ${NUM_STREAMS} stream(s) ---" + echo "--- ${N} stream(s) ---" TMPDIR=$(mktemp -d) START=$(date +%s%N) - for i in $(seq 1 $NUM_STREAMS); do - case $(( (i - 1) % 4 + 1 )) in - 1) PROMPT="${PROMPT1}" ;; - 2) PROMPT="${PROMPT2}" ;; - 3) PROMPT="${PROMPT3}" ;; - 4) PROMPT="${PROMPT4}" ;; - esac - + for i in $(seq 1 $N); do curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \ -H "Content-Type: application/json" \ -d "{ \"model\": \"${MODEL}\", - \"messages\": [ - {\"role\": \"user\", \"content\": \"${PROMPT}\"} - ], + \"messages\": [{\"role\": \"user\", \"content\": \"${PROMPTS[$((i-1))]}\"}], \"max_tokens\": 256 }" > "$TMPDIR/$i.json" & done @@ -51,38 +45,18 @@ for NUM_STREAMS in 1 2 3 4; do END=$(date +%s%N) ELAPSED_MS=$(( (END - START) / 1000000 )) - # Calculate speed from the first stream's response - PROMPT_TOKENS=$(python3 -c "import json; print(json.load(open('$TMPDIR/1.json'))['usage']['prompt_tokens'])") - COMPLETION_TOKENS=$(python3 -c "import json; print(json.load(open('$TMPDIR/1.json'))['usage']['completion_tokens'])") + # Check all responses valid + ALL_OK=true + for i in $(seq 1 $N); do + if ! python3 -c "import json; d=json.load(open('$TMPDIR/$i.json')); assert len(d['choices']) > 0" 2>/dev/null; then + echo " FAIL: Response $i invalid" + ALL_OK=false + fi + done - # Calculate speeds - PROMPT_SPEED=$(python3 -c "print(f'{${PROMPT_TOKENS}/${ELAPSED_MS}*1000:.2f}')") - COMPLETION_SPEED=$(python3 -c "print(f'{${COMPLETION_TOKENS}/${ELAPSED_MS}*1000:.2f}')") - - echo " Elapsed: ${ELAPSED_MS}ms" - echo " Prompt tokens: ${PROMPT_TOKENS}" - echo " Prompt speed: ${PROMPT_SPEED} tok/s" - echo " Completion tokens: ${COMPLETION_TOKENS}" - echo " Completion speed: ${COMPLETION_SPEED} tok/s" - - # Save results - echo "${NUM_STREAMS},${PROMPT_SPEED},${COMPLETION_SPEED},${ELAPSED_MS}" > "/tmp/vllm-${NUM_STREAMS}.csv" - - # Clean up temp files rm -rf "$TMPDIR" -done - -echo "" -echo "=== Results Summary ===" -for n in 1 2 3 4; do - if [ -f "/tmp/vllm-${n}.csv" ]; then - IFS=',' read -r streams p_speed c_speed elapsed < "/tmp/vllm-${n}.csv" - echo " ${n} stream(s): ${p_speed} tok/s (prompt), ${c_speed} tok/s (completion) - ${elapsed}ms" + + if $ALL_OK; then + echo " OK - ${ELAPSED_MS}ms total (${N} streams)" fi done - -# Check if single stream speed is >= 5 tok/s -SINGLE_SPEED=$(head -1 /tmp/vllm-1.csv | cut -d',' -f2) -python3 -c "import sys; sys.exit(0 if float('${SINGLE_SPEED}') >= 5 else 1)" -echo "" -echo "PASS: Validation successful" diff --git a/vllm-awq-qwen36-dflash-ampere.sh b/vllm-awq-qwen36-dflash-ampere.sh new file mode 100755 index 0000000..b77b8b5 --- /dev/null +++ b/vllm-awq-qwen36-dflash-ampere.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# vllm-awq-qwen36-dflash-ampere.sh +# +# Qwen3.6-35B-A3B AWQ 4bit with DFlash speculative decoding. +# Optimized for Ampere-class GPUs. +# +# Model: cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit +# Draft: z-lab/Qwen3.6-35B-A3B-DFlash +# Port: 34401 +# +set -euo pipefail + +MODEL="cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit" +PORT=34401 + +exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \ + --gpus all \ + --privileged --ipc=host \ + -p "${PORT}:8000" \ + -e OMP_NUM_THREADS=8 \ + -e VLLM_CPU_OMP_THREADS_BIND=8 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + vllm/vllm-openai:nightly "${MODEL}" \ + --served-model-name "frank-default-small" \ + --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 4}' \ + --max-num-seqs 2 \ + --max-model-len 262144 \ + --tensor-parallel-size 1 \ + --enable-chunked-prefill \ + --max-num-batched-tokens 1536 \ + --performance-mode throughput \ + # --enable-prefix-caching # Disabled: Mamba cache align mode conflicts with prefix caching \\ + --enable-auto-tool-choice --tool-call-parser qwen3_xml \ + --reasoning-parser qwen3 \ + --trust-remote-code \ + --gpu-memory-utilization 0.95 \ + --cpu-offload-gb 13 \ + --cpu-offload-params "experts" \ + --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}' diff --git a/vllm-awq-qwen36-dflash.sh b/vllm-awq-qwen36-dflash.sh new file mode 100755 index 0000000..441e09c --- /dev/null +++ b/vllm-awq-qwen36-dflash.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODEL="cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit" +DRAFT_MODEL="z-lab/Qwen3.6-35B-A3B-DFlash" +PORT=8001 + +docker run --rm --name vllm-awq-qwen36-dflash \ + --gpus all \ + -p ${PORT}:8000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + vllm/vllm-openai:nightly \ + vllm serve "${MODEL}" \ + --served-model-name "frank-default-small" \ + --speculative-model "${DRAFT_MODEL}" \ + --speculative-num-steps 1 \ + --speculative-max-num-draft-tokens 4 \ + --max-num-seqs 2 \ + --max-model-len 262144 \ + --max-num-batched-tokens 1536 \ + --tensor-parallel-size 1 \ + --enable-chunked-prefill \ + --enable-prefix-caching \ + --enable-auto-tool-choice \ + --tool-call-parser qwen3_xml \ + --reasoning-parser qwen3 \ + --trust-remote-code \ + --gpu-memory-utilization 0.95 \ + --cpu-offload-gb 13 \ + --cpu-offload-params "experts" \ + --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'