Increase max-model-len to 32768, cpu-offload to 40GB, add reasoning-parser and tool-call-parser
This commit is contained in:
Frank committed
1 parent
0f97dcf449
commit
4edcada2a8
2 files changed
+103
-68
No files matched your search
+71
-68
@@ -1,11 +1,15 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# End-to-end validation test for vLLM Qwen3.6-35B-A3B-FP8
|
|
||||||
# Tests single stream and 4 parallel streams
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
HOST="${1:-127.0.0.1}"
|
HOST="127.0.0.1"
|
||||||
PORT="${2:-8001}"
|
PORT=8001
|
||||||
NUM_STREAMS=4
|
MODEL="frank-default-small"
|
||||||
|
|
||||||
|
# Different prompts for each test to avoid prefix caching
|
||||||
|
PROMPT1="Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations."
|
||||||
|
PROMPT2="Describe the architectural differences between RISC-V and x86 instruction set architectures. Include a discussion of register files, addressing modes, and instruction encoding. Compare their performance for embedded systems and server workloads."
|
||||||
|
PROMPT3="Analyze the economic impact of blockchain technology on supply chain management. Discuss the key benefits of transparency, immutability, and smart contracts. Compare implementation challenges of Bitcoin, Ethereum, and Hyperledger for enterprise use cases."
|
||||||
|
PROMPT4="Explain the principles of evolutionary algorithms in artificial intelligence. Describe how genetic algorithms, genetic programming, and evolutionary strategies differ in their approach to problem solving. Compare their performance on optimization problems and neural network architecture search."
|
||||||
|
|
||||||
echo "=== Validating vLLM on ${HOST}:${PORT} ==="
|
echo "=== Validating vLLM on ${HOST}:${PORT} ==="
|
||||||
|
|
||||||
@@ -16,70 +20,69 @@ if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Single stream test
|
# Test N streams sequentially (1, 2, 3, 4)
|
||||||
echo "Single stream test..."
|
for NUM_STREAMS in 1 2 3 4; do
|
||||||
START=$(date +%s%N)
|
echo ""
|
||||||
RESPONSE=$(curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
echo "--- Testing ${NUM_STREAMS} stream(s) ---"
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
TMPDIR=$(mktemp -d)
|
||||||
"model": "Qwen/Qwen3.6-35B-A3B-FP8",
|
START=$(date +%s%N)
|
||||||
"messages": [
|
|
||||||
{"role": "user", "content": "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone."}
|
for i in $(seq 1 $NUM_STREAMS); do
|
||||||
],
|
case $(( (i - 1) % 4 + 1 )) in
|
||||||
"max_tokens": 256
|
1) PROMPT="${PROMPT1}" ;;
|
||||||
}')
|
2) PROMPT="${PROMPT2}" ;;
|
||||||
END=$(date +%s%N)
|
3) PROMPT="${PROMPT3}" ;;
|
||||||
ELAPSED_MS=$(( (END - START) / 1000000 ))
|
4) PROMPT="${PROMPT4}" ;;
|
||||||
PROMPT_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['usage']['prompt_tokens'])")
|
esac
|
||||||
COMPLETION_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['usage']['completion_tokens'])")
|
|
||||||
SINGLE_SPEED=$(python3 -c "print(f'{${PROMPT_TOKENS}/${ELAPSED_MS}*1000:.2f}')")
|
curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
||||||
echo "Single stream: ${SINGLE_SPEED} tok/s (prompt)"
|
-H "Content-Type: application/json" \
|
||||||
|
-d "{
|
||||||
# Parallel streams test
|
\"model\": \"${MODEL}\",
|
||||||
echo "Parallel streams test (${NUM_STREAMS} streams)..."
|
\"messages\": [
|
||||||
START=$(date +%s%N)
|
{\"role\": \"user\", \"content\": \"${PROMPT}\"}
|
||||||
TMPDIR=$(mktemp -d)
|
],
|
||||||
for i in $(seq 1 $NUM_STREAMS); do
|
\"max_tokens\": 256
|
||||||
curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
}" > "$TMPDIR/$i.json" &
|
||||||
-H "Content-Type: application/json" \
|
done
|
||||||
-d '{
|
wait
|
||||||
"model": "Qwen/Qwen3.6-35B-A3B-FP8",
|
|
||||||
"messages": [
|
END=$(date +%s%N)
|
||||||
{"role": "user", "content": "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone."}
|
ELAPSED_MS=$(( (END - START) / 1000000 ))
|
||||||
],
|
|
||||||
"max_tokens": 256
|
# Calculate speed from the first stream's response
|
||||||
}' > "$TMPDIR/$i.json" &
|
PROMPT_TOKENS=$(python3 -c "import json; print(json.load(open('$TMPDIR/1.json'))['usage']['prompt_tokens'])")
|
||||||
|
COMPLETION_TOKENS=$(python3 -c "import json; print(json.load(open('$TMPDIR/1.json'))['usage']['completion_tokens'])")
|
||||||
|
|
||||||
|
# Calculate speeds
|
||||||
|
PROMPT_SPEED=$(python3 -c "print(f'{${PROMPT_TOKENS}/${ELAPSED_MS}*1000:.2f}')")
|
||||||
|
COMPLETION_SPEED=$(python3 -c "print(f'{${COMPLETION_TOKENS}/${ELAPSED_MS}*1000:.2f}')")
|
||||||
|
|
||||||
|
echo " Elapsed: ${ELAPSED_MS}ms"
|
||||||
|
echo " Prompt tokens: ${PROMPT_TOKENS}"
|
||||||
|
echo " Prompt speed: ${PROMPT_SPEED} tok/s"
|
||||||
|
echo " Completion tokens: ${COMPLETION_TOKENS}"
|
||||||
|
echo " Completion speed: ${COMPLETION_SPEED} tok/s"
|
||||||
|
|
||||||
|
# Save results
|
||||||
|
echo "${NUM_STREAMS},${PROMPT_SPEED},${COMPLETION_SPEED},${ELAPSED_MS}" > "/tmp/vllm-${NUM_STREAMS}.csv"
|
||||||
|
|
||||||
|
# Clean up temp files
|
||||||
|
rm -rf "$TMPDIR"
|
||||||
done
|
done
|
||||||
wait
|
|
||||||
END=$(date +%s%N)
|
|
||||||
ELAPSED_MS=$(( (END - START) / 1000000 ))
|
|
||||||
|
|
||||||
# Aggregate results from all streams
|
|
||||||
TOTAL_PROMPT=0
|
|
||||||
TOTAL_COMPLETION=0
|
|
||||||
for i in $(seq 1 $NUM_STREAMS); do
|
|
||||||
PT=$(python3 -c "import json; print(json.load(open('$TMPDIR/$i.json'))['usage']['prompt_tokens'])")
|
|
||||||
CT=$(python3 -c "import json; print(json.load(open('$TMPDIR/$i.json'))['usage']['completion_tokens'])")
|
|
||||||
TOTAL_PROMPT=$((TOTAL_PROMPT + PT))
|
|
||||||
TOTAL_COMPLETION=$((TOTAL_COMPLETION + CT))
|
|
||||||
done
|
|
||||||
rm -rf "$TMPDIR"
|
|
||||||
|
|
||||||
PARALLEL_SPEED=$(python3 -c "print(f'{${TOTAL_PROMPT}/${ELAPSED_MS}*1000:.2f}')")
|
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Results ==="
|
echo "=== Results Summary ==="
|
||||||
echo "Single stream prompt speed: ${SINGLE_SPEED} tok/s"
|
for n in 1 2 3 4; do
|
||||||
echo "Parallel streams prompt speed: ${PARALLEL_SPEED} tok/s (${NUM_STREAMS} streams, ${TOTAL_PROMPT} total prompt tokens)"
|
if [ -f "/tmp/vllm-${n}.csv" ]; then
|
||||||
echo "Parallel completion speed: $(python3 -c "print(f'{${TOTAL_COMPLETION}/${ELAPSED_MS}*1000:.2f}')") tok/s"
|
IFS=',' read -r streams p_speed c_speed elapsed < "/tmp/vllm-${n}.csv"
|
||||||
echo "Total parallel time: ${ELAPSED_MS}ms"
|
echo " ${n} stream(s): ${p_speed} tok/s (prompt), ${c_speed} tok/s (completion) - ${elapsed}ms"
|
||||||
echo ""
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
# Fail if parallel speed < 5 tok/s
|
# Check if single stream speed is >= 5 tok/s
|
||||||
if python3 -c "import sys; sys.exit(0 if ${TOTAL_PROMPT}/${ELAPSED_MS}*1000 >= 5 else 1)"; then
|
SINGLE_SPEED=$(head -1 /tmp/vllm-1.csv | cut -d',' -f2)
|
||||||
echo "PASS: Validation successful"
|
python3 -c "import sys; sys.exit(0 if float('${SINGLE_SPEED}') >= 5 else 1)"
|
||||||
echo "$PARALLEL_SPEED" > /tmp/vllm_prompt_speed.txt
|
echo ""
|
||||||
else
|
echo "PASS: Validation successful"
|
||||||
echo "FAIL: Parallel prompt speed ${PARALLEL_SPEED} tok/s is below 5 tok/s threshold"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
Executable
+32
@@ -0,0 +1,32 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
MODEL="Qwen/Qwen3.6-35B-A3B-FP8"
|
||||||
|
PORT=8001
|
||||||
|
HOST="0.0.0.0"
|
||||||
|
|
||||||
|
exec docker run --rm --name vllm-qwen36 \
|
||||||
|
--gpus all \
|
||||||
|
--privileged \
|
||||||
|
--ipc=host \
|
||||||
|
-p "${PORT}:${PORT}" \
|
||||||
|
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
|
||||||
|
-v /home/dank/.cache/huggingface:/root/.cache/huggingface \
|
||||||
|
docker.io/vllm/vllm-openai:latest \
|
||||||
|
"${MODEL}" \
|
||||||
|
--port "${PORT}" \
|
||||||
|
--host "${HOST}" \
|
||||||
|
--kv-cache-dtype fp8 \
|
||||||
|
--gpu-memory-utilization 0.95 \
|
||||||
|
--max-num-seqs 4 \
|
||||||
|
--max-model-len 32768 \
|
||||||
|
--max-num-batched-tokens 8192 \
|
||||||
|
--trust-remote-code \
|
||||||
|
--cpu-offload-gb 40 \
|
||||||
|
--cpu-offload-params "experts" \
|
||||||
|
--enable-chunked-prefill \
|
||||||
|
--enable-prefix-caching \
|
||||||
|
--served-model-name frank-default-small \
|
||||||
|
--enable-auto-tool-choice \
|
||||||
|
--reasoning-parser qwen3 \
|
||||||
|
--tool-call-parser qwen3_xml
|
||||||
Reference in new issue
Block a user