diff --git a/scripts/validate-vllm.sh b/scripts/validate-vllm.sh index 50920c7..76283f6 100755 --- a/scripts/validate-vllm.sh +++ b/scripts/validate-vllm.sh @@ -1,93 +1,85 @@ #!/usr/bin/env bash -# End-to-end validation test for vLLM Qwen3.6-35B-A3B-FP8 on dankserver -# Sends a chat completion request and measures prompt processing speed +# End-to-end validation test for vLLM Qwen3.6-35B-A3B-FP8 +# Tests single stream and 4 parallel streams set -euo pipefail -PORT="${PORT:-8000}" -HOST="${HOST:-127.0.0.1}" -URL="http://${HOST}:${PORT}" -MIN_SPEED="${MIN_SPEED:-5.0}" -MAX_TOKENS="${MAX_TOKENS:-256}" +HOST="${1:-127.0.0.1}" +PORT="${2:-8001}" +NUM_STREAMS=4 -echo "=== vLLM Validation Test ===" -echo "Target: ${URL}" -echo "Min prompt speed: ${MIN_SPEED} tok/s" -echo "" +echo "=== Validating vLLM on ${HOST}:${PORT} ===" -# Wait for server to be ready -echo "Waiting for server..." -for i in $(seq 1 300); do - if curl -sf "${URL}/health" > /dev/null 2>&1; then - echo "Server ready after $((i * 2))s" - break - fi - if [ "$i" -eq 300 ]; then - echo "FAIL: Server did not become ready in 600s" - exit 1 - fi - sleep 2 -done +# Test health endpoint +echo "Health check..." +if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then + echo "FAIL: Health check failed" + exit 1 +fi -# Send chat completion and measure prompt processing speed -echo "" -echo "--- Sending validation prompt ---" -RESPONSE=$(curl -s -X POST "${URL}/v1/chat/completions" \ +# Single stream test +echo "Single stream test..." +START=$(date +%s%N) +RESPONSE=$(curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \ -H "Content-Type: application/json" \ - -d "{ - \"model\": \"frank-default-large\", - \"messages\": [ - {\"role\": \"user\", \"content\": \"Explain the difference between supervised learning and unsupervised learning in two sentences.\"} + -d '{ + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "messages": [ + {"role": "user", "content": "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone."} ], - \"max_tokens\": ${MAX_TOKENS} - }") + "max_tokens": 256 + }') +END=$(date +%s%N) +ELAPSED_MS=$(( (END - START) / 1000000 )) +PROMPT_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['usage']['prompt_tokens'])") +COMPLETION_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['usage']['completion_tokens'])") +SINGLE_SPEED=$(python3 -c "print(f'{${PROMPT_TOKENS}/${ELAPSED_MS}*1000:.2f}')") +echo "Single stream: ${SINGLE_SPEED} tok/s (prompt)" -echo "Response received." -echo "" +# Parallel streams test +echo "Parallel streams test (${NUM_STREAMS} streams)..." +START=$(date +%s%N) +TMPDIR=$(mktemp -d) +for i in $(seq 1 $NUM_STREAMS); do + curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "messages": [ + {"role": "user", "content": "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone."} + ], + "max_tokens": 256 + }' > "$TMPDIR/$i.json" & +done +wait +END=$(date +%s%N) +ELAPSED_MS=$(( (END - START) / 1000000 )) -# Parse timing from response -echo "$RESPONSE" | python3 -c " -import sys, json +# Aggregate results from all streams +TOTAL_PROMPT=0 +TOTAL_COMPLETION=0 +for i in $(seq 1 $NUM_STREAMS); do + PT=$(python3 -c "import json; print(json.load(open('$TMPDIR/$i.json'))['usage']['prompt_tokens'])") + CT=$(python3 -c "import json; print(json.load(open('$TMPDIR/$i.json'))['usage']['completion_tokens'])") + TOTAL_PROMPT=$((TOTAL_PROMPT + PT)) + TOTAL_COMPLETION=$((TOTAL_COMPLETION + CT)) +done +rm -rf "$TMPDIR" -data = json.load(sys.stdin) - -# Check for errors -if 'error' in data: - print(f'ERROR: {data[\"error\"]}') - sys.exit(1) - -# Extract usage info -usage = data.get('usage', {}) -prompt_tokens = usage.get('prompt_tokens', 0) -completion_tokens = usage.get('completion_tokens', 0) - -print(f'Prompt tokens: {prompt_tokens}') -print(f'Completion tokens: {completion_tokens}') - -# Extract timing details -prompt_details = usage.get('prompt_details', {}) -if prompt_details: - prompt_time = prompt_details.get('total_time', 0) - speed = prompt_tokens / prompt_time if prompt_time > 0 else 0 - print(f'Prompt time: {prompt_time:.4f}s') - print(f'Prompt speed: {speed:.1f} tok/s') - - if speed < float('${MIN_SPEED}'): - print(f'FAIL: Speed {speed:.1f} tok/s < {\"${MIN_SPEED}\"} tok/s') - sys.exit(1) - else: - print(f'PASS: Speed {speed:.1f} tok/s >= {\"${MIN_SPEED}\"} tok/s') -else: - # Fallback: use total_time from usage - total_time = usage.get('total_time', 0) - if total_time > 0 and prompt_tokens > 0: - speed = prompt_tokens / total_time - print(f'Total time: {total_time:.4f}s') - print(f'Est. prompt speed: {speed:.1f} tok/s') - print('PASS (response received)') - else: - print('No timing details available') - print('PASS (response received)') -" +PARALLEL_SPEED=$(python3 -c "print(f'{${TOTAL_PROMPT}/${ELAPSED_MS}*1000:.2f}')") echo "" -echo "=== Validation complete ===" +echo "=== Results ===" +echo "Single stream prompt speed: ${SINGLE_SPEED} tok/s" +echo "Parallel streams prompt speed: ${PARALLEL_SPEED} tok/s (${NUM_STREAMS} streams, ${TOTAL_PROMPT} total prompt tokens)" +echo "Parallel completion speed: $(python3 -c "print(f'{${TOTAL_COMPLETION}/${ELAPSED_MS}*1000:.2f}')") tok/s" +echo "Total parallel time: ${ELAPSED_MS}ms" +echo "" + +# Fail if parallel speed < 5 tok/s +if python3 -c "import sys; sys.exit(0 if ${TOTAL_PROMPT}/${ELAPSED_MS}*1000 >= 5 else 1)"; then + echo "PASS: Validation successful" + echo "$PARALLEL_SPEED" > /tmp/vllm_prompt_speed.txt +else + echo "FAIL: Parallel prompt speed ${PARALLEL_SPEED} tok/s is below 5 tok/s threshold" + exit 1 +fi