Update validation script to test parallel streams (4 concurrent requests)
This commit is contained in:
Frank
+72
-80
@@ -1,93 +1,85 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# End-to-end validation test for vLLM Qwen3.6-35B-A3B-FP8 on dankserver
|
# End-to-end validation test for vLLM Qwen3.6-35B-A3B-FP8
|
||||||
# Sends a chat completion request and measures prompt processing speed
|
# Tests single stream and 4 parallel streams
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
PORT="${PORT:-8000}"
|
HOST="${1:-127.0.0.1}"
|
||||||
HOST="${HOST:-127.0.0.1}"
|
PORT="${2:-8001}"
|
||||||
URL="http://${HOST}:${PORT}"
|
NUM_STREAMS=4
|
||||||
MIN_SPEED="${MIN_SPEED:-5.0}"
|
|
||||||
MAX_TOKENS="${MAX_TOKENS:-256}"
|
|
||||||
|
|
||||||
echo "=== vLLM Validation Test ==="
|
echo "=== Validating vLLM on ${HOST}:${PORT} ==="
|
||||||
echo "Target: ${URL}"
|
|
||||||
echo "Min prompt speed: ${MIN_SPEED} tok/s"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# Wait for server to be ready
|
# Test health endpoint
|
||||||
echo "Waiting for server..."
|
echo "Health check..."
|
||||||
for i in $(seq 1 300); do
|
if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
|
||||||
if curl -sf "${URL}/health" > /dev/null 2>&1; then
|
echo "FAIL: Health check failed"
|
||||||
echo "Server ready after $((i * 2))s"
|
exit 1
|
||||||
break
|
fi
|
||||||
fi
|
|
||||||
if [ "$i" -eq 300 ]; then
|
|
||||||
echo "FAIL: Server did not become ready in 600s"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
sleep 2
|
|
||||||
done
|
|
||||||
|
|
||||||
# Send chat completion and measure prompt processing speed
|
# Single stream test
|
||||||
echo ""
|
echo "Single stream test..."
|
||||||
echo "--- Sending validation prompt ---"
|
START=$(date +%s%N)
|
||||||
RESPONSE=$(curl -s -X POST "${URL}/v1/chat/completions" \
|
RESPONSE=$(curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
||||||
-H "Content-Type: application/json" \
|
-H "Content-Type: application/json" \
|
||||||
-d "{
|
-d '{
|
||||||
\"model\": \"frank-default-large\",
|
"model": "Qwen/Qwen3.6-35B-A3B-FP8",
|
||||||
\"messages\": [
|
"messages": [
|
||||||
{\"role\": \"user\", \"content\": \"Explain the difference between supervised learning and unsupervised learning in two sentences.\"}
|
{"role": "user", "content": "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone."}
|
||||||
],
|
],
|
||||||
\"max_tokens\": ${MAX_TOKENS}
|
"max_tokens": 256
|
||||||
}")
|
}')
|
||||||
|
END=$(date +%s%N)
|
||||||
|
ELAPSED_MS=$(( (END - START) / 1000000 ))
|
||||||
|
PROMPT_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['usage']['prompt_tokens'])")
|
||||||
|
COMPLETION_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['usage']['completion_tokens'])")
|
||||||
|
SINGLE_SPEED=$(python3 -c "print(f'{${PROMPT_TOKENS}/${ELAPSED_MS}*1000:.2f}')")
|
||||||
|
echo "Single stream: ${SINGLE_SPEED} tok/s (prompt)"
|
||||||
|
|
||||||
echo "Response received."
|
# Parallel streams test
|
||||||
echo ""
|
echo "Parallel streams test (${NUM_STREAMS} streams)..."
|
||||||
|
START=$(date +%s%N)
|
||||||
|
TMPDIR=$(mktemp -d)
|
||||||
|
for i in $(seq 1 $NUM_STREAMS); do
|
||||||
|
curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{
|
||||||
|
"model": "Qwen/Qwen3.6-35B-A3B-FP8",
|
||||||
|
"messages": [
|
||||||
|
{"role": "user", "content": "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone."}
|
||||||
|
],
|
||||||
|
"max_tokens": 256
|
||||||
|
}' > "$TMPDIR/$i.json" &
|
||||||
|
done
|
||||||
|
wait
|
||||||
|
END=$(date +%s%N)
|
||||||
|
ELAPSED_MS=$(( (END - START) / 1000000 ))
|
||||||
|
|
||||||
# Parse timing from response
|
# Aggregate results from all streams
|
||||||
echo "$RESPONSE" | python3 -c "
|
TOTAL_PROMPT=0
|
||||||
import sys, json
|
TOTAL_COMPLETION=0
|
||||||
|
for i in $(seq 1 $NUM_STREAMS); do
|
||||||
|
PT=$(python3 -c "import json; print(json.load(open('$TMPDIR/$i.json'))['usage']['prompt_tokens'])")
|
||||||
|
CT=$(python3 -c "import json; print(json.load(open('$TMPDIR/$i.json'))['usage']['completion_tokens'])")
|
||||||
|
TOTAL_PROMPT=$((TOTAL_PROMPT + PT))
|
||||||
|
TOTAL_COMPLETION=$((TOTAL_COMPLETION + CT))
|
||||||
|
done
|
||||||
|
rm -rf "$TMPDIR"
|
||||||
|
|
||||||
data = json.load(sys.stdin)
|
PARALLEL_SPEED=$(python3 -c "print(f'{${TOTAL_PROMPT}/${ELAPSED_MS}*1000:.2f}')")
|
||||||
|
|
||||||
# Check for errors
|
|
||||||
if 'error' in data:
|
|
||||||
print(f'ERROR: {data[\"error\"]}')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# Extract usage info
|
|
||||||
usage = data.get('usage', {})
|
|
||||||
prompt_tokens = usage.get('prompt_tokens', 0)
|
|
||||||
completion_tokens = usage.get('completion_tokens', 0)
|
|
||||||
|
|
||||||
print(f'Prompt tokens: {prompt_tokens}')
|
|
||||||
print(f'Completion tokens: {completion_tokens}')
|
|
||||||
|
|
||||||
# Extract timing details
|
|
||||||
prompt_details = usage.get('prompt_details', {})
|
|
||||||
if prompt_details:
|
|
||||||
prompt_time = prompt_details.get('total_time', 0)
|
|
||||||
speed = prompt_tokens / prompt_time if prompt_time > 0 else 0
|
|
||||||
print(f'Prompt time: {prompt_time:.4f}s')
|
|
||||||
print(f'Prompt speed: {speed:.1f} tok/s')
|
|
||||||
|
|
||||||
if speed < float('${MIN_SPEED}'):
|
|
||||||
print(f'FAIL: Speed {speed:.1f} tok/s < {\"${MIN_SPEED}\"} tok/s')
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print(f'PASS: Speed {speed:.1f} tok/s >= {\"${MIN_SPEED}\"} tok/s')
|
|
||||||
else:
|
|
||||||
# Fallback: use total_time from usage
|
|
||||||
total_time = usage.get('total_time', 0)
|
|
||||||
if total_time > 0 and prompt_tokens > 0:
|
|
||||||
speed = prompt_tokens / total_time
|
|
||||||
print(f'Total time: {total_time:.4f}s')
|
|
||||||
print(f'Est. prompt speed: {speed:.1f} tok/s')
|
|
||||||
print('PASS (response received)')
|
|
||||||
else:
|
|
||||||
print('No timing details available')
|
|
||||||
print('PASS (response received)')
|
|
||||||
"
|
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Validation complete ==="
|
echo "=== Results ==="
|
||||||
|
echo "Single stream prompt speed: ${SINGLE_SPEED} tok/s"
|
||||||
|
echo "Parallel streams prompt speed: ${PARALLEL_SPEED} tok/s (${NUM_STREAMS} streams, ${TOTAL_PROMPT} total prompt tokens)"
|
||||||
|
echo "Parallel completion speed: $(python3 -c "print(f'{${TOTAL_COMPLETION}/${ELAPSED_MS}*1000:.2f}')") tok/s"
|
||||||
|
echo "Total parallel time: ${ELAPSED_MS}ms"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Fail if parallel speed < 5 tok/s
|
||||||
|
if python3 -c "import sys; sys.exit(0 if ${TOTAL_PROMPT}/${ELAPSED_MS}*1000 >= 5 else 1)"; then
|
||||||
|
echo "PASS: Validation successful"
|
||||||
|
echo "$PARALLEL_SPEED" > /tmp/vllm_prompt_speed.txt
|
||||||
|
else
|
||||||
|
echo "FAIL: Parallel prompt speed ${PARALLEL_SPEED} tok/s is below 5 tok/s threshold"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|||||||
Reference in New Issue
Block a user