Update validation script to use frank-default-small model name
This commit is contained in:
Frank
+50
-33
@@ -1,10 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
# End-to-end validation test for vLLM Qwen3.6-35B-A3B-FP8 on dankserver
|
||||
# Measures prompt processing speed (tokens/second)
|
||||
# End-to-end validation test for vLLM Qwen3.6-35B-A3B-FP8
|
||||
# Tests single stream and 4 parallel streams
|
||||
set -euo pipefail
|
||||
|
||||
HOST="${1:-127.0.0.1}"
|
||||
PORT="${2:-8001}"
|
||||
NUM_STREAMS=4
|
||||
|
||||
echo "=== Validating vLLM on ${HOST}:${PORT} ==="
|
||||
|
||||
@@ -15,54 +16,70 @@ if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test chat completions with timing
|
||||
echo "Running prompt test..."
|
||||
# Single stream test
|
||||
echo "Single stream test..."
|
||||
START=$(date +%s%N)
|
||||
RESPONSE=$(curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "Qwen/Qwen3.6-35B-A3B-FP8",
|
||||
"model": "frank-default-small",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone."}
|
||||
],
|
||||
"max_tokens": 256
|
||||
}')
|
||||
END=$(date +%s%N)
|
||||
|
||||
ELAPSED_MS=$(( (END - START) / 1000000 ))
|
||||
|
||||
# Extract metrics from response
|
||||
PROMPT_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['usage']['prompt_tokens'])")
|
||||
COMPLETION_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['usage']['completion_tokens'])")
|
||||
CONTENT=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['choices'][0]['message']['content'][:100])")
|
||||
SINGLE_SPEED=$(python3 -c "print(f'{${PROMPT_TOKENS}/${ELAPSED_MS}*1000:.2f}')")
|
||||
echo "Single stream: ${SINGLE_SPEED} tok/s (prompt)"
|
||||
|
||||
# Calculate speed
|
||||
if [ "$ELAPSED_MS" -gt 0 ]; then
|
||||
PROMPT_SPEED=$(python3 -c "print(f'{${PROMPT_TOKENS}/${ELAPSED_MS}*1000:.2f}')")
|
||||
COMPLETION_SPEED=$(python3 -c "print(f'{${COMPLETION_TOKENS}/${ELAPSED_MS}*1000:.2f}')")
|
||||
else
|
||||
PROMPT_SPEED="N/A"
|
||||
COMPLETION_SPEED="N/A"
|
||||
fi
|
||||
# Parallel streams test
|
||||
echo "Parallel streams test (${NUM_STREAMS} streams)..."
|
||||
START=$(date +%s%N)
|
||||
TMPDIR=$(mktemp -d)
|
||||
for i in $(seq 1 $NUM_STREAMS); do
|
||||
curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "frank-default-small",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone."}
|
||||
],
|
||||
"max_tokens": 256
|
||||
}' > "$TMPDIR/$i.json" &
|
||||
done
|
||||
wait
|
||||
END=$(date +%s%N)
|
||||
ELAPSED_MS=$(( (END - START) / 1000000 ))
|
||||
|
||||
# Aggregate results from all streams
|
||||
TOTAL_PROMPT=0
|
||||
TOTAL_COMPLETION=0
|
||||
for i in $(seq 1 $NUM_STREAMS); do
|
||||
PT=$(python3 -c "import json; print(json.load(open('$TMPDIR/$i.json'))['usage']['prompt_tokens'])")
|
||||
CT=$(python3 -c "import json; print(json.load(open('$TMPDIR/$i.json'))['usage']['completion_tokens'])")
|
||||
TOTAL_PROMPT=$((TOTAL_PROMPT + PT))
|
||||
TOTAL_COMPLETION=$((TOTAL_COMPLETION + CT))
|
||||
done
|
||||
rm -rf "$TMPDIR"
|
||||
|
||||
PARALLEL_SPEED=$(python3 -c "print(f'{${TOTAL_PROMPT}/${ELAPSED_MS}*1000:.2f}')")
|
||||
|
||||
echo ""
|
||||
echo "=== Results ==="
|
||||
echo "Prompt tokens: ${PROMPT_TOKENS}"
|
||||
echo "Completion tokens: ${COMPLETION_TOKENS}"
|
||||
echo "Total time: ${ELAPSED_MS}ms"
|
||||
echo "Prompt speed: ${PROMPT_SPEED} tok/s"
|
||||
echo "Completion speed: ${COMPLETION_SPEED} tok/s"
|
||||
echo "Sample output: ${CONTENT}..."
|
||||
echo "Single stream prompt speed: ${SINGLE_SPEED} tok/s"
|
||||
echo "Parallel streams prompt speed: ${PARALLEL_SPEED} tok/s (${NUM_STREAMS} streams, ${TOTAL_PROMPT} total prompt tokens)"
|
||||
echo "Parallel completion speed: $(python3 -c "print(f'{${TOTAL_COMPLETION}/${ELAPSED_MS}*1000:.2f}')") tok/s"
|
||||
echo "Total parallel time: ${ELAPSED_MS}ms"
|
||||
echo ""
|
||||
|
||||
# Fail if prompt speed < 5 tok/s
|
||||
if [ "$ELAPSED_MS" -gt 0 ] && [ "$PROMPT_TOKENS" -gt 0 ]; then
|
||||
TOO_SLOW=$(python3 -c "print('yes' if ${PROMPT_TOKENS}/${ELAPSED_MS}*1000 < 5 else 'no')")
|
||||
if [ "$TOO_SLOW" = "yes" ]; then
|
||||
echo "FAIL: Prompt speed ${PROMPT_SPEED} tok/s is below 5 tok/s threshold"
|
||||
exit 1
|
||||
fi
|
||||
# Fail if parallel speed < 5 tok/s
|
||||
if python3 -c "import sys; sys.exit(0 if ${TOTAL_PROMPT}/${ELAPSED_MS}*1000 >= 5 else 1)"; then
|
||||
echo "PASS: Validation successful"
|
||||
echo "$PARALLEL_SPEED" > /tmp/vllm_prompt_speed.txt
|
||||
else
|
||||
echo "FAIL: Parallel prompt speed ${PARALLEL_SPEED} tok/s is below 5 tok/s threshold"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "PASS: Validation successful"
|
||||
echo "$PROMPT_SPEED" > /tmp/vllm_prompt_speed.txt
|
||||
|
||||
Reference in New Issue
Block a user