Add validator script with target selection and test suite
This commit is contained in:
Frank
+64
-19
@@ -1,21 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
HOST="${VLLM_HOST:-127.0.0.1}"
|
||||
PORT="${VLLM_PORT:-34401}"
|
||||
MODEL="frank-default-small"
|
||||
# Configuration
|
||||
TARGET="${VLLM_TARGET:-dankserver}"
|
||||
HOST="${VLLM_HOST:-}"
|
||||
PORT="${VLLM_PORT:-}"
|
||||
MODEL="${VLLM_MODEL:-}"
|
||||
|
||||
# Target presets
|
||||
case "$TARGET" in
|
||||
dankserver)
|
||||
HOST="${VLLM_HOST:-127.0.0.1}"
|
||||
PORT="${VLLM_PORT:-34401}"
|
||||
MODEL="${VLLM_MODEL:-frank-default-small}"
|
||||
;;
|
||||
dankclaw)
|
||||
HOST="${VLLM_HOST:-127.0.0.1}"
|
||||
PORT="${VLLM_PORT:-3090}"
|
||||
MODEL="${VLLM_MODEL:-frank-default-large}"
|
||||
;;
|
||||
*)
|
||||
echo "Unknown target: $TARGET"
|
||||
echo "Usage: $0 [-t dankserver|dankclaw] [-H host] [-P port] [-M model]"
|
||||
echo ""
|
||||
echo "Targets:"
|
||||
echo " dankserver - frank-default-small on port 34401"
|
||||
echo " dankclaw - frank-default-large on port 3090"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -t TARGET Target preset (dankserver or dankclaw)"
|
||||
echo " -H HOST Override host"
|
||||
echo " -P PORT Override port"
|
||||
echo " -M MODEL Override model name"
|
||||
echo " -n N Number of streams to test (default: 4)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Parse arguments
|
||||
while getopts "t:H:P:M:n:" opt; do
|
||||
case $opt in
|
||||
t) TARGET="$OPTARG" ;;
|
||||
H) HOST="$OPTARG" ;;
|
||||
P) PORT="$OPTARG" ;;
|
||||
M) MODEL="$OPTARG" ;;
|
||||
n) MAX_STREAMS="$OPTARG" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Default max streams if not set
|
||||
MAX_STREAMS="${MAX_STREAMS:-4}"
|
||||
|
||||
# Different prompts for each stream to avoid prefix caching effects
|
||||
PROMPT1="Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations."
|
||||
PROMPT2="Describe the architectural differences between RISC-V and x86 instruction set architectures. Include a discussion of register files, addressing modes, and instruction encoding."
|
||||
PROMPT3="Analyze the economic impact of blockchain technology on supply chain management. Discuss the key benefits of transparency, immutability, and smart contracts."
|
||||
PROMPT4="Explain the principles of evolutionary algorithms in artificial intelligence. Describe how genetic algorithms, genetic programming, and evolutionary strategies differ in their approach to problem solving."
|
||||
PROMPTS=(
|
||||
"Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations."
|
||||
"Describe the architectural differences between RISC-V and x86 instruction set architectures. Include a discussion of register files, addressing modes, and instruction encoding. Compare their performance for embedded systems and server workloads."
|
||||
"Analyze the economic impact of blockchain technology on supply chain management. Discuss the key benefits of transparency, immutability, and smart contracts. Compare implementation challenges of Bitcoin, Ethereum, and Hyperledger for enterprise use cases."
|
||||
"Explain the principles of evolutionary algorithms in artificial intelligence. Describe how genetic algorithms, genetic programming, and evolutionary strategies differ in their approach to problem solving. Compare their performance on optimization problems and neural network architecture search."
|
||||
)
|
||||
|
||||
PROMPTS=("$PROMPT1" "$PROMPT2" "$PROMPT3" "$PROMPT4")
|
||||
echo "=== Validating vLLM on ${HOST}:${PORT} (${TARGET}) ==="
|
||||
|
||||
echo "=== Validating vLLM on ${HOST}:${PORT} ==="
|
||||
|
||||
# Test health
|
||||
# Test health endpoint
|
||||
echo "Health check..."
|
||||
if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
|
||||
echo "FAIL: Health check failed"
|
||||
@@ -23,18 +69,17 @@ if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
|
||||
fi
|
||||
echo "OK"
|
||||
|
||||
# Test N streams sequentially (separate test for each N)
|
||||
# Each test uses different prompts to avoid caching
|
||||
for N in 1 2 3 4; do
|
||||
# Test each stream count separately
|
||||
for N in $(seq 1 "$MAX_STREAMS"); do
|
||||
echo ""
|
||||
echo "--- ${N} stream(s) ---"
|
||||
|
||||
TMPDIR=$(mktemp -d)
|
||||
START=$(date +%s%N)
|
||||
|
||||
for i in $(seq 1 $N); do
|
||||
# Use a different prompt for each stream to avoid caching
|
||||
PROMPT_IDX=$(( (i - 1) % 4 ))
|
||||
# Launch N concurrent requests with different prompts
|
||||
for i in $(seq 1 "$N"); do
|
||||
PROMPT_IDX=$(( (i - 1) % ${#PROMPTS[@]} ))
|
||||
PROMPT="${PROMPTS[$PROMPT_IDX]}"
|
||||
|
||||
curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
||||
@@ -50,9 +95,9 @@ for N in 1 2 3 4; do
|
||||
END=$(date +%s%N)
|
||||
ELAPSED_MS=$(( (END - START) / 1000000 ))
|
||||
|
||||
# Check all responses valid
|
||||
# Validate all responses
|
||||
ALL_OK=true
|
||||
for i in $(seq 1 $N); do
|
||||
for i in $(seq 1 "$N"); do
|
||||
if ! python3 -c "import json; d=json.load(open('$TMPDIR/$i.json')); assert len(d['choices']) > 0" 2>/dev/null; then
|
||||
echo " FAIL: Response $i invalid"
|
||||
ALL_OK=false
|
||||
|
||||
Reference in New Issue
Block a user