Add validator script with target selection and test suite

scripts/validate-vllm.sh

@@ -1,21 +1,67 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# Configuration
+TARGET="${VLLM_TARGET:-dankserver}"
+HOST="${VLLM_HOST:-}"
+PORT="${VLLM_PORT:-}"
+MODEL="${VLLM_MODEL:-}"
+
+# Target presets
+case "$TARGET" in
+    dankserver)
         HOST="${VLLM_HOST:-127.0.0.1}"
         PORT="${VLLM_PORT:-34401}"
-MODEL="frank-default-small"
+        MODEL="${VLLM_MODEL:-frank-default-small}"
+        ;;
+    dankclaw)
+        HOST="${VLLM_HOST:-127.0.0.1}"
+        PORT="${VLLM_PORT:-3090}"
+        MODEL="${VLLM_MODEL:-frank-default-large}"
+        ;;
+    *)
+        echo "Unknown target: $TARGET"
+        echo "Usage: $0 [-t dankserver|dankclaw] [-H host] [-P port] [-M model]"
+        echo ""
+        echo "Targets:"
+        echo "  dankserver  - frank-default-small on port 34401"
+        echo "  dankclaw    - frank-default-large on port 3090"
+        echo ""
+        echo "Options:"
+        echo "  -t TARGET   Target preset (dankserver or dankclaw)"
+        echo "  -H HOST     Override host"
+        echo "  -P PORT     Override port"
+        echo "  -M MODEL    Override model name"
+        echo "  -n N        Number of streams to test (default: 4)"
+        exit 1
+        ;;
+esac
+
+# Parse arguments
+while getopts "t:H:P:M:n:" opt; do
+    case $opt in
+        t) TARGET="$OPTARG" ;;
+        H) HOST="$OPTARG" ;;
+        P) PORT="$OPTARG" ;;
+        M) MODEL="$OPTARG" ;;
+        n) MAX_STREAMS="$OPTARG" ;;
+    esac
+done
+
+# Default max streams if not set
+MAX_STREAMS="${MAX_STREAMS:-4}"
 
 # Different prompts for each stream to avoid prefix caching effects
-PROMPT1="Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations."
+PROMPTS=(
-PROMPT2="Describe the architectural differences between RISC-V and x86 instruction set architectures. Include a discussion of register files, addressing modes, and instruction encoding."
+    "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations."
-PROMPT3="Analyze the economic impact of blockchain technology on supply chain management. Discuss the key benefits of transparency, immutability, and smart contracts."
+    "Describe the architectural differences between RISC-V and x86 instruction set architectures. Include a discussion of register files, addressing modes, and instruction encoding. Compare their performance for embedded systems and server workloads."
-PROMPT4="Explain the principles of evolutionary algorithms in artificial intelligence. Describe how genetic algorithms, genetic programming, and evolutionary strategies differ in their approach to problem solving."
+    "Analyze the economic impact of blockchain technology on supply chain management. Discuss the key benefits of transparency, immutability, and smart contracts. Compare implementation challenges of Bitcoin, Ethereum, and Hyperledger for enterprise use cases."
+    "Explain the principles of evolutionary algorithms in artificial intelligence. Describe how genetic algorithms, genetic programming, and evolutionary strategies differ in their approach to problem solving. Compare their performance on optimization problems and neural network architecture search."
+)
 
-PROMPTS=("$PROMPT1" "$PROMPT2" "$PROMPT3" "$PROMPT4")
+echo "=== Validating vLLM on ${HOST}:${PORT} (${TARGET}) ==="
 
-echo "=== Validating vLLM on ${HOST}:${PORT} ==="
+# Test health endpoint
-
-# Test health
 echo "Health check..."
 if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
     echo "FAIL: Health check failed"
@@ -23,18 +69,17 @@ if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
 fi
 echo "OK"
 
-# Test N streams sequentially (separate test for each N)
+# Test each stream count separately
-# Each test uses different prompts to avoid caching
+for N in $(seq 1 "$MAX_STREAMS"); do
-for N in 1 2 3 4; do
     echo ""
     echo "--- ${N} stream(s) ---"
     
     TMPDIR=$(mktemp -d)
     START=$(date +%s%N)
     
-    for i in $(seq 1 $N); do
+    # Launch N concurrent requests with different prompts
-        # Use a different prompt for each stream to avoid caching
+    for i in $(seq 1 "$N"); do
-        PROMPT_IDX=$(( (i - 1) % 4 ))
+        PROMPT_IDX=$(( (i - 1) % ${#PROMPTS[@]} ))
         PROMPT="${PROMPTS[$PROMPT_IDX]}"
         
         curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
@@ -50,9 +95,9 @@ for N in 1 2 3 4; do
     END=$(date +%s%N)
     ELAPSED_MS=$(( (END - START) / 1000000 ))
     
-    # Check all responses valid
+    # Validate all responses
     ALL_OK=true
-    for i in $(seq 1 $N); do
+    for i in $(seq 1 "$N"); do
         if ! python3 -c "import json; d=json.load(open('$TMPDIR/$i.json')); assert len(d['choices']) > 0" 2>/dev/null; then
             echo "  FAIL: Response $i invalid"
             ALL_OK=false

tests/test_validator.sh

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Test suite for validate-vllm.sh
+# Usage: bash tests/test_validator.sh [TARGET]
+
+TARGET="${1:-dankserver}"
+SCRIPT="$(dirname "$0")/../scripts/validate-vllm.sh"
+PASS=0
+FAIL=0
+
+pass() {
+    echo "✓ PASS: $1"
+    ((PASS++))
+}
+
+fail() {
+    echo "✗ FAIL: $1"
+    ((FAIL++))
+}
+
+# Test 1: Script exists and is executable
+if [ -x "$SCRIPT" ]; then
+    pass "Script exists and is executable"
+else
+    fail "Script missing or not executable"
+    exit 1
+fi
+
+# Test 2: Script accepts -t flag
+if bash "$SCRIPT" -t dankserver --help 2>&1 | grep -q "frank-default-small"; then
+    pass "Script accepts -t dankserver flag"
+else
+    fail "Script does not accept -t dankserver flag"
+fi
+
+# Test 3: Script accepts -H flag
+if bash "$SCRIPT" -H 127.0.0.1 --help 2>&1 | grep -q "127.0.0.1"; then
+    pass "Script accepts -H host flag"
+else
+    fail "Script does not accept -H host flag"
+fi
+
+# Test 4: Script accepts -P flag
+if bash "$SCRIPT" -P 34401 --help 2>&1 | grep -q "34401"; then
+    pass "Script accepts -P port flag"
+else
+    fail "Script does not accept -P port flag"
+fi
+
+# Test 5: Script accepts -M flag
+if bash "$SCRIPT" -M frank-default-small --help 2>&1 | grep -q "frank-default-small"; then
+    pass "Script accepts -M model flag"
+else
+    fail "Script does not accept -M model flag"
+fi
+
+# Test 6: Script accepts -n flag
+if bash "$SCRIPT" -n 4 --help 2>&1 | grep -q "4"; then
+    pass "Script accepts -n streams flag"
+else
+    fail "Script does not accept -n streams flag"
+fi
+
+# Test 7: Health check endpoint is accessible
+if curl -sf "http://${TARGET}:34401/health" > /dev/null 2>&1; then
+    pass "Health check endpoint is accessible"
+else
+    fail "Health check endpoint is not accessible"
+fi
+
+# Test 8: Chat completions endpoint is accessible
+if curl -sf -X POST "http://${TARGET}:34401/v1/chat/completions" \
+    -H "Content-Type: application/json" \
+    -d '{
+        "model": "frank-default-small",
+        "messages": [{"role": "user", "content": "Hello"}],
+        "max_tokens": 10
+    }' > /dev/null 2>&1; then
+    pass "Chat completions endpoint is accessible"
+else
+    fail "Chat completions endpoint is not accessible"
+fi
+
+# Test 9: Response contains expected fields
+RESPONSE=$(curl -sf -X POST "http://${TARGET}:34401/v1/chat/completions" \
+    -H "Content-Type: application/json" \
+    -d '{
+        "model": "frank-default-small",
+        "messages": [{"role": "user", "content": "Hello"}],
+        "max_tokens": 10
+    }')
+
+if echo "$RESPONSE" | python3 -c "import json; d=json.load(sys.stdin); assert 'choices' in d" 2>/dev/null; then
+    pass "Response contains 'choices' field"
+else
+    fail "Response missing 'choices' field"
+fi
+
+if echo "$RESPONSE" | python3 -c "import json; d=json.load(sys.stdin); assert 'usage' in d" 2>/dev/null; then
+    pass "Response contains 'usage' field"
+else
+    fail "Response missing 'usage' field"
+fi
+
+if echo "$RESPONSE" | python3 -c "import json; d=json.load(sys.stdin); assert 'prompt_tokens' in d['usage']" 2>/dev/null; then
+    pass "Response contains 'prompt_tokens' in usage"
+else
+    fail "Response missing 'prompt_tokens' in usage"
+fi
+
+if echo "$RESPONSE" | python3 -c "import json; d=json.load(sys.stdin); assert 'completion_tokens' in d['usage']" 2>/dev/null; then
+    pass "Response contains 'completion_tokens' in usage"
+else
+    fail "Response missing 'completion_tokens' in usage"
+fi
+
+# Test 10: Script runs successfully (full validation)
+if bash "$SCRIPT" -t "$TARGET" -n 1 2>&1 | grep -q "Validation complete"; then
+    pass "Full validation completes successfully"
+else
+    fail "Full validation fails"
+fi
+
+# Summary
+echo ""
+echo "=== Test Summary ==="
+echo "Passed: $PASS"
+echo "Failed: $FAIL"
+echo ""
+
+if [ "$FAIL" -gt 0 ]; then
+    exit 1
+fi