135 lines
3.7 KiB
Bash
Executable File
135 lines
3.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Test suite for validate-vllm.sh
|
|
# Usage: bash tests/test_validator.sh [TARGET]
|
|
|
|
TARGET="${1:-dankserver}"
|
|
SCRIPT="$(dirname "$0")/../scripts/validate-vllm.sh"
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
pass() {
|
|
echo "✓ PASS: $1"
|
|
((PASS++))
|
|
}
|
|
|
|
fail() {
|
|
echo "✗ FAIL: $1"
|
|
((FAIL++))
|
|
}
|
|
|
|
# Test 1: Script exists and is executable
|
|
if [ -x "$SCRIPT" ]; then
|
|
pass "Script exists and is executable"
|
|
else
|
|
fail "Script missing or not executable"
|
|
exit 1
|
|
fi
|
|
|
|
# Test 2: Script accepts -t flag
|
|
if bash "$SCRIPT" -t dankserver --help 2>&1 | grep -q "frank-default-small"; then
|
|
pass "Script accepts -t dankserver flag"
|
|
else
|
|
fail "Script does not accept -t dankserver flag"
|
|
fi
|
|
|
|
# Test 3: Script accepts -H flag
|
|
if bash "$SCRIPT" -H 127.0.0.1 --help 2>&1 | grep -q "127.0.0.1"; then
|
|
pass "Script accepts -H host flag"
|
|
else
|
|
fail "Script does not accept -H host flag"
|
|
fi
|
|
|
|
# Test 4: Script accepts -P flag
|
|
if bash "$SCRIPT" -P 34401 --help 2>&1 | grep -q "34401"; then
|
|
pass "Script accepts -P port flag"
|
|
else
|
|
fail "Script does not accept -P port flag"
|
|
fi
|
|
|
|
# Test 5: Script accepts -M flag
|
|
if bash "$SCRIPT" -M frank-default-small --help 2>&1 | grep -q "frank-default-small"; then
|
|
pass "Script accepts -M model flag"
|
|
else
|
|
fail "Script does not accept -M model flag"
|
|
fi
|
|
|
|
# Test 6: Script accepts -n flag
|
|
if bash "$SCRIPT" -n 4 --help 2>&1 | grep -q "4"; then
|
|
pass "Script accepts -n streams flag"
|
|
else
|
|
fail "Script does not accept -n streams flag"
|
|
fi
|
|
|
|
# Test 7: Health check endpoint is accessible
|
|
if curl -sf "http://${TARGET}:34401/health" > /dev/null 2>&1; then
|
|
pass "Health check endpoint is accessible"
|
|
else
|
|
fail "Health check endpoint is not accessible"
|
|
fi
|
|
|
|
# Test 8: Chat completions endpoint is accessible
|
|
if curl -sf -X POST "http://${TARGET}:34401/v1/chat/completions" \
|
|
-H "Content-Type: application/json" \
|
|
-d '{
|
|
"model": "frank-default-small",
|
|
"messages": [{"role": "user", "content": "Hello"}],
|
|
"max_tokens": 10
|
|
}' > /dev/null 2>&1; then
|
|
pass "Chat completions endpoint is accessible"
|
|
else
|
|
fail "Chat completions endpoint is not accessible"
|
|
fi
|
|
|
|
# Test 9: Response contains expected fields
|
|
RESPONSE=$(curl -sf -X POST "http://${TARGET}:34401/v1/chat/completions" \
|
|
-H "Content-Type: application/json" \
|
|
-d '{
|
|
"model": "frank-default-small",
|
|
"messages": [{"role": "user", "content": "Hello"}],
|
|
"max_tokens": 10
|
|
}')
|
|
|
|
if echo "$RESPONSE" | python3 -c "import json; d=json.load(sys.stdin); assert 'choices' in d" 2>/dev/null; then
|
|
pass "Response contains 'choices' field"
|
|
else
|
|
fail "Response missing 'choices' field"
|
|
fi
|
|
|
|
if echo "$RESPONSE" | python3 -c "import json; d=json.load(sys.stdin); assert 'usage' in d" 2>/dev/null; then
|
|
pass "Response contains 'usage' field"
|
|
else
|
|
fail "Response missing 'usage' field"
|
|
fi
|
|
|
|
if echo "$RESPONSE" | python3 -c "import json; d=json.load(sys.stdin); assert 'prompt_tokens' in d['usage']" 2>/dev/null; then
|
|
pass "Response contains 'prompt_tokens' in usage"
|
|
else
|
|
fail "Response missing 'prompt_tokens' in usage"
|
|
fi
|
|
|
|
if echo "$RESPONSE" | python3 -c "import json; d=json.load(sys.stdin); assert 'completion_tokens' in d['usage']" 2>/dev/null; then
|
|
pass "Response contains 'completion_tokens' in usage"
|
|
else
|
|
fail "Response missing 'completion_tokens' in usage"
|
|
fi
|
|
|
|
# Test 10: Script runs successfully (full validation)
|
|
if bash "$SCRIPT" -t "$TARGET" -n 1 2>&1 | grep -q "Validation complete"; then
|
|
pass "Full validation completes successfully"
|
|
else
|
|
fail "Full validation fails"
|
|
fi
|
|
|
|
# Summary
|
|
echo ""
|
|
echo "=== Test Summary ==="
|
|
echo "Passed: $PASS"
|
|
echo "Failed: $FAIL"
|
|
echo ""
|
|
|
|
if [ "$FAIL" -gt 0 ]; then
|
|
exit 1
|
|
fi
|