Frank
9227e1a8b9
- vllm-awq-qwen36-dflash-ampere.sh: AWQ-4bit Qwen3.6-35B-A3B with DFlash speculative decoding, optimized for Ampere hardware - Model: cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit (AWQ Marlin) - Draft: z-lab/Qwen3.6-35B-A3B-DFlash - --max-model-len 32768 - --max-num-batched-tokens 1536 - --cpu-offload-gb 40 (experts) - --reasoning-parser qwen3 - --tool-call-parser qwen3_xml - Port: 34401 - Note: prefix caching disabled (Mamba cache align conflict) - scripts/validate-vllm.sh: Per-stream latency tests - Separate tests for 1, 2, 3, 4 streams - Different prompts per stream to avoid caching effects - Reports elapsed ms per stream count
63 lines
2.2 KiB
Bash
Executable File
63 lines
2.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
HOST="${VLLM_HOST:-127.0.0.1}"
|
|
PORT="${VLLM_PORT:-34401}"
|
|
MODEL="frank-default-small"
|
|
|
|
# Different prompts per stream to avoid prefix caching effects
|
|
PROMPTS=(
|
|
"Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations."
|
|
"Describe the architectural differences between RISC-V and x86 instruction set architectures. Include a discussion of register files, addressing modes, and instruction encoding."
|
|
"Analyze the economic impact of blockchain technology on supply chain management. Discuss the key benefits of transparency, immutability, and smart contracts."
|
|
"Explain the principles of evolutionary algorithms in artificial intelligence. Describe how genetic algorithms, genetic programming, and evolutionary strategies differ."
|
|
)
|
|
|
|
echo "=== Validating vLLM on ${HOST}:${PORT} ==="
|
|
|
|
# Test health
|
|
echo "Health check..."
|
|
if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
|
|
echo "FAIL: Health check failed"
|
|
exit 1
|
|
fi
|
|
echo "OK"
|
|
|
|
# Test N streams (separate test for each N)
|
|
for N in 1 2 3 4; do
|
|
echo ""
|
|
echo "--- ${N} stream(s) ---"
|
|
|
|
TMPDIR=$(mktemp -d)
|
|
START=$(date +%s%N)
|
|
|
|
for i in $(seq 1 $N); do
|
|
curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{
|
|
\"model\": \"${MODEL}\",
|
|
\"messages\": [{\"role\": \"user\", \"content\": \"${PROMPTS[$((i-1))]}\"}],
|
|
\"max_tokens\": 256
|
|
}" > "$TMPDIR/$i.json" &
|
|
done
|
|
wait
|
|
|
|
END=$(date +%s%N)
|
|
ELAPSED_MS=$(( (END - START) / 1000000 ))
|
|
|
|
# Check all responses valid
|
|
ALL_OK=true
|
|
for i in $(seq 1 $N); do
|
|
if ! python3 -c "import json; d=json.load(open('$TMPDIR/$i.json')); assert len(d['choices']) > 0" 2>/dev/null; then
|
|
echo " FAIL: Response $i invalid"
|
|
ALL_OK=false
|
|
fi
|
|
done
|
|
|
|
rm -rf "$TMPDIR"
|
|
|
|
if $ALL_OK; then
|
|
echo " OK - ${ELAPSED_MS}ms total (${N} streams)"
|
|
fi
|
|
done
|