Frank
d9af7f1d19
- Move all previous scripts to old/scripts/ (retired PP/NPU-era) - frank-default-large: llama.cpp FastMTP on dankserver (HauhauCS Qwen3.8-27B IQ4_XS + FastMTP 32K draft + ngram-mod, K q8_0/V f16, 128K ctx, batch 2048/ubatch 1024, port 7770) - frank-default-small: vLLM ROCm on dankclaw (Avesed Qwen3.6-35B-A3B INT8 W8A8, full 262144 ctx, no KV quant, 4 seqs, batch 16384, ngram spec, port 52625) - retarget deploy.sh (dankserver/dankclaw, inline sudo, no helpers) - docs: deployment + index + AGENTS.md + README for new topology, superseded banners on retired docs, add frank-redo-2026-08-17 runbook
89 lines
3.7 KiB
Bash
Executable File
89 lines
3.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
HOST="127.0.0.1"
|
|
PORT=8001
|
|
MODEL="frank-default-small"
|
|
|
|
# Different prompts for each test to avoid prefix caching
|
|
PROMPT1="Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations."
|
|
PROMPT2="Describe the architectural differences between RISC-V and x86 instruction set architectures. Include a discussion of register files, addressing modes, and instruction encoding. Compare their performance for embedded systems and server workloads."
|
|
PROMPT3="Analyze the economic impact of blockchain technology on supply chain management. Discuss the key benefits of transparency, immutability, and smart contracts. Compare implementation challenges of Bitcoin, Ethereum, and Hyperledger for enterprise use cases."
|
|
PROMPT4="Explain the principles of evolutionary algorithms in artificial intelligence. Describe how genetic algorithms, genetic programming, and evolutionary strategies differ in their approach to problem solving. Compare their performance on optimization problems and neural network architecture search."
|
|
|
|
echo "=== Validating vLLM on ${HOST}:${PORT} ==="
|
|
|
|
# Test health endpoint
|
|
echo "Health check..."
|
|
if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
|
|
echo "FAIL: Health check failed"
|
|
exit 1
|
|
fi
|
|
|
|
# Test N streams sequentially (1, 2, 3, 4)
|
|
for NUM_STREAMS in 1 2 3 4; do
|
|
echo ""
|
|
echo "--- Testing ${NUM_STREAMS} stream(s) ---"
|
|
|
|
TMPDIR=$(mktemp -d)
|
|
START=$(date +%s%N)
|
|
|
|
for i in $(seq 1 $NUM_STREAMS); do
|
|
case $(( (i - 1) % 4 + 1 )) in
|
|
1) PROMPT="${PROMPT1}" ;;
|
|
2) PROMPT="${PROMPT2}" ;;
|
|
3) PROMPT="${PROMPT3}" ;;
|
|
4) PROMPT="${PROMPT4}" ;;
|
|
esac
|
|
|
|
curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{
|
|
\"model\": \"${MODEL}\",
|
|
\"messages\": [
|
|
{\"role\": \"user\", \"content\": \"${PROMPT}\"}
|
|
],
|
|
\"max_tokens\": 256
|
|
}" > "$TMPDIR/$i.json" &
|
|
done
|
|
wait
|
|
|
|
END=$(date +%s%N)
|
|
ELAPSED_MS=$(( (END - START) / 1000000 ))
|
|
|
|
# Calculate speed from the first stream's response
|
|
PROMPT_TOKENS=$(python3 -c "import json; print(json.load(open('$TMPDIR/1.json'))['usage']['prompt_tokens'])")
|
|
COMPLETION_TOKENS=$(python3 -c "import json; print(json.load(open('$TMPDIR/1.json'))['usage']['completion_tokens'])")
|
|
|
|
# Calculate speeds
|
|
PROMPT_SPEED=$(python3 -c "print(f'{${PROMPT_TOKENS}/${ELAPSED_MS}*1000:.2f}')")
|
|
COMPLETION_SPEED=$(python3 -c "print(f'{${COMPLETION_TOKENS}/${ELAPSED_MS}*1000:.2f}')")
|
|
|
|
echo " Elapsed: ${ELAPSED_MS}ms"
|
|
echo " Prompt tokens: ${PROMPT_TOKENS}"
|
|
echo " Prompt speed: ${PROMPT_SPEED} tok/s"
|
|
echo " Completion tokens: ${COMPLETION_TOKENS}"
|
|
echo " Completion speed: ${COMPLETION_SPEED} tok/s"
|
|
|
|
# Save results
|
|
echo "${NUM_STREAMS},${PROMPT_SPEED},${COMPLETION_SPEED},${ELAPSED_MS}" > "/tmp/vllm-${NUM_STREAMS}.csv"
|
|
|
|
# Clean up temp files
|
|
rm -rf "$TMPDIR"
|
|
done
|
|
|
|
echo ""
|
|
echo "=== Results Summary ==="
|
|
for n in 1 2 3 4; do
|
|
if [ -f "/tmp/vllm-${n}.csv" ]; then
|
|
IFS=',' read -r streams p_speed c_speed elapsed < "/tmp/vllm-${n}.csv"
|
|
echo " ${n} stream(s): ${p_speed} tok/s (prompt), ${c_speed} tok/s (completion) - ${elapsed}ms"
|
|
fi
|
|
done
|
|
|
|
# Check if single stream speed is >= 5 tok/s
|
|
SINGLE_SPEED=$(head -1 /tmp/vllm-1.csv | cut -d',' -f2)
|
|
python3 -c "import sys; sys.exit(0 if float('${SINGLE_SPEED}') >= 5 else 1)"
|
|
echo ""
|
|
echo "PASS: Validation successful"
|