Frank
764a65080f
- Add second parallel prompt (Byzantine Empire history vs ML) - Both hit 1024 tokens, combined 2048 in 111.6s = 18.34 tok/s - Parallel throughput matches single-stream baseline
98 lines
3.8 KiB
Bash
Executable File
98 lines
3.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Validation test for llama.cpp DFlash+MTP server
|
|
# Sends a simple prompt and measures generation speed (tokens/sec)
|
|
set -euo pipefail
|
|
|
|
PORT="${LLAMA_PORT:-8080}"
|
|
HOST="${LLAMA_HOST:-127.0.0.1}"
|
|
MAX_TOKENS="${MAX_TOKENS:-1024}"
|
|
|
|
echo "=== llama.cpp Validation Test ==="
|
|
echo "Target: http://${HOST}:${PORT}"
|
|
echo "Max tokens: ${MAX_TOKENS}"
|
|
echo ""
|
|
|
|
# Wait for server to be ready
|
|
echo "Waiting for server..."
|
|
for i in $(seq 1 300); do
|
|
if curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
|
|
echo "Server ready after $((i * 2))s"
|
|
break
|
|
fi
|
|
if [ "$i" -eq 300 ]; then
|
|
echo "FAIL: Server did not become ready in 600s"
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
# Send two parallel requests and capture timing
|
|
echo ""
|
|
echo "Sending 2 parallel validation prompts..."
|
|
START_TIME=$(date +%s%N)
|
|
|
|
RESPONSE1=$(curl -s \
|
|
-X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{
|
|
\"messages\": [
|
|
{\"role\": \"user\", \"content\": \"Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone.\"}
|
|
],
|
|
\"max_tokens\": ${MAX_TOKENS},
|
|
\"temperature\": 0.7,
|
|
\"top_p\": 0.95
|
|
}" &)
|
|
|
|
RESPONSE2=$(curl -s \
|
|
-X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{
|
|
\"messages\": [
|
|
{\"role\": \"user\", \"content\": \"Write a detailed analysis of the geopolitical consequences of the fall of the Byzantine Empire in 1453. Trace the chain of events from the Ottoman conquest of Constantinople through the next two centuries, covering at minimum: the shift in Mediterranean trade routes, the Italian Renaissance's relationship with fleeing Greek scholars, the Russian claim to Roman succession, and the papal response. Then compare how the region might have developed differently had Constantinople remained independent.\"}
|
|
],
|
|
\"max_tokens\": ${MAX_TOKENS},
|
|
\"temperature\": 0.7,
|
|
\"top_p\": 0.95
|
|
}" &)
|
|
|
|
wait
|
|
|
|
END_TIME=$(date +%s%N)
|
|
|
|
# Extract token counts and timing from responses
|
|
COMPLETION_TOKENS1=$(echo "$RESPONSE1" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0")
|
|
COMPLETION_TOKENS2=$(echo "$RESPONSE2" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0")
|
|
TOTAL_TOKENS=$((COMPLETION_TOKENS1 + COMPLETION_TOKENS2))
|
|
TOTAL_TIME_MS=$(( (END_TIME - START_TIME) / 1000000 ))
|
|
|
|
if [ "$TOTAL_TOKENS" -eq 0 ]; then
|
|
echo "FAIL: No tokens generated or response parse error"
|
|
echo "Response1: $RESPONSE1"
|
|
echo "Response2: $RESPONSE2"
|
|
exit 1
|
|
fi
|
|
|
|
# Calculate tokens/sec
|
|
TOK_PER_SEC=$(python3 -c "print(f'{${TOTAL_TOKENS} * 1000 / ${TOTAL_TIME_MS}:.2f}')")
|
|
|
|
echo ""
|
|
echo "=== Results ==="
|
|
echo "Prompt 1 tokens: ${COMPLETION_TOKENS1}"
|
|
echo "Prompt 2 tokens: ${COMPLETION_TOKENS2}"
|
|
echo "Total tokens: ${TOTAL_TOKENS}"
|
|
echo "Total time: ${TOTAL_TIME_MS}ms"
|
|
echo "Speed: ${TOK_PER_SEC} tok/s"
|
|
echo ""
|
|
|
|
# Check minimum speed threshold
|
|
MIN_SPEED="${MIN_SPEED:-5.0}"
|
|
PASS=$(python3 -c "print('yes' if ${TOK_PER_SEC} >= ${MIN_SPEED} else 'no')")
|
|
|
|
if [ "$PASS" = "yes" ]; then
|
|
echo "PASS: Speed ${TOK_PER_SEC} tok/s >= ${MIN_SPEED} tok/s threshold"
|
|
exit 0
|
|
else
|
|
echo "FAIL: Speed ${TOK_PER_SEC} tok/s < ${MIN_SPEED} tok/s threshold"
|
|
exit 1
|
|
fi
|