Use --alias frank-default-large, multi-stream validation script
- Replace --override-kv with --alias for model naming - Validation script tests 1, 2, 3 parallel streams sequentially - Model name confirmed: frank-default-large
This commit is contained in:
Frank committed
1 parent
471e963350
commit
a5f72627bb
3 files changed
+63
-63
No files matched your search
+61
-62
@@ -1,11 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# Validation test for llama.cpp DFlash server
|
||||
# Sends 2 parallel prompts and measures generation speed (tokens/sec)
|
||||
# Tests 1, 2, and 3 parallel prompts sequentially
|
||||
set -euo pipefail
|
||||
|
||||
PORT="${LLAMA_PORT:-8080}"
|
||||
HOST="${LLAMA_HOST:-127.0.0.1}"
|
||||
MAX_TOKENS="${MAX_TOKENS:-1024}"
|
||||
MIN_SPEED="${MIN_SPEED:-5.0}"
|
||||
|
||||
echo "=== llama.cpp Validation Test ==="
|
||||
echo "Target: http://${HOST}:${PORT}"
|
||||
@@ -26,78 +27,76 @@ for i in $(seq 1 300); do
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Send two parallel requests and capture timing
|
||||
echo ""
|
||||
echo "Sending 2 parallel validation prompts..."
|
||||
START_TIME=$(date +%s%N)
|
||||
# Test function: send N parallel prompts and measure speed
|
||||
run_test() {
|
||||
local num_streams=$1
|
||||
local tmpfiles=()
|
||||
local pids=()
|
||||
|
||||
TMP1=$(mktemp)
|
||||
TMP2=$(mktemp)
|
||||
trap "rm -f $TMP1 $TMP2" EXIT
|
||||
echo ""
|
||||
echo "--- Testing ${num_streams} parallel stream(s) ---"
|
||||
START_TIME=$(date +%s%N)
|
||||
|
||||
curl -s \
|
||||
-X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"messages\": [
|
||||
{\"role\": \"user\", \"content\": \"Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone.\"}
|
||||
],
|
||||
\"max_tokens\": ${MAX_TOKENS},
|
||||
\"temperature\": 0.7,
|
||||
\"top_p\": 0.95
|
||||
}" > "$TMP1" &
|
||||
PID1=$!
|
||||
for s in $(seq 1 "$num_streams"); do
|
||||
tmpf=$(mktemp)
|
||||
tmpfiles+=("$tmpf")
|
||||
|
||||
curl -s \
|
||||
-X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"messages\": [
|
||||
{\"role\": \"user\", \"content\": \"Write a detailed analysis of the geopolitical consequences of the fall of the Byzantine Empire in 1453. Trace the chain of events from the Ottoman conquest of Constantinople through the next two centuries, covering at minimum: the shift in Mediterranean trade routes, the Italian Renaissance's relationship with fleeing Greek scholars, the Russian claim to Roman succession, and the papal response. Then compare how the region might have developed differently had Constantinople remained independent.\"}
|
||||
],
|
||||
\"max_tokens\": ${MAX_TOKENS},
|
||||
\"temperature\": 0.7,
|
||||
\"top_p\": 0.95
|
||||
}" > "$TMP2" &
|
||||
PID2=$!
|
||||
curl -s \
|
||||
-X POST "http://${HOST}:${PORT}/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"messages\": [
|
||||
{\"role\": \"user\", \"content\": \"Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone.\"}
|
||||
],
|
||||
\"max_tokens\": ${MAX_TOKENS},
|
||||
\"temperature\": 0.7,
|
||||
\"top_p\": 0.95
|
||||
}" > "$tmpf" &
|
||||
pids+=($!)
|
||||
done
|
||||
|
||||
wait $PID1 $PID2
|
||||
wait "${pids[@]}"
|
||||
|
||||
END_TIME=$(date +%s%N)
|
||||
END_TIME=$(date +%s%N)
|
||||
TOTAL_TIME_MS=$(( (END_TIME - START_TIME) / 1000000 ))
|
||||
|
||||
# Extract token counts and timing from responses
|
||||
COMPLETION_TOKENS1=$(python3 -c "import json; d=json.load(open('$TMP1')); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0")
|
||||
COMPLETION_TOKENS2=$(python3 -c "import json; d=json.load(open('$TMP2')); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0")
|
||||
TOTAL_TOKENS=$((COMPLETION_TOKENS1 + COMPLETION_TOKENS2))
|
||||
TOTAL_TIME_MS=$(( (END_TIME - START_TIME) / 1000000 ))
|
||||
TOTAL_TOKENS=0
|
||||
for tmpf in "${tmpfiles[@]}"; do
|
||||
tokens=$(python3 -c "import json; d=json.load(open('$tmpf')); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0")
|
||||
TOTAL_TOKENS=$((TOTAL_TOKENS + tokens))
|
||||
rm -f "$tmpf"
|
||||
done
|
||||
|
||||
if [ "$TOTAL_TOKENS" -eq 0 ]; then
|
||||
echo "FAIL: No tokens generated or response parse error"
|
||||
echo "Response1: $(cat $TMP1)"
|
||||
echo "Response2: $(cat $TMP2)"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$TOTAL_TOKENS" -eq 0 ]; then
|
||||
echo "FAIL: No tokens generated"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Calculate tokens/sec
|
||||
TOK_PER_SEC=$(python3 -c "print(f'{${TOTAL_TOKENS} * 1000 / ${TOTAL_TIME_MS}:.2f}')")
|
||||
TOK_PER_SEC=$(python3 -c "print(f'{${TOTAL_TOKENS} * 1000 / ${TOTAL_TIME_MS}:.2f}')")
|
||||
echo " Tokens: ${TOTAL_TOKENS} / ${TOTAL_TIME_MS}ms = ${TOK_PER_SEC} tok/s"
|
||||
|
||||
PASS=$(python3 -c "print('yes' if ${TOK_PER_SEC} >= ${MIN_SPEED} else 'no')")
|
||||
if [ "$PASS" = "yes" ]; then
|
||||
echo " PASS: >= ${MIN_SPEED} tok/s"
|
||||
else
|
||||
echo " FAIL: < ${MIN_SPEED} tok/s"
|
||||
fi
|
||||
}
|
||||
|
||||
# Run tests with 1, 2, and 3 parallel streams
|
||||
ALL_PASS="yes"
|
||||
for n in 1 2 3; do
|
||||
if ! run_test "$n"; then
|
||||
ALL_PASS="no"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Results ==="
|
||||
echo "Prompt 1 tokens: ${COMPLETION_TOKENS1}"
|
||||
echo "Prompt 2 tokens: ${COMPLETION_TOKENS2}"
|
||||
echo "Total tokens: ${TOTAL_TOKENS}"
|
||||
echo "Total time: ${TOTAL_TIME_MS}ms"
|
||||
echo "Speed: ${TOK_PER_SEC} tok/s"
|
||||
echo ""
|
||||
|
||||
# Check minimum speed threshold
|
||||
MIN_SPEED="${MIN_SPEED:-5.0}"
|
||||
PASS=$(python3 -c "print('yes' if ${TOK_PER_SEC} >= ${MIN_SPEED} else 'no')")
|
||||
|
||||
if [ "$PASS" = "yes" ]; then
|
||||
echo "PASS: Speed ${TOK_PER_SEC} tok/s >= ${MIN_SPEED} tok/s threshold"
|
||||
echo "=== Summary ==="
|
||||
if [ "$ALL_PASS" = "yes" ]; then
|
||||
echo "ALL TESTS PASSED"
|
||||
exit 0
|
||||
else
|
||||
echo "FAIL: Speed ${TOK_PER_SEC} tok/s < ${MIN_SPEED} tok/s threshold"
|
||||
echo "SOME TESTS FAILED"
|
||||
exit 1
|
||||
fi
|
||||
Reference in new issue
Block a user