dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Use --alias frank-default-large, multi-stream validation script

Browse Source 
- Replace --override-kv with --alias for model naming
- Validation script tests 1, 2, 3 parallel streams sequentially
- Model name confirmed: frank-default-large
This commit is contained in:
Frank committed 2026-08-07 17:15:09 -07:00
1 parent 471e963350
commit a5f72627bb
3 files changed
+63 -63
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/llama-server.service
+1 -1
View File
@@ -6,7 +6,7 @@ After=network.target
Type=simple Type=simple
User=dank User=dank
Group=dank Group=dank
ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-type draft-dflash --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --load-mode none ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-type draft-dflash --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --alias frank-default-large --load-mode none
Restart=on-failure Restart=on-failure
RestartSec=10 RestartSec=10
scripts/serve-llamacpp-dflash.sh
+1
View File
@@ -39,4 +39,5 @@ exec "${SERVER_BIN}" \
--port "${PORT}" \ --port "${PORT}" \
--host "0.0.0.0" \ --host "0.0.0.0" \
--threads 16 \ --threads 16 \
--alias frank-default-large \
--load-mode none --load-mode none
scripts/validate.sh
+61 -62
View File
@@ -1,11 +1,12 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Validation test for llama.cpp DFlash server # Validation test for llama.cpp DFlash server
# Sends 2 parallel prompts and measures generation speed (tokens/sec) # Tests 1, 2, and 3 parallel prompts sequentially
set -euo pipefail set -euo pipefail
PORT="${LLAMA_PORT:-8080}" PORT="${LLAMA_PORT:-8080}"
HOST="${LLAMA_HOST:-127.0.0.1}" HOST="${LLAMA_HOST:-127.0.0.1}"
MAX_TOKENS="${MAX_TOKENS:-1024}" MAX_TOKENS="${MAX_TOKENS:-1024}"
MIN_SPEED="${MIN_SPEED:-5.0}"
echo "=== llama.cpp Validation Test ===" echo "=== llama.cpp Validation Test ==="
echo "Target: http://${HOST}:${PORT}" echo "Target: http://${HOST}:${PORT}"
@@ -26,78 +27,76 @@ for i in $(seq 1 300); do
sleep 2 sleep 2
done done
# Send two parallel requests and capture timing # Test function: send N parallel prompts and measure speed
echo "" run_test() {
echo "Sending 2 parallel validation prompts..." local num_streams=$1
START_TIME=$(date +%s%N) local tmpfiles=()
local pids=()
TMP1=$(mktemp) echo ""
TMP2=$(mktemp) echo "--- Testing ${num_streams} parallel stream(s) ---"
trap "rm -f $TMP1 $TMP2" EXIT START_TIME=$(date +%s%N)
curl -s \ for s in $(seq 1 "$num_streams"); do
-X POST "http://${HOST}:${PORT}/v1/chat/completions" \ tmpf=$(mktemp)
-H "Content-Type: application/json" \ tmpfiles+=("$tmpf")
-d "{
\"messages\": [
{\"role\": \"user\", \"content\": \"Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone.\"}
],
\"max_tokens\": ${MAX_TOKENS},
\"temperature\": 0.7,
\"top_p\": 0.95
}" > "$TMP1" &
PID1=$!
curl -s \ curl -s \
-X POST "http://${HOST}:${PORT}/v1/chat/completions" \ -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
-H "Content-Type: application/json" \ -H "Content-Type: application/json" \
-d "{ -d "{
\"messages\": [ \"messages\": [
{\"role\": \"user\", \"content\": \"Write a detailed analysis of the geopolitical consequences of the fall of the Byzantine Empire in 1453. Trace the chain of events from the Ottoman conquest of Constantinople through the next two centuries, covering at minimum: the shift in Mediterranean trade routes, the Italian Renaissance's relationship with fleeing Greek scholars, the Russian claim to Roman succession, and the papal response. Then compare how the region might have developed differently had Constantinople remained independent.\"} {\"role\": \"user\", \"content\": \"Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone.\"}
], ],
\"max_tokens\": ${MAX_TOKENS}, \"max_tokens\": ${MAX_TOKENS},
\"temperature\": 0.7, \"temperature\": 0.7,
\"top_p\": 0.95 \"top_p\": 0.95
}" > "$TMP2" & }" > "$tmpf" &
PID2=$! pids+=($!)
done
wait $PID1 $PID2 wait "${pids[@]}"
END_TIME=$(date +%s%N) END_TIME=$(date +%s%N)
TOTAL_TIME_MS=$(( (END_TIME - START_TIME) / 1000000 ))
# Extract token counts and timing from responses TOTAL_TOKENS=0
COMPLETION_TOKENS1=$(python3 -c "import json; d=json.load(open('$TMP1')); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0") for tmpf in "${tmpfiles[@]}"; do
COMPLETION_TOKENS2=$(python3 -c "import json; d=json.load(open('$TMP2')); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0") tokens=$(python3 -c "import json; d=json.load(open('$tmpf')); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0")
TOTAL_TOKENS=$((COMPLETION_TOKENS1 + COMPLETION_TOKENS2)) TOTAL_TOKENS=$((TOTAL_TOKENS + tokens))
TOTAL_TIME_MS=$(( (END_TIME - START_TIME) / 1000000 )) rm -f "$tmpf"
done
if [ "$TOTAL_TOKENS" -eq 0 ]; then if [ "$TOTAL_TOKENS" -eq 0 ]; then
echo "FAIL: No tokens generated or response parse error" echo "FAIL: No tokens generated"
echo "Response1: $(cat $TMP1)" return 1
echo "Response2: $(cat $TMP2)" fi
exit 1
fi
# Calculate tokens/sec TOK_PER_SEC=$(python3 -c "print(f'{${TOTAL_TOKENS} * 1000 / ${TOTAL_TIME_MS}:.2f}')")
TOK_PER_SEC=$(python3 -c "print(f'{${TOTAL_TOKENS} * 1000 / ${TOTAL_TIME_MS}:.2f}')") echo " Tokens: ${TOTAL_TOKENS} / ${TOTAL_TIME_MS}ms = ${TOK_PER_SEC} tok/s"
PASS=$(python3 -c "print('yes' if ${TOK_PER_SEC} >= ${MIN_SPEED} else 'no')")
if [ "$PASS" = "yes" ]; then
echo " PASS: >= ${MIN_SPEED} tok/s"
else
echo " FAIL: < ${MIN_SPEED} tok/s"
fi
}
# Run tests with 1, 2, and 3 parallel streams
ALL_PASS="yes"
for n in 1 2 3; do
if ! run_test "$n"; then
ALL_PASS="no"
fi
done
echo "" echo ""
echo "=== Results ===" echo "=== Summary ==="
echo "Prompt 1 tokens: ${COMPLETION_TOKENS1}" if [ "$ALL_PASS" = "yes" ]; then
echo "Prompt 2 tokens: ${COMPLETION_TOKENS2}" echo "ALL TESTS PASSED"
echo "Total tokens: ${TOTAL_TOKENS}"
echo "Total time: ${TOTAL_TIME_MS}ms"
echo "Speed: ${TOK_PER_SEC} tok/s"
echo ""
# Check minimum speed threshold
MIN_SPEED="${MIN_SPEED:-5.0}"
PASS=$(python3 -c "print('yes' if ${TOK_PER_SEC} >= ${MIN_SPEED} else 'no')")
if [ "$PASS" = "yes" ]; then
echo "PASS: Speed ${TOK_PER_SEC} tok/s >= ${MIN_SPEED} tok/s threshold"
exit 0 exit 0
else else
echo "FAIL: Speed ${TOK_PER_SEC} tok/s < ${MIN_SPEED} tok/s threshold" echo "SOME TESTS FAILED"
exit 1 exit 1
fi fi