From 841d872aa7cc972e09dbb427dfd3b7b145d174c2 Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 10 Aug 2026 19:48:46 -0700 Subject: [PATCH] sync scripts from remotes: ROCmFPX llama-server.service (dankclaw), dankserver vLLM serve/validate; drop failed-attempt scripts - llama-server.service: dankclaw migrated to ROCmFPX runtime (Ling-3.0-flash, draft-mtp, no-mmap/direct-io) from regular llama.cpp Qwen3.6-27B-DFlash - serve-vllm-qwen36-dflash.sh: dankserver version (relies on image 'vllm serve' entrypoint; HF cache mounted read-write) - validate-vllm.sh: dankserver multi-stream vLLM validation (4 unique prompts, 1-4 streams, prompt+completion speed) - remove superseded scripts: serve-llamacpp-dflash.sh, validate.sh, vllm-awq-qwen36-dflash-ampere.sh, vllm-qwen36.service --- scripts/llama-server.service | 4 +- scripts/serve-llamacpp-dflash.sh | 43 ------- scripts/serve-vllm-qwen36-dflash.sh | 5 +- scripts/validate-vllm.sh | 139 ++++++++++++----------- scripts/validate.sh | 102 ----------------- scripts/vllm-awq-qwen36-dflash-ampere.sh | 44 ------- scripts/vllm-qwen36.service | 24 ---- 7 files changed, 76 insertions(+), 285 deletions(-) delete mode 100755 scripts/serve-llamacpp-dflash.sh delete mode 100755 scripts/validate.sh delete mode 100755 scripts/vllm-awq-qwen36-dflash-ampere.sh delete mode 100644 scripts/vllm-qwen36.service diff --git a/scripts/llama-server.service b/scripts/llama-server.service index c9a9a23..9fad688 100644 --- a/scripts/llama-server.service +++ b/scripts/llama-server.service @@ -1,12 +1,12 @@ [Unit] -Description=llama.cpp DFlash+MTP Inference Server (Qwen3.6-27B on ROCm) +Description=llama.cpp Inference Runtime After=network.target [Service] Type=simple User=dank Group=dank -ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-type draft-dflash --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 600000 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --alias frank-default-large --load-mode none +ExecStart=/home/dank/GitPackages/ROCmFPX/build-strix-rocmfp4/bin/llama-server -hf raulvidis/Ling-3.0-flash-ROCmFP4-STRIX-MTP-GGUF:Q4_0 --spec-type draft-mtp --spec-draft-n-max 2 --device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 2 --ctx-size 401400 --batch-size 8192 --ubatch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --alias frank-default-large --no-mmap --direct-io -lv 4 --timeout 7200 --chat-template-file /home/dank/LingFlash/chat_template.jinja Restart=on-failure RestartSec=10 diff --git a/scripts/serve-llamacpp-dflash.sh b/scripts/serve-llamacpp-dflash.sh deleted file mode 100755 index 0519852..0000000 --- a/scripts/serve-llamacpp-dflash.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Serve Qwen3.6-27B with DFlash + MTP speculative decoding via llama.cpp on AMD Strix Halo (gfx1151) -# Uses llama.cpp built with ROCm support -# 3 parallel streams, 131072 context each -set -euo pipefail - -SERVER_BIN="/home/dank/llama.cpp/build/bin/llama-server" -PORT="${LLAMA_PORT:-8080}" -N_PARALLEL="${LLAMA_N_PARALLEL:-3}" -CTX_SIZE="${LLAMA_CTX_SIZE:-600000}" -BATCH_SIZE="${LLAMA_BATCH_SIZE:-4096}" -DRAFT_N_MAX="${LLAMA_DRAFT_N_MAX:-16}" - -# Models via Hugging Face -TARGET_REPO="unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0" -DRAFT_REPO="Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0" - -echo "Starting llama.cpp server..." -echo " Target model: ${TARGET_REPO}" -echo " Draft model: ${DRAFT_REPO}" -echo " Parallel streams: ${N_PARALLEL}" -echo " Context/stream: ${CTX_SIZE}" -echo " Draft tokens: ${DRAFT_N_MAX}" -echo "" - -exec "${SERVER_BIN}" \ - --hf-repo "${TARGET_REPO}" \ - --spec-draft-hf "${DRAFT_REPO}" \ - --spec-type draft-dflash \ - --spec-draft-n-max "${DRAFT_N_MAX}" \ - --device ROCm0 \ - --spec-draft-device ROCm0 \ - --gpu-layers all \ - --fit on \ - --fit-target 512 \ - --parallel "${N_PARALLEL}" \ - --ctx-size "${CTX_SIZE}" \ - --batch-size "${BATCH_SIZE}" \ - --port "${PORT}" \ - --host "0.0.0.0" \ - --threads 16 \ - --alias frank-default-large \ - --load-mode none diff --git a/scripts/serve-vllm-qwen36-dflash.sh b/scripts/serve-vllm-qwen36-dflash.sh index a2811dc..04a29c5 100755 --- a/scripts/serve-vllm-qwen36-dflash.sh +++ b/scripts/serve-vllm-qwen36-dflash.sh @@ -32,6 +32,7 @@ echo "" podman rm -f "${CONTAINER_NAME}" 2>/dev/null || true # Run vLLM in Podman container (foreground mode for systemd) +# Note: vllm/vllm-openai image has entrypoint "vllm serve", so we pass args directly exec podman run --name "${CONTAINER_NAME}" \ --device /dev/nvidia0 \ --device /dev/nvidiactl \ @@ -39,10 +40,10 @@ exec podman run --name "${CONTAINER_NAME}" \ --device /dev/nvidia-uvm-tools \ --device /dev/nvidia-modeset \ -v "${GPU_LIBS}:/usr/lib/x86_64-linux-gnu:ro" \ - -v "${HF_CACHE}:/root/.cache/huggingface:ro" \ + -v "${HF_CACHE}:/root/.cache/huggingface" \ -p "${PORT}:${PORT}" \ docker.io/vllm/vllm-openai:latest \ - vllm serve "${MODEL}" \ + "${MODEL}" \ --port "${PORT}" \ --host "0.0.0.0" \ --kv-cache-dtype "${KV_CACHE_DTYPE}" \ diff --git a/scripts/validate-vllm.sh b/scripts/validate-vllm.sh index 76283f6..8fa4905 100755 --- a/scripts/validate-vllm.sh +++ b/scripts/validate-vllm.sh @@ -1,11 +1,15 @@ #!/usr/bin/env bash -# End-to-end validation test for vLLM Qwen3.6-35B-A3B-FP8 -# Tests single stream and 4 parallel streams set -euo pipefail -HOST="${1:-127.0.0.1}" -PORT="${2:-8001}" -NUM_STREAMS=4 +HOST="127.0.0.1" +PORT=8001 +MODEL="frank-default-small" + +# Different prompts for each test to avoid prefix caching +PROMPT1="Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations." +PROMPT2="Describe the architectural differences between RISC-V and x86 instruction set architectures. Include a discussion of register files, addressing modes, and instruction encoding. Compare their performance for embedded systems and server workloads." +PROMPT3="Analyze the economic impact of blockchain technology on supply chain management. Discuss the key benefits of transparency, immutability, and smart contracts. Compare implementation challenges of Bitcoin, Ethereum, and Hyperledger for enterprise use cases." +PROMPT4="Explain the principles of evolutionary algorithms in artificial intelligence. Describe how genetic algorithms, genetic programming, and evolutionary strategies differ in their approach to problem solving. Compare their performance on optimization problems and neural network architecture search." echo "=== Validating vLLM on ${HOST}:${PORT} ===" @@ -16,70 +20,69 @@ if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then exit 1 fi -# Single stream test -echo "Single stream test..." -START=$(date +%s%N) -RESPONSE=$(curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -d '{ - "model": "Qwen/Qwen3.6-35B-A3B-FP8", - "messages": [ - {"role": "user", "content": "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone."} - ], - "max_tokens": 256 - }') -END=$(date +%s%N) -ELAPSED_MS=$(( (END - START) / 1000000 )) -PROMPT_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['usage']['prompt_tokens'])") -COMPLETION_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['usage']['completion_tokens'])") -SINGLE_SPEED=$(python3 -c "print(f'{${PROMPT_TOKENS}/${ELAPSED_MS}*1000:.2f}')") -echo "Single stream: ${SINGLE_SPEED} tok/s (prompt)" - -# Parallel streams test -echo "Parallel streams test (${NUM_STREAMS} streams)..." -START=$(date +%s%N) -TMPDIR=$(mktemp -d) -for i in $(seq 1 $NUM_STREAMS); do - curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -d '{ - "model": "Qwen/Qwen3.6-35B-A3B-FP8", - "messages": [ - {"role": "user", "content": "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone."} - ], - "max_tokens": 256 - }' > "$TMPDIR/$i.json" & +# Test N streams sequentially (1, 2, 3, 4) +for NUM_STREAMS in 1 2 3 4; do + echo "" + echo "--- Testing ${NUM_STREAMS} stream(s) ---" + + TMPDIR=$(mktemp -d) + START=$(date +%s%N) + + for i in $(seq 1 $NUM_STREAMS); do + case $(( (i - 1) % 4 + 1 )) in + 1) PROMPT="${PROMPT1}" ;; + 2) PROMPT="${PROMPT2}" ;; + 3) PROMPT="${PROMPT3}" ;; + 4) PROMPT="${PROMPT4}" ;; + esac + + curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d "{ + \"model\": \"${MODEL}\", + \"messages\": [ + {\"role\": \"user\", \"content\": \"${PROMPT}\"} + ], + \"max_tokens\": 256 + }" > "$TMPDIR/$i.json" & + done + wait + + END=$(date +%s%N) + ELAPSED_MS=$(( (END - START) / 1000000 )) + + # Calculate speed from the first stream's response + PROMPT_TOKENS=$(python3 -c "import json; print(json.load(open('$TMPDIR/1.json'))['usage']['prompt_tokens'])") + COMPLETION_TOKENS=$(python3 -c "import json; print(json.load(open('$TMPDIR/1.json'))['usage']['completion_tokens'])") + + # Calculate speeds + PROMPT_SPEED=$(python3 -c "print(f'{${PROMPT_TOKENS}/${ELAPSED_MS}*1000:.2f}')") + COMPLETION_SPEED=$(python3 -c "print(f'{${COMPLETION_TOKENS}/${ELAPSED_MS}*1000:.2f}')") + + echo " Elapsed: ${ELAPSED_MS}ms" + echo " Prompt tokens: ${PROMPT_TOKENS}" + echo " Prompt speed: ${PROMPT_SPEED} tok/s" + echo " Completion tokens: ${COMPLETION_TOKENS}" + echo " Completion speed: ${COMPLETION_SPEED} tok/s" + + # Save results + echo "${NUM_STREAMS},${PROMPT_SPEED},${COMPLETION_SPEED},${ELAPSED_MS}" > "/tmp/vllm-${NUM_STREAMS}.csv" + + # Clean up temp files + rm -rf "$TMPDIR" done -wait -END=$(date +%s%N) -ELAPSED_MS=$(( (END - START) / 1000000 )) - -# Aggregate results from all streams -TOTAL_PROMPT=0 -TOTAL_COMPLETION=0 -for i in $(seq 1 $NUM_STREAMS); do - PT=$(python3 -c "import json; print(json.load(open('$TMPDIR/$i.json'))['usage']['prompt_tokens'])") - CT=$(python3 -c "import json; print(json.load(open('$TMPDIR/$i.json'))['usage']['completion_tokens'])") - TOTAL_PROMPT=$((TOTAL_PROMPT + PT)) - TOTAL_COMPLETION=$((TOTAL_COMPLETION + CT)) -done -rm -rf "$TMPDIR" - -PARALLEL_SPEED=$(python3 -c "print(f'{${TOTAL_PROMPT}/${ELAPSED_MS}*1000:.2f}')") echo "" -echo "=== Results ===" -echo "Single stream prompt speed: ${SINGLE_SPEED} tok/s" -echo "Parallel streams prompt speed: ${PARALLEL_SPEED} tok/s (${NUM_STREAMS} streams, ${TOTAL_PROMPT} total prompt tokens)" -echo "Parallel completion speed: $(python3 -c "print(f'{${TOTAL_COMPLETION}/${ELAPSED_MS}*1000:.2f}')") tok/s" -echo "Total parallel time: ${ELAPSED_MS}ms" -echo "" +echo "=== Results Summary ===" +for n in 1 2 3 4; do + if [ -f "/tmp/vllm-${n}.csv" ]; then + IFS=',' read -r streams p_speed c_speed elapsed < "/tmp/vllm-${n}.csv" + echo " ${n} stream(s): ${p_speed} tok/s (prompt), ${c_speed} tok/s (completion) - ${elapsed}ms" + fi +done -# Fail if parallel speed < 5 tok/s -if python3 -c "import sys; sys.exit(0 if ${TOTAL_PROMPT}/${ELAPSED_MS}*1000 >= 5 else 1)"; then - echo "PASS: Validation successful" - echo "$PARALLEL_SPEED" > /tmp/vllm_prompt_speed.txt -else - echo "FAIL: Parallel prompt speed ${PARALLEL_SPEED} tok/s is below 5 tok/s threshold" - exit 1 -fi +# Check if single stream speed is >= 5 tok/s +SINGLE_SPEED=$(head -1 /tmp/vllm-1.csv | cut -d',' -f2) +python3 -c "import sys; sys.exit(0 if float('${SINGLE_SPEED}') >= 5 else 1)" +echo "" +echo "PASS: Validation successful" diff --git a/scripts/validate.sh b/scripts/validate.sh deleted file mode 100755 index 0e31336..0000000 --- a/scripts/validate.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env bash -# Validation test for llama.cpp DFlash server -# Tests 1, 2, and 3 parallel prompts sequentially -set -euo pipefail - -PORT="${LLAMA_PORT:-8080}" -HOST="${LLAMA_HOST:-127.0.0.1}" -MAX_TOKENS="${MAX_TOKENS:-1024}" -MIN_SPEED="${MIN_SPEED:-5.0}" - -echo "=== llama.cpp Validation Test ===" -echo "Target: http://${HOST}:${PORT}" -echo "Max tokens: ${MAX_TOKENS}" -echo "" - -# Wait for server to be ready -echo "Waiting for server..." -for i in $(seq 1 300); do - if curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then - echo "Server ready after $((i * 2))s" - break - fi - if [ "$i" -eq 300 ]; then - echo "FAIL: Server did not become ready in 600s" - exit 1 - fi - sleep 2 -done - -# Test function: send N parallel prompts and measure speed -run_test() { - local num_streams=$1 - local tmpfiles=() - local pids=() - - echo "" - echo "--- Testing ${num_streams} parallel stream(s) ---" - START_TIME=$(date +%s%N) - - for s in $(seq 1 "$num_streams"); do - tmpf=$(mktemp) - tmpfiles+=("$tmpf") - - curl -s \ - -X POST "http://${HOST}:${PORT}/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -d "{ - \"messages\": [ - {\"role\": \"user\", \"content\": \"Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone.\"} - ], - \"max_tokens\": ${MAX_TOKENS}, - \"temperature\": 0.7, - \"top_p\": 0.95 - }" > "$tmpf" & - pids+=($!) - done - - wait "${pids[@]}" - - END_TIME=$(date +%s%N) - TOTAL_TIME_MS=$(( (END_TIME - START_TIME) / 1000000 )) - - TOTAL_TOKENS=0 - for tmpf in "${tmpfiles[@]}"; do - tokens=$(python3 -c "import json; d=json.load(open('$tmpf')); print(d['usage']['completion_tokens'])" 2>/dev/null || echo "0") - TOTAL_TOKENS=$((TOTAL_TOKENS + tokens)) - rm -f "$tmpf" - done - - if [ "$TOTAL_TOKENS" -eq 0 ]; then - echo "FAIL: No tokens generated" - return 1 - fi - - TOK_PER_SEC=$(python3 -c "print(f'{${TOTAL_TOKENS} * 1000 / ${TOTAL_TIME_MS}:.2f}')") - echo " Tokens: ${TOTAL_TOKENS} / ${TOTAL_TIME_MS}ms = ${TOK_PER_SEC} tok/s" - - PASS=$(python3 -c "print('yes' if ${TOK_PER_SEC} >= ${MIN_SPEED} else 'no')") - if [ "$PASS" = "yes" ]; then - echo " PASS: >= ${MIN_SPEED} tok/s" - else - echo " FAIL: < ${MIN_SPEED} tok/s" - fi -} - -# Run tests with 1, 2, and 3 parallel streams -ALL_PASS="yes" -for n in 1 2 3; do - if ! run_test "$n"; then - ALL_PASS="no" - fi -done - -echo "" -echo "=== Summary ===" -if [ "$ALL_PASS" = "yes" ]; then - echo "ALL TESTS PASSED" - exit 0 -else - echo "SOME TESTS FAILED" - exit 1 -fi diff --git a/scripts/vllm-awq-qwen36-dflash-ampere.sh b/scripts/vllm-awq-qwen36-dflash-ampere.sh deleted file mode 100755 index 0d77df0..0000000 --- a/scripts/vllm-awq-qwen36-dflash-ampere.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# vllm-awq-qwen36-dflash-ampere.sh -# -# Qwen3.6-35B-A3B AWQ 4bit with DFlash speculative decoding. -# Optimized for Ampere-class GPUs. -# -# Model: cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit -# Draft: z-lab/Qwen3.6-35B-A3B-DFlash -# Port: 34401 -# -set -euo pipefail - -MODEL="Avesed/Qwen3.6-35B-A3B-INT8-W8A8" -PORT=34401 - -exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \ - --gpus all \ - --privileged --ipc=host \ - -p "${PORT}:8000" \ - -e OMP_NUM_THREADS=8 \ - -e VLLM_CPU_OMP_THREADS_BIND=8 \ - -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ - -e VLLM_USE_V2_MODEL_RUNNER=0 \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - vllm/vllm-openai:nightly "${MODEL}" \ - --served-model-name "frank-default-small" \ - --max-num-seqs 3 \ - --max-model-len 200700 \ - --tensor-parallel-size 1 \ - --enable-chunked-prefill \ - --enable-prefix-caching \ - --max-num-batched-tokens 3072 \ - --performance-mode throughput \ - --enable-auto-tool-choice --tool-call-parser qwen3_xml \ - --reasoning-parser qwen3 \ - --trust-remote-code \ - --gpu-memory-utilization 0.97 \ - --kv-cache-dtype auto \ - --cpu-offload-gb 30 \ - --cpu-offload-params "experts" \ - --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}' \ - -O3 \ - --speculative-config '{"method": "mtp", "num_speculative_tokens": 2}' - diff --git a/scripts/vllm-qwen36.service b/scripts/vllm-qwen36.service deleted file mode 100644 index 6b4c9b5..0000000 --- a/scripts/vllm-qwen36.service +++ /dev/null @@ -1,24 +0,0 @@ -[Unit] -Description=vLLM Qwen3.6-35B-A3B-FP8 with DFlash Drafting (Podman) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=dank -Group=dank -WorkingDirectory=/home/dank/Projects/vllmScripts -Environment="PORT=8000" -Environment="GPU_MEM_UTIL=0.95" -Environment="MAX_NUM_SEQS=4" -Environment="MAX_MODEL_LEN=262144" -Environment="KV_CACHE_DTYPE=fp8" -Environment="NUM_SPEC_TOKENS=8" -ExecStart=/home/dank/Projects/vllmScripts/scripts/serve-vllm-qwen36-dflash.sh -ExecStop=/usr/bin/podman stop vllm-qwen36-dflash -ExecStopPost=/usr/bin/podman rm -f vllm-qwen36-dflash -Restart=on-failure -RestartSec=5 - -[Install] -WantedBy=multi-user.target