Merge PR #1: Update AGENTS.md with target-specific branching and service status

2026-08-09 11:03:12 -07:00
parent 0f97dcf449 69bb41057c
commit ad96733340
2 changed files with 116 additions and 0 deletions
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# End-to-end validation test for vLLM Qwen3.6-35B-A3B-FP8
+# Tests single stream and 4 parallel streams
+set -euo pipefail
+
+HOST="${1:-127.0.0.1}"
+PORT="${2:-8001}"
+NUM_STREAMS=4
+
+echo "=== Validating vLLM on ${HOST}:${PORT} ==="
+
+# Test health endpoint
+echo "Health check..."
+if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
+    echo "FAIL: Health check failed"
+    exit 1
+fi
+
+# Single stream test
+echo "Single stream test..."
+START=$(date +%s%N)
+RESPONSE=$(curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
+    -H "Content-Type: application/json" \
+    -d '{
+        "model": "frank-default-small",
+        "messages": [
+            {"role": "user", "content": "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone."}
+        ],
+        "max_tokens": 256
+    }')
+END=$(date +%s%N)
+ELAPSED_MS=$(( (END - START) / 1000000 ))
+PROMPT_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['usage']['prompt_tokens'])")
+COMPLETION_TOKENS=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin)['usage']['completion_tokens'])")
+SINGLE_SPEED=$(python3 -c "print(f'{${PROMPT_TOKENS}/${ELAPSED_MS}*1000:.2f}')")
+echo "Single stream: ${SINGLE_SPEED} tok/s (prompt)"
+
+# Parallel streams test
+echo "Parallel streams test (${NUM_STREAMS} streams)..."
+START=$(date +%s%N)
+TMPDIR=$(mktemp -d)
+for i in $(seq 1 $NUM_STREAMS); do
+    curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
+        -H "Content-Type: application/json" \
+        -d '{
+            "model": "frank-default-small",
+            "messages": [
+                {"role": "user", "content": "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations. Then compare them across three dimensions: data requirements, interpretability of results, and computational cost. Finally, describe a hybrid scenario where combining two of these approaches would outperform using either alone."}
+            ],
+            "max_tokens": 256
+        }' > "$TMPDIR/$i.json" &
+done
+wait
+END=$(date +%s%N)
+ELAPSED_MS=$(( (END - START) / 1000000 ))
+
+# Aggregate results from all streams
+TOTAL_PROMPT=0
+TOTAL_COMPLETION=0
+for i in $(seq 1 $NUM_STREAMS); do
+    PT=$(python3 -c "import json; print(json.load(open('$TMPDIR/$i.json'))['usage']['prompt_tokens'])")
+    CT=$(python3 -c "import json; print(json.load(open('$TMPDIR/$i.json'))['usage']['completion_tokens'])")
+    TOTAL_PROMPT=$((TOTAL_PROMPT + PT))
+    TOTAL_COMPLETION=$((TOTAL_COMPLETION + CT))
+done
+rm -rf "$TMPDIR"
+
+PARALLEL_SPEED=$(python3 -c "print(f'{${TOTAL_PROMPT}/${ELAPSED_MS}*1000:.2f}')")
+
+echo ""
+echo "=== Results ==="
+echo "Single stream prompt speed:   ${SINGLE_SPEED} tok/s"
+echo "Parallel streams prompt speed: ${PARALLEL_SPEED} tok/s (${NUM_STREAMS} streams, ${TOTAL_PROMPT} total prompt tokens)"
+echo "Parallel completion speed:    $(python3 -c "print(f'{${TOTAL_COMPLETION}/${ELAPSED_MS}*1000:.2f}')") tok/s"
+echo "Total parallel time:          ${ELAPSED_MS}ms"
+echo ""
+
+# Fail if parallel speed < 5 tok/s
+if python3 -c "import sys; sys.exit(0 if ${TOTAL_PROMPT}/${ELAPSED_MS}*1000 >= 5 else 1)"; then
+    echo "PASS: Validation successful"
+    echo "$PARALLEL_SPEED" > /tmp/vllm_prompt_speed.txt
+else
+    echo "FAIL: Parallel prompt speed ${PARALLEL_SPEED} tok/s is below 5 tok/s threshold"
+    exit 1
+fi
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MODEL="Qwen/Qwen3.6-35B-A3B-FP8"
+PORT=8001
+HOST="0.0.0.0"
+
+exec docker run --rm --name vllm-qwen36 \
+    --gpus all \
+    --privileged \
+    --ipc=host \
+    -p "${PORT}:${PORT}" \
+    -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+    -v /home/dank/.cache/huggingface:/root/.cache/huggingface \
+    docker.io/vllm/vllm-openai:latest \
+    "${MODEL}" \
+    --port "${PORT}" \
+    --host "${HOST}" \
+    --kv-cache-dtype fp8 \
+    --gpu-memory-utilization 0.95 \
+    --max-num-seqs 4 \
+    --max-model-len 16384 \
+    --max-num-batched-tokens 4096 \
+    --trust-remote-code \
+    --cpu-offload-gb 30 \
+    --cpu-offload-params "experts" \
+    --enable-chunked-prefill \
+    --enable-prefix-caching \
+    --served-model-name frank-default-small \
+    --enable-auto-tool-choice \
+    --tool-call-parser qwen3_xml