From 5ed9bf0410662b9d65a2c71dc5a40ee83bcc0ca0 Mon Sep 17 00:00:00 2001 From: Frank Date: Sat, 8 Aug 2026 21:29:23 -0700 Subject: [PATCH] Update documentation with AWQ+DFlash lessons learned and improved validation script --- README.md | 36 +++++++++++++++++++++++- docs/awq-dflash-ampere-notes.md | 49 +++++++++++++++++++++++++++++++++ scripts/validate-vllm.sh | 43 +++++++++++++++++++++-------- 3 files changed, 115 insertions(+), 13 deletions(-) create mode 100644 docs/awq-dflash-ampere-notes.md diff --git a/README.md b/README.md index 4c1ace2..df0bd64 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,46 @@ Scripts for running large language models via vLLM on two local machines. |-------|--------|--------| | Qwen3.6 35B A3B FP8 | dankserver | FP8 | | Qwen3.6 27B FP8 | dankclaw | FP8 | +| Qwen3.6 35B A3B AWQ 4bit | dankserver | AWQ 4bit | +| Qwen3.6 35B A3B DFlash | dankserver | Draft model | ## Structure - `docs/` — Project knowledge base with research notes on vLLM configuration, hardware specifics, and deployment notes -- `scripts/` — Executable vLLM launch scripts (coming soon) +- `scripts/` — Executable vLLM launch scripts +- `vllm-qwen36.sh` — FP8 inference script for dankserver +- `vllm-awq-qwen36-dflash.sh` — AWQ 4bit with DFlash speculative decoding (docker) +- `vllm-awq-qwen36-dflash-ampere.sh` — AWQ 4bit with DFlash speculative decoding (Ampere-optimized, systemd) ## Knowledge Base See [[docs/README.md]] for the project knowledge base index. + +## Lessons Learned + +### AWQ + DFlash on Ampere Hardware (2026-08-08) + +**What worked:** +- `cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit` loads successfully on Ampere GPUs +- AWQ Marlin kernel engaged automatically for weight conversion +- DFlash draft model `z-lab/Qwen3.6-35B-A3B-DFlash` detected as EAGLE model +- torch.compile with cudagraphs works with `--max-num-batched-tokens 1536` + +**What didn't work:** +- `--enable-prefix-caching` conflicts with Mamba cache "align" mode — causes `AssertionError` in `unify_kv_cache_spec_page_size` +- `--max-model-len 262144` causes OOM during AWQ→Marlin weight conversion on 24GB GPU +- Nightly vLLM `--speculative-config` requires `num_speculative_tokens` when model is provided +- `--speculative-model` flag removed in nightly; use `--speculative-config` JSON instead + +**Working configuration:** +- `--max-model-len 32768` (32k context) +- `--cpu-offload-gb 40` with `--cpu-offload-params "experts"` +- `--gpu-memory-utilization 0.95` +- `--performance-mode throughput` +- `--enable-chunked-prefill` +- `--max-num-batched-tokens 1536` +- Thread binding: `OMP_NUM_THREADS=8`, `VLLM_CPU_OMP_THREADS_BIND=8` + +**Still needs work:** +- DFlash speculative decoding integration (KV cache assertion bug in nightly) +- Full context length support (32k is temporary workaround for OOM) diff --git a/docs/awq-dflash-ampere-notes.md b/docs/awq-dflash-ampere-notes.md new file mode 100644 index 0000000..d32918b --- /dev/null +++ b/docs/awq-dflash-ampere-notes.md @@ -0,0 +1,49 @@ +# AWQ + DFlash on Ampere Hardware + +## Date: 2026-08-08 + +## Objective +Deploy Qwen3.6-35B-A3B with AWQ 4bit quantization and DFlash speculative decoding on NVIDIA Ampere-class GPU (24GB VRAM). + +## Scripts Created +- `vllm-awq-qwen36-dflash.sh` — Docker-based AWQ + DFlash +- `vllm-awq-qwen36-dflash-ampere.sh` — Systemd service, Ampere-optimized + +## Key Findings + +### vLLM CLI Changes (Nightly) +- `--speculative-model` removed; use `--speculative-config` JSON instead +- Format: `--speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 4}'` +- `num_speculative_tokens` is required when model is provided + +### Memory Issues +- **OOM with 262144 context**: AWQ→Marlin weight conversion requires extra GPU memory +- **Solution**: Lower to 32768 context temporarily +- **CPU offload**: `--cpu-offload-gb 40` + `--cpu-offload-params "experts"` helps + +### Conflicts +- **Prefix caching + Mamba cache**: Causes `AssertionError` in `unify_kv_cache_spec_page_size` +- **DFlash speculative decoding**: KV cache assertion bug in nightly vLLM +- Workaround: Disable speculative decoding or prefix caching + +### Performance Observations (no speculative decoding) +| Streams | Prompt Speed | Completion Speed | +|---------|-------------|------------------| +| 1 | 4.27 tok/s | 21.42 tok/s | +| 2 | 2.44 tok/s | 12.25 tok/s | +| 3 | 1.35 tok/s | 6.76 tok/s | +| 4 | 1.43 tok/s | 7.19 tok/s | + +Note: Speeds decrease with more streams due to KV cache and compute contention. + +## Systemd Service +- Service: `vllm-awq-qwen36.service` +- ExecStart: `/home/dank/vllm/vllm-awq-qwen36-dflash-ampere.sh` +- Port: 34401 +- Status: Disabled (needs speculative decoding fix) + +## Next Steps +1. Fix DFlash speculative decoding (upstream vLLM bug or workaround) +2. Test with lower context length for full functionality +3. Benchmark with and without speculative decoding +4. Investigate `max_num_scheduled_tokens` warning (set to 1530 due to speculative settings) diff --git a/scripts/validate-vllm.sh b/scripts/validate-vllm.sh index 750f1cb..3b42a6c 100755 --- a/scripts/validate-vllm.sh +++ b/scripts/validate-vllm.sh @@ -5,13 +5,13 @@ HOST="${VLLM_HOST:-127.0.0.1}" PORT="${VLLM_PORT:-34401}" MODEL="frank-default-small" -# Different prompts per stream to avoid prefix caching effects -PROMPTS=( - "Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations." - "Describe the architectural differences between RISC-V and x86 instruction set architectures. Include a discussion of register files, addressing modes, and instruction encoding." - "Analyze the economic impact of blockchain technology on supply chain management. Discuss the key benefits of transparency, immutability, and smart contracts." - "Explain the principles of evolutionary algorithms in artificial intelligence. Describe how genetic algorithms, genetic programming, and evolutionary strategies differ." -) +# Different prompts for each stream to avoid prefix caching effects +PROMPT1="Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations." +PROMPT2="Describe the architectural differences between RISC-V and x86 instruction set architectures. Include a discussion of register files, addressing modes, and instruction encoding." +PROMPT3="Analyze the economic impact of blockchain technology on supply chain management. Discuss the key benefits of transparency, immutability, and smart contracts." +PROMPT4="Explain the principles of evolutionary algorithms in artificial intelligence. Describe how genetic algorithms, genetic programming, and evolutionary strategies differ in their approach to problem solving." + +PROMPTS=("$PROMPT1" "$PROMPT2" "$PROMPT3" "$PROMPT4") echo "=== Validating vLLM on ${HOST}:${PORT} ===" @@ -23,7 +23,8 @@ if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then fi echo "OK" -# Test N streams (separate test for each N) +# Test N streams sequentially (separate test for each N) +# Each test uses different prompts to avoid caching for N in 1 2 3 4; do echo "" echo "--- ${N} stream(s) ---" @@ -32,11 +33,15 @@ for N in 1 2 3 4; do START=$(date +%s%N) for i in $(seq 1 $N); do + # Use a different prompt for each stream to avoid caching + PROMPT_IDX=$(( (i - 1) % 4 )) + PROMPT="${PROMPTS[$PROMPT_IDX]}" + curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \ -H "Content-Type: application/json" \ -d "{ \"model\": \"${MODEL}\", - \"messages\": [{\"role\": \"user\", \"content\": \"${PROMPTS[$((i-1))]}\"}], + \"messages\": [{\"role\": \"user\", \"content\": \"${PROMPT}\"}], \"max_tokens\": 256 }" > "$TMPDIR/$i.json" & done @@ -54,9 +59,23 @@ for N in 1 2 3 4; do fi done - rm -rf "$TMPDIR" - + # Calculate speeds from first response if $ALL_OK; then - echo " OK - ${ELAPSED_MS}ms total (${N} streams)" + PROMPT_TOKENS=$(python3 -c "import json; print(json.load(open('$TMPDIR/1.json'))['usage']['prompt_tokens'])") + COMPLETION_TOKENS=$(python3 -c "import json; print(json.load(open('$TMPDIR/1.json'))['usage']['completion_tokens'])") + TOTAL_TOKENS=$((PROMPT_TOKENS + COMPLETION_TOKENS)) + PROMPT_SPEED=$(python3 -c "print(f'{${PROMPT_TOKENS}/${ELAPSED_MS}*1000:.2f}')") + COMPLETION_SPEED=$(python3 -c "print(f'{${COMPLETION_TOKENS}/${ELAPSED_MS}*1000:.2f}')") + TOTAL_SPEED=$(python3 -c "print(f'{${TOTAL_TOKENS}/${ELAPSED_MS}*1000:.2f}')") + + echo " OK - ${ELAPSED_MS}ms total" + echo " Prompt tokens: ${PROMPT_TOKENS} (${PROMPT_SPEED} tok/s)" + echo " Completion tokens: ${COMPLETION_TOKENS} (${COMPLETION_SPEED} tok/s)" + echo " Total: ${TOTAL_TOKENS} tokens (${TOTAL_SPEED} tok/s)" fi + + rm -rf "$TMPDIR" done + +echo "" +echo "=== Validation complete ==="