dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Lower context to 32k, fix duplicate lines, remove prefix caching

Browse Source
This commit is contained in:
Frank committed 2026-08-08 17:03:16 -07:00
1 parent 9227e1a8b9
commit 9d00f36d2b
1 file changed
+1 -2
Download Patch File Download Diff File Expand all files Collapse all files
vllm-awq-qwen36-dflash-ampere.sh
+1 -2
View File
@@ -24,12 +24,11 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
--served-model-name "frank-default-small" \ --served-model-name "frank-default-small" \
--speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 4}' \ --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 4}' \
--max-num-seqs 2 \ --max-num-seqs 2 \
--max-model-len 262144 \ --max-model-len 32768 \
--tensor-parallel-size 1 \ --tensor-parallel-size 1 \
--enable-chunked-prefill \ --enable-chunked-prefill \
--max-num-batched-tokens 1536 \ --max-num-batched-tokens 1536 \
--performance-mode throughput \ --performance-mode throughput \
# --enable-prefix-caching # Disabled: Mamba cache align mode conflicts with prefix caching \\
--enable-auto-tool-choice --tool-call-parser qwen3_xml \ --enable-auto-tool-choice --tool-call-parser qwen3_xml \
--reasoning-parser qwen3 \ --reasoning-parser qwen3 \
--trust-remote-code \ --trust-remote-code \