Baseline: ThinkingCap 27B FP8 on dankclaw with enforce-eager and validation timeout
This commit is contained in:
1 parent
e4fba35e89
commit
2bb1b2ae1d
2 files changed
+7
-1
No files matched your search
@@ -21,6 +21,7 @@ exec "${VENV}/bin/vllm" serve "${MODEL}" \
|
||||
--host 0.0.0.0 \
|
||||
--kv-cache-dtype fp8 \
|
||||
--gpu-memory-utilization 0.80 \
|
||||
--enforce-eager \
|
||||
--max-model-len 32768 \
|
||||
--max-num-batched-tokens 2048 \
|
||||
--max-num-seqs 32 \
|
||||
|
||||
Reference in new issue
Block a user