frank-default-large: ubatch 512, disable speculative decoding
- UBATCH_SIZE: 4096 -> 512 (reduce GPU memory pressure) - SPEC_TYPE: ngram-mod -> none (disable speculative decoding entirely)
This commit is contained in:
1 parent
e3bceb0506
commit
1ca3db4ae2
2 files changed
+8
-8
No files matched your search
@@ -59,7 +59,7 @@ BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7770
|
||||
|
||||
# Serving: 401408-token context, TWO slots (parallel 2), batch 4096 /
|
||||
# ubatch 4096, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB
|
||||
# ubatch 512, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB
|
||||
# prompt cache. --fit off: auto-fit segfaults on this model's
|
||||
# hyper-connection/indexer config; -ngl 999 puts every layer on the
|
||||
# SVM-backed GPU pool.
|
||||
@@ -71,7 +71,7 @@ CTX_SIZE=401408
|
||||
PARALLEL=2
|
||||
|
||||
BATCH_SIZE=4096
|
||||
UBATCH_SIZE=4096
|
||||
UBATCH_SIZE=512
|
||||
NUM_THREADS=16
|
||||
THREAD_BATCH=16
|
||||
CACHE_TYPE_K=f16
|
||||
@@ -104,10 +104,10 @@ HSA_OVERRIDE_GFX_VERSION=11.5.1
|
||||
VERBOSITY=4
|
||||
|
||||
|
||||
# Speculative decoding: ngram-mod only. The MTP draft head is retained
|
||||
# Speculative decoding: disabled.
|
||||
DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST
|
||||
DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf
|
||||
SPEC_TYPE=ngram-mod
|
||||
SPEC_TYPE=none
|
||||
SPEC_DRAFT_N_MAX=3
|
||||
SPEC_DRAFT_N_MIN=0
|
||||
SPEC_DRAFT_P_MIN=0.75
|
||||
|
||||
Reference in new issue
Block a user