frank-default-large: ubatch 512, disable speculative decoding

- UBATCH_SIZE: 4096 -> 512 (reduce GPU memory pressure)
- SPEC_TYPE: ngram-mod -> none (disable speculative decoding entirely)
This commit is contained in:
Frank committed 2026-08-30 01:24:40 -07:00
1 parent e3bceb0506
commit 1ca3db4ae2
2 files changed
+8 -8

No files matched your search

+4 -4
View File
@@ -59,7 +59,7 @@ BIND_ADDR=0.0.0.0
LISTEN_PORT=7770
# Serving: 401408-token context, TWO slots (parallel 2), batch 4096 /
# ubatch 4096, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB
# ubatch 512, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB
# prompt cache. --fit off: auto-fit segfaults on this model's
# hyper-connection/indexer config; -ngl 999 puts every layer on the
# SVM-backed GPU pool.
@@ -71,7 +71,7 @@ CTX_SIZE=401408
PARALLEL=2
BATCH_SIZE=4096
UBATCH_SIZE=4096
UBATCH_SIZE=512
NUM_THREADS=16
THREAD_BATCH=16
CACHE_TYPE_K=f16
@@ -104,10 +104,10 @@ HSA_OVERRIDE_GFX_VERSION=11.5.1
VERBOSITY=4
# Speculative decoding: ngram-mod only. The MTP draft head is retained
# Speculative decoding: disabled.
DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST
DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf
SPEC_TYPE=ngram-mod
SPEC_TYPE=none
SPEC_DRAFT_N_MAX=3
SPEC_DRAFT_N_MIN=0
SPEC_DRAFT_P_MIN=0.75