frank-default-large: ubatch 512, disable speculative decoding

- UBATCH_SIZE: 4096 -> 512 (reduce GPU memory pressure)
- SPEC_TYPE: ngram-mod -> none (disable speculative decoding entirely)
This commit is contained in:
Frank committed 2026-08-30 01:24:40 -07:00
1 parent e3bceb0506
commit 1ca3db4ae2
2 files changed
+8 -8

No files matched your search

+4 -4
View File
@@ -47,8 +47,8 @@ fi
: "${LISTEN_PORT:=7770}"
: "${CTX_SIZE:=401408}"
: "${PARALLEL:=2}"
: "${BATCH_SIZE:=4096}"
: "${UBATCH_SIZE:=4096}"
:: "${BATCH_SIZE:=4096}"
:: "${UBATCH_SIZE:=512}"
: "${NUM_THREADS:=16}"
: "${THREAD_BATCH:=16}"
: "${CACHE_TYPE_K:=f16}"
@@ -67,8 +67,8 @@ fi
: "${PRESENCE_PENALTY:=1.5}"
: "${REASONING:=on}"
: "${REASONING_FORMAT:=deepseek}"
: "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}"
: "${SPEC_TYPE:=ngram-mod}"
:: "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}"
:: "${SPEC_TYPE:=none}"
: "${DRAFT_MODEL_DIR:=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST}"
: "${DRAFT_MODEL_FILE:=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf}"
: "${SPEC_DRAFT_N_MAX:=3}"