dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-default-large: ubatch 512, disable speculative decoding

Browse Source 
- UBATCH_SIZE: 4096 -> 512 (reduce GPU memory pressure)
- SPEC_TYPE: ngram-mod -> none (disable speculative decoding entirely)
This commit is contained in:
Frank committed 2026-08-30 01:24:40 -07:00
1 parent e3bceb0506
commit 1ca3db4ae2
2 files changed
+8 -8
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/frank-default-large.env
+4 -4
View File
@@ -59,7 +59,7 @@ BIND_ADDR=0.0.0.0
LISTEN_PORT=7770 LISTEN_PORT=7770
# Serving: 401408-token context, TWO slots (parallel 2), batch 4096 / # Serving: 401408-token context, TWO slots (parallel 2), batch 4096 /
# ubatch 4096, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB # ubatch 512, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB
# prompt cache. --fit off: auto-fit segfaults on this model's # prompt cache. --fit off: auto-fit segfaults on this model's
# hyper-connection/indexer config; -ngl 999 puts every layer on the # hyper-connection/indexer config; -ngl 999 puts every layer on the
# SVM-backed GPU pool. # SVM-backed GPU pool.
@@ -71,7 +71,7 @@ CTX_SIZE=401408
PARALLEL=2 PARALLEL=2
BATCH_SIZE=4096 BATCH_SIZE=4096
UBATCH_SIZE=4096 UBATCH_SIZE=512
NUM_THREADS=16 NUM_THREADS=16
THREAD_BATCH=16 THREAD_BATCH=16
CACHE_TYPE_K=f16 CACHE_TYPE_K=f16
@@ -104,10 +104,10 @@ HSA_OVERRIDE_GFX_VERSION=11.5.1
VERBOSITY=4 VERBOSITY=4
# Speculative decoding: ngram-mod only. The MTP draft head is retained # Speculative decoding: disabled.
DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST
DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf
SPEC_TYPE=ngram-mod SPEC_TYPE=none
SPEC_DRAFT_N_MAX=3 SPEC_DRAFT_N_MAX=3
SPEC_DRAFT_N_MIN=0 SPEC_DRAFT_N_MIN=0
SPEC_DRAFT_P_MIN=0.75 SPEC_DRAFT_P_MIN=0.75
scripts/serve-frank-large.sh
+4 -4
View File
@@ -47,8 +47,8 @@ fi
: "${LISTEN_PORT:=7770}" : "${LISTEN_PORT:=7770}"
: "${CTX_SIZE:=401408}" : "${CTX_SIZE:=401408}"
: "${PARALLEL:=2}" : "${PARALLEL:=2}"
: "${BATCH_SIZE:=4096}" :: "${BATCH_SIZE:=4096}"
: "${UBATCH_SIZE:=4096}" :: "${UBATCH_SIZE:=512}"
: "${NUM_THREADS:=16}" : "${NUM_THREADS:=16}"
: "${THREAD_BATCH:=16}" : "${THREAD_BATCH:=16}"
: "${CACHE_TYPE_K:=f16}" : "${CACHE_TYPE_K:=f16}"
@@ -67,8 +67,8 @@ fi
: "${PRESENCE_PENALTY:=1.5}" : "${PRESENCE_PENALTY:=1.5}"
: "${REASONING:=on}" : "${REASONING:=on}"
: "${REASONING_FORMAT:=deepseek}" : "${REASONING_FORMAT:=deepseek}"
: "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}" :: "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}"
: "${SPEC_TYPE:=ngram-mod}" :: "${SPEC_TYPE:=none}"
: "${DRAFT_MODEL_DIR:=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST}" : "${DRAFT_MODEL_DIR:=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST}"
: "${DRAFT_MODEL_FILE:=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf}" : "${DRAFT_MODEL_FILE:=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf}"
: "${SPEC_DRAFT_N_MAX:=3}" : "${SPEC_DRAFT_N_MAX:=3}"