dank/serverAppConfigs
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-default-small: ncpu-moe=32(8GPU, ~51.1 t/s gen)

Browse Source
This commit is contained in:
Frank
2026-08-20 17:09:23 -07:00
parent 90a8b1c562
commit 99c7ec531f
2 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/frank-default-small.env
+2 -2
View File
@@ -31,8 +31,8 @@ NGRAM_MOD_N_MIN=48
NGRAM_MOD_N_MAX=64 NGRAM_MOD_N_MAX=64
NGRAM_MOD_N_MATCH=24 NGRAM_MOD_N_MATCH=24
# Expert offload: 35 of 40 layers keep MoE weights on CPU; 5 on GPU. # Expert offload: 32 of 40 layers keep MoE weights on CPU; 8 on GPU.
N_CPU_MOE=35 N_CPU_MOE=32
N_GPU_LAYERS=all N_GPU_LAYERS=all
# KV cache on GPU, both K and V at f16. # KV cache on GPU, both K and V at f16.
scripts/serve-frank-small.sh
+1 -1
View File
@@ -36,7 +36,7 @@ fi
: "${CACHE_TYPE_K:=f16}" : "${CACHE_TYPE_K:=f16}"
: "${CACHE_TYPE_V:=f16}" : "${CACHE_TYPE_V:=f16}"
: "${VERBOSITY:=4}" : "${VERBOSITY:=4}"
: "${N_CPU_MOE:=35}" : "${N_CPU_MOE:=32}"
: "${N_GPU_LAYERS:=all}" : "${N_GPU_LAYERS:=all}"
: "${BATCH_SIZE:=4096}" : "${BATCH_SIZE:=4096}"
: "${UBATCH_SIZE:=2048}" : "${UBATCH_SIZE:=2048}"