frank-default-small: ctx=786432 max, parallel=3, batch=6144

scripts/serve-frank-small.sh

@@ -27,8 +27,8 @@ fi
 : "${ALIAS:=frank-default-small}"
 : "${BIND_ADDR:=0.0.0.0}"
 : "${LISTEN_PORT:=7770}"
-: "${CTX_SIZE:=700000}"
-: "${PARALLEL:=4}"
+: "${CTX_SIZE:=786432}"
+: "${PARALLEL:=3}"
 : "${SPEC_TYPE:=ngram-mod}"
 : "${NGRAM_MOD_N_MIN:=48}"
 : "${NGRAM_MOD_N_MAX:=64}"
@@ -38,7 +38,7 @@ fi
 : "${VERBOSITY:=4}"
 : "${N_CPU_MOE:=36}"
 : "${N_GPU_LAYERS:=all}"
-: "${BATCH_SIZE:=8192}"
+: "${BATCH_SIZE:=6144}"
 : "${UBATCH_SIZE:=3072}"
 : "${CTX_CHECKPOINTS:=4}"
 : "${NO_MMAP:=1}"