Frank
e45f457ad4
Bigger logical/physical batches for prefill throughput. Enable --ctx-checkpoints to park attention checkpoints in RAM, holding peak GPU memory down so the larger ubatch fits the 24GB VRAM alongside 3x262144 KV without OOM. Weight split unchanged (experts CPU, non-experts GPU).
25 lines
876 B
Bash
25 lines
876 B
Bash
# Tunables for frank-default-small (HauhauCS Q6_K_P + DFlash draft, dankserver)
|
|
# Runtime: llama-server b10108; RTX 3090 24GB; Ryzen 7 7700X; 62GB RAM.
|
|
MODEL_DIR=/home/dank/models
|
|
ALIAS=frank-default-small
|
|
HOST=0.0.0.0
|
|
PORT=7770
|
|
# 3 parallel slots x 262144 ctx each = 786432 total context pool.
|
|
CTX_SIZE=786432
|
|
PARALLEL=3
|
|
SPEC_TYPE=draft-dflash
|
|
SPEC_N_MAX=7
|
|
CACHE_TYPE_K=q8_0
|
|
CACHE_TYPE_V=f16
|
|
# All 40 layers keep their MoE expert weights on CPU (targeted expert offload).
|
|
N_CPU_MOE=40
|
|
# Every non-expert tensor (attention, norms, embedding, LM head) goes to GPU.
|
|
N_GPU_LAYERS=all
|
|
# Larger batches for prefill throughput. The bigger physical batch grows GPU
|
|
# activation buffers; ctx-checkpoints parks attention checkpoints in RAM to
|
|
# hold peak GPU memory down so the larger ubatch fits without OOM.
|
|
BATCH_SIZE=3072
|
|
UBATCH_SIZE=1536
|
|
CTX_CHECKPOINTS=4
|
|
THREADS=8
|
|
THREADS_HTTP=8 |