diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index cf5a547..099133d 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -4,29 +4,29 @@ MODEL_DIR=/home/dank/models ALIAS=frank-default-small HOST=0.0.0.0 PORT=7770 -# 3 parallel slots x 200000 ctx each = 600000 total context pool. -# (Reduced from 262144/stream per operator: allows the larger batch to fit VRAM.) -CTX_SIZE=600000 -PARALLEL=3 +# 4 parallel slots x 200000 ctx each = 800000 total context pool. +# (Increased from 3×200k to add a 4th parallel stream.) +CTX_SIZE=800000 +PARALLEL=4 +# Spec: ngram-mod. n_min=48, n_max=64, n_match=24 (defaults). 8 expert +# layers back on GPU; rest on CPU. Batch 8192 / ubatch 2048. SPEC_TYPE=ngram-mod -# Ngram-mod speculative decoding: look back this many tokens to guess the -# next tokens. Defaults (48/64/24) are a reasonable starting point; tune if -# you want more or fewer draft lookahead tokens. NGRAM_MOD_N_MIN=48 NGRAM_MOD_N_MAX=64 NGRAM_MOD_N_MATCH=24 +# Expert offload: 32 of 40 layers keep MoE weights on CPU; remaining +# 8 layers have their expert weights resident on GPU for faster routing. +N_CPU_MOE=32 +N_GPU_LAYERS=all +# KV cache on GPU, K quantized to q8_0, V stays f16 (user-locked decision). CACHE_TYPE_K=q8_0 CACHE_TYPE_V=f16 -# All 40 layers keep their MoE expert weights on CPU (targeted expert offload). -N_CPU_MOE=40 -# Every non-expert tensor (attention, norms, embedding, LM head) goes to GPU. -N_GPU_LAYERS=all -# Larger batches for prefill throughput. The bigger physical batch grows GPU -# activation buffers; ctx-checkpoints parks attention checkpoints in RAM to -# hold peak GPU memory down so the larger ubatch fits without OOM. -BATCH_SIZE=3072 -UBATCH_SIZE=1536 +# Store SWA checkpoints in RAM to hold peak GPU memory down for the +# larger batch/ubatch (leaves headroom for ctx-checkpoints operation). CTX_CHECKPOINTS=4 +# Larger batches / ubatch for big prefill workloads. +BATCH_SIZE=8192 +UBATCH_SIZE=2048 # Load the model into RAM rather than lazy-mapping it. The MoE experts live on # CPU, so eager RAM load avoids page faults during decode (uses ~31GB RAM). NO_MMAP=1