diff --git a/scripts/frank-small.env b/scripts/frank-small.env index acdcc31..dd23504 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -75,24 +75,27 @@ PARALLEL=4 CACHE_TYPE_K=q8_0 CACHE_TYPE_V=q8_0 -# Batch geometry: llama.cpp's own defaults (2048 logical / 512 physical). -BATCH_SIZE=2048 -UBATCH_SIZE=512 +# Batch geometry: 8192 logical / 1024 physical. Measured on this card at +# max length (254k-token prefill / 253k-ctx decode, 4 slots, q8 KV): +# ubatch 512 -> pp 915 t/s, tg@250k 90-204; ubatch 1024 -> pp 962, +# tg 257-302. Larger ubatches (2048) OOM the MTP draft context's pp +# compute buffer (~3.3 GiB against 2.2 GiB headroom) and must not be +# tried. Idle VRAM is 23,300 MiB at this geometry. +BATCH_SIZE=8192 +UBATCH_SIZE=1024 # CPU threads: this service runs entirely on the GPU (NGL=999), so threads # cover tokenization, sampling and the HTTP layer only. Deliberately modest # -- this host's CPU is shared with halogen's engine, which is CPU-hungry. NUM_THREADS=8 THREAD_BATCH=8 - -# Every layer on the 3090, with flash attention (required for q8 KV). -NGL=999 -FLASH_ATTN=on - # Speculative decoding: the MTP sidecar (draft context) chained with the # draftless ngram-mod matcher. Draft depth and busy-wait level are made -# explicit here for tuning (both were the implicit build defaults: 3 and 50; -# the Gemma4 sidecar carries 4 nextn layers). +# explicit here (both were the implicit build defaults: 3 and 50). +# Measured at 250k ctx: depth 4 + poll 100 DECODED WORSE (230-257 t/s) +# than depth 3 + poll 50 (257-302 t/s) — the longer verification round +# costs more attention re-read at long context than the extra accepted +# token earns. Keep 3/50. SPEC_TYPE=draft-mtp,ngram-mod SPEC_DRAFT_N_MAX=3 POLL_LEVEL=50