diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 756f295..0a0cb37 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -25,12 +25,13 @@ LISTEN_PORT=7770 RPC_SERVERS=192.168.0.235:50052 # Model split fractions, one per device in order [RPC0, iGPU]. -# 0.50,0.50 = ~33 of 66 layers on the 3090 - the maximum that fits its -# 24 GB card at the 262144 f16 KV pool (VRAM math 2026-08-16, from live -# journal buffers: 33 layers -> ~23.9 GB used, 714 MiB margin; 34 layers -# -> 244 MiB margin; 35 -> OOM). The 3090 is 2-3x faster per layer, so -# maxing its share minimizes the per-step device time in the pipeline. -TENSOR_SPLIT=0.50,0.50 +# 0.55,0.45 = 35 of 65 blocks on the 3090 (blocks 0-34) - the maximum +# that fits its 24 GB card at the 262144 f16 KV pool. VRAM math from the +# live load (2026-08-17): RPC0 buffer ~14.4 GiB + KV 8192 MiB (8 full- +# attention layers) + RS/compute/overhead -> ~0.8 GiB free. Block 35 is a +# full-attention layer: adding it costs +1 GiB KV -> OOM. GGUF block scan: +# FA at 3,7,...,63 (every 4th, offset 3). +TENSOR_SPLIT=0.55,0.45 # Context pool. 262144 total = 1 parallel slot x 262144 per slot # (n_ctx_seq = n_ctx / n_seq_max, llama-context.cpp:293).