diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 7226efc..844451f 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -25,14 +25,16 @@ LISTEN_PORT=7770 RPC_SERVERS=192.168.0.235:50052 # Model split fractions, one per device in order [RPC0, iGPU]. -# 0.54,0.46 = 35 of 65 blocks on the 3090 (blocks 0-34) - the maximum. -# Split math (exact GGUF sizes, 2026-08-17): layer pool 27720 MB + MTP -# head; llama.cpp assigns RPC0 the blocks while cumulative weight <= the -# fraction of the pool. cum(34) = 53.83%, cum(35) = 55.31%. Measured: -# 0.50 -> 32 blocks, 0.55 -> 36 blocks (9th FA block 35 -> KV 9216 MiB -# -> cudaMalloc OOM by ~740 MiB). FA blocks at 3,7,...,63; KV stays -# 8192 MiB (8 FA) at 35 blocks. Predicted free ~0.65 GiB. -TENSOR_SPLIT=0.54,0.46 +# 0.53,0.47 = 34 of 65 blocks on the 3090 (blocks 0-33) - the maximum +# provably-safe split, measured 2026-08-17 (two failed attempts): +# - 0.50: 32 blocks, KV 8192 MiB (8 FA) -> healthy, 1477 MiB free. +# - 0.54: weights 35 blocks but KV jumps to 9216 MiB (9th FA block) -> +# cudaMalloc OOM by ~740 MiB. The KV-per-device count rounds the +# split fraction over 66 layers, independent of the weight boundary; +# the flip to 9 FA sits at f ~= 0.538, just below the weight +# threshold for block 35. 0.53 keeps 8 FA under every rounding rule. +# GGUF block scan: FA at 3,7,...,63; linear blocks 431 MB, FA 410 MB. +TENSOR_SPLIT=0.53,0.47 # Context pool. 262144 total = 1 parallel slot x 262144 per slot # (n_ctx_seq = n_ctx / n_seq_max, llama-context.cpp:293).