frank-small: double the compute-buffer pair to 4096/2048
BATCH_SIZE 2048 -> 4096, UBATCH_SIZE 1024 -> 2048. Why the ubatch move is right on THIS card: the prefill curve is card-specific. On the 2x R9700 layer-split sibling it peaks at ubatch 512 and every step up costs GiB for no speed; on this CUDA 3090 it climbs monotonically to 2048 (710 -> 1188 t/s on the 35B MoE sweep, 1944 -> 2423 t/s on the Ornith 8k sweep) with decode flat to under 1 t/s on every rung. The logical batch is free either way: two sweeps agree that at a fixed ubatch, 2048/4096/8192 give identical prefill speed and identical peak VRAM, because the compute buffers are sized by the ubatch alone. The risk is all in the ubatch, and it is documented rather than assumed: the step cost ~1,244 MiB of peak graph VRAM on the 35B MoE, the shape this replaces is 2,815 MiB free, and the Q5_K_M predecessor recorded "ubatch 2048 failed the scratch boot" with MORE free than that. Compute buffers are allocated at graph capture during boot, so a failure is loud and fast; the step-down rungs (4096/1536, then 4096/1024) are named in the env. The seam stops hard-coding a rung: serve_argv now carries BATCH_SIZE and UBATCH_SIZE the way it already carried CTX_SIZE et al., and the pair test asserts the argv follows the shipped env plus the llama.cpp invariant ubatch <= batch. Verified both ways: retuning the env to 8192/4096 keeps the seam green, inverting the pair to 2048/4096 turns it red.
This commit is contained in:
@@ -52,6 +52,8 @@ BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7771
|
||||
CTX_SIZE=$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
PARALLEL=$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
BATCH_SIZE=$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
UBATCH_SIZE=$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
CACHE_TYPE_K=$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")
|
||||
CACHE_TYPE_V=$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")
|
||||
NGL=$(sed -nE 's/^NGL=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
@@ -150,14 +152,31 @@ argv_value() {
|
||||
[ $((native * 33 / 1048576 + 9)) -gt 24 ]
|
||||
}
|
||||
|
||||
@test "the compute-buffer pair is explicit" {
|
||||
# This endpoint ran on llama.cpp's own defaults until 2026-10-03; the
|
||||
# fleet now pins 2048/1024 so the number a benchmark attributes the
|
||||
# prefill rate to is a repo value, not an engine default nobody wrote down.
|
||||
@test "the compute-buffer pair follows the env file and stays self-consistent" {
|
||||
# This endpoint ran on llama.cpp's own defaults until 2026-10-03, so the
|
||||
# contract is: whatever rung the repo carries is the rung the engine is
|
||||
# exec'd with — the number a benchmark attributes a prefill rate to must be
|
||||
# a repo value, never an engine default nobody wrote down. Asserted against
|
||||
# the env rather than a literal so a retune moves one file, not two.
|
||||
local env_batch env_ubatch
|
||||
env_batch="$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
env_ubatch="$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --batch-size)" = "2048" ]
|
||||
[ "$(argv_value --ubatch-size)" = "1024" ]
|
||||
[ "$(argv_value --batch-size)" = "$env_batch" ]
|
||||
[ "$(argv_value --ubatch-size)" = "$env_ubatch" ]
|
||||
# llama.cpp refuses ubatch > batch, so a retune that inverts the pair has
|
||||
# to fail here instead of at boot on a live endpoint.
|
||||
[ "$env_batch" -ge "$env_ubatch" ]
|
||||
|
||||
# And an env that names neither flag still ships an explicit pair: the
|
||||
# serve script's fallbacks are written-down values, not blanks the engine
|
||||
# fills in silently.
|
||||
sed -i '/^BATCH_SIZE=/d;/^UBATCH_SIZE=/d' "$STAGE/test.env"
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
argv_value --batch-size | grep -qE '^[0-9]+$'
|
||||
argv_value --ubatch-size | grep -qE '^[0-9]+$'
|
||||
}
|
||||
|
||||
@test "no speculative draft ships by default" {
|
||||
|
||||
Reference in New Issue
Block a user