frank-medium: double the host-RAM prompt cache to 16 GiB
--cache-ram caps the total bytes of saved prompt states in HOST RAM, and alloc() refuses any single state larger than the whole cap. That refusal was firing in production: 137 times in the two days from 2026-09-16, at 11.0-11.4 GiB per state, so every conversation past ~170k tokens was uncacheable and re-prefilled from scratch on its next turn. The state law at this shape is 68 KiB/token (16 full-attention layers x 4 kv heads x 256 dim x 2 B x 2 for K/V, plus the MTP draft context's 4096 B), so 8192 MiB held ~120k tokens and 16384 holds ~240k: a FULL 196608-token slot state (~12.8 GiB) now fits with ~3 GiB spare. Host RAM only, allocated on demand and never pinned, so the VRAM fit is untouched. New seam: the cap must clear one full slot's state and stay under half the host's RAM (it fails on the old 8192 and passes on 16384). Also corrects the adjacent claim that 0 is the build's default for --ctx-checkpoints -- this binary's --help says 32.
This commit is contained in:
1 parent
cfabe07959
commit
1acca430a7
3 files changed
+100
-4
No files matched your search
@@ -248,6 +248,38 @@ print('draft-model wiring ok')
|
||||
[ "$ctx" -le 393216 ]
|
||||
}
|
||||
|
||||
@test "the host-RAM prompt cache can hold a full slot's state" {
|
||||
# --cache-ram is the one cache tunable that is NOT VRAM: it caps the total
|
||||
# bytes of saved prompt states in host RAM (server_prompt_cache, in
|
||||
# tools/server/server-task.cpp). alloc() REFUSES any single state larger
|
||||
# than the whole cap -- silently, with one WRN line -- so a budget below
|
||||
# one slot's full context makes every deep conversation uncacheable and
|
||||
# every turn past it re-prefill from scratch. That was the deployed state
|
||||
# on 2026-09-18: 137 refusals of 11.0-11.4 GiB states against the 8192 MiB
|
||||
# cap, in two days of journal.
|
||||
#
|
||||
# The state law at this shape, per token of sequence: 16 full-attention
|
||||
# layers x 4 kv heads x 256 dim x 2 B x 2 (K,V) = 64 KiB, plus 4096 B for
|
||||
# the MTP draft context's own KV. The DeltaNet state is constant-size and
|
||||
# is left out of the sum -- which only makes this gate more conservative.
|
||||
local ctx parallel cache_mib per_token slot_state host_mib
|
||||
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
cache_mib="$(sed -nE 's/^CACHE_RAM=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
[ -n "$ctx" ] && [ -n "$parallel" ] && [ -n "$cache_mib" ]
|
||||
per_token=$(( (16 * 4 * 256 * 2 * 2) + (4 * 256 * 2 * 2) ))
|
||||
slot_state=$(( per_token * (ctx / parallel) ))
|
||||
# one full slot must fit, or the cache is decorative
|
||||
[ "$cache_mib" -ge $(( slot_state / 1024 / 1024 )) ]
|
||||
# ...and this budget is host RAM: never more than half of dankserver's
|
||||
# 62220 MiB, which the media-stack co-tenants and the page cache share.
|
||||
host_mib=62220
|
||||
[ "$cache_mib" -le $(( host_mib / 2 )) ]
|
||||
# the arithmetic above is about nothing unless the flag goes out
|
||||
grep -q -- '--cache-ram "$CACHE_RAM"' "$SERVE"
|
||||
grep -q -- '--cache-prompt' "$SERVE"
|
||||
}
|
||||
|
||||
@test "vision is enabled: the BF16 projector loads with the model" {
|
||||
grep -q '^NO_MMPROJ=0$' "$ENV_FILE"
|
||||
grep -q '^MM_PROJ_FILE=mmproj-BF16.gguf$' "$ENV_FILE"
|
||||
|
||||
Reference in new issue
Block a user