diff --git a/scripts/frank-medium.env b/scripts/frank-medium.env index 62d88c3..cfcdc14 100644 --- a/scripts/frank-medium.env +++ b/scripts/frank-medium.env @@ -245,11 +245,39 @@ THREAD_BATCH=8 # f16 K/V: an agentic checkpoint is the wrong place to spend KV precision. CACHE_TYPE_K=f16 CACHE_TYPE_V=f16 -# Context checkpoints: 0 is the build's default -- no per-slot KV snapshots, so -# a re-sent or rewound conversation re-prefills instead of rewinding. Stated +# Context checkpoints: 0 -- no per-slot KV snapshots at all. The build's +# default is 32 (this binary's --help), so the 0 below is a deliberate +# deviation, not an inherited default: a checkpoint is a VRAM->host-RAM copy +# of part of a slot's KV, taken so a partially-matching prompt can be restored +# instead of re-prefilled. This endpoint trades that away -- its reuse comes +# from the slot prompt and from the RAM-side prompt cache below. Stated # explicitly so the tunable stays visible rather than merely absent. CTX_CHECKPOINTS=0 -CACHE_RAM=8192 +# PROMPT CACHE, HOST RAM -- this is the one cache tunable that is NOT VRAM. +# --cache-ram caps the total bytes of saved prompt states +# (server_prompt_cache, tools/server/server-task.cpp): update() evicts +# oldest-first past the cap, and alloc() REFUSES outright any single state +# bigger than the whole cap. A state is the serialized KV of a whole sequence +# plus its DeltaNet state plus the MTP draft context's own KV, so at this shape +# it costs 68 KiB per token -- +# 16 full-attention layers x 4 kv heads x 256 dim x 2 B x 2 (K,V) = 64 KiB +# plus 4096 B for the draft context (one nextn layer) +# -- which makes the cap a TOKEN budget in disguise: 8192 MiB held ~120k +# tokens, 16384 MiB holds ~240k. +# +# DOUBLED 8192 -> 16384 on 2026-09-18 because the old cap was BINDING, not +# theoretical: alloc() refused 137 states in the two days from 2026-09-16 with +# - prompt state size 11388.990 MiB exceeds cache size limit 8192.000 MiB, skipping +# i.e. 11.0-11.4 GiB states -- conversations of ~170k tokens at the law above -- +# so every deep conversation was silently uncacheable and re-prefilled from +# scratch on its next turn. At 16384 a FULL 196608-token slot (~12.8 GiB) fits +# with ~3 GiB of the budget spare for a second entry. +# +# The cost is host RAM, allocated on demand and never pinned (plain heap +# vectors), so no VRAM number above moves. dankserver has 62 GiB beside the +# media-stack co-tenants, so the cache is bounded rather than unlimited (-1): +# the seam test refuses a budget over half the host's RAM. +CACHE_RAM=16384 # Sampling defaults for requests that send none -- the values the vLLM # predecessor served on this endpoint (`--override-generation-config`), so a diff --git a/serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md b/serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md index ba887c3..dbb75bc 100644 --- a/serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md +++ b/serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md @@ -149,7 +149,7 @@ Deployed flags: | `--gpu-layers` / `--fit` | `999` / `off` | everything on the cards; auto-fit must not resize a context we set | | `--ctx-size` / `--parallel` | `393216` / `2` | the pool over two slots of 196608; `--parallel` **divides** the context. 1.5x the trained 262144 window, and the GGUF has no rope scaling — see the 2026-09-17 section | | `--batch-size` / `--ubatch-size` | `6144` / `512` | measured: ubatch 512 is the prefill optimum *and* the cheapest point on the curve; the logical batch is throughput- and VRAM-neutral. See [[frank-medium-batch-ubatch]] | -| `--ctx-checkpoints` / `--cache-ram` | `0` / `8192` | no per-slot KV snapshots (the build's default); the RAM-side prompt cache is capped at 8 GiB | +| `--ctx-checkpoints` / `--cache-ram` | `0` / `16384` | no per-slot KV snapshots (the build's default is **32** — this is a deliberate deviation, not an inherited default); the RAM-side prompt cache is capped at 16 GiB, which is what lets a full 196608-token slot state be cached at all (2026-09-18, see below) | | `--flash-attn` / cache types | `on` / `f16` K and V | an agentic checkpoint is the wrong place to spend KV precision | | `--spec-type` / `--spec-draft-n-max` | `draft-mtp` / `3` | the model's own head, `shares_model` — no sidecar file, no second model copy; n_max 3 is the measured MTP winner (32.4 tok/s vs 26.7 at n_max 7). The DFlash2 sidecar stays on disk: `SPEC_TYPE=draft-dflash` brings the 43.5 tok/s chain back at ~1.1 GiB on ROCm1 | | `--spec-draft-device` | `ROCm1` | the drafter rides the card that ends up holding the target's last layers | @@ -380,6 +380,42 @@ to give. A chat completion answers in 1.4 s end-to-end; MTP acceptance runs 0.51–0.91 (mean len 2.7–3.7) on the live traffic. +## 2026-09-18: the prompt cache doubles to 16 GiB + +`--cache-ram` is the one cache tunable on this service that is **not VRAM**: it +caps the total bytes of saved prompt states in host RAM (`server_prompt_cache`, +`tools/server/server-task.cpp`). `update()` evicts oldest-first past the cap, +and `alloc()` **refuses outright** any single state larger than the whole cap — +with one `WRN` line and nothing else: + +``` + - prompt state size 11388.990 MiB exceeds cache size limit 8192.000 MiB, skipping +``` + +That line appeared **137 times** in the two days from 2026-09-16. At 8192 MiB +every conversation past ~170k tokens was uncacheable — the state was refused, +the slot's KV went with it, and the next turn re-prefilled the whole thing. The +cap is a token budget in disguise, because a state is the serialized KV of a +whole sequence plus its constant-size DeltaNet state plus the MTP draft +context's own KV: + +| Component | Per token | +|---|---| +| 16 full-attention layers x 4 kv heads x 256 dim x 2 B x 2 (K,V) | 64 KiB | +| MTP draft context (one nextn layer, same arithmetic) | 4096 B | +| **state law** | **68 KiB** | + +| Cap | Tokens storable | Fits one 196608-token slot (~12.8 GiB)? | +|---|---|---| +| 8192 MiB (until 2026-09-18) | ~120k | no — refused, then re-prefilled | +| **16384 MiB (deployed)** | ~240k | **yes**, with ~3 GiB of budget spare | + +The cost is host RAM, allocated on demand and never pinned, so no VRAM figure +above moves. dankserver has 62 GiB beside the media-stack co-tenants, which is +why the cache is bounded rather than set to `-1` (unlimited): the seam test in +`tests/frank_medium_unit.bats` gates both ends — the cap must clear one full +slot's state, and it may not claim more than half the host's RAM. + ## Verified serving behaviour (2026-09-16) - Load: `model loaded` ~25 s after start with a warm page cache, then diff --git a/tests/frank_medium_unit.bats b/tests/frank_medium_unit.bats index 727f3c2..188db47 100755 --- a/tests/frank_medium_unit.bats +++ b/tests/frank_medium_unit.bats @@ -248,6 +248,38 @@ print('draft-model wiring ok') [ "$ctx" -le 393216 ] } +@test "the host-RAM prompt cache can hold a full slot's state" { + # --cache-ram is the one cache tunable that is NOT VRAM: it caps the total + # bytes of saved prompt states in host RAM (server_prompt_cache, in + # tools/server/server-task.cpp). alloc() REFUSES any single state larger + # than the whole cap -- silently, with one WRN line -- so a budget below + # one slot's full context makes every deep conversation uncacheable and + # every turn past it re-prefill from scratch. That was the deployed state + # on 2026-09-18: 137 refusals of 11.0-11.4 GiB states against the 8192 MiB + # cap, in two days of journal. + # + # The state law at this shape, per token of sequence: 16 full-attention + # layers x 4 kv heads x 256 dim x 2 B x 2 (K,V) = 64 KiB, plus 4096 B for + # the MTP draft context's own KV. The DeltaNet state is constant-size and + # is left out of the sum -- which only makes this gate more conservative. + local ctx parallel cache_mib per_token slot_state host_mib + ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" + parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")" + cache_mib="$(sed -nE 's/^CACHE_RAM=([0-9]+)$/\1/p' "$ENV_FILE")" + [ -n "$ctx" ] && [ -n "$parallel" ] && [ -n "$cache_mib" ] + per_token=$(( (16 * 4 * 256 * 2 * 2) + (4 * 256 * 2 * 2) )) + slot_state=$(( per_token * (ctx / parallel) )) + # one full slot must fit, or the cache is decorative + [ "$cache_mib" -ge $(( slot_state / 1024 / 1024 )) ] + # ...and this budget is host RAM: never more than half of dankserver's + # 62220 MiB, which the media-stack co-tenants and the page cache share. + host_mib=62220 + [ "$cache_mib" -le $(( host_mib / 2 )) ] + # the arithmetic above is about nothing unless the flag goes out + grep -q -- '--cache-ram "$CACHE_RAM"' "$SERVE" + grep -q -- '--cache-prompt' "$SERVE" +} + @test "vision is enabled: the BF16 projector loads with the model" { grep -q '^NO_MMPROJ=0$' "$ENV_FILE" grep -q '^MM_PROJ_FILE=mmproj-BF16.gguf$' "$ENV_FILE"