diff --git a/scripts/frank-small.env b/scripts/frank-small.env index 42d8431..9304e47 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -74,6 +74,12 @@ CTX_SIZE=1048576 PARALLEL=4 CACHE_TYPE_K=q8_0 CACHE_TYPE_V=q8_0 +# CPU RAM cache cap in MiB (build default 8192). This host is memory-tight +# (frank-large's 70 GiB GTT plus the rest of the stack), and Gemma4 runs +# fully on the 3090 (-ngl 999), so the CPU-side cache is not the hot path; +# cap it at 2048 to give back ~6 GiB of LPDDR. Gemma4 is not a DeltaNet +# model, so the >12.8 GiB CACHE_RAM floor does not apply. +CACHE_RAM=2048 # Batch geometry: 8192 logical / 1024 physical. Measured on this card at # max length (254k-token prefill / 253k-ctx decode, 4 slots, q8 KV): diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 0a9aade..5b37367 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -58,6 +58,7 @@ fi : "${PARALLEL:=1}" : "${CACHE_TYPE_K:=f16}" : "${CACHE_TYPE_V:=f16}" +: "${CACHE_RAM:=8192}" : "${BATCH_SIZE:=2048}" : "${UBATCH_SIZE:=512}" # Decode-shape knobs: the MTP drafter's per-round proposal length (build @@ -130,6 +131,7 @@ exec "$LLAMA_BIN" \ --poll "$POLL_LEVEL" \ --cache-type-k "$CACHE_TYPE_K" \ --cache-type-v "$CACHE_TYPE_V" \ + --cache-ram "$CACHE_RAM" \ "${SPEC_ARGS[@]}" \ --jinja \ --metrics \ diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 6b77526..1f729f4 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -45,6 +45,12 @@ ENV_LARGE="$REPO_ROOT/scripts/frank-large.env" grep -q -- '--metrics' "$SERVE" } +@test "frank-small caps its CPU RAM cache below the build default" { + # Without the flag the 8192 MiB default returns and the env cap is a + # silent no-op; this memory-tight host needs the cap to hold. + grep -q -- '--cache-ram "$CACHE_RAM"' "$SERVE" +} + @test "frank-small drafts with the MTP sidecar chained behind ngram-mod" { grep -q '^SPEC_TYPE=draft-mtp,ngram-mod$' "$ENV_FILE" grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE"