From df260dd4199adf402ffc23be9710c673cc0af2b9 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 16 Sep 2026 03:54:18 -0700 Subject: [PATCH] frank-medium: pass KV_MEM=0 through as --kv-cache-memory 0 The single-GPU replicas died with torch.OutOfMemoryError during KV allocation (2.16 GiB requested, 116 MiB free, 29.34 GiB already held by PyTorch) because the entrypoint blanked KV_MEM=0 into an omitted flag. The image's patched KV sizing reads an explicit 0 as 'profile this shape'; with the flag absent vLLM sizes the cache from --gpu-memory-utilization alone and overshoots. Passing 0 literally is the shape the bench numbers were measured on, and MAXSEQS gets its default back under set -u. --- dankserver/frank-medium/radiance-entrypoint.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dankserver/frank-medium/radiance-entrypoint.sh b/dankserver/frank-medium/radiance-entrypoint.sh index cb12dab..e6e60b2 100755 --- a/dankserver/frank-medium/radiance-entrypoint.sh +++ b/dankserver/frank-medium/radiance-entrypoint.sh @@ -27,11 +27,14 @@ set -euo pipefail : "${PORT:=7770}" : "${SERVED_NAMES:=frank-medium}" : "${MAXLEN:=262144}" -: "${KV_MEM:=18563072000}" -if [ "$KV_MEM" = "0" ]; then KV_MEM=""; fi # 0 = let vLLM profile (the launcher's rule) -: "${CHUNK:=8192}" -: "${GPU_UTIL:=0.98}" -: "${TP:=2}" +: "${MAXSEQS:=8}" +# KV_MEM=0 is passed THROUGH as --kv-cache-memory 0, which the image's +# patched KV sizing reads as "profile the shape" (the launcher's rule). +# Blanking the variable instead omits the flag and makes vLLM size the cache +# from --gpu-memory-utilization alone: measured on the single-GPU replicas +# that overshoots and the allocation dies with torch.OutOfMemoryError right +# after the weights load (2.16 GiB requested, 116 MiB free). Do not "fix" +# this into an empty expansion. : "${KV_MEM:=18563072000}" # shipped kv-profiles.tsv row 2x7551-32624 @ 8/8192/262144/dflash : "${SPEC_METHOD:=dflash}" if [ "$SPEC_METHOD" = dflash ]; then : "${SPEC:=7}"; else : "${SPEC:=4}"; fi