diff --git a/dankserver/frank-medium/radiance-entrypoint.sh b/dankserver/frank-medium/radiance-entrypoint.sh index cb12dab..e6e60b2 100755 --- a/dankserver/frank-medium/radiance-entrypoint.sh +++ b/dankserver/frank-medium/radiance-entrypoint.sh @@ -27,11 +27,14 @@ set -euo pipefail : "${PORT:=7770}" : "${SERVED_NAMES:=frank-medium}" : "${MAXLEN:=262144}" -: "${KV_MEM:=18563072000}" -if [ "$KV_MEM" = "0" ]; then KV_MEM=""; fi # 0 = let vLLM profile (the launcher's rule) -: "${CHUNK:=8192}" -: "${GPU_UTIL:=0.98}" -: "${TP:=2}" +: "${MAXSEQS:=8}" +# KV_MEM=0 is passed THROUGH as --kv-cache-memory 0, which the image's +# patched KV sizing reads as "profile the shape" (the launcher's rule). +# Blanking the variable instead omits the flag and makes vLLM size the cache +# from --gpu-memory-utilization alone: measured on the single-GPU replicas +# that overshoots and the allocation dies with torch.OutOfMemoryError right +# after the weights load (2.16 GiB requested, 116 MiB free). Do not "fix" +# this into an empty expansion. : "${KV_MEM:=18563072000}" # shipped kv-profiles.tsv row 2x7551-32624 @ 8/8192/262144/dflash : "${SPEC_METHOD:=dflash}" if [ "$SPEC_METHOD" = dflash ]; then : "${SPEC:=7}"; else : "${SPEC:=4}"; fi