frank-default-large: cap CTX_SIZE at 32768 (author quick-start) — 87 GiB fully-resident model + context buffer OOMs HIP heap (50.8 GiB at 131072, 103 GiB at 262144)

scripts/frank-default-large.env

@@ -60,11 +60,14 @@ LISTEN_PORT=7770
 # hyper-connection/indexer config; -ngl 999 puts every layer on the
 # SVM-backed GPU pool.
 #
-# CTX_SIZE is capped at 131072: the fully GPU-resident 87 GiB model plus a
-# 262144-token f16 KV cache and the worst-case compute buffer exceed the
-# Strix Halo SVM pool — verified live (cudaMalloc 103176 MiB OOM at
-# 262144). 131072 is the proven STRIX-family value on this box.
-CTX_SIZE=131072
+# CTX_SIZE is capped at 32768 (the author's own quick-start). This model is
+# fully GPU-resident (87.06 GiB) — the PLE n-gram table no longer streams
+# from RAM like STRIX_LEAN did, so there is no room for a huge context
+# buffer: HIP heap is 119.3 GiB, and the context buffer alone is 50.8 GiB
+# at 131072 and 103 GiB at 262144 (verified live:
+# `cudaMalloc … failed: out of memory` on first graph alloc). At 32768 the
+# context buffer drops to ~13 GiB (~100 GiB total) and loads clean.
+CTX_SIZE=32768
 PARALLEL=1
 BATCH_SIZE=4096
 UBATCH_SIZE=4096

scripts/serve-frank-large.sh

@@ -45,7 +45,7 @@ fi
 : "${ALIAS:=frank-default-large}"
 : "${BIND_ADDR:=0.0.0.0}"
 : "${LISTEN_PORT:=7770}"
-: "${CTX_SIZE:=131072}"
+: "${CTX_SIZE:=32768}"
 : "${PARALLEL:=1}"
 : "${BATCH_SIZE:=4096}"
 : "${UBATCH_SIZE:=4096}"