diff --git a/dankserver/frank-primary/frank-primary-entrypoint.sh b/dankserver/frank-primary/frank-primary-entrypoint.sh index 22cfdd6..2ad4b49 100755 --- a/dankserver/frank-primary/frank-primary-entrypoint.sh +++ b/dankserver/frank-primary/frank-primary-entrypoint.sh @@ -15,13 +15,20 @@ # that branch — see serverAppConfigs-docs/LLM/frank-primary-lucebox-ds41.md. # # LAUNCH SHAPE (server/docs/DS41.md — the V4.1 page, not the V4 DS4.md one): -# --ds4-prefill dense the V4.1 batched prefill. The default `exact` walks -# the prompt token by token, and the V4-only modes are -# REFUSED for V4.1: `sparse`, --ds4-fused-decode, -# --ds4-fused-verify-f16-kv, layer split and --mmproj -# all fail validate_model_features at load. -# --chunk 4096 the chunk the V4.1 dense-prefill numbers were taken -# at; the runtime caps it to what the device holds. +# --ds4-prefill exact THE ONLY prefill this placement can run. Any +# approximate mode (dense, sparse) makes +# DeepSeek4Backend::requires_monolithic_model() true, +# and a monolithic load asks ggml for the whole 187 GiB +# expert set as ONE device-0 buffer — measured +# 2026-09-26: "allocating 191932.25 MiB on device 0: +# cudaMalloc failed: out of memory", then +# "monolithic HIP load required for dense prefill". +# DS41.md's dense-prefill section measures that mode +# where the experts fit (in-process owners); on a +# 32 GiB target it cannot. exact feeds the prompt token +# by token, which is what the hybrid tier streams for. +# --chunk 1024 DS41.md's cold-expert chunk. The runtime caps the +# chunk to what fits after headroom regardless. # --max-ctx 131072 the V4.1 default window. The MLA compressed caches # are F16 by design and stay small; --cache-type-k/v # are ignored by the DeepSeek4 backend, so they are @@ -68,7 +75,7 @@ exec "$BIN" "$MODEL" \ --model-name frank-primary \ --target-device hip:0 \ --max-ctx 131072 \ - --chunk 4096 \ - --ds4-prefill dense \ + --chunk 1024 \ + --ds4-prefill exact \ --prefix-cache-slots 0 \ --prefill-cache-slots 0