diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 6ba6896..93d1ef4 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -58,21 +58,18 @@ ALIAS=frank-default-large BIND_ADDR=0.0.0.0 LISTEN_PORT=7770 -# Serving: 32768-token context on ONE slot (parallel 1), batch 4096 / +# Serving: 401408-token context, TWO slots (parallel 2), batch 4096 / # ubatch 4096, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB # prompt cache. --fit off: auto-fit segfaults on this model's # hyper-connection/indexer config; -ngl 999 puts every layer on the # SVM-backed GPU pool. # -# CTX_SIZE is capped at 32768 (the author's own quick-start). This model is -# fully GPU-resident (87.06 GiB) — the PLE n-gram table no longer streams -# from RAM like STRIX_LEAN did, so there is no room for a huge context -# buffer: HIP heap is 119.3 GiB, and the context buffer alone is 50.8 GiB -# at 131072 and 103 GiB at 262144 (verified live: -# `cudaMalloc … failed: out of memory` on first graph alloc). At 32768 the -# context buffer drops to ~13 GiB (~100 GiB total) and loads clean. -CTX_SIZE=32768 -PARALLEL=1 +# CTX_SIZE=401408 / PARALLEL=2: untested experiment. At 32768×1 the +# context buffer was ~13 GiB and total ~100 GiB in the 119.3 GiB HIP +# heap. Doubling slots × 12× context could OOM — verify load. +CTX_SIZE=401408 +PARALLEL=2 + BATCH_SIZE=4096 UBATCH_SIZE=4096 NUM_THREADS=16 @@ -106,13 +103,8 @@ HSA_OVERRIDE_GFX_VERSION=11.5.1 VERBOSITY=4 + # Speculative decoding: ngram-mod only. The MTP draft head is retained -# (agentionai Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST-GGUF, 2.28 GiB) but does -# NOT load on build-qwen38 (loader requires blk.0.hc_attn_norm.weight the -# head lacks — see KNOWN CONSTRAINTS). If a compatible binary appears, set -# SPEC_TYPE=draft-mtp,ngram-mod (n-max 3 measured best at 32.4 t/s / 0.612 -# acceptance; adaptive p-split measured worse, stays off). SPEC_TYPE=none -# disables speculative decoding entirely. DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf SPEC_TYPE=ngram-mod diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index a136c1d..751982b 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -45,8 +45,8 @@ fi : "${ALIAS:=frank-default-large}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7770}" -: "${CTX_SIZE:=32768}" -: "${PARALLEL:=1}" +: "${CTX_SIZE:=401408}" +: "${PARALLEL:=2}" : "${BATCH_SIZE:=4096}" : "${UBATCH_SIZE:=4096}" : "${NUM_THREADS:=16}"