From bd9d75af3f25f7fba31f8096e72bfc78233870c0 Mon Sep 17 00:00:00 2001 From: Frank Date: Sat, 22 Aug 2026 03:02:16 -0700 Subject: [PATCH] large: ubatch 4096 (match v3), add --no-mmap; sync bench harness prod config --- scripts/bench-kairic-ab.sh | 7 ++++--- scripts/frank-default-large.env | 5 +++-- scripts/serve-frank-large.sh | 19 ++++++++++++------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/scripts/bench-kairic-ab.sh b/scripts/bench-kairic-ab.sh index 211c9e1..841d48c 100755 --- a/scripts/bench-kairic-ab.sh +++ b/scripts/bench-kairic-ab.sh @@ -7,9 +7,10 @@ # greedy argmax fast path ON) # nommap runner + --no-mmap # nofast runner + fast path OFF -# prod production profile (b4096/ub2048, reasoning on/deepseek, +# prod production profile (b4096/ub4096, --no-mmap, reasoning on/deepseek, # fast path OFF, preserve_thinking) -# prodnommap prod + --no-mmap +# prodnommap (obsolete: identical to prod since ubatch 4096 + --no-mmap +# moved into the production profile) # # Usage: ./scripts/bench-kairic-ab.sh [ctx] [prompt_chars] # Example: ./scripts/bench-kairic-ab.sh prod 16384 53000 @@ -61,7 +62,7 @@ case "$CFG" in runner) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512; REA=off; RFMT=none; EXTRA="" ;; nommap) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512; REA=off; RFMT=none; EXTRA=""; MMAP="--no-mmap" ;; nofast) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=2048; UB=512; REA=off; RFMT=none; EXTRA="" ;; - prod) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=2048; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}" ;; + prod) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=4096; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;; prodnommap) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=2048; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;; esac diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 55dadef..cdef6da 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -51,9 +51,10 @@ LISTEN_PORT=52625 CTX_SIZE=262144 PARALLEL=1 # Local override: the pinned runner uses 2048/512; production chat uses a -# larger 4096/2048 batch (unified memory has headroom). +# larger 4096/4096 batch (unified memory has headroom; matches the retired +# v3 profile). BATCH_SIZE=4096 -UBATCH_SIZE=2048 +UBATCH_SIZE=4096 TARGET_THREADS=16 BATCH_THREADS=32 CACHE_TYPE_K=f16 diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 3024ddd..9d9a7ae 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -12,11 +12,13 @@ # See docs/kairic-edge.md for the build runbook and binary identity. # # This script replicates the release pinned runner -# (ROCmFPX-kairic/scripts/run-kairic-edge-gfx1151.sh) verbatim. The only -# deliberate differences: --host/--port come from BIND_ADDR/LISTEN_PORT -# (safe names; the ROCm HIP runtime reads a bare HOST env var, see the -# 2026-08-13 TEMP incident) and LD_LIBRARY_PATH uses this machine's split -# 7.15 SDK layout. +# (ROCmFPX-kairic/scripts/run-kairic-edge-gfx1151.sh). The deliberate +# differences: --host/--port come from BIND_ADDR/LISTEN_PORT (safe names; +# the ROCm HIP runtime reads a bare HOST env var, see the 2026-08-13 TEMP +# incident), LD_LIBRARY_PATH uses this machine's split 7.15 SDK layout, +# the profile runs --no-mmap, batch 4096 / ubatch 4096, reasoning on with +# deepseek format and preserve_thinking, and the target greedy argmax fast +# path is disabled for non-greedy chat sampling. # # Tunables come from frank-default-large.env in this directory unless # FRANK_LARGE_ENV_FILE overrides that path. @@ -48,7 +50,7 @@ fi : "${CTX_SIZE:=262144}" : "${PARALLEL:=1}" : "${BATCH_SIZE:=4096}" -: "${UBATCH_SIZE:=2048}" +: "${UBATCH_SIZE:=4096}" : "${TARGET_THREADS:=16}" : "${BATCH_THREADS:=32}" : "${CACHE_TYPE_K:=f16}" @@ -126,7 +128,9 @@ export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0 # 4, deterministic sampler). Local overrides for production chat: # - reasoning on + deepseek format (thinking goes to message.reasoning_content, # matching the retired v3 service so the fleet's reasoning clients work) -# - batch 4096 / ubatch 2048 (larger than the runner's 2048/512) +# - batch 4096 / ubatch 4096 (larger than the runner's 2048/512; matches v3) +# - --no-mmap (local choice; benchmarks show no perf difference versus +# mmap, kept per operator preference) # - chat_template_kwargs preserve_thinking=true (preserve thinking tokens # in the conversation history across turns) # Sampling is overridden per request by clients; the card recommends Qwen @@ -152,6 +156,7 @@ exec "$LLAMA_BIN" \ --cache-ram "$CACHE_RAM" \ --cache-prompt \ --cache-idle-slots \ + --no-mmap \ --metrics \ --kairic-edge \ --spec-type draft-mtp \