dank/serverAppConfigs
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

large: ubatch 4096 (match v3), add --no-mmap; sync bench harness prod config

Browse Source
This commit is contained in:
Frank
2026-08-22 03:02:16 -07:00
parent 0f2bb636d7
commit bd9d75af3f
3 changed files with 19 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bench-kairic-ab.sh
+4 -3
View File
@@ -7,9 +7,10 @@
# greedy argmax fast path ON)
# nommap runner + --no-mmap
# nofast runner + fast path OFF
# prod production profile (b4096/ub2048, reasoning on/deepseek,
# prod production profile (b4096/ub4096, --no-mmap, reasoning on/deepseek,
# fast path OFF, preserve_thinking)
# prodnommap prod + --no-mmap
# prodnommap (obsolete: identical to prod since ubatch 4096 + --no-mmap
# moved into the production profile)
#
# Usage: ./scripts/bench-kairic-ab.sh <cfg> [ctx] [prompt_chars]
# Example: ./scripts/bench-kairic-ab.sh prod 16384 53000
@@ -61,7 +62,7 @@ case "$CFG" in
runner) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512; REA=off; RFMT=none; EXTRA="" ;;
nommap) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512; REA=off; RFMT=none; EXTRA=""; MMAP="--no-mmap" ;;
nofast) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=2048; UB=512; REA=off; RFMT=none; EXTRA="" ;;
prod) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=2048; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}" ;;
prod) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=4096; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;;
prodnommap) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=2048; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;;
esac
scripts/frank-default-large.env
+3 -2
View File
@@ -51,9 +51,10 @@ LISTEN_PORT=52625
CTX_SIZE=262144
PARALLEL=1
# Local override: the pinned runner uses 2048/512; production chat uses a
# larger 4096/2048 batch (unified memory has headroom).
# larger 4096/4096 batch (unified memory has headroom; matches the retired
# v3 profile).
BATCH_SIZE=4096
UBATCH_SIZE=2048
UBATCH_SIZE=4096
TARGET_THREADS=16
BATCH_THREADS=32
CACHE_TYPE_K=f16
scripts/serve-frank-large.sh
+12 -7
View File
@@ -12,11 +12,13 @@
# See docs/kairic-edge.md for the build runbook and binary identity.
#
# This script replicates the release pinned runner
# (ROCmFPX-kairic/scripts/run-kairic-edge-gfx1151.sh) verbatim. The only
# deliberate differences: --host/--port come from BIND_ADDR/LISTEN_PORT
# (safe names; the ROCm HIP runtime reads a bare HOST env var, see the
# 2026-08-13 TEMP incident) and LD_LIBRARY_PATH uses this machine's split
# 7.15 SDK layout.
# (ROCmFPX-kairic/scripts/run-kairic-edge-gfx1151.sh). The deliberate
# differences: --host/--port come from BIND_ADDR/LISTEN_PORT (safe names;
# the ROCm HIP runtime reads a bare HOST env var, see the 2026-08-13 TEMP
# incident), LD_LIBRARY_PATH uses this machine's split 7.15 SDK layout,
# the profile runs --no-mmap, batch 4096 / ubatch 4096, reasoning on with
# deepseek format and preserve_thinking, and the target greedy argmax fast
# path is disabled for non-greedy chat sampling.
#
# Tunables come from frank-default-large.env in this directory unless
# FRANK_LARGE_ENV_FILE overrides that path.
@@ -48,7 +50,7 @@ fi
: "${CTX_SIZE:=262144}"
: "${PARALLEL:=1}"
: "${BATCH_SIZE:=4096}"
: "${UBATCH_SIZE:=2048}"
: "${UBATCH_SIZE:=4096}"
: "${TARGET_THREADS:=16}"
: "${BATCH_THREADS:=32}"
: "${CACHE_TYPE_K:=f16}"
@@ -126,7 +128,9 @@ export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0
# 4, deterministic sampler). Local overrides for production chat:
# - reasoning on + deepseek format (thinking goes to message.reasoning_content,
# matching the retired v3 service so the fleet's reasoning clients work)
# - batch 4096 / ubatch 2048 (larger than the runner's 2048/512)
# - batch 4096 / ubatch 4096 (larger than the runner's 2048/512; matches v3)
# - --no-mmap (local choice; benchmarks show no perf difference versus
# mmap, kept per operator preference)
# - chat_template_kwargs preserve_thinking=true (preserve thinking tokens
# in the conversation history across turns)
# Sampling is overridden per request by clients; the card recommends Qwen
@@ -152,6 +156,7 @@ exec "$LLAMA_BIN" \
--cache-ram "$CACHE_RAM" \
--cache-prompt \
--cache-idle-slots \
--no-mmap \
--metrics \
--kairic-edge \
--spec-type draft-mtp \