large: ubatch 4096 (match v3), add --no-mmap; sync bench harness prod config

This commit is contained in:
Frank committed 2026-08-22 03:02:16 -07:00
1 parent 0f2bb636d7
commit bd9d75af3f
3 files changed
+19 -12

No files matched your search

+12 -7
View File
@@ -12,11 +12,13 @@
# See docs/kairic-edge.md for the build runbook and binary identity.
#
# This script replicates the release pinned runner
# (ROCmFPX-kairic/scripts/run-kairic-edge-gfx1151.sh) verbatim. The only
# deliberate differences: --host/--port come from BIND_ADDR/LISTEN_PORT
# (safe names; the ROCm HIP runtime reads a bare HOST env var, see the
# 2026-08-13 TEMP incident) and LD_LIBRARY_PATH uses this machine's split
# 7.15 SDK layout.
# (ROCmFPX-kairic/scripts/run-kairic-edge-gfx1151.sh). The deliberate
# differences: --host/--port come from BIND_ADDR/LISTEN_PORT (safe names;
# the ROCm HIP runtime reads a bare HOST env var, see the 2026-08-13 TEMP
# incident), LD_LIBRARY_PATH uses this machine's split 7.15 SDK layout,
# the profile runs --no-mmap, batch 4096 / ubatch 4096, reasoning on with
# deepseek format and preserve_thinking, and the target greedy argmax fast
# path is disabled for non-greedy chat sampling.
#
# Tunables come from frank-default-large.env in this directory unless
# FRANK_LARGE_ENV_FILE overrides that path.
@@ -48,7 +50,7 @@ fi
: "${CTX_SIZE:=262144}"
: "${PARALLEL:=1}"
: "${BATCH_SIZE:=4096}"
: "${UBATCH_SIZE:=2048}"
: "${UBATCH_SIZE:=4096}"
: "${TARGET_THREADS:=16}"
: "${BATCH_THREADS:=32}"
: "${CACHE_TYPE_K:=f16}"
@@ -126,7 +128,9 @@ export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0
# 4, deterministic sampler). Local overrides for production chat:
# - reasoning on + deepseek format (thinking goes to message.reasoning_content,
# matching the retired v3 service so the fleet's reasoning clients work)
# - batch 4096 / ubatch 2048 (larger than the runner's 2048/512)
# - batch 4096 / ubatch 4096 (larger than the runner's 2048/512; matches v3)
# - --no-mmap (local choice; benchmarks show no perf difference versus
# mmap, kept per operator preference)
# - chat_template_kwargs preserve_thinking=true (preserve thinking tokens
# in the conversation history across turns)
# Sampling is overridden per request by clients; the card recommends Qwen
@@ -152,6 +156,7 @@ exec "$LLAMA_BIN" \
--cache-ram "$CACHE_RAM" \
--cache-prompt \
--cache-idle-slots \
--no-mmap \
--metrics \
--kairic-edge \
--spec-type draft-mtp \