Frank
918fe4eaec
The vllmScripts repository (LLM inference fleet: serve scripts, systemd units, env tunables, deployment runbook, research docs) is merged into this repository. Its full history is grafted under llm/ with a two-parent merge commit so no commit is lost. A follow-up commit reorganizes llm/ into the repo layout: llm/docs -> docs/LLM, llm/scripts -> scripts, llm/old -> old.
154 lines
6.8 KiB
Bash
Executable File
154 lines
6.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# bench-kairic-ab.sh — A/B perf isolation for the Kairic Edge deployment on
|
|
# Dankclaw. Run this ON DANKCLAW (paths are absolute target paths).
|
|
#
|
|
# Configs:
|
|
# runner pinned runner profile (b2048/ub512, reasoning off, target
|
|
# greedy argmax fast path ON)
|
|
# nommap runner + --no-mmap
|
|
# nofast runner + fast path OFF
|
|
# prod production profile (b4096/ub4096, --no-mmap, reasoning on/deepseek,
|
|
# fast path OFF, preserve_thinking)
|
|
# prod2048 prod with the pinned runner's batch sizes (b2048/ub512)
|
|
# prodnommap (obsolete: identical to prod since ubatch 4096 + --no-mmap
|
|
# moved into the production profile)
|
|
#
|
|
# Usage: ./scripts/bench-kairic-ab.sh <cfg> [ctx] [prompt_chars]
|
|
# Example: ./scripts/bench-kairic-ab.sh prod 16384 53000
|
|
#
|
|
# The server runs on scratch port 7893. STOP frank-default-large before
|
|
# benchmarking (see AGENTS.md). The server is killed on exit.
|
|
set -uo pipefail
|
|
CFG="${1:?runner|nofast|prod}"
|
|
CTX="${2:-32768}"
|
|
PCHARS="${3:-11000}"
|
|
PORT=7893
|
|
BIN=/home/dank/ciru-qwen38/ROCmFPX-kairic/build-kairic/bin/llama-server
|
|
MDIR=/home/dank/models/Kairic-Qwen3.8-27B-IU4-Edge
|
|
SDK=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages
|
|
LOG=/tmp/kairic-ab-$CFG.log
|
|
: > "$LOG"
|
|
|
|
export ROCM_PATH="$SDK/_rocm_sdk_devel"
|
|
export LD_LIBRARY_PATH="$(dirname "$BIN"):$SDK/_rocm_sdk_libraries/lib:$SDK/_rocm_sdk_core/lib:$SDK/_rocm_sdk_devel/lib"
|
|
export HIP_VISIBLE_DEVICES=0
|
|
export HSA_OVERRIDE_GFX_VERSION=11.5.1
|
|
export GGML_CUDA_GRAPH_OPT=0
|
|
export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1
|
|
export PROMPTFORGE_TARGET_ONLY=0
|
|
export PROMPTFORGE_MODE=iu4_ffn
|
|
export PROMPTFORGE_IU4_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs"
|
|
export PROMPTFORGE_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs"
|
|
export PROMPTFORGE_IU4_HADAMARD=1
|
|
export PROMPTFORGE_IU4_SEGMENTED=0
|
|
export PROMPTFORGE_ENABLE_FFN_KEEPERS=0
|
|
export PROMPTFORGE_FFN_KEEPERS=late6
|
|
export PROMPTFORGE_ENABLE_GDN=1
|
|
export PROMPTFORGE_ENABLE_GDN_W8=0
|
|
export PROMPTFORGE_GDN_SIDECAR_OVERRIDE="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs"
|
|
export PROMPTFORGE_GDN_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs"
|
|
export PROMPTFORGE_GDN_IU4_HADAMARD=1
|
|
export PROMPTFORGE_ENABLE_GDN_KEEPERS=0
|
|
export PROMPTFORGE_ENABLE_GDN_OUTPUT=1
|
|
export PROMPTFORGE_GDN_OUTPUT_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs"
|
|
export PROMPTFORGE_GDN_OUTPUT_KEEPERS=v3_lateq6
|
|
export PROMPTFORGE_ENABLE_SMALLM_IU4=1
|
|
export PROMPTFORGE_ENABLE_SMALLM_GDN_IU4=1
|
|
export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0
|
|
export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0
|
|
export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0
|
|
|
|
MMAP=""
|
|
case "$CFG" in
|
|
runner) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512; REA=off; RFMT=none; EXTRA="" ;;
|
|
nommap) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512; REA=off; RFMT=none; EXTRA=""; MMAP="--no-mmap" ;;
|
|
nofast) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=2048; UB=512; REA=off; RFMT=none; EXTRA="" ;;
|
|
prod) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=4096; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;;
|
|
prod2048) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=2048; UB=512; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;;
|
|
prodnommap) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=2048; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;;
|
|
esac
|
|
|
|
"$BIN" -m "$MDIR/Qwen3.8-27B-IU4-Kairic-Edge.gguf" \
|
|
--alias bench --host 127.0.0.1 --port "$PORT" --jinja \
|
|
-dev ROCm0 -ngl 999 -c "$CTX" -b "$B" -ub "$UB" -fa on -ctk f16 -ctv f16 \
|
|
-t 16 -tb 32 -np 1 -ctxcp 32 --cache-ram 8192 --cache-prompt --cache-idle-slots $MMAP \
|
|
--metrics --kairic-edge \
|
|
--spec-type draft-mtp --spec-draft-device ROCm0 --spec-draft-ngl 999 \
|
|
--spec-draft-type-k f16 --spec-draft-type-v f16 \
|
|
--spec-draft-n-max 4 --spec-draft-n-min 0 --spec-draft-p-min 0.0 --spec-draft-p-split 0.10 \
|
|
--spec-draft-backend-sampling \
|
|
--temp 0 --top-p 1.0 --top-k 0 --min-p 0.0 \
|
|
--reasoning "$REA" --reasoning-format "$RFMT" --reasoning-budget -1 \
|
|
$EXTRA >> "$LOG" 2>&1 &
|
|
SPID=$!
|
|
trap 'kill $SPID 2>/dev/null; wait $SPID 2>/dev/null' EXIT
|
|
|
|
for i in $(seq 1 300); do
|
|
kill -0 $SPID 2>/dev/null || { echo "SERVER DIED"; tail -20 "$LOG"; exit 2; }
|
|
curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1 && break
|
|
sleep 2
|
|
done
|
|
echo "=== CFG=$CFG ready ==="
|
|
|
|
python3 - "$PORT" "$CFG" "$PCHARS" <<'PY'
|
|
import sys, json, time, urllib.request
|
|
port, cfg, pchars = sys.argv[1], sys.argv[2], int(sys.argv[3])
|
|
VAR = [
|
|
"narrative nonfiction blends observation with reflection, ",
|
|
"and a careful writer lets each clause earn its place ",
|
|
"in the sentence's rising line, so meaning accumulates slowly. ",
|
|
"The reader is not hurried; the reader is led. Loose and periodic, ",
|
|
"the style turns on concrete nouns and verbs that carry weight. ",
|
|
"A paragraph is not a list but a single turn of thought that ",
|
|
"opens, develops, and lands, and clarity is a courtesy owed ",
|
|
"to the willing reader who gives the writer their attention. ",
|
|
]
|
|
parts = []
|
|
t = 0
|
|
while len("".join(parts)) < pchars:
|
|
for k in range(3):
|
|
parts.append(VAR[(t + k) % len(VAR)])
|
|
parts.append(f" Note {t}. ")
|
|
t += 1
|
|
longtext = "".join(parts)
|
|
prompt = longtext + "\n\nNow write a long, detailed continuation of this essay (at least five paragraphs) about the philosophy of language.\n"
|
|
body = {
|
|
"model": "bench",
|
|
"prompt": prompt,
|
|
"max_tokens": 512,
|
|
"temperature": 0,
|
|
"top_p": 1.0,
|
|
"top_k": 0,
|
|
"min_p": 0.0,
|
|
"stream": False,
|
|
"cache_prompt": True,
|
|
}
|
|
# ignore_eos is rejected by the exact-Q8 greedy argmax fast path (runner cfg).
|
|
if cfg not in ("runner",):
|
|
body["ignore_eos"] = True
|
|
def run(tag):
|
|
req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",
|
|
data=json.dumps(body).encode(),
|
|
headers={"Content-Type": "application/json"})
|
|
t0 = time.time()
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=3600) as resp:
|
|
out = json.load(resp)
|
|
except Exception as e:
|
|
body_txt = getattr(e, "read", lambda: b"")()
|
|
print(f"[{tag}] REQUEST FAILED: {e}")
|
|
print(f"[{tag}] body: {body_txt.decode('utf-8', 'ignore')[:500]}")
|
|
raise SystemExit(3)
|
|
wall = time.time() - t0
|
|
u = out.get("usage", {})
|
|
tm = out.get("timings", {})
|
|
pt = u.get("prompt_tokens", 0); ct = u.get("completion_tokens", 0)
|
|
pps = tm.get("prompt_per_second", 0); tps = tm.get("predicted_per_second", 0)
|
|
print(f"[{tag}] cfg={cfg} prompt={pt} completion={ct} prefill={pps:.1f} t/s decode={tps:.2f} t/s wall={wall:.1f}s")
|
|
run("cold")
|
|
run("warm")
|
|
PY
|
|
|
|
grep -E "print_timing|statistics" "$LOG" | tail -6
|
|
exit 0
|