bench: add Kairic A/B config harness; make v3 spec-stack harness self-contained (survives the Kairic env swap)
This commit is contained in:
Frank
Executable
+150
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env bash
|
||||
# bench-kairic-ab.sh — A/B perf isolation for the Kairic Edge deployment on
|
||||
# Dankclaw. Run this ON DANKCLAW (paths are absolute target paths).
|
||||
#
|
||||
# Configs:
|
||||
# runner pinned runner profile (b2048/ub512, reasoning off, target
|
||||
# greedy argmax fast path ON)
|
||||
# nommap runner + --no-mmap
|
||||
# nofast runner + fast path OFF
|
||||
# prod production profile (b4096/ub2048, reasoning on/deepseek,
|
||||
# fast path OFF, preserve_thinking)
|
||||
# prodnommap prod + --no-mmap
|
||||
#
|
||||
# Usage: ./scripts/bench-kairic-ab.sh <cfg> [ctx] [prompt_chars]
|
||||
# Example: ./scripts/bench-kairic-ab.sh prod 16384 53000
|
||||
#
|
||||
# The server runs on scratch port 7893. STOP frank-default-large before
|
||||
# benchmarking (see AGENTS.md). The server is killed on exit.
|
||||
set -uo pipefail
|
||||
CFG="${1:?runner|nofast|prod}"
|
||||
CTX="${2:-32768}"
|
||||
PCHARS="${3:-11000}"
|
||||
PORT=7893
|
||||
BIN=/home/dank/ciru-qwen38/ROCmFPX-kairic/build-kairic/bin/llama-server
|
||||
MDIR=/home/dank/models/Kairic-Qwen3.8-27B-IU4-Edge
|
||||
SDK=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages
|
||||
LOG=/tmp/kairic-ab-$CFG.log
|
||||
: > "$LOG"
|
||||
|
||||
export ROCM_PATH="$SDK/_rocm_sdk_devel"
|
||||
export LD_LIBRARY_PATH="$(dirname "$BIN"):$SDK/_rocm_sdk_libraries/lib:$SDK/_rocm_sdk_core/lib:$SDK/_rocm_sdk_devel/lib"
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_OVERRIDE_GFX_VERSION=11.5.1
|
||||
export GGML_CUDA_GRAPH_OPT=0
|
||||
export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1
|
||||
export PROMPTFORGE_TARGET_ONLY=0
|
||||
export PROMPTFORGE_MODE=iu4_ffn
|
||||
export PROMPTFORGE_IU4_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs"
|
||||
export PROMPTFORGE_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs"
|
||||
export PROMPTFORGE_IU4_HADAMARD=1
|
||||
export PROMPTFORGE_IU4_SEGMENTED=0
|
||||
export PROMPTFORGE_ENABLE_FFN_KEEPERS=0
|
||||
export PROMPTFORGE_FFN_KEEPERS=late6
|
||||
export PROMPTFORGE_ENABLE_GDN=1
|
||||
export PROMPTFORGE_ENABLE_GDN_W8=0
|
||||
export PROMPTFORGE_GDN_SIDECAR_OVERRIDE="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs"
|
||||
export PROMPTFORGE_GDN_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs"
|
||||
export PROMPTFORGE_GDN_IU4_HADAMARD=1
|
||||
export PROMPTFORGE_ENABLE_GDN_KEEPERS=0
|
||||
export PROMPTFORGE_ENABLE_GDN_OUTPUT=1
|
||||
export PROMPTFORGE_GDN_OUTPUT_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs"
|
||||
export PROMPTFORGE_GDN_OUTPUT_KEEPERS=v3_lateq6
|
||||
export PROMPTFORGE_ENABLE_SMALLM_IU4=1
|
||||
export PROMPTFORGE_ENABLE_SMALLM_GDN_IU4=1
|
||||
export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0
|
||||
export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0
|
||||
export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0
|
||||
|
||||
MMAP=""
|
||||
case "$CFG" in
|
||||
runner) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512; REA=off; RFMT=none; EXTRA="" ;;
|
||||
nommap) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512; REA=off; RFMT=none; EXTRA=""; MMAP="--no-mmap" ;;
|
||||
nofast) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=2048; UB=512; REA=off; RFMT=none; EXTRA="" ;;
|
||||
prod) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=2048; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}" ;;
|
||||
prodnommap) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=2048; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;;
|
||||
esac
|
||||
|
||||
"$BIN" -m "$MDIR/Qwen3.8-27B-IU4-Kairic-Edge.gguf" \
|
||||
--alias bench --host 127.0.0.1 --port "$PORT" --jinja \
|
||||
-dev ROCm0 -ngl 999 -c "$CTX" -b "$B" -ub "$UB" -fa on -ctk f16 -ctv f16 \
|
||||
-t 16 -tb 32 -np 1 -ctxcp 32 --cache-ram 8192 --cache-prompt --cache-idle-slots $MMAP \
|
||||
--metrics --kairic-edge \
|
||||
--spec-type draft-mtp --spec-draft-device ROCm0 --spec-draft-ngl 999 \
|
||||
--spec-draft-type-k f16 --spec-draft-type-v f16 \
|
||||
--spec-draft-n-max 4 --spec-draft-n-min 0 --spec-draft-p-min 0.0 --spec-draft-p-split 0.10 \
|
||||
--spec-draft-backend-sampling \
|
||||
--temp 0 --top-p 1.0 --top-k 0 --min-p 0.0 \
|
||||
--reasoning "$REA" --reasoning-format "$RFMT" --reasoning-budget -1 \
|
||||
$EXTRA >> "$LOG" 2>&1 &
|
||||
SPID=$!
|
||||
trap 'kill $SPID 2>/dev/null; wait $SPID 2>/dev/null' EXIT
|
||||
|
||||
for i in $(seq 1 300); do
|
||||
kill -0 $SPID 2>/dev/null || { echo "SERVER DIED"; tail -20 "$LOG"; exit 2; }
|
||||
curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
echo "=== CFG=$CFG ready ==="
|
||||
|
||||
python3 - "$PORT" "$CFG" "$PCHARS" <<'PY'
|
||||
import sys, json, time, urllib.request
|
||||
port, cfg, pchars = sys.argv[1], sys.argv[2], int(sys.argv[3])
|
||||
VAR = [
|
||||
"narrative nonfiction blends observation with reflection, ",
|
||||
"and a careful writer lets each clause earn its place ",
|
||||
"in the sentence's rising line, so meaning accumulates slowly. ",
|
||||
"The reader is not hurried; the reader is led. Loose and periodic, ",
|
||||
"the style turns on concrete nouns and verbs that carry weight. ",
|
||||
"A paragraph is not a list but a single turn of thought that ",
|
||||
"opens, develops, and lands, and clarity is a courtesy owed ",
|
||||
"to the willing reader who gives the writer their attention. ",
|
||||
]
|
||||
parts = []
|
||||
t = 0
|
||||
while len("".join(parts)) < pchars:
|
||||
for k in range(3):
|
||||
parts.append(VAR[(t + k) % len(VAR)])
|
||||
parts.append(f" Note {t}. ")
|
||||
t += 1
|
||||
longtext = "".join(parts)
|
||||
prompt = longtext + "\n\nNow write a long, detailed continuation of this essay (at least five paragraphs) about the philosophy of language.\n"
|
||||
body = {
|
||||
"model": "bench",
|
||||
"prompt": prompt,
|
||||
"max_tokens": 512,
|
||||
"temperature": 0,
|
||||
"top_p": 1.0,
|
||||
"top_k": 0,
|
||||
"min_p": 0.0,
|
||||
"stream": False,
|
||||
"cache_prompt": True,
|
||||
}
|
||||
# ignore_eos is rejected by the exact-Q8 greedy argmax fast path (runner cfg).
|
||||
if cfg not in ("runner",):
|
||||
body["ignore_eos"] = True
|
||||
def run(tag):
|
||||
req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",
|
||||
data=json.dumps(body).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
t0 = time.time()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=3600) as resp:
|
||||
out = json.load(resp)
|
||||
except Exception as e:
|
||||
body_txt = getattr(e, "read", lambda: b"")()
|
||||
print(f"[{tag}] REQUEST FAILED: {e}")
|
||||
print(f"[{tag}] body: {body_txt.decode('utf-8', 'ignore')[:500]}")
|
||||
raise SystemExit(3)
|
||||
wall = time.time() - t0
|
||||
u = out.get("usage", {})
|
||||
tm = out.get("timings", {})
|
||||
pt = u.get("prompt_tokens", 0); ct = u.get("completion_tokens", 0)
|
||||
pps = tm.get("prompt_per_second", 0); tps = tm.get("predicted_per_second", 0)
|
||||
print(f"[{tag}] cfg={cfg} prompt={pt} completion={ct} prefill={pps:.1f} t/s decode={tps:.2f} t/s wall={wall:.1f}s")
|
||||
run("cold")
|
||||
run("warm")
|
||||
PY
|
||||
|
||||
grep -E "print_timing|statistics" "$LOG" | tail -6
|
||||
exit 0
|
||||
@@ -19,6 +19,15 @@ OUT_TOKENS="${3:-256}"
|
||||
ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-large.env
|
||||
source "$ENV_FILE"
|
||||
|
||||
# This harness benchmarks the retired v3 stack. Force the v3 values so the
|
||||
# harness works regardless of the live service env (now Kairic).
|
||||
MODEL_DIR=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX-v3
|
||||
MODEL_FILE=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v3-Q8-FFNEQ-LateQ6.gguf
|
||||
FFN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs
|
||||
GDN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs
|
||||
OUTPUT_K8_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-Output-K8.pfs
|
||||
MMPROJ_FILE=Qwen3.8-27B-mmproj-BF16.mmproj
|
||||
|
||||
# Derive the PromptForge sidecar paths from the env base values.
|
||||
export PROMPTFORGE_SIDECAR="$MODEL_DIR/$FFN_PFS"
|
||||
export PROMPTFORGE_GDN_SIDECAR="$MODEL_DIR/$GDN_PFS"
|
||||
|
||||
Reference in New Issue
Block a user