diff --git a/scripts/bench-kairic-ab.sh b/scripts/bench-kairic-ab.sh new file mode 100755 index 0000000..211c9e1 --- /dev/null +++ b/scripts/bench-kairic-ab.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# bench-kairic-ab.sh — A/B perf isolation for the Kairic Edge deployment on +# Dankclaw. Run this ON DANKCLAW (paths are absolute target paths). +# +# Configs: +# runner pinned runner profile (b2048/ub512, reasoning off, target +# greedy argmax fast path ON) +# nommap runner + --no-mmap +# nofast runner + fast path OFF +# prod production profile (b4096/ub2048, reasoning on/deepseek, +# fast path OFF, preserve_thinking) +# prodnommap prod + --no-mmap +# +# Usage: ./scripts/bench-kairic-ab.sh [ctx] [prompt_chars] +# Example: ./scripts/bench-kairic-ab.sh prod 16384 53000 +# +# The server runs on scratch port 7893. STOP frank-default-large before +# benchmarking (see AGENTS.md). The server is killed on exit. +set -uo pipefail +CFG="${1:?runner|nofast|prod}" +CTX="${2:-32768}" +PCHARS="${3:-11000}" +PORT=7893 +BIN=/home/dank/ciru-qwen38/ROCmFPX-kairic/build-kairic/bin/llama-server +MDIR=/home/dank/models/Kairic-Qwen3.8-27B-IU4-Edge +SDK=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages +LOG=/tmp/kairic-ab-$CFG.log +: > "$LOG" + +export ROCM_PATH="$SDK/_rocm_sdk_devel" +export LD_LIBRARY_PATH="$(dirname "$BIN"):$SDK/_rocm_sdk_libraries/lib:$SDK/_rocm_sdk_core/lib:$SDK/_rocm_sdk_devel/lib" +export HIP_VISIBLE_DEVICES=0 +export HSA_OVERRIDE_GFX_VERSION=11.5.1 +export GGML_CUDA_GRAPH_OPT=0 +export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 +export PROMPTFORGE_TARGET_ONLY=0 +export PROMPTFORGE_MODE=iu4_ffn +export PROMPTFORGE_IU4_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs" +export PROMPTFORGE_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs" +export PROMPTFORGE_IU4_HADAMARD=1 +export PROMPTFORGE_IU4_SEGMENTED=0 +export PROMPTFORGE_ENABLE_FFN_KEEPERS=0 +export PROMPTFORGE_FFN_KEEPERS=late6 +export PROMPTFORGE_ENABLE_GDN=1 +export PROMPTFORGE_ENABLE_GDN_W8=0 +export PROMPTFORGE_GDN_SIDECAR_OVERRIDE="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs" +export PROMPTFORGE_GDN_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs" +export PROMPTFORGE_GDN_IU4_HADAMARD=1 +export PROMPTFORGE_ENABLE_GDN_KEEPERS=0 +export PROMPTFORGE_ENABLE_GDN_OUTPUT=1 +export PROMPTFORGE_GDN_OUTPUT_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs" +export PROMPTFORGE_GDN_OUTPUT_KEEPERS=v3_lateq6 +export PROMPTFORGE_ENABLE_SMALLM_IU4=1 +export PROMPTFORGE_ENABLE_SMALLM_GDN_IU4=1 +export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0 +export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0 +export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0 + +MMAP="" +case "$CFG" in + runner) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512; REA=off; RFMT=none; EXTRA="" ;; + nommap) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512; REA=off; RFMT=none; EXTRA=""; MMAP="--no-mmap" ;; + nofast) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=2048; UB=512; REA=off; RFMT=none; EXTRA="" ;; + prod) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=2048; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}" ;; + prodnommap) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=2048; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;; +esac + +"$BIN" -m "$MDIR/Qwen3.8-27B-IU4-Kairic-Edge.gguf" \ + --alias bench --host 127.0.0.1 --port "$PORT" --jinja \ + -dev ROCm0 -ngl 999 -c "$CTX" -b "$B" -ub "$UB" -fa on -ctk f16 -ctv f16 \ + -t 16 -tb 32 -np 1 -ctxcp 32 --cache-ram 8192 --cache-prompt --cache-idle-slots $MMAP \ + --metrics --kairic-edge \ + --spec-type draft-mtp --spec-draft-device ROCm0 --spec-draft-ngl 999 \ + --spec-draft-type-k f16 --spec-draft-type-v f16 \ + --spec-draft-n-max 4 --spec-draft-n-min 0 --spec-draft-p-min 0.0 --spec-draft-p-split 0.10 \ + --spec-draft-backend-sampling \ + --temp 0 --top-p 1.0 --top-k 0 --min-p 0.0 \ + --reasoning "$REA" --reasoning-format "$RFMT" --reasoning-budget -1 \ + $EXTRA >> "$LOG" 2>&1 & +SPID=$! +trap 'kill $SPID 2>/dev/null; wait $SPID 2>/dev/null' EXIT + +for i in $(seq 1 300); do + kill -0 $SPID 2>/dev/null || { echo "SERVER DIED"; tail -20 "$LOG"; exit 2; } + curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1 && break + sleep 2 +done +echo "=== CFG=$CFG ready ===" + +python3 - "$PORT" "$CFG" "$PCHARS" <<'PY' +import sys, json, time, urllib.request +port, cfg, pchars = sys.argv[1], sys.argv[2], int(sys.argv[3]) +VAR = [ + "narrative nonfiction blends observation with reflection, ", + "and a careful writer lets each clause earn its place ", + "in the sentence's rising line, so meaning accumulates slowly. ", + "The reader is not hurried; the reader is led. Loose and periodic, ", + "the style turns on concrete nouns and verbs that carry weight. ", + "A paragraph is not a list but a single turn of thought that ", + "opens, develops, and lands, and clarity is a courtesy owed ", + "to the willing reader who gives the writer their attention. ", +] +parts = [] +t = 0 +while len("".join(parts)) < pchars: + for k in range(3): + parts.append(VAR[(t + k) % len(VAR)]) + parts.append(f" Note {t}. ") + t += 1 +longtext = "".join(parts) +prompt = longtext + "\n\nNow write a long, detailed continuation of this essay (at least five paragraphs) about the philosophy of language.\n" +body = { + "model": "bench", + "prompt": prompt, + "max_tokens": 512, + "temperature": 0, + "top_p": 1.0, + "top_k": 0, + "min_p": 0.0, + "stream": False, + "cache_prompt": True, +} +# ignore_eos is rejected by the exact-Q8 greedy argmax fast path (runner cfg). +if cfg not in ("runner",): + body["ignore_eos"] = True +def run(tag): + req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + t0 = time.time() + try: + with urllib.request.urlopen(req, timeout=3600) as resp: + out = json.load(resp) + except Exception as e: + body_txt = getattr(e, "read", lambda: b"")() + print(f"[{tag}] REQUEST FAILED: {e}") + print(f"[{tag}] body: {body_txt.decode('utf-8', 'ignore')[:500]}") + raise SystemExit(3) + wall = time.time() - t0 + u = out.get("usage", {}) + tm = out.get("timings", {}) + pt = u.get("prompt_tokens", 0); ct = u.get("completion_tokens", 0) + pps = tm.get("prompt_per_second", 0); tps = tm.get("predicted_per_second", 0) + print(f"[{tag}] cfg={cfg} prompt={pt} completion={ct} prefill={pps:.1f} t/s decode={tps:.2f} t/s wall={wall:.1f}s") +run("cold") +run("warm") +PY + +grep -E "print_timing|statistics" "$LOG" | tail -6 +exit 0 diff --git a/scripts/bench-spec-stack.sh b/scripts/bench-spec-stack.sh index 0f7893d..005b089 100644 --- a/scripts/bench-spec-stack.sh +++ b/scripts/bench-spec-stack.sh @@ -19,6 +19,15 @@ OUT_TOKENS="${3:-256}" ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-large.env source "$ENV_FILE" +# This harness benchmarks the retired v3 stack. Force the v3 values so the +# harness works regardless of the live service env (now Kairic). +MODEL_DIR=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX-v3 +MODEL_FILE=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v3-Q8-FFNEQ-LateQ6.gguf +FFN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs +GDN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs +OUTPUT_K8_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-Output-K8.pfs +MMPROJ_FILE=Qwen3.8-27B-mmproj-BF16.mmproj + # Derive the PromptForge sidecar paths from the env base values. export PROMPTFORGE_SIDECAR="$MODEL_DIR/$FFN_PFS" export PROMPTFORGE_GDN_SIDECAR="$MODEL_DIR/$GDN_PFS"