Reorganize: llm/scripts -> scripts (LLM serve scripts, units, env tunables)

The LLM fleet serve scripts, systemd units, env tunables, and bench harnesses join the deploy scripts. The vllmScripts deploy.sh is removed here; its functionality is folded into the unified scripts/deploy.sh in a follow-up commit. bench-spec-stack.sh gains the executable bit (required by the scripts_syntax test seam).
2026-08-22 15:07:31 -07:00
parent 506614525b
commit aa410f4439
13 changed files with 0 additions and 167 deletions
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# Bench the Kairic Edge "fast greedy (default)" profile.
+# Usage: ./bench-fast-greedy.sh
+#
+# Config: reasoning off, argmax fastpath=1, deterministic sampler.
+# Sends a coding prompt, captures full stream, computes decode tokens and time.
+set -euo pipefail
+
+HOST="127.0.0.1"
+PORT=52625
+ALIAS="frank-default-large"
+
+# A coding prompt representative of the card's 164-task suite.
+PROMPT='### Instruction
+Write a Python class that implements a priority queue using a binary heap. Include methods for enqueue, dequeue, peek, and is_empty. Add type hints, docstrings, and a usage example. The heap should support arbitrary comparable elements.
+### Response'
+
+DECODE_TOKENS=0
+START_SEC=0
+END_SEC=0
+
+echo "=== Fast-greedy benchmark (FASTGREEDY=1) ==="
+echo "Model: $ALIAS on $HOST:$PORT"
+echo "Prompt tokens: $(echo "$PROMPT" | wc -c) chars"
+echo ""
+
+# Use /v1/completions with streaming, capture tokens+time from usage.
+RESPONSE=$(curl -sf http://$HOST:$PORT/v1/completions \
+  -H 'Content-Type: application/json' \
+  -d "{
+    \"model\": \"$ALIAS\",
+    \"prompt\": $(python3 -c "import json,sys; print(json.dumps(sys.stdin.read()))" <<< "$PROMPT"),
+    \"max_tokens\": 512,
+    \"temperature\": 0,
+    \"top_p\": 1.0,
+    \"top_k\": 0,
+    \"min_p\": 0.0,
+    \"n\": 1,
+    \"stream\": true
+  }" 2>/dev/null)
+
+# Extract final usage from the last JSON line (finish_reason=stop or length).
+# Also count output text tokens.
+LAST_LINE=$(echo "$RESPONSE" | grep 'finish_reason' | tail -1)
+
+# Parse timing from the last chunk (has full usage)
+if echo "$LAST_LINE" | grep -q 'timings'; then
+    COMP_TOKENS=$(echo "$LAST_LINE" | python3 -c "
+import sys, json
+data = json.loads(sys.stdin.read())
+print(data.get('usage', {}).get('completion_tokens', 0))
+print(data.get('timings', {}).get('predicted_per_second', 0))
+print(data.get('timings', {}).get('prompt_per_second', 0))
+" 2>/dev/null)
+    echo "$COMP_TOKENS" | {
+        read CT
+        read TPS
+        read PPS
+        echo "Completion tokens: $CT"
+        echo "Predicted (decode) tokens/sec: $TPS"
+        echo "Prompt tokens/sec: $PPS"
+    }
+else
+    echo "WARNING: No timing data found in response"
+    echo "$LAST_LINE" | head -5
+fi
+
+# Count the actual text output
+OUTPUT_TEXT=$(echo "$RESPONSE" | grep '"text"' | head -1 | sed 's/.*"text":"//;s/","index.*//')
+OUTPUT_TOKENS=$(echo -n "$OUTPUT_TEXT" | wc -w)
+echo "Output text (rough token count): ~$OUTPUT_TOKENS words"
+echo ""
+
+# Check journal for draft acceptance and final timing
+echo "=== Draft acceptance & final timing ==="
+sleep 2
+ssh -p 2222 dank@dankclaw "journalctl -u frank-default-large --since '1 minute ago' --no-pager 2>/dev/null | grep -E 'draft acceptance|print_timing.*n_decoded|statistics.*draft-mtp' | tail -4" 2>/dev/null || echo "(could not read journal)"
@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+# bench-kairic-ab.sh — A/B perf isolation for the Kairic Edge deployment on
+# Dankclaw. Run this ON DANKCLAW (paths are absolute target paths).
+#
+# Configs:
+#   runner        pinned runner profile (b2048/ub512, reasoning off, target
+#                 greedy argmax fast path ON)
+#   nommap        runner + --no-mmap
+#   nofast        runner + fast path OFF
+#   prod          production profile (b4096/ub4096, --no-mmap, reasoning on/deepseek,
+#                 fast path OFF, preserve_thinking)
+#   prod2048      prod with the pinned runner's batch sizes (b2048/ub512)
+#   prodnommap    (obsolete: identical to prod since ubatch 4096 + --no-mmap
+#                 moved into the production profile)
+#
+# Usage: ./scripts/bench-kairic-ab.sh <cfg> [ctx] [prompt_chars]
+# Example: ./scripts/bench-kairic-ab.sh prod 16384 53000
+#
+# The server runs on scratch port 7893. STOP frank-default-large before
+# benchmarking (see AGENTS.md). The server is killed on exit.
+set -uo pipefail
+CFG="${1:?runner|nofast|prod}"
+CTX="${2:-32768}"
+PCHARS="${3:-11000}"
+PORT=7893
+BIN=/home/dank/ciru-qwen38/ROCmFPX-kairic/build-kairic/bin/llama-server
+MDIR=/home/dank/models/Kairic-Qwen3.8-27B-IU4-Edge
+SDK=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages
+LOG=/tmp/kairic-ab-$CFG.log
+: > "$LOG"
+
+export ROCM_PATH="$SDK/_rocm_sdk_devel"
+export LD_LIBRARY_PATH="$(dirname "$BIN"):$SDK/_rocm_sdk_libraries/lib:$SDK/_rocm_sdk_core/lib:$SDK/_rocm_sdk_devel/lib"
+export HIP_VISIBLE_DEVICES=0
+export HSA_OVERRIDE_GFX_VERSION=11.5.1
+export GGML_CUDA_GRAPH_OPT=0
+export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1
+export PROMPTFORGE_TARGET_ONLY=0
+export PROMPTFORGE_MODE=iu4_ffn
+export PROMPTFORGE_IU4_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs"
+export PROMPTFORGE_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs"
+export PROMPTFORGE_IU4_HADAMARD=1
+export PROMPTFORGE_IU4_SEGMENTED=0
+export PROMPTFORGE_ENABLE_FFN_KEEPERS=0
+export PROMPTFORGE_FFN_KEEPERS=late6
+export PROMPTFORGE_ENABLE_GDN=1
+export PROMPTFORGE_ENABLE_GDN_W8=0
+export PROMPTFORGE_GDN_SIDECAR_OVERRIDE="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs"
+export PROMPTFORGE_GDN_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs"
+export PROMPTFORGE_GDN_IU4_HADAMARD=1
+export PROMPTFORGE_ENABLE_GDN_KEEPERS=0
+export PROMPTFORGE_ENABLE_GDN_OUTPUT=1
+export PROMPTFORGE_GDN_OUTPUT_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs"
+export PROMPTFORGE_GDN_OUTPUT_KEEPERS=v3_lateq6
+export PROMPTFORGE_ENABLE_SMALLM_IU4=1
+export PROMPTFORGE_ENABLE_SMALLM_GDN_IU4=1
+export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0
+export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0
+export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0
+
+MMAP=""
+case "$CFG" in
+  runner)  export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512;  REA=off;  RFMT=none;     EXTRA="" ;;
+  nommap)  export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512;  REA=off;  RFMT=none;     EXTRA=""; MMAP="--no-mmap" ;;
+  nofast)  export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=2048; UB=512;  REA=off;  RFMT=none;     EXTRA="" ;;
+  prod)    export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=4096; REA=on;   RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;;
+  prod2048)  export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=2048; UB=512;  REA=on;   RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;;
+  prodnommap) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=2048; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;;
+esac
+
+"$BIN" -m "$MDIR/Qwen3.8-27B-IU4-Kairic-Edge.gguf" \
+  --alias bench --host 127.0.0.1 --port "$PORT" --jinja \
+  -dev ROCm0 -ngl 999 -c "$CTX" -b "$B" -ub "$UB" -fa on -ctk f16 -ctv f16 \
+  -t 16 -tb 32 -np 1 -ctxcp 32 --cache-ram 8192 --cache-prompt --cache-idle-slots $MMAP \
+  --metrics --kairic-edge \
+  --spec-type draft-mtp --spec-draft-device ROCm0 --spec-draft-ngl 999 \
+  --spec-draft-type-k f16 --spec-draft-type-v f16 \
+  --spec-draft-n-max 4 --spec-draft-n-min 0 --spec-draft-p-min 0.0 --spec-draft-p-split 0.10 \
+  --spec-draft-backend-sampling \
+  --temp 0 --top-p 1.0 --top-k 0 --min-p 0.0 \
+  --reasoning "$REA" --reasoning-format "$RFMT" --reasoning-budget -1 \
+  $EXTRA >> "$LOG" 2>&1 &
+SPID=$!
+trap 'kill $SPID 2>/dev/null; wait $SPID 2>/dev/null' EXIT
+
+for i in $(seq 1 300); do
+  kill -0 $SPID 2>/dev/null || { echo "SERVER DIED"; tail -20 "$LOG"; exit 2; }
+  curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1 && break
+  sleep 2
+done
+echo "=== CFG=$CFG ready ==="
+
+python3 - "$PORT" "$CFG" "$PCHARS" <<'PY'
+import sys, json, time, urllib.request
+port, cfg, pchars = sys.argv[1], sys.argv[2], int(sys.argv[3])
+VAR = [
+    "narrative nonfiction blends observation with reflection, ",
+    "and a careful writer lets each clause earn its place ",
+    "in the sentence's rising line, so meaning accumulates slowly. ",
+    "The reader is not hurried; the reader is led. Loose and periodic, ",
+    "the style turns on concrete nouns and verbs that carry weight. ",
+    "A paragraph is not a list but a single turn of thought that ",
+    "opens, develops, and lands, and clarity is a courtesy owed ",
+    "to the willing reader who gives the writer their attention. ",
+]
+parts = []
+t = 0
+while len("".join(parts)) < pchars:
+    for k in range(3):
+        parts.append(VAR[(t + k) % len(VAR)])
+    parts.append(f" Note {t}. ")
+    t += 1
+longtext = "".join(parts)
+prompt = longtext + "\n\nNow write a long, detailed continuation of this essay (at least five paragraphs) about the philosophy of language.\n"
+body = {
+    "model": "bench",
+    "prompt": prompt,
+    "max_tokens": 512,
+    "temperature": 0,
+    "top_p": 1.0,
+    "top_k": 0,
+    "min_p": 0.0,
+    "stream": False,
+    "cache_prompt": True,
+}
+# ignore_eos is rejected by the exact-Q8 greedy argmax fast path (runner cfg).
+if cfg not in ("runner",):
+    body["ignore_eos"] = True
+def run(tag):
+    req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",
+                                 data=json.dumps(body).encode(),
+                                 headers={"Content-Type": "application/json"})
+    t0 = time.time()
+    try:
+        with urllib.request.urlopen(req, timeout=3600) as resp:
+            out = json.load(resp)
+    except Exception as e:
+        body_txt = getattr(e, "read", lambda: b"")()
+        print(f"[{tag}] REQUEST FAILED: {e}")
+        print(f"[{tag}] body: {body_txt.decode('utf-8', 'ignore')[:500]}")
+        raise SystemExit(3)
+    wall = time.time() - t0
+    u = out.get("usage", {})
+    tm = out.get("timings", {})
+    pt = u.get("prompt_tokens", 0); ct = u.get("completion_tokens", 0)
+    pps = tm.get("prompt_per_second", 0); tps = tm.get("predicted_per_second", 0)
+    print(f"[{tag}] cfg={cfg} prompt={pt} completion={ct} prefill={pps:.1f} t/s decode={tps:.2f} t/s wall={wall:.1f}s")
+run("cold")
+run("warm")
+PY
+
+grep -E "print_timing|statistics" "$LOG" | tail -6
+exit 0
@@ -0,0 +1,191 @@
+#!/usr/bin/env bash
+# bench-spec-stack.sh — compare MTP-only vs MTP+ngram-mod spec on the v3
+# CIRU prompt-forge stack (Dankclaw, gfx1151), at 8k / 16k / 64k context.
+#
+# Reads authoritative prefill/generation tok/s from llama-server's own
+# `print_timing` lines (prompt eval time / eval time). Runs one server
+# instance per (spec, ctx) on a scratch port.
+#
+# Usage:
+#   ./scripts/bench-spec-stack.sh <draft-mtp|draft-mtp,ngram-mod> <ctx> [out_tokens]
+# Example:
+#   ./scripts/bench-spec-stack.sh draft-mtp,ngram-mod 16384 256
+set -euo pipefail
+
+SPEC="${1:?usage: bench-spec-stack.sh <spec> <ctx> [out_tokens]}"
+CTX="${2:?usage: bench-spec-stack.sh <spec> <ctx> [out_tokens]}"
+OUT_TOKENS="${3:-256}"
+
+ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-large.env
+source "$ENV_FILE"
+
+# This harness benchmarks the retired v3 stack. Force the v3 values so the
+# harness works regardless of the live service env (now Kairic).
+MODEL_DIR=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX-v3
+MODEL_FILE=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v3-Q8-FFNEQ-LateQ6.gguf
+FFN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs
+GDN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs
+OUTPUT_K8_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-Output-K8.pfs
+MMPROJ_FILE=Qwen3.8-27B-mmproj-BF16.mmproj
+
+# Derive the PromptForge sidecar paths from the env base values.
+export PROMPTFORGE_SIDECAR="$MODEL_DIR/$FFN_PFS"
+export PROMPTFORGE_GDN_SIDECAR="$MODEL_DIR/$GDN_PFS"
+export PROMPTFORGE_MTP_OUTPUT_K8_PROXY="$MODEL_DIR/$OUTPUT_K8_PFS"
+
+MODEL="$MODEL_DIR/$MODEL_FILE"
+LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server
+PORT=7893
+PARALLEL=1
+BATCH=4096
+UBATCH=4096
+LOG="/tmp/specstack-$(echo "$SPEC" | tr ',' '-')-ctx$CTX.log"
+: > "$LOG"
+
+for f in "$MODEL" "$PROMPTFORGE_SIDECAR" "$PROMPTFORGE_GDN_SIDECAR" "$PROMPTFORGE_MTP_OUTPUT_K8_PROXY" "$MODEL_DIR/$MMPROJ_FILE"; do
+    [[ -f "$f" ]] || { echo "missing: $f" >&2; exit 1; }
+done
+[[ -x "$LLAMA_BIN" ]] || { echo "missing: $LLAMA_BIN" >&2; exit 1; }
+
+export PROMPTFORGE_MODE="m2048_fused_tail1476"
+export GGML_CUDA_GRAPH_OPT=0
+export PROMPTFORGE_ENABLE_SMALLM_W8=1
+export PROMPTFORGE_MTP_OUTPUT_K8=1
+export PROMPTFORGE_MTP_OUTPUT_K8_VALIDATE=0
+export PROMPTFORGE_MTP_OUTPUT_K8_PROXY
+unset LLAMA_MTP_CPU_ARGMAX_FASTPATH LLAMA_MTP_BACKEND_GREEDY || true
+unset PROMPTFORGE_IU4_SIDECAR PROMPTFORGE_GDN_IU4_SIDECAR || true
+unset PROMPTFORGE_ENABLE_NGRAM_MOD PROMPTFORGE_ENABLE_NGRAM_M65_IU4 || true
+
+export ROCM_PATH="$ROCM_DEVEL"
+export LD_LIBRARY_PATH="${ROCM_LIB}/lib:${ROCM_CORE}/lib:${ROCM_DEVEL}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+
+echo "=== spec-stack bench: spec=$SPEC ctx=$CTX model=$MODEL_FILE ==="
+echo "=== env: OUTPUT_K8=$PROMPTFORGE_MTP_OUTPUT_K8 smallm=$PROMPTFORGE_ENABLE_SMALLM_W8 ==="
+
+# ngram-mod tuning matches the retired fallback line: n_min 48 / n_max 64 / n_match 24
+"$LLAMA_BIN" \
+    --model "$MODEL" \
+    --mmproj "$MODEL_DIR/$MMPROJ_FILE" \
+    --alias bench \
+    --host 127.0.0.1 \
+    --port "$PORT" \
+    -dev ROCm0 \
+    -ngl 999 \
+    -c "$CTX" \
+    -b "$BATCH" \
+    -ub "$UBATCH" \
+    -fa on \
+    -ctk "$CACHE_TYPE_K" \
+    -ctv "$CACHE_TYPE_V" \
+    -np "$PARALLEL" \
+    -ctxcp "$CTX_CHECKPOINTS" \
+    --cache-ram "$CACHE_RAM" \
+    --cache-prompt \
+    --no-mmap \
+    --spec-type "$SPEC" \
+    --spec-draft-device ROCm0 \
+    --spec-draft-ngl 999 \
+    --spec-draft-type-k f16 \
+    --spec-draft-type-v f16 \
+    --spec-draft-n-max 4 \
+    --spec-draft-n-min 0 \
+    --spec-draft-p-min 0.0 \
+    --spec-draft-p-split 0.10 \
+    --spec-draft-backend-sampling \
+    --spec-ngram-mod-n-min 48 \
+    --spec-ngram-mod-n-max 64 \
+    --spec-ngram-mod-n-match 24 \
+    --temp 0.7 \
+    --top-p 0.95 \
+    -lv 4 \
+    >> "$LOG" 2>&1 &
+SERVER_PID=$!
+trap 'kill $SERVER_PID 2>/dev/null; wait $SERVER_PID 2>/dev/null' EXIT
+
+echo "server pid $SERVER_PID on 127.0.0.1:$PORT"
+echo -n "waiting for server"
+ready=0
+crashed=0
+for i in $(seq 1 900); do
+    if ! kill -0 "$SERVER_PID" 2>/dev/null; then
+        crashed=1
+        break
+    fi
+    if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
+        echo " [ready]"
+        ready=1
+        break
+    fi
+    [ $((i % 10)) = 0 ] && echo -n .
+    sleep 2
+done
+if [ "$crashed" = "1" ]; then
+    echo " BACKEND_CRASHED (server died before ready)"
+    tail -30 "$LOG"
+    exit 2
+fi
+[ "$ready" = "1" ] || { echo " FAIL (timeout)"; tail -25 "$LOG"; exit 1; }
+
+# Print the implementations that initialized (feasibility evidence)
+grep -E "adding speculative implementation|ngram_mod|draft-mtp" "$LOG" | head -6
+
+python3 - "$SPEC" "$CTX" "$PORT" "$OUT_TOKENS" "$LOG" <<'PY'
+import sys, time, json, urllib.request
+spec, ctx, port, out_tokens, log = sys.argv[1], int(sys.argv[2]), sys.argv[3], int(sys.argv[4]), sys.argv[5]
+
+# Flowing prose (varied) at ~75% of ctx so MTP acceptance is realistic.
+VAR = (
+    "narrative nonfiction blends observation with reflection, ",
+    "and a careful writer lets each clause earn its place ",
+    "in the sentence's rising line, so meaning accumulates slowly. ",
+    "The reader is not hurried; the reader is led. Loose and periodic, ",
+    "the style turns on concrete nouns and verbs that carry weight, ",
+    "and it resists the abstract noun that hides a vague claim. ",
+    "A paragraph is not a list. It is a single turn of thought that ",
+    "opens, develops, and lands. This is the craft of prose. Choose, ",
+    "arrange, and above all omit. Omission is the discipline that ",
+    "keeps the whole clear, and clarity is the courtesy we owe the ",
+    "willing reader."
+)
+n = len(VAR)
+parts = []
+t = 0
+while t * 76 < (ctx * 3) // 4:
+    for k in range(5):
+        parts.append(VAR[(t + k) % n])
+    parts.append(f" This is cued by note {t}. ")
+    t += 1
+prompt = "".join(parts)
+prompt += ("\n\nNow, in a detailed and flowing paragraph, continue this essay about "
+           "the philosophy of language and its influence on modern computation. ")
+
+body = json.dumps({
+    "prompt": prompt,
+    "max_tokens": out_tokens,
+    "temperature": 0.0,
+    "stream": False,
+}).encode()
+req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",
+                             data=body, headers={"Content-Type": "application/json"})
+t0 = time.time()
+with urllib.request.urlopen(req, timeout=7200) as resp:
+    out = json.load(resp)
+dt = time.time() - t0
+pt = out.get("usage", {}).get("prompt_tokens", 0)
+ct = out.get("usage", {}).get("completion_tokens", 0)
+print(f"request done: prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s", flush=True)
+
+lines = open(log, encoding="utf-8", errors="ignore").read().splitlines()
+prompt_evals = [l for l in lines if "prompt eval time" in l]
+evals       = [l for l in lines if "eval time" in l and "prompt eval" not in l]
+
+def grab(lines, label):
+    if lines:
+        print(label + ": " + lines[-1].strip())
+
+print(f"== RESULT spec={spec} ctx={ctx} ==")
+grab(prompt_evals, "PREFILL")
+grab(evals, "GENERATION")
+print(f"prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s")
+PY
@@ -0,0 +1,80 @@
+# Tunables for frank-default-large (CIRU llama.cpp, dankclaw, Strix Halo
+# gfx1151).
+#
+# Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag
+# qwen3.8-activefpx-promptforge-v2.3 plus the v3 output-K8 runtime patch
+# serving jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge (v3). v3 tensor
+# allocation (FFN-equalized, late-Q6 writers, full-Q8 output) with the
+# full-W8 PromptForge companions, the Output-K8 draft proxy, and the BF16
+# vision projector. Alias frank-default-large, port 52625.
+#
+# This runtime replaced the retired vLLM service that previously occupied
+# this slot. See docs/ciru-activefpx-promptforge.md.
+#
+# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm
+# HIP runtime reads those bare env vars (see the 2026-08-13 TEMP incident).
+# llama.cpp reads --host/--port/--temp as CLI flags, never as env vars.
+
+# CIRU llama-server binary (ROCmFPX v2.3 + v3 output-K8 patch build, see
+# docs/ciru runtime page). Built against the private TheRock ROCm 7.15 SDK
+# kept in the retired ling3-ciru venv. ROCm 7.15 is required for the
+# advertised Strix Halo performance; the system ROCm 7.2.4 selects
+# slower/fallback kernels.
+LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server
+
+# Model files (v3). The five-file payload (GGUF + FFN/GDN .pfs + Output-K8
+# proxy + BF16 vision projector) is the full v3 configuration. The v3 files
+# live in a separate directory so the v2 artifacts stay for rollback.
+MODEL_DIR=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX-v3
+MODEL_FILE=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v3-Q8-FFNEQ-LateQ6.gguf
+FFN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs
+GDN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs
+OUTPUT_K8_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-Output-K8.pfs
+MMPROJ_FILE=Qwen3.8-27B-mmproj-BF16.mmproj
+
+# ROCm 7.15 SDK paths (inside the retired ling3-ciru venv). The serve script
+# exports ROCM_PATH and LD_LIBRARY_PATH from these so the 7.15 libraries
+# precede any system ROCm 7.2.4.
+ROCM_DEVEL=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel
+ROCM_LIB=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries
+ROCM_CORE=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core
+
+ALIAS=frank-default-large
+BIND_ADDR=0.0.0.0
+LISTEN_PORT=52625
+
+# Serving: full native 262144 context (n_ctx_train), 1 slot (user request
+# 2026-08-19: parallel slots degraded performance). -c is the TOTAL
+# context; with PARALLEL=1 it equals the per-slot context. Concurrent
+# requests queue on the single slot; the server never preempts a running
+# generation. f16 KV, context checkpoints, prompt cache, native MTP
+# depth-4. The Strix Halo is unified memory; rocm-smi reports the 64 GiB
+# budget. KV at 262144 x 64 KiB/cell is ~16 GiB.
+CTX_SIZE=262144
+PARALLEL=1
+BATCH_SIZE=4096
+UBATCH_SIZE=4096
+CACHE_TYPE_K=f16
+CACHE_TYPE_V=f16
+CTX_CHECKPOINTS=16
+CACHE_RAM=8192
+
+# Native MTP draft (depth-4, p_min 0, backend sampling). The qualified v3
+# profile drives the draft via the Output-K8 proxy; the CPU argmax fast
+# path and IU4/ngram lanes are disabled in the serve script.
+SPEC_DRAFT_N_MAX=4
+SPEC_DRAFT_N_MIN=0
+SPEC_DRAFT_P_MIN=0.0
+SPEC_DRAFT_P_SPLIT=0.10
+
+# Sampling (model card defaults).
+TEMPERATURE=0.7
+TOP_P=0.95
+
+# Thinking enabled (user request, 2026-08-18). deepseek format moves the
+# <think> reasoning into message.reasoning_content (OpenAI-compatible);
+# none would leave the thought tags unparsed in content.
+REASONING=on
+REASONING_FORMAT=deepseek
+
+VERBOSITY=4
@@ -0,0 +1,16 @@
+[Unit]
+Description=CIRU llama.cpp Qwen3.8-27B ActiveFPX PromptForge MTP (dankclaw, frank-default-large)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=dank
+Group=dank
+Environment="FRANK_LARGE_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-large.env"
+ExecStart=/home/dank/vllmScripts/scripts/serve-frank-large.sh
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,17 @@
+# Tunables for frank-default-npu (FastFlowLM NPU endpoint, Dankclaw)
+# Runtime: FLM 1.0.1 on the XDNA2 NPU (50 TOPS). Model: Gemma4-E4B-IT
+# (8B, Q4_1, vision + audio + reasoning). The NPU allocates from the
+# unified pool shared with frank-default-large (the CIRU llama.cpp
+# service on the iGPU). See docs/FastFlowLM NPU Inference.md.
+#
+# 2026-08-20: gemma4-it:e4b deployed. Footprint ~9.1 GB. 64k context.
+
+FLM_MODEL=gemma4-it:e4b
+FLM_HOST=0.0.0.0
+FLM_PORT=52626
+FLM_PMODE=performance
+FLM_ASR=0
+# 64K context (native max for gemma4-it is 32768, but NPU supports up to 256k).
+# The model's max_prefill_len is 4096, so prompts longer than that get
+# truncated at prefill time.
+FLM_CTX_LEN=65536
@@ -0,0 +1,17 @@
+[Unit]
+Description=FastFlowLM NPU inference (frank-default-npu) on Dankclaw
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=dank
+Group=dank
+Environment="FLM_NPU_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-npu.env"
+ExecStart=/home/dank/vllmScripts/scripts/serve-frank-default-npu.sh
+LimitMEMLOCK=infinity
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,61 @@
+# Tunables for frank-default-small (HauhauCS Qwen3.6-35B-A3B Q6_K_P,
+# ngram-mod spec, Dankserver, RTX 3090).
+#
+# Runtime: upstream llama.cpp llama-server (b10121). RTX 3090 24 GB;
+# Ryzen 7 7700X; 62 GB RAM. MoE experts offloaded to CPU (--n-cpu-moe 36).
+# Restored 2026-08-18 (was the pre-2026-08-17-redo frank-default-small).
+#
+# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The
+# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
+# incident). This box is NVIDIA/CUDA, but the rule keeps the env portable.
+
+# Model files (already on disk; Q6_K_P GGUF + f16 mmproj vision encoder).
+MODEL_DIR=/home/dank/models
+MODEL_FILE=Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q6_K_P.gguf
+MMPROJ_FILE=mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf
+# Upstream llama.cpp binary (b10121).
+LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
+
+ALIAS=frank-default-small
+BIND_ADDR=0.0.0.0
+LISTEN_PORT=7770
+
+# 3 parallel slots x 262144 ctx each = ~786432 total context pool (max per-slot).
+# Dual-f16 KV at 786432 cells leaves ~15 GiB free on the 3090.
+CTX_SIZE=786432
+PARALLEL=3
+
+# Spec: ngram-mod (n_min=48, n_max=64, n_match=24).
+SPEC_TYPE=ngram-mod
+NGRAM_MOD_N_MIN=48
+NGRAM_MOD_N_MAX=64
+NGRAM_MOD_N_MATCH=24
+
+# Expert offload: 38 of 40 layers keep MoE weights on CPU; 2 on GPU.
+N_CPU_MOE=38
+N_GPU_LAYERS=all
+
+# KV cache on GPU, both K and V at f16.
+CACHE_TYPE_K=f16
+CACHE_TYPE_V=f16
+
+VERBOSITY=4
+# Store SWA/context checkpoints in RAM to hold peak GPU memory down.
+CTX_CHECKPOINTS=16
+
+# Big prefill batches (ubatch 2048, batch 4096).
+BATCH_SIZE=4096
+UBATCH_SIZE=2048
+
+# Eager RAM load (MoE experts on CPU; ~31 GB RAM).
+NO_MMAP=1
+
+# Sampling (HauhauCS Qwen3.6-35B-A3B model card, thinking-general).
+TEMPERATURE=1.0
+TOP_K=20
+TOP_P=0.95
+MIN_P=0.0
+
+# Compute threads lowered to 7 so other processes keep a core.
+THREADS=7
+THREADS_HTTP=8
@@ -0,0 +1,16 @@
+[Unit]
+Description=llama.cpp Qwen3.6-35B-A3B Q6_K_P + ngram-mod (dankserver, frank-default-small)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=dank
+Group=dank
+Environment="FRANK_SMALL_ENV_FILE=/home/dank/vLLM/scripts/frank-default-small.env"
+ExecStart=/home/dank/vLLM/scripts/serve-frank-small.sh
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Serve the FastFlowLM (FLM) NPU endpoint — frank-default-npu.
+#
+# Runs flm serve with the model tag in frank-default-npu.env. The NPU
+# (XDNA2, 50 TOPS) is a single-lane device: generations serialize in FIFO
+# order, so this is a background/triage endpoint. The NPU shares the ~30 GiB
+# unified pool with frank-default-large (the CIRU llama.cpp service on the
+# iGPU); see docs/FastFlowLM NPU Inference.md for the memory budget.
+#
+# NOTE: FLM requires unlimited memlock. The unit sets LimitMEMLOCK=infinity.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FLM_NPU_ENV_FILE:-$SCRIPT_DIR/frank-default-npu.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -------------------
+: "${FLM_BIN:=/usr/bin/flm}"
+: "${FLM_MODEL:=gemma4-it:e4b}"
+: "${FLM_HOST:=0.0.0.0}"
+: "${FLM_PORT:=52626}"
+: "${FLM_PMODE:=performance}"
+: "${FLM_ASR:=0}"
+: "${FLM_CTX_LEN:=-1}"
+
+if [[ ! -x "$FLM_BIN" ]]; then
+    echo "missing flm binary: $FLM_BIN" >&2
+    exit 1
+fi
+
+# --- optional flags ----------------------------------------------------------
+OPT_ARGS=()
+if [[ "${FLM_ASR}" != "0" ]]; then
+    OPT_ARGS+=(-a "${FLM_ASR}")
+fi
+if [[ "${FLM_CTX_LEN}" != "-1" ]]; then
+    OPT_ARGS+=(-c "${FLM_CTX_LEN}")
+fi
+
+exec "$FLM_BIN" serve "$FLM_MODEL" \
+    --host "$FLM_HOST" \
+    -p "$FLM_PORT" \
+    --pmode "$FLM_PMODE" \
+    "${OPT_ARGS[@]}"
@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+# Serve jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge (v3) as
+# frank-default-large via the CIRU ROCmFPX llama.cpp runtime on Dankclaw
+# (Strix Halo gfx1151).
+#
+# Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag
+# qwen3.8-activefpx-promptforge-v2.3 (commit 103af8b41) plus the v3 output-K8
+# runtime patch (runtime/qwen38-v3-output-k8-runtime.patch, commit af0eac6 on
+# Dankclaw) against the private TheRock ROCm 7.15 SDK (kept in the retired
+# ling3-ciru venv). The model is the v3 GGUF with the full-W8 PromptForge
+# companion views (FFN/GDN .pfs), the Output-K8 draft proxy, and the BF16
+# vision projector. Native depth-4 MTP speculative decoding.
+#
+# The model card requires ROCm 7.15 (not the system ROCm 7.2.4) for the
+# advertised Strix Halo performance. LD_LIBRARY_PATH therefore points at the
+# 7.15 SDK libraries first. An older system ROCm must not precede them.
+#
+# Tunables come from frank-default-large.env in this directory unless
+# FRANK_LARGE_ENV_FILE overrides that path.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -------------------
+: "${LLAMA_BIN:=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server}"
+: "${MODEL_DIR:=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX-v3}"
+: "${MODEL_FILE:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v3-Q8-FFNEQ-LateQ6.gguf}"
+: "${FFN_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs}"
+: "${GDN_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs}"
+: "${OUTPUT_K8_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-Output-K8.pfs}"
+: "${MMPROJ_FILE:=Qwen3.8-27B-mmproj-BF16.mmproj}"
+: "${ROCM_DEVEL:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel}"
+: "${ROCM_LIB:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries}"
+: "${ROCM_CORE:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core}"
+: "${ALIAS:=frank-default-large}"
+: "${BIND_ADDR:=0.0.0.0}"
+: "${LISTEN_PORT:=52625}"
+: "${CTX_SIZE:=524288}"
+: "${PARALLEL:=2}"
+: "${BATCH_SIZE:=4096}"
+: "${UBATCH_SIZE:=4096}"
+: "${CACHE_TYPE_K:=f16}"
+: "${CACHE_TYPE_V:=f16}"
+: "${CTX_CHECKPOINTS:=16}"
+: "${CACHE_RAM:=8192}"
+: "${SPEC_DRAFT_N_MAX:=4}"
+: "${SPEC_DRAFT_N_MIN:=0}"
+: "${SPEC_DRAFT_P_MIN:=0.0}"
+: "${SPEC_DRAFT_P_SPLIT:=0.10}"
+: "${TEMPERATURE:=0.7}"
+: "${TOP_P:=0.95}"
+: "${REASONING:=on}"
+: "${REASONING_FORMAT:=deepseek}"
+: "${VERBOSITY:=4}"
+
+MODEL="$MODEL_DIR/$MODEL_FILE"
+export PROMPTFORGE_SIDECAR="$MODEL_DIR/$FFN_PFS"
+export PROMPTFORGE_GDN_SIDECAR="$MODEL_DIR/$GDN_PFS"
+# v3 qualified runtime: Output-K8 draft proxy + small-row W8 prompt route.
+export PROMPTFORGE_MODE="m2048_fused_tail1476"
+export GGML_CUDA_GRAPH_OPT=0
+export PROMPTFORGE_ENABLE_SMALLM_W8=1
+export PROMPTFORGE_MTP_OUTPUT_K8=1
+export PROMPTFORGE_MTP_OUTPUT_K8_VALIDATE=0
+export PROMPTFORGE_MTP_OUTPUT_K8_PROXY="$MODEL_DIR/$OUTPUT_K8_PFS"
+# The qualified v3 profile disables the CPU argmax draft fast path; the
+# Output-K8 proxy + backend sampling replace it. Also drop any IU4/ngram
+# companions from the non-IU4 development lane.
+unset LLAMA_MTP_CPU_ARGMAX_FASTPATH LLAMA_MTP_BACKEND_GREEDY || true
+unset PROMPTFORGE_IU4_SIDECAR PROMPTFORGE_GDN_IU4_SIDECAR || true
+unset PROMPTFORGE_ENABLE_NGRAM_MOD PROMPTFORGE_ENABLE_NGRAM_M65_IU4 || true
+
+for f in "$MODEL" "$PROMPTFORGE_SIDECAR" "$PROMPTFORGE_GDN_SIDECAR" "$PROMPTFORGE_MTP_OUTPUT_K8_PROXY" "$MODEL_DIR/$MMPROJ_FILE"; do
+    if [[ ! -f "$f" ]]; then
+        echo "missing required file: $f" >&2
+        exit 1
+    fi
+done
+if [[ ! -x "$LLAMA_BIN" ]]; then
+    echo "missing llama-server binary: $LLAMA_BIN" >&2
+    exit 1
+fi
+
+# ROCm 7.15 SDK libraries must precede any system ROCm in LD_LIBRARY_PATH.
+export ROCM_PATH="$ROCM_DEVEL"
+export LD_LIBRARY_PATH="${ROCM_LIB}/lib:${ROCM_CORE}/lib:${ROCM_DEVEL}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+
+exec "$LLAMA_BIN" \
+    --model "$MODEL" \
+    --mmproj "$MODEL_DIR/$MMPROJ_FILE" \
+    --alias "$ALIAS" \
+    --host "$BIND_ADDR" \
+    --port "$LISTEN_PORT" \
+    -dev ROCm0 \
+    -ngl 999 \
+    -c "$CTX_SIZE" \
+    -b "$BATCH_SIZE" \
+    -ub "$UBATCH_SIZE" \
+    -fa on \
+    -ctk "$CACHE_TYPE_K" \
+    -ctv "$CACHE_TYPE_V" \
+    -np "$PARALLEL" \
+    -ctxcp "$CTX_CHECKPOINTS" \
+    --cache-ram "$CACHE_RAM" \
+    --cache-prompt \
+    --metrics \
+    --no-mmap \
+    --spec-type draft-mtp \
+    --spec-draft-device ROCm0 \
+    --spec-draft-ngl 999 \
+    --spec-draft-type-k f16 \
+    --spec-draft-type-v f16 \
+    --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
+    --spec-draft-n-min "$SPEC_DRAFT_N_MIN" \
+    --spec-draft-p-min "$SPEC_DRAFT_P_MIN" \
+    --spec-draft-p-split "$SPEC_DRAFT_P_SPLIT" \
+    --spec-draft-backend-sampling \
+    --temp "$TEMPERATURE" \
+    --top-p "$TOP_P" \
+    --reasoning "$REASONING" \
+    --reasoning-format "$REASONING_FORMAT" \
+    --reasoning-budget -1 \
+    -lv "$VERBOSITY"
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+# Serve HauhauCS Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q6_K_P with
+# ngram-mod speculative decoding on Dankserver (RTX 3090, MoE experts on CPU).
+#
+# Runtime: upstream llama.cpp llama-server (b10121). The model is a 35B-total /
+# 3B-active MoE vision-language model; --n-cpu-moe keeps most expert weights on
+# CPU so the non-expert tensors + KV cache fit the 24 GB 3090.
+#
+# Tunables come from frank-default-small.env in this directory unless the
+# FRANK_SMALL_ENV_FILE environment variable overrides that path.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -----------------
+: "${MODEL_DIR:=/home/dank/models}"
+: "${MODEL_FILE:=Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q6_K_P.gguf}"
+: "${MMPROJ_FILE:=mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf}"
+: "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}"
+: "${ALIAS:=frank-default-small}"
+: "${BIND_ADDR:=0.0.0.0}"
+: "${LISTEN_PORT:=7770}"
+: "${CTX_SIZE:=786432}"
+: "${PARALLEL:=3}"
+: "${SPEC_TYPE:=ngram-mod}"
+: "${NGRAM_MOD_N_MIN:=48}"
+: "${NGRAM_MOD_N_MAX:=64}"
+: "${NGRAM_MOD_N_MATCH:=24}"
+: "${CACHE_TYPE_K:=f16}"
+: "${CACHE_TYPE_V:=f16}"
+: "${VERBOSITY:=4}"
+: "${N_CPU_MOE:=38}"
+: "${N_GPU_LAYERS:=all}"
+: "${BATCH_SIZE:=4096}"
+: "${UBATCH_SIZE:=2048}"
+: "${CTX_CHECKPOINTS:=16}"
+: "${NO_MMAP:=1}"
+: "${TEMPERATURE:=1.0}"
+: "${TOP_K:=20}"
+: "${TOP_P:=0.95}"
+: "${MIN_P:=0.0}"
+: "${THREADS:=7}"
+: "${THREADS_HTTP:=8}"
+
+MODEL="$MODEL_DIR/$MODEL_FILE"
+MMPROJ="$MODEL_DIR/$MMPROJ_FILE"
+
+for f in "$MODEL" "$MMPROJ"; do
+    if [[ ! -f "$f" ]]; then
+        echo "missing required file: $f" >&2
+        exit 1
+    fi
+done
+if [[ ! -x "$LLAMA_SERVER" ]]; then
+    echo "missing llama-server binary: $LLAMA_SERVER" >&2
+    exit 1
+fi
+
+exec "$LLAMA_SERVER" \
+    --model "$MODEL" \
+    --mmproj "$MMPROJ" \
+    --jinja \
+    --spec-type "$SPEC_TYPE" \
+    --spec-ngram-mod-n-min "$NGRAM_MOD_N_MIN" \
+    --spec-ngram-mod-n-max "$NGRAM_MOD_N_MAX" \
+    --spec-ngram-mod-n-match "$NGRAM_MOD_N_MATCH" \
+    --cache-type-k "$CACHE_TYPE_K" \
+    --cache-type-v "$CACHE_TYPE_V" \
+    --verbosity "$VERBOSITY" \
+    --ctx-size "$CTX_SIZE" \
+    --parallel "$PARALLEL" \
+    --n-cpu-moe "$N_CPU_MOE" \
+    --gpu-layers "$N_GPU_LAYERS" \
+    --batch-size "$BATCH_SIZE" \
+    --ubatch-size "$UBATCH_SIZE" \
+    --ctx-checkpoints "$CTX_CHECKPOINTS" \
+    $([ "$NO_MMAP" = "1" ] && printf -- "--no-mmap") \
+    --temp "$TEMPERATURE" \
+    --top-k "$TOP_K" \
+    --top-p "$TOP_P" \
+    --min-p "$MIN_P" \
+    --threads "$THREADS" \
+    --threads-http "$THREADS_HTTP" \
+    --alias "$ALIAS" \
+    --host "$BIND_ADDR" \
+    --port "$LISTEN_PORT"