Reorganize: llm/old -> old (retired PP/NPU-era scripts, reference only)

2026-08-22 15:06:51 -07:00
parent 017d3704d2
commit 506614525b
28 changed files with 0 additions and 0 deletions
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Analyze the spec-matrix TSV: per-config median gen t/s, acceptance,
+sorted, with the top configs and per-spec-type bests."""
+import sys, statistics, collections
+
+path = sys.argv[1] if len(sys.argv) > 1 else "/tmp/spec_matrix_results.tsv"
+rows = []
+with open(path) as f:
+    header = f.readline().strip().split("\t")
+    for line in f:
+        p = line.rstrip("\n").split("\t")
+        if len(p) < 9:
+            continue
+        spec, n_max, p_min, run, tps, pps, acc, dn, stop = p[:9]
+        try:
+            tps = float(tps)
+        except ValueError:
+            continue
+        rows.append(dict(spec=spec, n_max=n_max, p_min=p_min, run=run,
+                         tps=tps, pps=float(pps or 0), acc=acc, dn=dn, stop=stop))
+
+by_cfg = collections.defaultdict(list)
+for r in rows:
+    by_cfg[(r["spec"], r["n_max"], r["p_min"])].append(r)
+
+print("%-22s %5s %5s %8s %8s %10s" % ("spec", "n_max", "p_min", "med_tps", "med_pps", "accept"))
+summary = []
+for (spec, n_max, p_min), runs in by_cfg.items():
+    tps_list = [r["tps"] for r in runs]
+    med = statistics.median(tps_list)
+    pps_list = [r["pps"] for r in runs]
+    accs = [r["acc"] for r in runs if r["acc"] not in ("", "None")]
+    acc_s = ""
+    if accs:
+        try:
+            nums = [float(a) for a in accs]
+            acc_s = "%.3f" % (sum(nums)/len(nums))
+        except ValueError:
+            pass
+    summary.append((med, spec, n_max, p_min,
+                    statistics.median(pps_list), acc_s, len(runs)))
+
+summary.sort(reverse=True)
+for med, spec, n_max, p_min, pps, acc, n in summary:
+    print("%-22s %5s %5s %8.2f %8.2f %10s" % (spec, n_max, p_min, med, pps, acc))
+
+print("\n=== TOP 12 CONFIGS ===")
+for med, spec, n_max, p_min, pps, acc, n in summary[:12]:
+    print("  %5.2f t/s  spec=%-22s n_max=%2s p_min=%s" % (med, spec, n_max, p_min))
+
+print("\n=== BEST PER SPEC TYPE ===")
+best = {}
+for med, spec, n_max, p_min, pps, acc, n in summary:
+    key = spec
+    if key not in best:
+        best[key] = (med, n_max, p_min, pps, acc)
+for k, v in best.items():
+    print("  %-22s %5.2f t/s n_max=%s p_min=%s (pps=%s)" % (k, v[0], v[1], v[2], v[3]))
+
+print("\n=== p_min 0.8 vs off (per spec/n_max, med delta) ===")
+from itertools import product
+for spec in sorted(set(r["spec"] for r in rows)):
+    print("  spec:", spec)
+    for n_max in sorted(set(int(r["n_max"]) for r in rows if r["spec"] == spec)):
+        on = [r["tps"] for r in rows if r["spec"] == spec and r["n_max"] == str(n_max) and r["p_min"] == "0.8"]
+        off = [r["tps"] for r in rows if r["spec"] == spec and r["n_max"] == str(n_max) and r["p_min"] == "none"]
+        if on and off:
+            d = statistics.median(on) - statistics.median(off)
+            print("    n_max=%-2s  on=%.2f off=%.2f delta=%+.2f" % (n_max,
+                  statistics.median(on), statistics.median(off), d))
@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+# bench-frank-large.sh — 64k-context benchmark for the frank-default-large stack.
+#
+# Measures generation throughput of Qwen3.6-27B Q8_K_P + DFlash + ngram-mod on
+# a single backend at 64k context. It starts llama-server on a scratch port,
+# drives one generation through the OpenAI-compatible API with a ~64k-token
+# prompt, and prints tokens/sec.
+#
+# Usage:
+#   ./scripts/bench-frank-large.sh <ROCm0|Vulkan0> [--generations N]
+#
+# Example:
+#   ./scripts/bench-frank-large.sh ROCm0
+#   ./scripts/bench-frank-large.sh Vulkan0 --generations 3
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+DEVICE="${1:?usage: bench-frank-large.sh <ROCm0|Vulkan0> [--generations N]}"
+GENERATIONS=1
+if [ "${2:-}" = "--generations" ]; then
+    GENERATIONS="${3:?--generations needs a count}"
+fi
+
+MODEL_DIR=/home/dank/models
+MODEL_FILE=Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf
+DRAFT_FILE=Qwen3.6-27B-DFlash-Q8_0.gguf
+LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
+MODEL="$MODEL_DIR/$MODEL_FILE"
+DRAFT="$MODEL_DIR/$DRAFT_FILE"
+
+PORT=7890
+CTX=65536
+PARALLEL=1
+BATCH=2048
+UBATCH=512
+OUT_TOKENS=256
+
+for f in "$MODEL" "$DRAFT"; do
+    if [[ ! -f "$f" ]]; then
+        echo "missing required file: $f" >&2
+        exit 1
+    fi
+done
+
+echo "=== 64k benchmark: device=$DEVICE model=$MODEL_FILE ==="
+
+# --- start the server ----------------------------------------------------
+"$LLAMA_SERVER" \
+    --model "$MODEL" \
+    --spec-draft-model "$DRAFT" \
+    --spec-type "draft-dflash,ngram-mod" \
+    --spec-draft-n-max 8 \
+    --spec-ngram-mod-n-min 48 \
+    --spec-ngram-mod-n-max 64 \
+    --spec-ngram-mod-n-match 24 \
+    --device "$DEVICE" \
+    --gpu-layers all \
+    --cache-type-k q8_0 \
+    --cache-type-v f16 \
+    --ctx-size "$CTX" \
+    --parallel "$PARALLEL" \
+    --batch-size "$BATCH" \
+    --ubatch-size "$UBATCH" \
+    --alias bench \
+    --host 127.0.0.1 \
+    --port "$PORT" \
+    > "/tmp/frankbench-$DEVICE.log" 2>&1 &
+SERVER_PID=$!
+echo "server pid $SERVER_PID on 127.0.0.1:$PORT"
+
+trap 'kill $SERVER_PID 2>/dev/null; wait $SERVER_PID 2>/dev/null' EXIT
+
+# --- wait for ready ------------------------------------------------------
+echo -n "waiting for server"
+ready=0
+for i in $(seq 1 400); do
+    if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
+        echo " [ready]"
+        ready=1
+        break
+    fi
+    echo -n "."
+    sleep 2
+done
+if [ "$ready" != "1" ]; then
+    echo ""
+    echo "FAIL: server did not become ready"
+    tail -25 "/tmp/frankbench-$DEVICE.log"
+    exit 1
+fi
+
+# --- drive N generations and measure --------------------------------------
+python3 - "$DEVICE" "$CTX" "$OUT_TOKENS" "$GENERATIONS" "$PORT" <<'PY'
+import sys, time, json, urllib.request, statistics
+
+device       = sys.argv[1]
+ctx          = int(sys.argv[2])
+out_tokens   = int(sys.argv[3])
+generations  = int(sys.argv[4])
+port         = sys.argv[5]
+
+# ~64k-token prompt. One prose unit is ~110 chars ≈ 28 tokens for this model.
+# We aim for a prompt of ~60k tokens so generation stays within the 64k ctx.
+unit = ("The careful study of natural language reveals patterns of meaning "
+        "that emerge from the deliberate arrangement of words and phrases. ")
+# approximate: unit is 160 chars, ~40 tokens; repeat to ~1500 units = ~60k tokens
+prompt = unit * 1500
+
+def req(prompt, n):
+    body = json.dumps({
+        "prompt": prompt,
+        "max_tokens": n,
+        "temperature": 0.0,
+        "stream": False,
+    }).encode()
+    r = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",
+                               data=body, headers={"Content-Type": "application/json"})
+    with urllib.request.urlopen(r, timeout=1800) as resp:
+        return json.load(resp)
+
+# prefill timing
+t0 = time.time()
+first = req(prompt, 0)
+t_pre = time.time() - t0
+used = first.get("usage", {})
+prompt_tokens = used.get("prompt_tokens", 0)
+print(f"prefill: {prompt_tokens} tokens in {t_pre:.2f}s "
+      f"({prompt_tokens/t_pre:.1f} tok/s)")
+
+# generation timing
+rates = []
+total_gentok = 0
+total_time = 0.0
+for i in range(generations):
+    t0 = time.time()
+    out = req(prompt, out_tokens)
+    dt = time.time() - t0
+    uit = out.get("usage", {})
+    comp = uit.get("completion_tokens", 0)
+    gt = dt - t_pre  # approximate generation time = total - prefill
+    r = comp / gt if gt > 0 else 0
+    rates.append(r)
+    total_gentok += comp
+    total_time += gt
+    print(f"gen {i+1}: {comp} tokens in {gt:.2f}s ({r:.1f} tok/s)")
+
+avg = statistics.mean(rates) if rates else 0
+print(f"\nRESULT device={device} prompt_tokens={prompt_tokens} "
+      f"avg_gen_tok_s={avg:.1f} over {generations} runs")
+PY
@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+# bench-high-ctx.sh — high-context prefill/generation bench for
+# frank-default-large (Qwen3.8-27B UD-Q8_K_XL, --spec-default) on a scratch
+# port.  Compares backends at very high context.
+#
+# Reads authoritative prefill / generation tok/s from the server's own
+# `slot print_timing` lines.
+#
+# Usage:
+#   ./bench-high-ctx.sh <ROCm0|Vulkan0> <ctx> [out_tokens]
+# Example:
+#   ./bench-high-ctx.sh ROCm0 262144
+set -euo pipefail
+
+DEVICE="${1:?usage: bench-high-ctx.sh <ROCm0|Vulkan0> <ctx> [out_tokens]}"
+CTX="${2:?usage: bench-high-ctx.sh <ROCm0|Vulkan0> <ctx> [out_tokens]}"
+OUT_TOKENS="${3:-256}"
+
+MODEL_DIR=/home/dank/models
+MODEL_FILE=Qwen3.8-27B-UD-Q8_K_XL.gguf
+LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
+MODEL="$MODEL_DIR/$MODEL_FILE"
+
+PORT=7892
+PARALLEL=1
+BATCH=8192
+UBATCH=256
+LOG="/tmp/highctx-$DEVICE.log"
+
+[[ -f "$MODEL" ]] || { echo "missing: $MODEL" >&2; exit 1; }
+: > "$LOG"
+
+echo "=== high-ctx bench: device=$DEVICE ctx=$CTX spec=--spec-default ==="
+
+"$LLAMA_SERVER" \
+    --model "$MODEL" \
+    --spec-default \
+    --device "$DEVICE" \
+    --gpu-layers all \
+    --fit off \
+    --cache-type-k q8_0 \
+    --cache-type-v q8_0 \
+    --ctx-size "$CTX" \
+    --parallel "$PARALLEL" \
+    --batch-size "$BATCH" \
+    --ubatch-size "$UBATCH" \
+    --alias bench \
+    --host 127.0.0.1 \
+    --port "$PORT" \
+    >> "$LOG" 2>&1 &
+SERVER_PID=$!
+trap 'kill $SERVER_PID 2>/dev/null; wait $SERVER_PID 2>/dev/null' EXIT
+
+echo "server pid $SERVER_PID on 127.0.0.1:$PORT"
+echo -n "waiting for server"
+ready=0
+crashed=0
+for i in $(seq 1 600); do
+    if ! kill -0 "$SERVER_PID" 2>/dev/null; then
+        crashed=1
+        break
+    fi
+    if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
+        echo " [ready]"
+        ready=1
+        break
+    fi
+    echo -n "."
+    sleep 2
+done
+if [ "$crashed" = "1" ]; then
+    echo " BACKEND_CRASHED (server died before ready)"
+    tail -30 "$LOG"
+    exit 2
+fi
+[ "$ready" = "1" ] || { echo " FAIL (timeout)"; tail -25 "$LOG"; exit 1; }
+
+python3 - "$DEVICE" "$CTX" "$PORT" "$OUT_TOKENS" "$LOG" <<'PY'
+import sys, time, json, urllib.request
+device, ctx, port, out_tokens, log = sys.argv[1], int(sys.argv[2]), sys.argv[3], int(sys.argv[4]), sys.argv[5]
+
+# Flowing, varied literary prose (no repeated walls of identical text, so the
+# model does NOT legitimately EOS early and ngram acceptance is realistic).
+VAR = (
+    "narrative nonfiction blends observation with reflection, ",
+    "and a careful writer lets each clause earn its place ",
+    "in the sentence's rising line, so meaning accumulates slowly. ",
+    "The reader is not hurried; the reader is led. Loose and periodic, ",
+    "the style turns on concrete nouns and verbs that carry weight, ",
+    "and it resists the abstract noun that hides a vague claim. ",
+    "A paragraph is not a list; it is a single turn of thought that ",
+    "opens, develops, and lands. This is the craft of prose: choose, ",
+    "arrange, and above all omit. Omission is the discipline that ",
+    "keeps the whole clear, and clarity is the courtesy we owe the ",
+    "willing reader."
+)
+# ~76 tokens per outer loop (measured on this model); target ~75% of ctx.
+n = len(VAR)
+parts = []
+t = 0
+while t * 76 < (ctx * 3) // 4:
+    for k in range(5):
+        parts.append(VAR[(t + k) % n])
+    parts.append(f" This is the {device} subject, cued by note {t}. ")
+    t += 1
+prompt = "".join(parts)
+# Explicit continuation request forces real generation.
+prompt += ("\n\nNow, in a detailed and flowing paragraph, continue this essay about "
+           "the philosophy of language and its influence on modern computation. ")
+
+body = json.dumps({
+    "prompt": prompt,
+    "max_tokens": out_tokens,
+    "temperature": 0.0,
+    "stream": False,
+}).encode()
+req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",
+                             data=body, headers={"Content-Type": "application/json"})
+t0 = time.time()
+with urllib.request.urlopen(req, timeout=7200) as resp:
+    out = json.load(resp)
+dt = time.time() - t0
+pt = out.get("usage", {}).get("prompt_tokens", 0)
+ct = out.get("usage", {}).get("completion_tokens", 0)
+print(f"request done: prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s", flush=True)
+
+lines = open(log, encoding="utf-8", errors="ignore").read().splitlines()
+prompt_evals = [l for l in lines if "prompt eval time" in l]
+evals       = [l for l in lines if "      eval time" in l and "prompt eval" not in l]
+
+def grab(lines, label):
+    if lines:
+        print(label + ": " + lines[-1].strip())
+
+print(f"== RESULT device={device} ctx={ctx} spec=--spec-default ==")
+grab(prompt_evals, "PREFILL")
+grab(evals, "GENERATION")
+print(f"prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s")
+PY
@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+# bench-pipeline-64k.sh — 4k-context benchmark through a speculative pipeline on
+# a single backend.
+#
+# Default spec chain is draft-dflash + ngram-mod. Pass a second argument to
+# override --spec-type (e.g. "ngram-mod" to bench ngram only, or "none" for no
+# speculation). The DFlash draft model is only referenced when the spec type
+# includes "dflash".
+#
+# Reads authoritative prefill / generation tok/s and draft acceptance from the
+# server's own `slot print_timing` lines.
+#
+# Usage:
+#   ./bench-pipeline-64k.sh <ROCm0|Vulkan0> [spec-type]
+set -euo pipefail
+
+DEVICE="${1:?usage: bench-pipeline-64k.sh <ROCm0|Vulkan0> [spec-type]}"
+SPEC_TYPE="${2:-draft-dflash,ngram-mod}"
+MODEL_DIR=/home/dank/models
+MODEL_FILE=Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf
+DRAFT_FILE=Qwen3.6-27B-DFlash-Q8_0.gguf
+LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
+MODEL="$MODEL_DIR/$MODEL_FILE"
+DRAFT="$MODEL_DIR/$DRAFT_FILE"
+
+PORT=7891
+CTX=8192        # 8k ctx: room for a ~4k prompt + generation
+PARALLEL=1
+BATCH=2048
+UBATCH=1024
+OUT_TOKENS=256
+LOG="/tmp/pipeline-$DEVICE.log"
+
+for f in "$MODEL" "$DRAFT"; do
+    [[ -f "$f" ]] || { echo "missing: $f" >&2; exit 1; }
+done
+
+: > "$LOG"
+echo "=== 4k pipeline bench: device=$DEVICE spec=$SPEC_TYPE ==="
+
+# Only add the draft-model argument when the spec type references dflash.
+DRAFT_ARGS=()
+if [[ "$SPEC_TYPE" == *dflash* ]]; then
+    DRAFT_ARGS+=(--spec-draft-model "$DRAFT")
+fi
+
+"$LLAMA_SERVER" \
+    --model "$MODEL" \
+    "${DRAFT_ARGS[@]}" \
+    --spec-type "$SPEC_TYPE" \
+    --spec-draft-n-max 8 \
+    --spec-ngram-mod-n-min 48 \
+    --spec-ngram-mod-n-max 64 \
+    --spec-ngram-mod-n-match 24 \
+    --device "$DEVICE" \
+    --gpu-layers all \
+    --cache-type-k q8_0 \
+    --cache-type-v f16 \
+    --ctx-size "$CTX" \
+    --parallel "$PARALLEL" \
+    --batch-size "$BATCH" \
+    --ubatch-size "$UBATCH" \
+    --alias bench \
+    --host 127.0.0.1 \
+    --port "$PORT" \
+    --ignore-eos \
+    >> "$LOG" 2>&1 &
+SERVER_PID=$!
+trap 'kill $SERVER_PID 2>/dev/null; wait $SERVER_PID 2>/dev/null' EXIT
+
+echo "server pid $SERVER_PID on 127.0.0.1:$PORT"
+echo -n "waiting for server"
+ready=0
+crashed=0
+for i in $(seq 1 200); do
+    if ! kill -0 "$SERVER_PID" 2>/dev/null; then
+        # server process died -> backend abort (e.g. Vulkan NONE-op crash)
+        crashed=1
+        break
+    fi
+    if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
+        echo " [ready]"
+        ready=1
+        break
+    fi
+    echo -n "."
+    sleep 2
+done
+if [ "$crashed" = "1" ]; then
+    echo " BACKEND_CRASHED (server died before ready)"
+    tail -30 "$LOG"
+    exit 2
+fi
+[ "$ready" = "1" ] || { echo " FAIL (timeout)"; tail -25 "$LOG"; exit 1; }
+
+# --- drive a ~4k-token generation, then pull timings from the server log ---
+python3 - "$DEVICE" "$SPEC_TYPE" "$PORT" "$OUT_TOKENS" "$LOG" <<'PY'
+import sys, time, json, urllib.request, os
+device, spec, port, out_tokens, log = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]), sys.argv[5]
+
+# Build a ~4k filler prompt, then append an incomplete continuation so the
+# model produces real generation (pure repetitive filler stops early/EOS).
+unit = ("The careful study of natural language reveals patterns of meaning that "
+        "emerge from the deliberate arrangement of words and phrases, and the "
+        "patient reader soon learns to recognize the structure beneath the surface. ")
+prompt = (unit * 120)  # ~16k chars -> ~4k tokens
+# Append a clear continuation target that forces real generation.
+prompt += ("\n\nNow, in a detailed and flowing paragraph, continue this essay about "
+           "the philosophy of language and its influence on modern computation. ")
+
+body = json.dumps({
+    "prompt": prompt,
+    "max_tokens": out_tokens,
+    "temperature": 0.0,
+    "stream": False,
+}).encode()
+req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",
+                             data=body, headers={"Content-Type": "application/json"})
+t0 = time.time()
+with urllib.request.urlopen(req, timeout=3600) as resp:
+    out = json.load(resp)
+dt = time.time() - t0
+pt = out.get("usage", {}).get("prompt_tokens", 0)
+ct = out.get("usage", {}).get("completion_tokens", 0)
+print(f"request done: prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s", flush=True)
+
+# The server log already printed authoritative per-slot timings; surface them.
+lines = open(log, encoding="utf-8", errors="ignore").read().splitlines()
+prompt_evals = [l for l in lines if "prompt eval time" in l]
+evals       = [l for l in lines if "      eval time" in l and "prompt eval" not in l]
+accepts     = [l for l in lines if "draft acceptance" in l]
+
+def grab(lines, label):
+    if lines:
+        print(label + ": " + lines[-1].strip())
+
+print(f"== RESULT spec={spec} device={device} ==")
+grab(prompt_evals, "PREFILL")
+grab(evals, "GENERATION")
+grab(accepts, "DRAFT_ACCEPTANCE")
+print(f"prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s")
+PY
@@ -0,0 +1,196 @@
+#!/usr/bin/env bash
+# deploy.sh — Deploy the vllmScripts repo to a target and restart its service.
+#
+# This script implements the second half of the git deployment workflow from
+# AGENTS.md. It assumes the local changes are committed and pushed, then:
+#   1. Pulls the repo on the target
+#   2. Installs / enables the systemd unit
+#   3. Drops page caches (dankclaw) and restarts the service
+#   4. Waits for the health endpoint and reports status
+#
+# Sudo: when FRANK_SUDO_PW is set, remote sudo pipes it via `sudo -S` for
+# headless runs. Otherwise plain `sudo` is used (tty or passwordless config).
+# Prefer the passwordless restart helpers installed on each target
+# (/usr/local/sbin/frank-restart-<large|small>.sh); they cover the unit
+# symlink, cache drop (dankclaw), daemon-reload, and restart in one root
+# script. See docs/deployment.md -> Passwordless restart.
+#
+# Usage:
+#   ./deploy.sh <target> [--dry-run]
+#   FRANK_SUDO_PW=... ./deploy.sh <target>
+#
+# Targets:
+#   dankclaw   - llama.cpp server (frank-default-large.service, port 7770)
+#   dankserver - llama.cpp server (frank-default-small.service, port 7770)
+#
+# Example:
+#   ./deploy.sh dankclaw
+#   ./deploy.sh dankserver --dry-run
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Target configuration
+# ---------------------------------------------------------------------------
+declare -A REMOTE_USER REMOTE_HOST REMOTE_DIR SERVICE SERVICE_HEALTH DROP_CACHES HELPER HEALTH_WAIT
+
+REMOTE_USER[dankclaw]=dank
+REMOTE_HOST[dankclaw]=dankclaw
+REMOTE_DIR[dankclaw]="/home/dank/vllmScripts"
+SERVICE[dankclaw]="frank-default-large"
+SERVICE_HEALTH[dankclaw]="http://127.0.0.1:7770/health"
+# Drop page caches before restart: with the model's pages fully cached, the
+# KV/graph allocation stalls at load (VRAM stuck at 28 GiB). See AGENTS.md.
+DROP_CACHES[dankclaw]=1
+# Passwordless restart helper (root-owned, installed via /etc/sudoers.d/frank-large).
+# When present, deploy.sh uses it instead of inline sudo commands.
+HELPER[dankclaw]="/usr/local/sbin/frank-restart-large.sh"
+# Health wait in iterations of 2s. Cold loads after the cache drop measured
+# at up to 17 minutes on 2026-08-13; 600 iterations = 20 minutes.
+HEALTH_WAIT[dankclaw]=600
+
+REMOTE_USER[dankserver]=dank
+REMOTE_HOST[dankserver]=dankserver
+REMOTE_DIR[dankserver]="/home/dank/vLLM"
+SERVICE[dankserver]="frank-default-small"
+SERVICE_HEALTH[dankserver]="http://127.0.0.1:7770/health"
+DROP_CACHES[dankserver]=0
+HELPER[dankserver]="/usr/local/sbin/frank-restart-small.sh"
+HEALTH_WAIT[dankserver]=300
+
+TARGET="${1:-}"
+DRY_RUN=0
+if [ "${2:-}" = "--dry-run" ]; then
+    DRY_RUN=1
+fi
+
+# ---------------------------------------------------------------------------
+# Validation
+# ---------------------------------------------------------------------------
+if [ -z "${TARGET:-}" ]; then
+    echo "Usage: $0 <dankclaw|dankserver> [--dry-run]"
+    exit 1
+fi
+if [ -z "${REMOTE_HOST[$TARGET]:-}" ]; then
+    echo "Unknown target: $TARGET"
+    exit 1
+fi
+
+# ---------------------------------------------------------------------------
+# Sudo wrapper (headless-friendly)
+# ---------------------------------------------------------------------------
+# When FRANK_SUDO_PW is set, remote sudo reads the password from stdin via
+# `sudo -S`. Otherwise plain `sudo` is used (tty or passwordless config).
+SUDO_PREFIX="sudo"
+if [ -n "${FRANK_SUDO_PW:-}" ]; then
+    SUDO_PREFIX="printf '%s\n' '$FRANK_SUDO_PW' | sudo -S"
+fi
+
+# ---------------------------------------------------------------------------
+# Local checks (pre-flight)
+# ---------------------------------------------------------------------------
+cd "$(dirname "$0")/.."
+if ! git diff --quiet HEAD; then
+    echo "FAIL: Local working tree has uncommitted changes."
+    echo "Commit or stash them before deploying."
+    exit 1
+fi
+if [ "$(git rev-parse --abbrev-ref HEAD)" != "main" ]; then
+    echo "WARN: Not on 'main'; deploying a non-main branch."
+fi
+
+echo "=== Deploying to ${TARGET} ==="
+echo "  host:    ${REMOTE_HOST[$TARGET]}"
+echo "  dir:     ${REMOTE_DIR[$TARGET]}"
+echo "  service: ${SERVICE[$TARGET]}"
+echo "  health:  ${SERVICE_HEALTH[$TARGET]}"
+echo ""
+
+# ---------------------------------------------------------------------------
+# Push (unless dry-run)
+# ---------------------------------------------------------------------------
+if [ "$DRY_RUN" = "1" ]; then
+    echo "[dry-run] Would push local main to origin"
+    echo "[dry-run] Would SSH to ${REMOTE_HOST[$TARGET]} and git pull"
+else
+    echo "[1/4] Pushing local main to origin..."
+    git push origin main
+fi
+
+REMOTE="$(
+    printf '%s@%s' "${REMOTE_USER[$TARGET]}" "${REMOTE_HOST[$TARGET]}"
+)"
+
+# ---------------------------------------------------------------------------
+# Remote: pull + install + restart
+# ---------------------------------------------------------------------------
+DROP_CACHES_CMD=""
+if [ "${DROP_CACHES[$TARGET]}" = "1" ]; then
+    DROP_CACHES_CMD="
+echo 'Dropping page caches for reliable model load...'
+${SUDO_PREFIX} sh -c 'sync; echo 3 > /proc/sys/vm/drop_caches'
+"
+fi
+
+HELPER_PATH="${HELPER[$TARGET]:-}"
+
+REMOTE_SCRIPT="
+set -euo pipefail
+cd '${REMOTE_DIR[$TARGET]}'
+
+echo '[2/4] Pulling repo on ${TARGET}...'
+git pull origin main
+
+if [ -x '${HELPER_PATH}' ]; then
+    # Passwordless restart helper (sudoers NOPASSWD) - preferred path.
+    echo '[3/4] Restarting via passwordless helper ${HELPER_PATH}...'
+    ${SUDO_PREFIX} '${HELPER_PATH}'
+else
+    echo '[3/4] Helper not installed; using inline sudo commands...'
+    # Install / symlink the systemd unit
+    case '${TARGET}' in
+      dankclaw)
+        ${SUDO_PREFIX} ln -sf '${REMOTE_DIR[$TARGET]}/scripts/frank-default-large.service' /etc/systemd/system/frank-default-large.service
+        ;;
+      dankserver)
+        ${SUDO_PREFIX} ln -sf '${REMOTE_DIR[$TARGET]}/scripts/frank-default-small.service' /etc/systemd/system/frank-default-small.service
+        ;;
+    esac
+    ${DROP_CACHES_CMD}
+    echo '[3/4] Reload and restart ${SERVICE[$TARGET]}...'
+    ${SUDO_PREFIX} systemctl daemon-reload
+    ${SUDO_PREFIX} systemctl enable '${SERVICE[$TARGET]}'
+    ${SUDO_PREFIX} systemctl restart '${SERVICE[$TARGET]}'
+fi
+"
+
+if [ "$DRY_RUN" = "1" ]; then
+    echo "[dry-run] Remote commands would run:"
+    echo "$REMOTE_SCRIPT"
+else
+    # shellcheck disable=SC2087
+    ssh "$REMOTE" bash -s <<REMOTE_EOF
+${REMOTE_SCRIPT}
+REMOTE_EOF
+fi
+
+# ---------------------------------------------------------------------------
+# Health check
+# ---------------------------------------------------------------------------
+if [ "$DRY_RUN" = "1" ]; then
+    echo "[dry-run] Would wait for health at ${SERVICE_HEALTH[$TARGET]}"
+else
+    echo "[4/4] Waiting for health endpoint ${SERVICE_HEALTH[$TARGET]} (via ssh)..."
+    HEALTH_URL="${SERVICE_HEALTH[$TARGET]}"
+    HEALTH_ITERS="${HEALTH_WAIT[$TARGET]:-300}"
+    for i in $(seq 1 "$HEALTH_ITERS"); do
+        if ssh "$REMOTE" "curl -sf '$HEALTH_URL'" > /dev/null 2>&1; then
+            echo "  Service healthy after approx $((i * 2))s"
+            echo ""
+            echo "=== Deploy complete: ${TARGET} ==="
+            exit 0
+        fi
+        sleep 2
+    done
+    echo "FAIL: Service did not become healthy in $((HEALTH_ITERS * 2))s"
+    exit 1
+fi
@@ -0,0 +1,88 @@
+# Head tunables — Qwen3.8-27B pipeline-parallel head (Dankclaw).
+#
+# The head is the LOCAL side of the split. It runs llama-server (b10433,
+# HIP+Vulkan+RPC build in /home/dank/llama.cpp/build) with --rpc pointing at
+# the worker on Dankserver. The model file lives on this machine only; the
+# worker receives its tensor share over the network at load time.
+#
+# CRITICAL: this script must NOT pass --device. When --device is set,
+# llama.cpp uses exactly that device list and silently drops the RPC device
+# (src/llama.cpp llama_prepare_model_devices). Default device selection
+# inserts RPC servers at the FRONT of the device list, so the split becomes
+# [RPC0, iGPU] and --tensor-split index 0 is the worker.
+#
+# WARNING: do not name any variable here TEMP, HOST, or GPU_MAX_HEAP_SIZE
+# (HIP runtime collision, see the 2026-08-13 TEMP incident).
+
+MODEL_DIR=/home/dank/models
+MODEL_FILE=Qwen3.8-27B-Uncensored-Q6_K.gguf
+HEAD_BIN=/home/dank/llama.cpp/build/bin/llama-server
+ALIAS=frank-default-large
+BIND_ADDR=0.0.0.0
+LISTEN_PORT=7770
+
+# Comma-separated RPC endpoints (host:port). One worker here.
+RPC_SERVERS=192.168.0.235:50052
+
+# Model split fractions, one per device in order [RPC0, iGPU].
+# 0.68,0.32 = 45 of 65 blocks on the 3090 (blocks 0-44) - the maximum
+# for the Q6_K + q8_0/f16 KV config, measured 2026-08-17 phase 2:
+#  - KV per FA layer = 784 MiB (K 1088 B/token + V 2048 B/token @262144)
+#  - weights(f) = 20334 MB layer pool x f (GGUF scan, exact); embd on
+#    the head side (RPC0 buffer observed 10429.88 MiB at f=0.53)
+#  - KV-per-device count = round(f x 66) layers -> 45 layers = 11 FA
+#    (3..43) -> 8624 MiB. 46 layers (f>=0.7045) still 11 FA but margin
+#    drops to ~0.4 GiB; 48 layers (f>=0.7197) adds FA block 47 -> KV
+#    9408 -> OOM by ~0.85 GiB.
+# Predicted: 13430 (weights) + 8624 (KV) + ~530 (RS) + 189 (compute) =
+# ~22773 MiB -> ~0.7 GiB free. Draft model runs on ROCm0 only (measured
+# phase 1: 1718 MiB, no worker impact).
+TENSOR_SPLIT=0.68,0.32
+
+# Context pool. 262144 total = 1 parallel slot x 262144 per slot
+# (n_ctx_seq = n_ctx / n_seq_max, llama-context.cpp:293).
+CTX_SIZE=262144
+PARALLEL=1
+
+# Speculative decoding. MTP with the STANDALONE draft head (the Q6_K
+# model has no bundled MTP head; the repo ships the draft sidecar) + the
+# ngram-mod fallback. Set SPEC_NONE=1 for a raw baseline.
+SPEC_NONE=0
+SPEC_DEFAULT=1
+MTP=1
+SPEC_DRAFT_N_MAX=4
+# Standalone MTP draft head (Q8_0 MTP-only sidecar from the same repo).
+SPEC_DRAFT_MODEL=/home/dank/models/Qwen3.8-27B-Uncensored-draft-Q8_0.gguf
+SPEC_NGRAM_MOD_N_MIN=48
+SPEC_NGRAM_MOD_N_MAX=64
+SPEC_NGRAM_MOD_N_MATCH=24
+
+# KV on GPU: K q8_0 + V f16 (mismatched, 16 full-attention layers x 4
+# heads x 256 dim: K = 1088 B/token, V = 2048 B/token -> 3136 B/token ->
+# 784 MiB per FA layer at 262144, 12.25 GiB total). Mismatched K/V with
+# Flash Attention requires the GGML_CUDA_FA_ALL_QUANTS rebuild (2026-08-17:
+# head HIP + worker CUDA at 9b05354ec). Do not revert the binaries without
+# also reverting this.
+CACHE_TYPE_K=q8_0
+CACHE_TYPE_V=f16
+
+# Batch sizing: production-verified values (2026-08-14).
+BATCH_SIZE=8192
+UBATCH_SIZE=256
+
+# Sampling — Qwen3.8 thinking-mode defaults.
+TEMPERATURE=1.0
+TOP_K=20
+TOP_P=0.95
+MIN_P=0.0
+
+THREADS=12
+THREADS_HTTP=1
+
+# 4 (info): clean journal for interactive use. Raise to 10 only when
+# debugging the layer split (the per-layer assignment lines are debug-level).
+VERBOSITY=4
+
+# 1: --direct-io model load. Required in practice for large models loaded
+# over RPC (load hangs otherwise; visorcraft Strix RPC finding).
+DIRECT_IO=1
@@ -0,0 +1,47 @@
+# Tunables for the Ling-3.0-flash TRAIL on dankclaw (frank-large-ling).
+# This REPLACES frank-default-large (Qwen3.6-27B) temporarily. To switch
+# back: systemctl stop frank-large-ling && systemctl start frank-default-large.
+# Runtime: llama.cpp FORK bailingmoe3-support; Vulkan + AMD GTT.
+#
+# Memory model (amdgpu Strix Halo gfx1151, 8x16GB LPDDR5 = 128 GiB physical):
+#   - firmware UMA carve-out gives the iGPU ~96 GiB VRAM (rocm-smi "VRAM total").
+#   - CPU system RAM is the remainder (~30 GiB; kernel MemTotal).
+#   - Vulkan (RADV) reports an ~111.2 GiB heap (113910 MiB) - larger than ROCm's
+#     96 GiB, because Vulkan can also reach system memory via GTT.
+#   - Q6_K is 98.26 GiB, leaving only ~13 GiB of heap for KV + graphs, so keep
+#     the context small (Ling uses MLA/KDA, so KV stays small). Tune CTX_SIZE.
+MODEL_DIR=/home/dank/models
+MODEL_FILE=Ling-3.0-flash-Q6_K.gguf
+ALIAS=frank-default-large   # keep the API alias so clients are unaffected
+HOST=0.0.0.0
+PORT=7770
+# Context: Q6_K uses ~98.7 GiB of the ~111.2 GiB Vulkan heap; Ling's MLA/KDA
+# KV is tiny (~2.5 GiB for 256k), so a large context fits. 262144 total = 2
+# slots x 131072, ~10 GiB heap headroom after load. Verified 2026-08-14.
+CTX_SIZE=262144
+PARALLEL=2
+# MTP is bundled in the Ling GGUF itself (block 42) - no separate draft file.
+# Layering ngram-mod under MTP (draft-mtp,ngram-mod) re-drafts via ngram.
+SPEC_TYPE=none
+SPEC_DRAFT_N_MAX=3
+SPEC_NGRAM_MOD_N_MIN=48
+SPEC_NGRAM_MOD_N_MAX=64
+SPEC_NGRAM_MOD_N_MATCH=24
+# Vulkan exposes the larger heap (UMA + GTT) vs ROCm. Use Vulkan0.
+DEVICE=Vulkan0
+N_GPU_LAYERS=all
+FLASH_ATTN=auto
+# Batch kept small-leaning for the tight memory budget; tune with CTX_SIZE.
+BATCH_SIZE=3072
+UBATCH_SIZE=3072
+
+# Diagnostics: high verbosity + a dedicated log file for investigating the
+# premature 'stop' reports. -lv 4 = verbose (ggml); 5 = debug.
+VERBOSITY=0
+# Sampling defaults (model card: reasoning-capable; leave temp/top to default)
+TEMPERATURE=1.0
+TOP_K=20
+TOP_P=0.95
+MIN_P=0.0
+THREADS=12
+THREADS_HTTP=1
@@ -0,0 +1,16 @@
+[Unit]
+Description=llama.cpp Qwen3.8-27B pipeline-parallel head (Strix + RPC to Dankserver) (frank-default-large)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=dank
+Group=dank
+Environment="FRANK_PP_HEAD_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-large.env"
+ExecStart=/home/dank/vllmScripts/scripts/serve-pp-head.sh
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,33 @@
+# Tunables for frank-default-small (FastFlowLM NPU endpoint, dankclaw)
+# Runtime: FLM 1.0.1 on the XDNA2 NPU (50 TOPS). Model choice, measured
+# results, and the NPU memory budget are in docs/FastFlowLM NPU Inference.md.
+# The NPU allocates from the unified pool shared with frank-default-large
+# (the PP head). Keep the resident footprint small enough that the head
+# keeps its headroom.
+#
+# 2026-08-17: qwythos:9b (Huihui-Qwythos-9B-Claude-Mythos-5-1M-abliterated,
+# Qwen3.5-9B hybrid linear-attention base, Q4NX, 7.76 GB) chosen after the
+# user's model swap (was qwen3.6-moe:35b-a3b, which stays installed for
+# rollback). FLM upgraded 0.9.45 -> 1.0.1: the pack declares flm_version
+# 1.0.0 and 0.9.45 refused to serve it.
+
+FLM_MODEL=qwythos:9b
+FLM_HOST=0.0.0.0
+FLM_PORT=52625
+# Power mode: turbo. Smoke decode ~5-7 t/s on the 9B dense at 262k ctx
+# (not benchmarked; see docs/FastFlowLM NPU Inference.md).
+FLM_PMODE=turbo
+# Load the ASR model (Whisper-V3-Turbo-NPU2) alongside the LLM. Exposes
+# /v1/audio/transcriptions. Adds ~1 GiB to the resident footprint.
+FLM_ASR=1
+# Maximum context length. Qwythos-9B native max is 262144 (Qwen3.5-9B
+# hybrid: 8 of 32 layers are full attention, so the KV cache stays small).
+# Verified 2026-08-17: loads at -c 262144 with turbo+asr on FLM 1.0.1;
+# KV occupancy 0.007% at 19 tokens.
+FLM_CTX_LEN=262144
+# Custom-model resolution: the qwythos:9b tag and its xclbin symlinks live
+# in the user config, not the system registry. Without these, flm treats
+# the tag as unknown and silently serves llama3.2:1b (see the unknown-tag
+# fallback in docs/FastFlowLM NPU Inference.md).
+FLM_CONFIG_PATH=/home/dank/.config/flm/model_list.json
+FLM_XCLBIN_PATH=/home/dank/.config/flm
@@ -0,0 +1,17 @@
+[Unit]
+Description=FastFlowLM NPU inference (frank-default-small) on Dankclaw
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=dank
+Group=dank
+Environment="FLM_SMALL_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-small.env"
+ExecStart=/home/dank/vllmScripts/scripts/serve-flm-small.sh
+LimitMEMLOCK=infinity
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,17 @@
+[Unit]
+Description=llama.cpp Ling-3.0-flash Q6_K Vulkan+MTP (frank-large-ling TRIAL)
+After=network.target
+
+[Service]
+Type=simple
+User=dank
+Group=dank
+Environment="HOST=0.0.0.0"
+Environment="PORT=7770"
+Environment="FRANK_LING_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-large.ling.env"
+ExecStart=/home/dank/vllmScripts/scripts/serve-ling-flash.sh
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,30 @@
+# Worker tunables — Qwen3.8-27B pipeline-parallel RPC worker (Dankserver).
+#
+# The worker is the REMOTE side of the split. It exposes the RTX 3090 to the
+# head on Dankclaw as an RPC device. It runs from the b10433 CUDA+RPC build
+# in /home/dank/llama-cpp-rpc (worktree of /home/dank/llama.cpp).
+#
+# WARNING: do not name any variable here TEMP, HOST, or GPU_MAX_HEAP_SIZE.
+# The ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
+# incident in docs/deployment.md).
+
+# Bind address and port for ggml-rpc-server. Must be the LAN address the head
+# can reach; the default bind (127.0.0.1) refuses remote clients.
+RPC_BIND=192.168.0.235
+RPC_PORT=50052
+
+# Device to expose. CUDA0 = the RTX 3090.
+DEVICE=CUDA0
+
+# Threads for the worker's CPU device (server has 8 cores; leave 1 for the OS
+# and the running frank-default-small service).
+THREADS=7
+
+# 1: enable the local tensor file cache (~/.cache/llama.cpp/rpc/). The head
+# streams the 31.5 GB model to the worker on first load; with the cache, later
+# loads skip the transfer.
+CACHE=1
+
+# Worker binary. Must be the SAME llama.cpp build as the head (b10433 /
+# 9b05354ec). Version skew hangs the RPC handshake.
+WORKER_BIN=/home/dank/llama-cpp-rpc/build/bin/ggml-rpc-server
@@ -0,0 +1,15 @@
+[Unit]
+Description=llama.cpp Qwen3.8-27B pipeline-parallel RPC worker (RTX 3090)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=dank
+Group=dank
+ExecStart=/home/dank/vLLM/scripts/serve-pp-worker.sh
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,14 @@
+[Unit]
+Description=llama.cpp Inference Runtime
+After=network.target
+
+[Service]
+Type=simple
+User=dank
+Group=dank
+ExecStart=/home/dank/GitPackages/ROCmFPX/build-strix-rocmfp4/bin/llama-server -hf raulvidis/Ling-3.0-flash-ROCmFP4-STRIX-MTP-GGUF:Q4_0 --spec-type draft-mtp --spec-draft-n-max 2 --device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 2 --ctx-size 401400 --batch-size 8192 --ubatch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --alias frank-default-large --no-mmap --direct-io -lv 4 --timeout 7200 --chat-template-file /home/dank/LingFlash/chat_template.jinja
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""pp-bench.py — drive the PP head API and report server-side timings.
+
+Runs on the HEAD machine (Dankclaw). Targets the running PP head
+(serve-pp-head.sh, port from argument). Builds a ~Nk-token prose prompt,
+sends one warmup completion, then REPS measured completions, and reads the
+authoritative `slot print_timing` lines from the head log.
+
+Usage:
+    python3 pp-bench.py <prompt_k> <out_tokens> <reps> <port> <head_log>
+"""
+import json
+import re
+import sys
+import time
+import urllib.request
+
+prompt_k, out_tokens, reps, port, head_log = sys.argv[1:6]
+out_tokens = int(out_tokens)
+reps = int(reps)
+target = int(prompt_k) * 1000
+
+# ~38 tokens per 160-char prose unit; size the filler to hit the target.
+unit = ("The careful study of natural language reveals patterns of meaning "
+        "that emerge from the deliberate arrangement of words and phrases. ")
+approx = max(1, target // 38)
+prompt = unit * approx + (
+    "\n\nNow, in a detailed and flowing paragraph, "
+    "continue this essay about the philosophy of "
+    "language and its influence on modern computation. "
+)
+
+
+def req(n):
+    body = json.dumps({"prompt": prompt, "max_tokens": n,
+                       "temperature": 0.0, "stream": False}).encode()
+    r = urllib.request.Request(
+        f"http://127.0.0.1:{port}/v1/completions",
+        data=body, headers={"Content-Type": "application/json"})
+    with urllib.request.urlopen(r, timeout=7200) as resp:
+        return json.load(resp)
+
+
+def log_timings():
+    try:
+        lines = open(head_log, encoding="utf-8", errors="ignore").read().splitlines()
+    except OSError:
+        return []
+    out = []
+    for l in lines:
+        if re.search(r"prompt eval time|eval time|draft acceptance", l):
+            out.append(l.strip())
+    return out
+
+
+GEN_RE = re.compile(
+    r"eval time =\s+([\d.]+) ms /\s+(\d+) tokens.*?([\d.]+) tokens per second")
+PRE_RE = re.compile(
+    r"prompt eval time =\s+([\d.]+) ms /\s+(\d+) tokens.*?([\d.]+) tokens per second")
+ACC_RE = re.compile(r"draft acceptance = ([\d.]+)")
+
+# warmup (first run pays graph-compile / JIT costs)
+warm = req(16)
+print(f"warmup: prompt_tokens={warm['usage']['prompt_tokens']} "
+      f"completion_tokens={warm['usage']['completion_tokens']}")
+
+gen_rates, prefill_rates, accepts = [], [], []
+for i in range(reps):
+    t0 = time.time()
+    out = req(out_tokens)
+    dt = time.time() - t0
+    u = out["usage"]
+    pt, ct = u["prompt_tokens"], u["completion_tokens"]
+    gen = pre = acc = None
+    for l in log_timings():
+        if "prompt eval" not in l:
+            m = GEN_RE.search(l)
+            if m:
+                gen = float(m.group(3))
+        m = PRE_RE.search(l)
+        if m:
+            pre = float(m.group(3))
+        m = ACC_RE.search(l)
+        if m:
+            acc = float(m.group(1))
+    gen_rates.append(gen if gen is not None else ct / dt)
+    if pre is not None:
+        prefill_rates.append(pre)
+    if acc is not None:
+        accepts.append(acc)
+    print(f"rep {i + 1}: prompt={pt} gen={ct} wall={dt:.1f}s "
+          f"gen={gen_rates[-1]:.2f} t/s "
+          f"prefill={f'{pre:.1f}' if pre is not None else 'n/a'} t/s "
+          f"acceptance={f'{acc:.3f}' if acc is not None else 'n/a'}")
+
+print("== RESULT pp ==")
+print(f"avg gen tok/s   = {sum(gen_rates) / len(gen_rates):.2f}")
+if prefill_rates:
+    print(f"avg prefill t/s = {sum(prefill_rates) / len(prefill_rates):.1f}")
+if accepts:
+    print(f"avg acceptance  = {sum(accepts) / len(accepts):.3f}")
+for l in log_timings()[-3:]:
+    print("  " + l)
@@ -0,0 +1,204 @@
+#!/usr/bin/env bash
+# pp-test.sh — Qwen3.8-27B pipeline-parallel (llama.cpp RPC) test orchestrator.
+#
+# Runs Qwen3.8-27B UD-Q8_K_XL split across Dankclaw (head, Strix iGPU) and
+# Dankserver (worker, RTX 3090) over the 2.5 GbE LAN. Both boxes run the same
+# llama.cpp build (b10433 / 9b05354ec); the worker build lives in
+# /home/dank/llama-cpp-rpc on Dankserver.
+#
+# BENCHMARKING POLICY (AGENTS.md): this test STOPS both production services
+# and restores them when done. Do not run it while users are mid-stream.
+#
+# Usage (from any box with ssh access to both targets, repo checked out):
+#   ./scripts/pp-test.sh preflight
+#   ./scripts/pp-test.sh start                # stop prod, start PP head+worker
+#   ./scripts/pp-test.sh verify               # show the layer split + VRAM
+#   ./scripts/pp-test.sh bench [prompt_k] [out_tokens] [reps]
+#   ./scripts/pp-test.sh stop-pp              # stop PP head+worker only
+#   ./scripts/pp-test.sh restore              # start production services
+#   ./scripts/pp-test.sh all                  # start + verify + bench + restore
+#
+# Sudo: set FRANK_SUDO_PW for headless runs (stop/start of system services).
+#   FRANK_SUDO_PW=... ./scripts/pp-test.sh start
+set -euo pipefail
+
+CLAW="dank@dankclaw"
+SERVER="dank@dankserver"
+CLAW_REPO="/home/dank/vllmScripts"
+SERVER_REPO="/home/dank/vLLM"
+HEAD_PORT=7770
+HEAD_LOG="/tmp/pp-head.log"
+WORKER_LOG="/tmp/pp-worker.log"
+RPC_PORT=50052
+
+SUDO="sudo"
+if [ -n "${FRANK_SUDO_PW:-}" ]; then
+    SUDO="printf '%s\n' '$FRANK_SUDO_PW' | sudo -S"
+fi
+
+log()  { echo "[pp-test] $*"; }
+die()  { echo "[pp-test] FAIL: $*" >&2; exit 1; }
+
+ssh_claw()   { ssh -o BatchMode=yes -o ConnectTimeout=8 "$CLAW"   "$@"; }
+ssh_server() { ssh -o BatchMode=yes -o ConnectTimeout=8 "$SERVER" "$@"; }
+
+# ---------------------------------------------------------------------------
+# preflight
+# ---------------------------------------------------------------------------
+do_preflight() {
+    log "preflight: build identity"
+    local h_v w_v
+    h_v=$(ssh_claw '/home/dank/llama.cpp/build/bin/llama-server --version 2>&1 | head -1')
+    w_v=$(ssh_server '/home/dank/llama-cpp-rpc/build/bin/llama-server --version 2>&1 | head -1')
+    log "  head:   $h_v"
+    log "  worker: $w_v"
+    [[ "$h_v" == *9b05354ec* && "$w_v" == *9b05354ec* ]] || die "build skew (need identical commit 9b05354ec on both sides)"
+
+    log "preflight: binaries"
+    ssh_claw   "[ -x $CLAW_REPO/scripts/serve-pp-head.sh ]"   || die "missing $CLAW_REPO/scripts/serve-pp-head.sh on claw (git pull?)"
+    ssh_server "[ -x $SERVER_REPO/scripts/serve-pp-worker.sh ]" || die "missing $SERVER_REPO/scripts/serve-pp-worker.sh on server (git pull?)"
+    ssh_server "[ -x /home/dank/llama-cpp-rpc/build/bin/ggml-rpc-server ]" || die "missing worker RPC binary on server"
+    ssh_claw   "[ -f /home/dank/models/Qwen3.8-27B-UD-Q8_K_XL.gguf ]" || die "missing model on claw"
+
+    log "preflight: network (claw -> server:50052 listener check needs the worker up; testing route + link)"
+    ssh_claw "ip -4 addr show eno1 | grep -q 192.168.0.143" || die "unexpected claw address"
+    ssh_server "ip -4 addr show eno1 | grep -q 192.168.0.235" || die "unexpected server address"
+    ssh_claw "ping -c 2 -W 2 192.168.0.235 >/dev/null" || die "claw cannot reach server"
+    local rtt
+    rtt=$(ssh_claw "ping -c 5 -W 2 192.168.0.235 | tail -1" | grep -oE '[0-9.]+/[0-9.]+ ms' | head -1 || true)
+    log "  rtt: ${rtt:-unknown}"
+
+    log "preflight: production services (health endpoint = no sudo needed)"
+    local h1 h2
+    h1=$(ssh_claw   "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:7770/health 2>/dev/null || true")
+    h2=$(ssh_server "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:7770/health 2>/dev/null || true")
+    log "  frank-default-large:  ${h1:-no-response}"
+    log "  frank-default-small:  ${h2:-no-response}"
+    log "preflight OK"
+}
+
+# ---------------------------------------------------------------------------
+# start
+# ---------------------------------------------------------------------------
+do_start() {
+    log "stopping production services (both boxes)"
+    ssh_claw   "$SUDO systemctl stop frank-default-large"  || die "could not stop frank-default-large"
+    ssh_server "$SUDO systemctl stop frank-default-small"  || die "could not stop frank-default-small"
+    sleep 5
+
+    log "dropping page caches on claw (reliable model load)"
+    ssh_claw "$SUDO sh -c 'sync; echo 3 > /proc/sys/vm/drop_caches'" || log "WARN: cache drop failed"
+
+    log "starting RPC worker on server"
+    ssh_server "cd $SERVER_REPO && nohup bash scripts/serve-pp-worker.sh > $WORKER_LOG 2>&1 & echo worker_pid=\$!"
+    for i in $(seq 1 30); do
+        if ssh_server "ss -tln | grep -q :$RPC_PORT"; then
+            log "worker listening on :$RPC_PORT"
+            break
+        fi
+        [ "$i" = "30" ] && { ssh_server "tail -20 $WORKER_LOG"; die "worker did not bind :$RPC_PORT"; }
+        sleep 1
+    done
+
+    log "starting PP head on claw (model load + weight stream: allow ~10 min)"
+    local head_pid
+    head_pid=$(ssh_claw "cd $CLAW_REPO && nohup bash scripts/serve-pp-head.sh > $HEAD_LOG 2>&1 & echo \$!")
+    log "head pid on claw: $head_pid"
+    for i in $(seq 1 360); do
+        local st
+        st=$(ssh_claw "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:$HEAD_PORT/health 2>/dev/null || true")
+        if [ "$st" = "200" ]; then
+            log "head healthy after ~$((i * 5))s"
+            return 0
+        fi
+        if ! ssh_claw "kill -0 $head_pid 2>/dev/null"; then
+            ssh_claw "tail -30 $HEAD_LOG"
+            die "head process died during load"
+        fi
+        [ $((i % 12)) = 0 ] && log "  still loading... ($((i * 5))s)"
+        sleep 5
+    done
+    ssh_claw "tail -30 $HEAD_LOG"
+    die "head did not become healthy in 30 min"
+}
+
+# ---------------------------------------------------------------------------
+# verify
+# ---------------------------------------------------------------------------
+do_verify() {
+    log "health + device split (from head log)"
+    ssh_claw "curl -s http://127.0.0.1:$HEAD_PORT/health; echo"
+    ssh_claw "grep -aE 'using device|assigned to device|MiB free' $HEAD_LOG | head -20 || true"
+    log "worker VRAM on server:"
+    ssh_server "nvidia-smi --query-gpu=memory.used --format=csv,noheader"
+    log "worker log tail:"
+    ssh_server "tail -5 $WORKER_LOG"
+}
+
+# ---------------------------------------------------------------------------
+# bench
+# ---------------------------------------------------------------------------
+do_bench() {
+    local PROMPT_K="${1:-4}"
+    local OUT_TOKENS="${2:-256}"
+    local REPS="${3:-2}"
+    log "bench: prompt ~${PROMPT_K}k tokens, out=${OUT_TOKENS}, reps=${REPS} (on claw, via :$HEAD_PORT)"
+    ssh_claw "python3 $CLAW_REPO/scripts/pp-bench.py $PROMPT_K $OUT_TOKENS $REPS $HEAD_PORT $HEAD_LOG"
+}
+
+# ---------------------------------------------------------------------------
+# stop / restore
+# ---------------------------------------------------------------------------
+do_stop_pp() {
+    log "stopping PP head (claw) and worker (server)"
+    ssh_claw   "pkill -f 'serve-pp-head.sh' 2>/dev/null; pkill -f 'llama-server.*frank-pp-large' 2>/dev/null; true"
+    ssh_server "pkill -x ggml-rpc-server 2>/dev/null; true"
+    sleep 3
+    ssh_server "nvidia-smi --query-gpu=memory.used --format=csv,noheader"
+}
+
+do_restore() {
+    # Warm-cache loads crash fast on the claw (AGENTS.md); cold loads survive.
+    log "dropping page caches on claw before production load"
+    ssh_claw "$SUDO sh -c 'sync; echo 3 > /proc/sys/vm/drop_caches'" || log "WARN: cache drop failed"
+    log "starting production services"
+    ssh_server "$SUDO systemctl start frank-default-small" || die "could not start frank-default-small"
+    ssh_claw   "$SUDO systemctl start frank-default-large" || die "could not start frank-default-large"
+    log "waiting for health (model load takes minutes; up to ~17 min cold on claw)"
+    for i in $(seq 1 300); do
+        local a b
+        a=$(ssh_claw   "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:7770/health 2>/dev/null || true")
+        b=$(ssh_server "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:7770/health 2>/dev/null || true")
+        if [ "$a" = "200" ] && [ "$b" = "200" ]; then
+            log "both production services healthy after ~$((i * 4))s"
+            return 0
+        fi
+        [ $((i % 15)) = 0 ] && log "  claw=$a server=$b ... ($((i * 4))s)"
+        sleep 4
+    done
+    die "production services did not both become healthy"
+}
+
+# ---------------------------------------------------------------------------
+case "${1:-}" in
+    preflight) do_preflight ;;
+    start)     do_start ;;
+    verify)    do_verify ;;
+    bench)     shift; do_bench "$@" ;;
+    stop-pp)   do_stop_pp ;;
+    restore)   do_restore ;;
+    all)
+        log "full run: any failure auto-restores production services"
+        do_preflight
+        trap 'do_restore || true' EXIT
+        do_start
+        do_verify
+        do_bench
+        do_restore
+        trap - EXIT
+        ;;
+    *)
+        echo "Usage: $0 {preflight|start|verify|bench [prompt_k] [out] [reps]|stop-pp|restore|all}"
+        exit 1
+        ;;
+esac
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Bound the ggml-rpc worker tensor cache.
+#
+# The worker (frank-pp-worker on Dankserver) keys cache entries per
+# session, so each model load writes a fresh full copy (~15 GB). Left
+# unbounded, the directory filled the root disk to 100% on 2026-08-17;
+# failed cache writes left 0-byte entries, and a later reload served
+# corrupt weights -> degenerate model output (see the cache incident in
+# docs/Pipeline Parallelism/Qwen3.8 PP Deployment.md).
+#
+# This script prunes the cache to a byte cap, oldest files first. Cache
+# entries are derived data: safe to delete. Run from cron on Dankserver.
+set -euo pipefail
+
+CACHE_DIR=/home/dank/.cache/llama.cpp/rpc
+MAX_BYTES=$((40 * 1024 * 1024 * 1024))   # 40 GiB
+
+if [[ ! -d "$CACHE_DIR" ]]; then
+    exit 0
+fi
+
+total=$(du -sb "$CACHE_DIR" | cut -f1)
+if (( total <= MAX_BYTES )); then
+    exit 0
+fi
+
+while (( total > MAX_BYTES )); do
+    oldest=$(ls -1t "$CACHE_DIR" | tail -1)
+    if [[ -z "$oldest" ]]; then
+        break
+    fi
+    rm -f -- "$CACHE_DIR/$oldest"
+    total=$(du -sb "$CACHE_DIR" | cut -f1)
+done
+
+echo "prune-rpc-cache: pruned to $(du -sh "$CACHE_DIR" | cut -f1)"
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Serve the FastFlowLM (FLM) NPU endpoint on Dankclaw — frank-default-small.
+#
+# Runs flm serve with the model tag in frank-default-small.env. The NPU
+# (XDNA2, 50 TOPS) is a single-lane device: generations serialize in FIFO
+# order, so this is a background/triage endpoint, not an interactive
+# multi-user chat backend. The NPU shares the 30 GiB unified pool with the
+# frank-default-large PP head; see docs/FastFlowLM NPU Inference.md for the
+# memory budget and the measured model matrix.
+#
+# NOTE: FLM requires unlimited memlock. The unit sets LimitMEMLOCK=infinity.
+# Running this script by hand from a login shell needs the pam_limits rule
+# in /etc/security/limits.d/99-flm-memlock.conf.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FLM_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -------------------
+: "${FLM_BIN:=/usr/bin/flm}"
+: "${FLM_MODEL:=gemma4-it:e4b}"
+: "${FLM_HOST:=0.0.0.0}"
+: "${FLM_PORT:=52625}"
+: "${FLM_PMODE:=performance}"
+: "${FLM_ASR:=0}"
+: "${FLM_CTX_LEN:=-1}"
+
+if [[ ! -x "$FLM_BIN" ]]; then
+    echo "missing flm binary: $FLM_BIN" >&2
+    exit 1
+fi
+
+# --- optional flags ----------------------------------------------------------
+OPT_ARGS=()
+if [[ "${FLM_ASR}" != "0" ]]; then
+    OPT_ARGS+=(-a "${FLM_ASR}")
+fi
+if [[ "${FLM_CTX_LEN}" != "-1" ]]; then
+    OPT_ARGS+=(-c "${FLM_CTX_LEN}")
+fi
+
+exec "$FLM_BIN" serve "$FLM_MODEL" \
+    --host "$FLM_HOST" \
+    -p "$FLM_PORT" \
+    --pmode "$FLM_PMODE" \
+    "${OPT_ARGS[@]}"
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Serve inclusionAI Ling-3.0-flash (Q6_K) on dankclaw as a TRIAL, replacing
+# frank-default-large temporarily.
+#
+# Runtime: llama.cpp FORK (aetherbird bailingmoe3-support) - stock llama.cpp
+# cannot load the bailingmoe3 architecture. Backend: Vulkan (RADV) on the
+# Strix Halo APU; memory is the shared 128 GB pool (96 GiB UMA + GTT that
+# Vulkan exposes as an ~111 GiB heap). MTP drafter is bundled in the same
+# GGUF (block 42) - no separate --spec-draft-model needed.
+#
+# Tie-back: to restore the Qwen3.6-27B service, stop frank-large-ling and
+# start frank-default-large (its env/unit are committed and unchanged).
+#
+# Tunables come from frank-default-large.ling.env in this directory unless
+# FRANK_LING_ENV_FILE overrides that path.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FRANK_LING_ENV_FILE:-$SCRIPT_DIR/frank-default-large.ling.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults -------------------------------------------------------------
+: "${MODEL_DIR:=/home/dank/models}"
+: "${MODEL_FILE:=Ling-3.0-flash-Q6_K.gguf}"
+: "${ALIAS:=frank-default-large}"   # keep the same API alias for clients
+: "${HOST:=0.0.0.0}"
+: "${PORT:=7770}"
+: "${CTX_SIZE:=32768}"
+: "${PARALLEL:=2}"
+: "${SPEC_TYPE:=draft-mtp}"         # MTP block is bundled in the model GGUF
+: "${SPEC_DRAFT_N_MAX:=3}"
+: "${SPEC_NGRAM_MOD_N_MIN:=48}"
+: "${SPEC_NGRAM_MOD_N_MAX:=64}"
+: "${SPEC_NGRAM_MOD_N_MATCH:=24}"
+: "${DEVICE:=Vulkan0}"
+: "${N_GPU_LAYERS:=all}"
+: "${BATCH_SIZE:=512}"
+: "${UBATCH_SIZE:=512}"
+: "${TEMPERATURE:=1.0}"
+: "${TOP_K:=20}"
+: "${TOP_P:=0.95}"
+: "${MIN_P:=0.0}"
+: "${THREADS:=12}"
+: "${THREADS_HTTP:=1}"
+: "${VERBOSITY:=0}"      # ggml log verbosity 0-5; 4 = verbose, 5 = debug
+: "${LOG_FILE:=}"         # optional; if set, --log-file path
+
+# Fork binary; stock llama.cpp cannot load bailingmoe3.
+LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp-ling/build/bin/llama-server}"
+MODEL="$MODEL_DIR/$MODEL_FILE"
+
+if [[ ! -x "$LLAMA_SERVER" ]]; then
+    echo "missing fork binary: $LLAMA_SERVER (build required)" >&2
+    exit 1
+fi
+if [[ ! -f "$MODEL" ]]; then
+    echo "missing model: $MODEL" >&2
+    exit 1
+fi
+
+exec "$LLAMA_SERVER" \
+    $(if [[ -n "$LOG_FILE" ]]; then echo "--log-file $LOG_FILE"; fi) \
+    --model "$MODEL" \
+    --spec-type "$SPEC_TYPE" \
+    --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
+    --spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN" \
+    --spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX" \
+    --spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH" \
+    --device "$DEVICE" \
+    --gpu-layers "$N_GPU_LAYERS" \
+    --flash-attn auto \
+    --ctx-size "$CTX_SIZE" \
+    --parallel "$PARALLEL" \
+    --batch-size "$BATCH_SIZE" \
+    --ubatch-size "$UBATCH_SIZE" \
+    --temp "$TEMPERATURE" \
+    --top-k "$TOP_K" \
+    --top-p "$TOP_P" \
+    --min-p "$MIN_P" \
+    --threads "$THREADS" \
+    --threads-http "$THREADS_HTTP" \
+    -lv "$VERBOSITY" \
+    --alias "$ALIAS" \
+    --host "$HOST" \
+    --port "$PORT"
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# serve-ling3-vllm.sh — Launch Ling-3.0-flash CIRU INT4 vLLM (gfx1151) on Dankclaw
+#
+# This wrapper sources the jcbtc ling3-runtime.env (sets VENV and MODEL_PATH),
+# applies our production settings (host, port, served-model-name), and hands
+# off to the validated run-256k.sh launcher (TRITON_MLA, MTP K1, 256k, 6 slots,
+# ROCr idle-fix LD_PRELOAD). Do not hand-roll the vllm serve command; the
+# launcher carries required env vars and the ROCr override.
+set -Eeuo pipefail
+
+PKG="/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native"
+
+if [ ! -f "$PKG/ling3-runtime.env" ]; then
+  echo "CIRU package runtime environment not found: $PKG/ling3-runtime.env" >&2
+  exit 1
+fi
+
+set -a
+# shellcheck disable=SC1091
+source "$PKG/ling3-runtime.env"
+set +a
+
+export HOST="${HOST:-0.0.0.0}"
+export PORT="${PORT:-7770}"
+export SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-frank-default-large}"
+# Raise the vLLM frame-buffer utilization so weights + 256k KV fit in VRAM
+# (the default 0.72 from native-256k.env OOMs; native-256k uses
+# ${GPU_MEMORY_UTILIZATION:-0.72}, so this exported value wins).
+export GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.95}"
+
+# Run from a directory with no top-level 'vllm/' dir.
+# Neither launcher changes directory, so the process cwd becomes sys.path[0]
+# for the 'python -m ...registry' inspection subprocess. From the package root,
+# the top-level vllm/ repo dir shadows the editable install as a namespace
+# package and breaks lazy imports (for example, SamplingParams).
+cd /home/dank/ling3-ciru
+
+exec bash "$PKG/scripts/run-256k.sh"
@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# Serve unsloth Qwen3.8-27B UD-Q8_K_XL on dankclaw with llama.cpp's
+# --spec-default preset (ngram-mod). ROCm backend.
+# Runtime: llama.cpp llama-server (>= b10411).
+#
+# Tunables come from frank-default-large.env in this directory unless the
+# FRANK_LARGE_ENV_FILE environment variable overrides that path.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -----------------
+: "${MODEL_DIR:=/home/dank/models}"
+: "${MODEL_FILE:=Qwen3.8-27B-UD-Q8_K_XL.gguf}"
+: "${DRAFT_FILE:=}"        # set: DSpark/DFlash/MTP/... sidecar GGUF in MODEL_DIR
+: "${ALIAS:=frank-default-large}"
+: "${HOST:=0.0.0.0}"
+: "${PORT:=7770}"
+: "${CTX_SIZE:=262144}"   # Qwen3.8 native context (env may raise for parallel≥2)
+: "${PARALLEL:=2}"
+: "${SPEC_DEFAULT:=0}"   # 1: use --spec-default (ngram-mod preset) instead of --spec-type
+: "${MTP:=0}"            # 1: prepend --spec-type draft-mtp before --spec-default (MTP drafts, ngram fallback)
+: "${SPEC_DRAFT_N_MAX:=10}"
+: "${SPEC_NGRAM_MOD_N_MIN:=48}"
+: "${SPEC_NGRAM_MOD_N_MAX:=64}"
+: "${SPEC_NGRAM_MOD_N_MATCH:=24}"
+: "${DEVICE:=ROCm0}"
+: "${N_GPU_LAYERS:=all}"
+: "${CACHE_TYPE_K:=q8_0}"
+: "${CACHE_TYPE_V:=q8_0}"
+: "${BATCH_SIZE:=8192}"
+: "${UBATCH_SIZE:=2048}"
+: "${TEMPERATURE:=1.0}"
+: "${TOP_K:=20}"
+: "${TOP_P:=0.95}"
+: "${MIN_P:=0.0}"
+: "${THREADS:=12}"
+: "${THREADS_HTTP:=1}"
+
+LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp/build/bin/llama-server}"
+MODEL="$MODEL_DIR/$MODEL_FILE"
+DRAFT="$MODEL_DIR/$DRAFT_FILE"
+
+if [[ ! -f "$MODEL" ]]; then
+    echo "missing required file: $MODEL" >&2
+    exit 1
+fi
+
+# Spec flags.  --spec-default is a preset that enables ngram-mod
+# (n_match=24, n_min=48, n_max=64) and takes no draft model.  The explicit
+# --spec-type / draft-model path is used when SPEC_DEFAULT is 0.
+SPEC_ARGS=()
+if [[ "${SPEC_DEFAULT}" == "1" ]]; then
+    if [[ "${MTP}" == "1" ]]; then
+        # MTP first (bundled in-model head), then the --spec-default ngram-mod
+        # fallback.  Order matters: the chain drafts through impls in types
+        # vector order.  --spec-type appends, so it must precede --spec-default.
+        SPEC_ARGS+=(--spec-type draft-mtp)
+    fi
+    SPEC_ARGS+=(--spec-default)
+else
+    if [[ -z "${SPEC_TYPE:-}" ]]; then
+        echo "SPEC_TYPE must be set when SPEC_DEFAULT=0" >&2
+        exit 1
+    fi
+    SPEC_ARGS+=(--spec-type "$SPEC_TYPE")
+    if [[ -n "$DRAFT_FILE" ]]; then
+        if [[ ! -f "$DRAFT" ]]; then
+            echo "missing required file: $DRAFT" >&2
+            exit 1
+        fi
+        SPEC_ARGS+=(--spec-draft-model "$DRAFT")
+    fi
+    if [[ -n "${SPEC_DRAFT_P_MIN:-}" ]]; then
+        SPEC_ARGS+=(--spec-draft-p-min "$SPEC_DRAFT_P_MIN")
+    fi
+    SPEC_ARGS+=(--spec-draft-n-max "$SPEC_DRAFT_N_MAX")
+    SPEC_ARGS+=(--spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN")
+    SPEC_ARGS+=(--spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX")
+    SPEC_ARGS+=(--spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH")
+fi
+
+exec "$LLAMA_SERVER" \
+    --model "$MODEL" \
+    "${SPEC_ARGS[@]}" \
+    --device "$DEVICE" \
+    --gpu-layers "$N_GPU_LAYERS" \
+    --fit off \
+    --cache-type-k "$CACHE_TYPE_K" \
+    --cache-type-v "$CACHE_TYPE_V" \
+    --ctx-size "$CTX_SIZE" \
+    --parallel "$PARALLEL" \
+    --batch-size "$BATCH_SIZE" \
+    --ubatch-size "$UBATCH_SIZE" \
+    --temp "$TEMPERATURE" \
+    --top-k "$TOP_K" \
+    --top-p "$TOP_P" \
+    --min-p "$MIN_P" \
+    --threads "$THREADS" \
+    --threads-http "$THREADS_HTTP" \
+    --alias "$ALIAS" \
+    --host "$HOST" \
+    --port "$PORT"
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Serve HauhauCS Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q6_K_P
+# with ngram-mod speculative decoding on dankserver (RTX 3090, MoE experts on CPU).
+# Runtime: upstream llama.cpp llama-server (b10108+).
+#
+# Tunables come from frank-default-small.env in this directory unless the
+# FRANK_SMALL_ENV_FILE environment variable overrides that path.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -----------------
+: "${MODEL_DIR:=/home/dank/models}"
+: "${MODEL_FILE:=Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q6_K_P.gguf}"
+: "${DRAFT_FILE:=}"   # not used — ngram-mod reuses the loaded model for drafts
+: "${MMPROJ_FILE:=mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf}"
+: "${ALIAS:=frank-default-small}"
+: "${HOST:=0.0.0.0}"
+: "${PORT:=7770}"
+: "${CTX_SIZE:=262144}"
+: "${PARALLEL:=3}"
+: "${SPEC_TYPE:=ngram-mod}"
+: "${NGRAM_MOD_N_MIN:=48}"
+: "${NGRAM_MOD_N_MAX:=64}"
+: "${NGRAM_MOD_N_MATCH:=24}"
+: "${CACHE_TYPE_K:=f16}"
+: "${CACHE_TYPE_V:=f16}"
+: "${VERBOSITY:=3}"
+: "${N_CPU_MOE:=36}"     # 36 of 40 layers: MoE expert weights stay on CPU; 4 on GPU
+: "${N_GPU_LAYERS:=all}" # every non-expert tensor goes to GPU
+: "${BATCH_SIZE:=2048}"
+: "${UBATCH_SIZE:=3072}"
+: "${CTX_CHECKPOINTS:=0}"
+: "${NO_MMAP:=0}"
+: "${TEMPERATURE:=1.0}"
+: "${TOP_K:=20}"
+: "${TOP_P:=0.95}"
+: "${MIN_P:=0.0}"
+: "${THREADS:=8}"        # 7700X physical cores
+: "${THREADS_HTTP:=8}"
+
+LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp/build/bin/llama-server}"
+MODEL="$MODEL_DIR/$MODEL_FILE"
+MMPROJ="$MODEL_DIR/$MMPROJ_FILE"
+
+for f in "$MODEL" "$MMPROJ"; do
+    if [[ ! -f "$f" ]]; then
+        echo "missing required file: $f" >&2
+        exit 1
+    fi
+done
+
+exec "$LLAMA_SERVER" \
+    --model "$MODEL" \
+    --mmproj "$MMPROJ" \
+    --jinja \
+    --spec-type "$SPEC_TYPE" \
+    --spec-ngram-mod-n-min "$NGRAM_MOD_N_MIN" \
+    --spec-ngram-mod-n-max "$NGRAM_MOD_N_MAX" \
+    --spec-ngram-mod-n-match "$NGRAM_MOD_N_MATCH" \
+    --cache-type-k "$CACHE_TYPE_K" \
+    --cache-type-v "$CACHE_TYPE_V" \
+    --verbosity "$VERBOSITY" \
+    --ctx-size "$CTX_SIZE" \
+    --parallel "$PARALLEL" \
+    --n-cpu-moe "$N_CPU_MOE" \
+    --gpu-layers "$N_GPU_LAYERS" \
+    --batch-size "$BATCH_SIZE" \
+    --ubatch-size "$UBATCH_SIZE" \
+    --ctx-checkpoints "$CTX_CHECKPOINTS" \
+    $([ "$NO_MMAP" = "1" ] && printf -- "--no-mmap") \
+    --temp "$TEMPERATURE" \
+    --top-k "$TOP_K" \
+    --top-p "$TOP_P" \
+    --min-p "$MIN_P" \
+    --threads "$THREADS" \
+    --threads-http "$THREADS_HTTP" \
+    --alias "$ALIAS" \
+    --host "$HOST" \
+    --port "$PORT"
@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+# Serve the Qwen3.8-27B pipeline-parallel head on Dankclaw.
+#
+# Runs llama-server (b10433) with --rpc pointing at the worker on
+# Dankserver. The model is split pipeline-style across [RPC0, iGPU]:
+# layers run in order, hidden states cross the ethernet boundary once per
+# token step.
+#
+# IMPORTANT: no --device flag. The RPC device only joins the model's device
+# list through default device selection; an explicit --device silently drops
+# it. See the header of frank-default-large.env.
+#
+# Tunables come from frank-default-large.env in this directory unless the
+# FRANK_PP_HEAD_ENV_FILE environment variable overrides that path.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FRANK_PP_HEAD_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -------------------
+: "${MODEL_DIR:=/home/dank/models}"
+: "${MODEL_FILE:=Qwen3.8-27B-Uncensored-Q6_K.gguf}"
+: "${HEAD_BIN:=/home/dank/llama.cpp/build/bin/llama-server}"
+: "${ALIAS:=frank-pp-large}"
+: "${BIND_ADDR:=0.0.0.0}"
+: "${LISTEN_PORT:=7770}"
+: "${RPC_SERVERS:=192.168.0.235:50052}"
+: "${TENSOR_SPLIT:=1,1}"
+: "${CTX_SIZE:=131072}"
+: "${PARALLEL:=1}"
+: "${SPEC_NONE:=0}"
+: "${SPEC_DEFAULT:=1}"
+: "${MTP:=0}"
+: "${SPEC_DRAFT_N_MAX:=4}"
+: "${SPEC_DRAFT_MODEL:=}"
+: "${SPEC_NGRAM_MOD_N_MIN:=48}"
+: "${SPEC_NGRAM_MOD_N_MAX:=64}"
+: "${SPEC_NGRAM_MOD_N_MATCH:=24}"
+: "${CACHE_TYPE_K:=q8_0}"
+: "${CACHE_TYPE_V:=f16}"
+: "${BATCH_SIZE:=8192}"
+: "${UBATCH_SIZE:=256}"
+: "${TEMPERATURE:=1.0}"
+: "${TOP_K:=20}"
+: "${TOP_P:=0.95}"
+: "${MIN_P:=0.0}"
+: "${THREADS:=12}"
+: "${THREADS_HTTP:=1}"
+: "${VERBOSITY:=4}"
+: "${DIRECT_IO:=1}"
+
+MODEL="$MODEL_DIR/$MODEL_FILE"
+
+if [[ ! -f "$MODEL" ]]; then
+    echo "missing required file: $MODEL" >&2
+    exit 1
+fi
+if [[ ! -x "$HEAD_BIN" ]]; then
+    echo "missing head binary: $HEAD_BIN" >&2
+    exit 1
+fi
+
+# --- RPC endpoints -----------------------------------------------------------
+RPC_ARGS=()
+for s in ${RPC_SERVERS//,/ }; do
+    RPC_ARGS+=(--rpc "$s")
+done
+
+# --- split fractions ---------------------------------------------------------
+SPLIT_ARGS=()
+if [[ -n "${TENSOR_SPLIT}" ]]; then
+    SPLIT_ARGS+=(--tensor-split "${TENSOR_SPLIT}")
+fi
+
+# --- speculative decoding -----------------------------------------------------
+SPEC_ARGS=()
+if [[ "${SPEC_NONE}" == "1" ]]; then
+    SPEC_ARGS+=(--spec-type none)
+else
+    if [[ "${SPEC_DEFAULT}" == "1" ]]; then
+        if [[ "${MTP}" == "1" ]]; then
+            # MTP first (bundled in-model head by default, or a standalone
+            # draft file when SPEC_DRAFT_MODEL is set), then the ngram-mod
+            # fallback. Order matters: --spec-type appends to the impl chain.
+            SPEC_ARGS+=(--spec-type draft-mtp)
+            SPEC_ARGS+=(--spec-draft-n-max "${SPEC_DRAFT_N_MAX}")
+            if [[ -n "${SPEC_DRAFT_MODEL}" ]]; then
+                if [[ ! -f "${SPEC_DRAFT_MODEL}" ]]; then
+                    echo "SPEC_DRAFT_MODEL file missing: ${SPEC_DRAFT_MODEL}" >&2
+                    exit 1
+                fi
+                SPEC_ARGS+=(--spec-draft-model "${SPEC_DRAFT_MODEL}")
+            fi
+        fi
+        SPEC_ARGS+=(--spec-default)
+        SPEC_ARGS+=(--spec-ngram-mod-n-min "${SPEC_NGRAM_MOD_N_MIN}")
+        SPEC_ARGS+=(--spec-ngram-mod-n-max "${SPEC_NGRAM_MOD_N_MAX}")
+        SPEC_ARGS+=(--spec-ngram-mod-n-match "${SPEC_NGRAM_MOD_N_MATCH}")
+    else
+        echo "SPEC_NONE=0 requires SPEC_DEFAULT=1" >&2
+        exit 1
+    fi
+fi
+
+# --- load mode ----------------------------------------------------------------
+LOAD_ARGS=()
+if [[ "${DIRECT_IO}" == "1" ]]; then
+    LOAD_ARGS+=(--direct-io)
+fi
+
+exec "$HEAD_BIN" \
+    --model "$MODEL" \
+    "${RPC_ARGS[@]}" \
+    "${SPLIT_ARGS[@]}" \
+    "${SPEC_ARGS[@]}" \
+    "${LOAD_ARGS[@]}" \
+    --gpu-layers all \
+    --fit off \
+    --cache-type-k "$CACHE_TYPE_K" \
+    --cache-type-v "$CACHE_TYPE_V" \
+    --ctx-size "$CTX_SIZE" \
+    --parallel "$PARALLEL" \
+    --batch-size "$BATCH_SIZE" \
+    --ubatch-size "$UBATCH_SIZE" \
+    --temp "$TEMPERATURE" \
+    --top-k "$TOP_K" \
+    --top-p "$TOP_P" \
+    --min-p "$MIN_P" \
+    --threads "$THREADS" \
+    --threads-http "$THREADS_HTTP" \
+    --alias "$ALIAS" \
+    -lv "$VERBOSITY" \
+    --host "$BIND_ADDR" \
+    --port "$LISTEN_PORT"
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Serve the Qwen3.8-27B pipeline-parallel RPC worker on Dankserver.
+#
+# Runs ggml-rpc-server from the b10433 CUDA+RPC build and exposes the RTX
+# 3090 as a remote device for the head on Dankclaw.
+#
+# Tunables come from frank-pp-worker.env in this directory unless the
+# FRANK_PP_WORKER_ENV_FILE environment variable overrides that path.
+#
+# Security note (upstream): the RPC server is "fragile and insecure". No
+# authentication, no encryption. Keep it on the trusted LAN only.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FRANK_PP_WORKER_ENV_FILE:-$SCRIPT_DIR/frank-pp-worker.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -------------------
+: "${RPC_BIND:=192.168.0.235}"
+: "${RPC_PORT:=50052}"
+: "${DEVICE:=CUDA0}"
+: "${THREADS:=7}"
+: "${CACHE:=1}"
+: "${WORKER_BIN:=/home/dank/llama-cpp-rpc/build/bin/ggml-rpc-server}"
+
+if [[ ! -x "$WORKER_BIN" ]]; then
+    echo "missing worker binary: $WORKER_BIN" >&2
+    echo "Build it: cd /home/dank/llama-cpp-rpc && cmake -B build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=ON -DGGML_CUDA_FA_ALL_QUANTS=ON -DGGML_RPC=ON -DCMAKE_BUILD_TYPE=Release && cmake --build build --config Release -- -j 8" >&2
+    exit 1
+fi
+
+ARGS=()
+if [[ "${CACHE}" == "1" ]]; then
+    ARGS+=(--cache)
+fi
+
+exec "$WORKER_BIN" \
+    --host "$RPC_BIND" \
+    --port "$RPC_PORT" \
+    --device "$DEVICE" \
+    --threads "$THREADS" \
+    "${ARGS[@]}"
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Serve Qwen3.6-35B-A3B-FP8 with DFlash speculative decoding via vLLM on NVIDIA RTX 3090
+# Uses Podman container with CUDA support
+# 4 parallel streams, full context (262K) each
+set -euo pipefail
+
+# Configuration
+MODEL="${MODEL:-Qwen/Qwen3.6-35B-A3B-FP8}"
+DRAFT_MODEL="${DRAFT_MODEL:-z-lab/Qwen3.6-35B-A3B-DFlash}"
+PORT="${PORT:-8000}"
+GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.95}"
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-4}"
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}"
+KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
+NUM_SPEC_TOKENS="${NUM_SPEC_TOKENS:-8}"
+CONTAINER_NAME="${CONTAINER_NAME:-vllm-qwen36-dflash}"
+HF_CACHE="${HF_CACHE:-/home/dank/.cache/huggingface}"
+GPU_LIBS="${GPU_LIBS:-/usr/lib/x86_64-linux-gnu}"
+
+echo "Starting vLLM server in Podman..."
+echo "  Target model: ${MODEL}"
+echo "  Draft model:  ${DRAFT_MODEL}"
+echo "  Port:         ${PORT}"
+echo "  GPU mem util: ${GPU_MEM_UTIL}"
+echo "  Max seqs:     ${MAX_NUM_SEQS}"
+echo "  Context len:  ${MAX_MODEL_LEN}"
+echo "  KV cache:     ${KV_CACHE_DTYPE}"
+echo "  Spec tokens:  ${NUM_SPEC_TOKENS}"
+echo ""
+
+# Remove existing container if present
+podman rm -f "${CONTAINER_NAME}" 2>/dev/null || true
+
+# Run vLLM in Podman container (foreground mode for systemd)
+# Note: vllm/vllm-openai image has entrypoint "vllm serve", so we pass args directly
+exec podman run --name "${CONTAINER_NAME}" \
+  --device /dev/nvidia0 \
+  --device /dev/nvidiactl \
+  --device /dev/nvidia-uvm \
+  --device /dev/nvidia-uvm-tools \
+  --device /dev/nvidia-modeset \
+  -v "${GPU_LIBS}:/usr/lib/x86_64-linux-gnu:ro" \
+  -v "${HF_CACHE}:/root/.cache/huggingface" \
+  -p "${PORT}:${PORT}" \
+  docker.io/vllm/vllm-openai:latest \
+  "${MODEL}" \
+    --port "${PORT}" \
+    --host "0.0.0.0" \
+    --kv-cache-dtype "${KV_CACHE_DTYPE}" \
+    --gpu-memory-utilization "${GPU_MEM_UTIL}" \
+    --max-num-seqs "${MAX_NUM_SEQS}" \
+    --max-model-len "${MAX_MODEL_LEN}" \
+    --spec-method dflash \
+    --spec-model "${DRAFT_MODEL}" \
+    --spec-tokens "${NUM_SPEC_TOKENS}" \
+    --trust-remote-code
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# bench-spec-matrix.sh - Qwen3.8-27B UD-Q8_K_XL speculative-decode matrix.
+# Matrix: spec_type {draft-mtp,ngram-mod | draft-mtp | ngram-mod}
+#         x --spec-draft-n-max {2..10} x --spec-draft-p-min {off, 0.8}
+# = 54 configs. Each: start llama-server, wait ready, 3 generations of a
+# 1100-token prompt (n_predict 200, default sampling), record timings.
+set -u
+
+MODEL=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL.gguf
+LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
+PORT=7890
+DEVICE=ROCm0
+OUT=/tmp/spec_matrix_results.tsv
+RUNLOG=/tmp/spec_matrix_run.log
+PROMPT=$(python3 -c "print(\"The fleet of inference servers must remain available at all times. Operators monitor memory, throughput, and queue depth. \"*50)")
+N_OUT=200
+RUNS=3
+
+: > "$OUT"
+printf 'spec_type\tn_max\tp_min\trun\ttps\tpps\tdraft_acc\tdraft_n\tstop\n' >> "$OUT"
+
+cfg=0
+for spec in "draft-mtp,ngram-mod" "draft-mtp" "ngram-mod"; do
+  for n_max in 2 3 4 5 6 7 8 9 10; do
+    for p_min in none 0.8; do
+      cfg=$((cfg+1))
+      echo "=== CONFIG $cfg/54: spec=$spec n_max=$n_max p_min=$p_min $(date +%T) ===" | tee -a "$RUNLOG"
+      ARGS=(--model "$MODEL" --spec-type "$spec" --spec-draft-n-max "$n_max")
+      # Qwen3.8 bundles its MTP drafter inside the target GGUF; no separate
+      # --spec-draft-model file (llama.cpp auto-detects, b10411+).
+      if [[ "$p_min" != none ]]; then
+        ARGS+=(--spec-draft-p-min "$p_min")
+      fi
+      ARGS+=(--device "$DEVICE" --gpu-layers all --cache-type-k q8_0 --cache-type-v f16
+             --ctx-size 2048 --parallel 1 --batch-size 512 --ubatch-size 128
+             --temp 1.0 --top-k 20 --top-p 0.95 --min-p 0.0
+             --host 127.0.0.1 --port "$PORT")
+      "$LLAMA_SERVER" "${ARGS[@]}" > /tmp/matrix_server.log 2>&1 &
+      SPID=$!
+      ready=0
+      for i in $(seq 1 150); do
+        if curl -sf "http://127.0.0.1:$PORT/health" > /dev/null 2>&1; then ready=1; break; fi
+        if ! kill -0 "$SPID" 2>/dev/null; then break; fi
+        sleep 2
+      done
+      if [[ "$ready" != 1 ]]; then
+        echo "NOT_READY: spec=$spec n_max=$n_max p_min=$p_min" | tee -a "$RUNLOG"
+        tail -4 /tmp/matrix_server.log >> "$RUNLOG"
+        kill "$SPID" 2>/dev/null; wait "$SPID" 2>/dev/null
+        continue
+      fi
+      for r in 1 2 3; do
+        curl -s -m 300 -X POST "http://127.0.0.1:$PORT/completion" -H "Content-Type: application/json" \
+          -d "{\"prompt\": \"$PROMPT\", \"n_predict\": $N_OUT, \"stream\": false, \"cache_prompt\": false}" \
+          | python3 -c "
+import json,sys
+try:
+    d=json.load(sys.stdin)
+    t=d.get('timings',{})
+    row=[\"$spec\",\"$n_max\",\"$p_min\",\"$r\",
+         str(round(t.get('predicted_per_second',0),2)),
+         str(round(t.get('prompt_per_second',0),2)),
+         str(t.get('draft_n_accepted','')),
+         str(t.get('draft_n','')),
+         str(d.get('stop_type',''))]
+    with open(\"$OUT\",\"a\") as f:
+        f.write('\t'.join(row)+'\n')
+except Exception as e:
+    with open(\"$OUT\",\"a\") as f:
+        f.write('\t'.join([\"$spec\",\"$n_max\",\"$p_min\",\"$r\",\"ERR\",str(e),\"\",\"\",\"\"])+'\n')
+"
+      done
+      kill "$SPID" 2>/dev/null; wait "$SPID" 2>/dev/null
+      sleep 3
+    done
+  done
+done
+echo "=== MATRIX DONE $(date) ===" | tee -a "$RUNLOG"
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""Accurate-size stream test for frank-default-large (DSpark, PARALLEL=1).
+
+Fires one prompt and reads back the server's OWN reported prompt_tokens so
+the actual stream size is never mislabeled.  Then, with parallel=1, fires a
+second concurrent stream to confirm it QUEUES behind the single slot and
+completes without crashing the ROCm backend.
+
+The prose is flowing/realistic, so a completion_tokens < max_tokens with
+finish_reason "stop" is NOT a degenerate-prompt artifact.
+"""
+import json, sys, threading, time, urllib.request
+
+HOST = sys.argv[1] if len(sys.argv) > 1 else "127.0.0.1"
+PORT = sys.argv[2] if len(sys.argv) > 2 else "7770"
+MAX_TOKENS = int(sys.argv[3]) if len(sys.argv) > 3 else 256
+TARGET_TOKENS = int(sys.argv[4]) if len(sys.argv) > 4 else 16000
+
+VAR = (
+    "narrative nonfiction blends observation with reflection, ",
+    "and a careful writer lets each clause earn its place ",
+    "in the sentence's rising line, so meaning accumulates slowly. ",
+    "The reader is not hurried; the reader is led. Loose and periodic, ",
+    "the style turns on concrete nouns and verbs that carry weight, ",
+    "and it resists the abstract noun that hides a vague claim. ",
+    "A paragraph is not a list; it is a single turn of thought that ",
+    "opens, develops, and lands. This is the craft of prose: choose, ",
+    "arrange, and above all omit. Omission is the discipline that ",
+    "keeps the whole clear, and clarity is the courtesy we owe the ",
+    "willing reader."
+)
+
+def build_prompt(topic, salt):
+    n = len(VAR)
+    parts = []
+    t = 0
+    while t * 76 < TARGET_TOKENS:     # ~76 tokens per outer loop (measured)
+        for k in range(5):
+            parts.append(VAR[(t + k + salt) % n])
+        parts.append(f" This is the {topic}, cued by note {salt} added later. ")
+        t += 1
+    return "".join(parts)
+
+def post(prompt, max_tokens, timeout=3600):
+    body = json.dumps({
+        "prompt": prompt,
+        "max_tokens": max_tokens,
+        "temperature": 0.0,
+        "stream": False,
+    }).encode()
+    req = urllib.request.Request(
+        f"http://{HOST}:{PORT}/v1/completions",
+        data=body, headers={"Content-Type": "application/json"})
+    t0 = time.time()
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        out = json.load(resp)
+    dt = time.time() - t0
+    return out, dt
+
+# ---- Stream 1: single 16k prompt -------------------------------
+p1 = build_prompt("first subject", 1)
+print(f"built prompt 1: {len(p1)} chars")
+o1, t1 = post(p1, MAX_TOKENS)
+u1 = o1.get("usage", {})
+print(f"STREAM1 single: prompt_tokens={u1.get('prompt_tokens')} "
+      f"completion_tokens={u1.get('completion_tokens')} "
+      f"finish={o1.get('choices',[{}])[0].get('finish_reason')} secs={t1:.1f}")
+
+# ---- Streams 2 & 3: concurrent, must queue (parallel=1) ---------
+results = {}
+def worker(key, prompt):
+    try:
+        out, dt = post(prompt, MAX_TOKENS, timeout=7200)
+        u = out.get("usage", {})
+        results[key] = {
+            "prompt_tokens": u.get("prompt_tokens"),
+            "completion_tokens": u.get("completion_tokens"),
+            "finish": out.get("choices", [{}])[0].get("finish_reason"),
+            "secs": round(dt, 1),
+        }
+    except Exception as e:
+        results[key] = {"error": str(e)}
+
+p2 = build_prompt("second subject", 2)
+p3 = build_prompt("third subject", 3)
+print(f"built prompt 2 ({len(p2)} chars) and prompt 3 ({len(p3)} chars); concurrent...")
+ta = threading.Thread(target=worker, args=("s2", p2))
+tb = threading.Thread(target=worker, args=("s3", p3))
+ta.start(); tb.start(); ta.join(); tb.join()
+for k in sorted(results):
+    print(f"  [{k}] {results[k]}")
+
+# verdict
+def suspicious(r):
+    if "error" in r:
+        return True
+    return r.get("finish") == "stop" and r.get("completion_tokens", 0) < MAX_TOKENS
+
+allr = [("stream1", o1.get("choices",[{}])[0].get("finish_reason"),
+         u1.get("completion_tokens")),
+        ("s2", results.get("s2",{}).get("finish"), results.get("s2",{}).get("completion_tokens")),
+        ("s3", results.get("s3",{}).get("finish"), results.get("s3",{}).get("completion_tokens"))]
+print("\nSUMMARY")
+ok = True
+for name, fin, comp in allr:
+    sig = fin == "stop" and comp is not None and comp < MAX_TOKENS
+    err = comp is None
+    if sig or err:
+        ok = False
+    print(f"  {name}: finish={fin} completion={comp}/{MAX_TOKENS} "
+          f"{'SIGNATURE' if sig else ('ERROR' if err else 'ok')}")
+print("  RESULT:", "PASS" if ok else "FAIL")
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+HOST="127.0.0.1"
+PORT=8001
+MODEL="frank-default-small"
+
+# Different prompts for each test to avoid prefix caching
+PROMPT1="Explain the difference between supervised, unsupervised, and reinforcement learning in machine learning. For each type, provide a concrete real-world example, describe the training process, and discuss the advantages and limitations."
+PROMPT2="Describe the architectural differences between RISC-V and x86 instruction set architectures. Include a discussion of register files, addressing modes, and instruction encoding. Compare their performance for embedded systems and server workloads."
+PROMPT3="Analyze the economic impact of blockchain technology on supply chain management. Discuss the key benefits of transparency, immutability, and smart contracts. Compare implementation challenges of Bitcoin, Ethereum, and Hyperledger for enterprise use cases."
+PROMPT4="Explain the principles of evolutionary algorithms in artificial intelligence. Describe how genetic algorithms, genetic programming, and evolutionary strategies differ in their approach to problem solving. Compare their performance on optimization problems and neural network architecture search."
+
+echo "=== Validating vLLM on ${HOST}:${PORT} ==="
+
+# Test health endpoint
+echo "Health check..."
+if ! curl -sf "http://${HOST}:${PORT}/health" > /dev/null 2>&1; then
+    echo "FAIL: Health check failed"
+    exit 1
+fi
+
+# Test N streams sequentially (1, 2, 3, 4)
+for NUM_STREAMS in 1 2 3 4; do
+    echo ""
+    echo "--- Testing ${NUM_STREAMS} stream(s) ---"
+    
+    TMPDIR=$(mktemp -d)
+    START=$(date +%s%N)
+    
+    for i in $(seq 1 $NUM_STREAMS); do
+        case $(( (i - 1) % 4 + 1 )) in
+            1) PROMPT="${PROMPT1}" ;;
+            2) PROMPT="${PROMPT2}" ;;
+            3) PROMPT="${PROMPT3}" ;;
+            4) PROMPT="${PROMPT4}" ;;
+        esac
+        
+        curl -s -X POST "http://${HOST}:${PORT}/v1/chat/completions" \
+            -H "Content-Type: application/json" \
+            -d "{
+                \"model\": \"${MODEL}\",
+                \"messages\": [
+                    {\"role\": \"user\", \"content\": \"${PROMPT}\"}
+                ],
+                \"max_tokens\": 256
+            }" > "$TMPDIR/$i.json" &
+    done
+    wait
+    
+    END=$(date +%s%N)
+    ELAPSED_MS=$(( (END - START) / 1000000 ))
+    
+    # Calculate speed from the first stream's response
+    PROMPT_TOKENS=$(python3 -c "import json; print(json.load(open('$TMPDIR/1.json'))['usage']['prompt_tokens'])")
+    COMPLETION_TOKENS=$(python3 -c "import json; print(json.load(open('$TMPDIR/1.json'))['usage']['completion_tokens'])")
+    
+    # Calculate speeds
+    PROMPT_SPEED=$(python3 -c "print(f'{${PROMPT_TOKENS}/${ELAPSED_MS}*1000:.2f}')")
+    COMPLETION_SPEED=$(python3 -c "print(f'{${COMPLETION_TOKENS}/${ELAPSED_MS}*1000:.2f}')")
+    
+    echo "  Elapsed: ${ELAPSED_MS}ms"
+    echo "  Prompt tokens: ${PROMPT_TOKENS}"
+    echo "  Prompt speed: ${PROMPT_SPEED} tok/s"
+    echo "  Completion tokens: ${COMPLETION_TOKENS}"
+    echo "  Completion speed: ${COMPLETION_SPEED} tok/s"
+    
+    # Save results
+    echo "${NUM_STREAMS},${PROMPT_SPEED},${COMPLETION_SPEED},${ELAPSED_MS}" > "/tmp/vllm-${NUM_STREAMS}.csv"
+    
+    # Clean up temp files
+    rm -rf "$TMPDIR"
+done
+
+echo ""
+echo "=== Results Summary ==="
+for n in 1 2 3 4; do
+    if [ -f "/tmp/vllm-${n}.csv" ]; then
+        IFS=',' read -r streams p_speed c_speed elapsed < "/tmp/vllm-${n}.csv"
+        echo "  ${n} stream(s): ${p_speed} tok/s (prompt), ${c_speed} tok/s (completion) - ${elapsed}ms"
+    fi
+done
+
+# Check if single stream speed is >= 5 tok/s
+SINGLE_SPEED=$(head -1 /tmp/vllm-1.csv | cut -d',' -f2)
+python3 -c "import sys; sys.exit(0 if float('${SINGLE_SPEED}') >= 5 else 1)"
+echo ""
+echo "PASS: Validation successful"