inference: frank-large returns to halogen, frank-small takes the 3090
Dankclaw's endpoints are re-split. frank-large is the halogen-flash-server quadlet again -- the 2026-09-13 unit file was never touched by the two-day ik_llama.cpp interlude, so the runtime was still in the repo and its image is still in local storage -- retuned to the shape asked for: HALOGEN_CTX 200192 with a 400,384-position pool and two slots, exactly two 200k conversations. frank-small is new: mainline llama.cpp on the RTX 3090 serving ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF at IQ3_S, using the -mtp build (it carries blk.64.nextn.*, which is what lets --spec-type draft-mtp run on the main model's own weights instead of a second 12 GiB copy) plus the BF16 vision projector, as one 131,072-token slot with f16 KV on port 7771. The host is shared now, and that is what the halogen retune pays for: HALOGEN_MAX_TOK 24576 (arena 20.6 -> 16.4 GiB, ~4-5% prefill) and the vision tower dropped (1.97 GiB) hand ~6 GiB back to the host, and the reserve rises to 20 to keep the fit check honest. Vision moved wholesale to frank-small. Retired to scripts/.AGENTTRASH/: the ik trio (env, service, serve script), the Qwen-Sharp template only that runtime read, and four benchmark harnesses bound to retired runtimes. Because retiring a file makes any test that greps it pass by absence, the seams assert the archive exists AND the live path does not.
This commit is contained in:
Frank
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Decode throughput benchmark for frank-large.
|
||||
Starts the server with a given --threads-batch value, sends an 8K+ token prompt,
|
||||
generates 256 tokens, measures generation tok/s.
|
||||
|
||||
Each --threads-batch value requires a cold server load (~3 min).
|
||||
"""
|
||||
import json, time, urllib.request, urllib.error, subprocess, os, sys
|
||||
|
||||
MODEL = "/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX/Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf"
|
||||
SERVE = "/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server"
|
||||
PORT = 7770
|
||||
N_GEN = 256
|
||||
PROMPT_FILE = "/tmp/bench-prompts/prompt_code.txt"
|
||||
ENV_FILE = os.path.join(os.path.dirname(__file__), "frank-large.env")
|
||||
SERVER_DIR = os.path.dirname(SERVE)
|
||||
ROCM_PATH = "/opt/rocm"
|
||||
|
||||
def make_env():
|
||||
"""Build the same env as serve-frank-large.sh."""
|
||||
env = {**os.environ, "HSA_OVERRIDE_GFX_VERSION": "11.5.1", "HIP_VISIBLE_DEVICES": "0"}
|
||||
|
||||
# Source env file for tunables
|
||||
if os.path.exists(ENV_FILE):
|
||||
with open(ENV_FILE) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, v = line.split("=", 1)
|
||||
env[k] = v
|
||||
|
||||
# Build LD_LIBRARY_PATH
|
||||
lib_path = f"{SERVER_DIR}:{ROCM_PATH}/lib:{ROCM_PATH}/lib/rocm_sysdeps/lib:{ROCM_PATH}/lib/llvm/lib"
|
||||
if os.environ.get("LD_LIBRARY_PATH"):
|
||||
lib_path += f":{os.environ['LD_LIBRARY_PATH']}"
|
||||
env["LD_LIBRARY_PATH"] = lib_path
|
||||
env["ROCM_PATH"] = ROCM_PATH
|
||||
|
||||
return env
|
||||
|
||||
ENV = make_env()
|
||||
|
||||
def kill_all():
|
||||
subprocess.run(["pkill", "-9", "llama-server"], capture_output=True)
|
||||
time.sleep(3)
|
||||
|
||||
def wait_ready():
|
||||
for i in range(600):
|
||||
try:
|
||||
r = urllib.request.urlopen(f"http://127.0.0.1:{PORT}/health", timeout=2)
|
||||
if json.loads(r.read()).get("status") == "ok":
|
||||
return True
|
||||
except:
|
||||
pass
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
def send_request(prompt, max_tokens=N_GEN):
|
||||
"""Send chat completion, measure generation throughput."""
|
||||
body = json.dumps({
|
||||
"model": "frank-large",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens,
|
||||
"stream": False,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.8
|
||||
}).encode()
|
||||
|
||||
t0 = time.time()
|
||||
req = urllib.request.Request(
|
||||
f"http://127.0.0.1:{PORT}/v1/chat/completions",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST"
|
||||
)
|
||||
r = urllib.request.urlopen(req, timeout=1200)
|
||||
resp = json.loads(r.read())
|
||||
wall = time.time() - t0
|
||||
|
||||
usage = resp.get("usage", {})
|
||||
pt = usage.get("prompt_tokens", 0)
|
||||
gt = usage.get("completion_tokens", 0)
|
||||
gtps = gt / wall if wall > 0 else 0
|
||||
return pt, gt, wall, gtps
|
||||
|
||||
def load_and_extend_prompt():
|
||||
"""Load prompt and extend to ~8K tokens if needed."""
|
||||
with open(PROMPT_FILE) as f:
|
||||
text = f.read()
|
||||
|
||||
# Count approximate tokens (rough: ~4 chars per token for code)
|
||||
approx_tokens = len(text) / 4.0
|
||||
needed = 8200
|
||||
|
||||
if approx_tokens < needed:
|
||||
repeats = needed // max(int(approx_tokens), 1) + 1
|
||||
text = (text + "\n") * repeats
|
||||
words = text.split()[:needed]
|
||||
text = " ".join(words)
|
||||
|
||||
return text
|
||||
|
||||
def main():
|
||||
tb_values = [int(x) for x in sys.argv[1:]] if len(sys.argv) > 1 else [16, 32, 64, 128]
|
||||
results = []
|
||||
|
||||
prompt = load_and_extend_prompt()
|
||||
n_words = len(prompt.split())
|
||||
print(f"Prompt: {n_words} words (~{n_words} tokens), {len(prompt)} chars\n", flush=True)
|
||||
|
||||
for tb in tb_values:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"threads-batch = {tb}")
|
||||
print(f"{'='*60}", flush=True)
|
||||
|
||||
kill_all()
|
||||
print(" Starting server...", flush=True)
|
||||
|
||||
lib_path = ENV.get("LD_LIBRARY_PATH", "")
|
||||
cmd = [
|
||||
"/usr/bin/env",
|
||||
"HSA_OVERRIDE_GFX_VERSION=11.5.1",
|
||||
"HIP_VISIBLE_DEVICES=0",
|
||||
f"LD_LIBRARY_PATH={lib_path}",
|
||||
f"ROCM_PATH={ROCM_PATH}",
|
||||
SERVE, "--model", MODEL, "--port", str(PORT), "--jinja",
|
||||
"--device", "ROCm0", "--gpu-layers", "999", "--fit", "off",
|
||||
"--threads", "16", "--threads-batch", str(tb),
|
||||
"--parallel", "2", "--ctx-size", "131072",
|
||||
"--batch-size", "4096", "--ubatch-size", "4096",
|
||||
"--flash-attn", "on",
|
||||
"--cache-type-k", "f16", "--cache-type-v", "f16",
|
||||
"--cache-prompt", "--cache-idle-slots", "--metrics"
|
||||
]
|
||||
|
||||
srv = subprocess.Popen(cmd, env=ENV, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
|
||||
ready = wait_ready()
|
||||
if not ready:
|
||||
print(" FAILED: server did not become ready", flush=True)
|
||||
srv.terminate()
|
||||
srv.wait()
|
||||
for line in (srv.stdout.readlines() or [])[-20:]:
|
||||
print(f" LOG: {line.rstrip()}", flush=True)
|
||||
continue
|
||||
|
||||
print(" Server ready.", flush=True)
|
||||
|
||||
# Warmup (small request)
|
||||
try:
|
||||
send_request("Hi", 16)
|
||||
except Exception as e:
|
||||
print(f" Warmup failed: {e}", flush=True)
|
||||
srv.terminate()
|
||||
srv.wait()
|
||||
continue
|
||||
|
||||
# Benchmark
|
||||
print(f" Generating {N_GEN} tokens...", flush=True)
|
||||
try:
|
||||
pt, gt, wall, gtps = send_request(prompt, N_GEN)
|
||||
print(f" {gt}/{N_GEN} gen tokens in {wall:.1f}s = {gtps:.1f} tok/s")
|
||||
print(f" Prompt tokens: {pt}")
|
||||
results.append({"tb": tb, "wall": wall, "gtps": gtps, "gt": gt, "pt": pt})
|
||||
except Exception as e:
|
||||
print(f" FAILED: {e}", flush=True)
|
||||
|
||||
srv.terminate()
|
||||
srv.wait()
|
||||
time.sleep(2)
|
||||
|
||||
# Summary
|
||||
print(f"\n{'='*60}")
|
||||
print("SUMMARY")
|
||||
print(f"{'='*60}")
|
||||
for r in results:
|
||||
print(f" tb={r['tb']:>3}: {r['gt']} gen tokens, {r['wall']:.1f}s, {r['gtps']:.1f} tok/s")
|
||||
|
||||
if results:
|
||||
best = max(results, key=lambda r: r['gtps'])
|
||||
print(f"\nBest: tb={best['tb']} at {best['gtps']:.1f} tok/s")
|
||||
|
||||
with open("/tmp/bench_decode_results.txt", "w") as f:
|
||||
f.write("tb,gen_tokens,wall_s,gen_tps,prompt_tokens\n")
|
||||
for r in results:
|
||||
f.write(f"{r['tb']},{r['gt']},{r['wall']:.1f},{r['gtps']:.1f},{r['pt']}\n")
|
||||
|
||||
print(f"\nResults: /tmp/bench_decode_results.txt")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+268
@@ -0,0 +1,268 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Benchmark n-gram speculative decoding vs no-spec for frank-large.
|
||||
|
||||
Tests prompt-processing throughput with 16K token prompts (manageable request size).
|
||||
Code-heavy prompt (high n-gram redundancy) vs prose prompt (low redundancy).
|
||||
|
||||
Usage:
|
||||
# Generate prompts: python3 scripts/bench-ngram-mod.py generate /tmp/prompts/
|
||||
# Run: python3 scripts/bench-ngram-mod.py /tmp/prompts/
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import subprocess
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
|
||||
MODEL = "/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX/Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf"
|
||||
CHAT_TEMPLATE = "/home/dank/serverAppConfigs/scripts/chat-template.jinja"
|
||||
BUILD_DIR = "/home/dank/GitPackages/ROCmFPX/build-qwen38/bin"
|
||||
LLAMA_BIN = os.path.join(BUILD_DIR, "llama-server")
|
||||
|
||||
# 16K tokens ~= 64KB text
|
||||
TARGET_PROMPT_TOKENS = 16000
|
||||
TOKEN_CHAR_RATIO = 4
|
||||
TARGET_PROMPT_BYTES = TARGET_PROMPT_TOKENS * TOKEN_CHAR_RATIO
|
||||
|
||||
CONTEXT = 65536
|
||||
BATCH = 4096
|
||||
UBATCH = 4096
|
||||
THREADS = 16
|
||||
THREAD_BATCH = 32
|
||||
NGRAM_N_MAX = 24
|
||||
NGRAM_N_MIN = 4
|
||||
TEST_TOKENS = 128
|
||||
SERVER_WAIT = 300
|
||||
|
||||
|
||||
def start_server(port, spec_type):
|
||||
cmd = [
|
||||
LLAMA_BIN,
|
||||
"--model", MODEL,
|
||||
"--chat-template-file", CHAT_TEMPLATE,
|
||||
"--jinja",
|
||||
"--device", "ROCm0",
|
||||
"--gpu-layers", "999",
|
||||
"--parallel", "1",
|
||||
"--ctx-size", str(CONTEXT),
|
||||
"--batch-size", str(BATCH),
|
||||
"--ubatch-size", str(UBATCH),
|
||||
"--fit", "off",
|
||||
"--flash-attn", "on",
|
||||
"--cache-type-k", "f16",
|
||||
"--cache-type-v", "f16",
|
||||
"--threads", str(THREADS),
|
||||
"--threads-batch", str(THREAD_BATCH),
|
||||
"--cache-prompt",
|
||||
"--cache-idle-slots",
|
||||
"--metrics",
|
||||
"--host", "127.0.0.1",
|
||||
"--port", str(port),
|
||||
"--temp", "0",
|
||||
]
|
||||
if spec_type == "ngram-mod":
|
||||
cmd.extend([
|
||||
"--spec-type", "ngram-mod",
|
||||
"--spec-ngram-mod-n-max", str(NGRAM_N_MAX),
|
||||
"--spec-ngram-mod-n-min", str(NGRAM_N_MIN),
|
||||
])
|
||||
else:
|
||||
cmd.extend(["--spec-type", "none"])
|
||||
|
||||
env = os.environ.copy()
|
||||
env["ROCM_PATH"] = "/opt/rocm"
|
||||
env["LD_LIBRARY_PATH"] = f"{BUILD_DIR}:/opt/rocm/lib:/opt/rocm/lib/rocm_sysdeps/lib:/opt/rocm/lib/llvm/lib"
|
||||
env["HSA_OVERRIDE_GFX_VERSION"] = "11.5.1"
|
||||
env["HIP_VISIBLE_DEVICES"] = "0"
|
||||
|
||||
proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
print(f" Waiting for server on port {port} (spec={spec_type})...")
|
||||
for i in range(SERVER_WAIT):
|
||||
try:
|
||||
req = urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=2)
|
||||
resp = json.loads(req.read().decode())
|
||||
if resp.get("status") == "ok":
|
||||
print(f" Server ready after {i+1}s")
|
||||
return proc
|
||||
except:
|
||||
time.sleep(1)
|
||||
|
||||
proc.terminate()
|
||||
out, err = proc.communicate()
|
||||
print(f"ERROR: Server failed to start in {SERVER_WAIT}s")
|
||||
print(err.decode()[-2000:])
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def run_test(prompt_text, spec_type, port):
|
||||
"""Send a request and extract prompt processing metrics."""
|
||||
data = json.dumps({
|
||||
"model": "frank-large",
|
||||
"messages": [{"role": "user", "content": prompt_text}],
|
||||
"max_tokens": TEST_TOKENS,
|
||||
"stream": False,
|
||||
"temperature": 0,
|
||||
}).encode()
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"http://127.0.0.1:{port}/v1/chat/completions",
|
||||
data=data,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
start = time.monotonic()
|
||||
resp = urllib.request.urlopen(req, timeout=600)
|
||||
elapsed = time.monotonic() - start
|
||||
|
||||
result = json.loads(resp.read().decode())
|
||||
timings = result.get("timings", {})
|
||||
usage = result.get("usage", {})
|
||||
|
||||
return {
|
||||
"prompt_tokens": usage.get("prompt_tokens", 0),
|
||||
"prompt_ms": timings.get("prompt_ms", 0),
|
||||
"prompt_per_second": timings.get("prompt_per_second", 0),
|
||||
"prompt_per_token_ms": timings.get("prompt_per_token_ms", 0),
|
||||
"predicted_per_second": timings.get("predicted_per_second", 0),
|
||||
"total_time_s": elapsed,
|
||||
"cache_n": timings.get("cache_n", 0),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print(f"Usage: {sys.argv[0]} <prompt_dir>")
|
||||
sys.exit(1)
|
||||
|
||||
prompt_dir = Path(sys.argv[1])
|
||||
prompt_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
code_prompt_file = prompt_dir / "prompt_code.txt"
|
||||
prose_prompt_file = prompt_dir / "prompt_prose.txt"
|
||||
|
||||
# Generate prompts if they don't exist
|
||||
if not code_prompt_file.exists() or not prose_prompt_file.exists():
|
||||
print("Generating prompts...")
|
||||
# Generate deterministic prompts
|
||||
# Code prompt: structured algorithms + data generation + config schemas
|
||||
code_lines = []
|
||||
for i in range(50):
|
||||
code_lines.append(f"# Algorithm {i}\n")
|
||||
code_lines.append(f"def algo_{i}(x, y):\n")
|
||||
code_lines.append(f" \"\"\"Algorithm {i} processing.\"\"\"\n")
|
||||
code_lines.append(f" result = 0\n")
|
||||
code_lines.append(f" for j in range({i * 10}):\n")
|
||||
code_lines.append(f" result += x * y + j\n")
|
||||
code_lines.append(f" return result\n\n")
|
||||
code_text = "".join(code_lines)
|
||||
# Pad/truncate to target
|
||||
while len(code_text) < TARGET_PROMPT_BYTES:
|
||||
code_text += "# Additional algorithm block\n"
|
||||
code_text += f"def extra_{len(code_text)//100}():\n"
|
||||
code_text += " x = 1\n"
|
||||
code_text += " y = 2\n"
|
||||
code_text += " return x + y\n\n"
|
||||
code_text = code_text[:TARGET_PROMPT_BYTES]
|
||||
|
||||
# Prose prompt: historical/scientific/philosophical text
|
||||
prose_text = ""
|
||||
topics = [
|
||||
"The Renaissance brought profound transformation to European civilization. Artists and scholars pushed the boundaries of human expression.",
|
||||
"Quantum mechanics describes the behavior of matter and energy at atomic scales. The uncertainty principle limits measurement precision.",
|
||||
"Ethical philosophy grapples with questions of right and wrong. Utilitarianism judges actions by their consequences and the happiness they produce.",
|
||||
"Mountains rise like ancient sentinels, their peaks lost in clouds that swirl and dance across the sky above the valleys below.",
|
||||
"The ocean whispers secrets to the shore, each wave carrying fragments of distant lands across the water before retreating back to sea.",
|
||||
]
|
||||
for _ in range(500):
|
||||
prose_text += f"{topics[len(prose_text) % len(topics)]} "
|
||||
prose_text = prose_text[:TARGET_PROMPT_BYTES]
|
||||
|
||||
code_prompt_file.write_text(code_text)
|
||||
prose_prompt_file.write_text(prose_text)
|
||||
print(f" Code: {code_prompt_file.stat().st_size:,} bytes")
|
||||
print(f" Prose: {prose_prompt_file.stat().st_size:,} bytes")
|
||||
|
||||
print("=" * 60)
|
||||
print(" N-gram Mod Benchmark — frank-large")
|
||||
print(f" STRIX ROCmFP4 · 64K context · 16K prompts")
|
||||
print("=" * 60)
|
||||
|
||||
results = []
|
||||
port_base = 7772
|
||||
port_ngram = 7773
|
||||
|
||||
tests = [
|
||||
("code", "none", port_base, code_prompt_file),
|
||||
("prose", "none", port_ngram, prose_prompt_file),
|
||||
("code", "ngram-mod", port_base, code_prompt_file),
|
||||
("prose", "ngram-mod", port_ngram, prose_prompt_file),
|
||||
]
|
||||
|
||||
for prompt_type, spec_type, port, pfile in tests:
|
||||
print(f"\n=== {spec_type} ({prompt_type}) ===")
|
||||
print(f" File: {pfile} ({pfile.stat().st_size:,} bytes)")
|
||||
print(f" Spec: {spec_type}")
|
||||
if spec_type == "ngram-mod":
|
||||
print(f" n_max={NGRAM_N_MAX}, n_min={NGRAM_N_MIN}")
|
||||
|
||||
proc = start_server(port, spec_type)
|
||||
|
||||
try:
|
||||
# Warm-up
|
||||
print(" Warm-up...")
|
||||
run_test("Hello test", spec_type, port)
|
||||
|
||||
# Full prompt
|
||||
print(f" Sending {TARGET_PROMPT_TOKENS} token prompt...")
|
||||
with open(pfile) as f:
|
||||
prompt_text = f.read()
|
||||
result = run_test(prompt_text, spec_type, port)
|
||||
|
||||
tps = result["prompt_per_second"]
|
||||
print(f" Tokens: {result['prompt_tokens']}")
|
||||
print(f" Prompt: {result['prompt_ms']:.0f}ms")
|
||||
print(f" TPS: {tps:.1f} tok/s")
|
||||
print(f" Total: {result['total_time_s']:.1f}s")
|
||||
|
||||
results.append({
|
||||
"prompt_type": prompt_type,
|
||||
"spec_type": spec_type,
|
||||
"prompt_tokens": result["prompt_tokens"],
|
||||
"prompt_ms": result["prompt_ms"],
|
||||
"tps": tps,
|
||||
"total_time_s": result["total_time_s"],
|
||||
})
|
||||
|
||||
finally:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except:
|
||||
proc.kill()
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 60)
|
||||
print(" RESULTS SUMMARY")
|
||||
print("=" * 60)
|
||||
|
||||
for ptype in ["code", "prose"]:
|
||||
baseline = next((r for r in results if r["prompt_type"] == ptype and r["spec_type"] == "none"), None)
|
||||
ngram = next((r for r in results if r["prompt_type"] == ptype and r["spec_type"] == "ngram-mod"), None)
|
||||
if baseline and ngram:
|
||||
gain = ((ngram["tps"] - baseline["tps"]) / baseline["tps"]) * 100 if baseline["tps"] > 0 else 0
|
||||
print(f"\n{ptype.upper()} PROMPT:")
|
||||
print(f" Baseline TPS: {baseline['tps']:.1f} tok/s ({baseline['prompt_ms']:.0f}ms)")
|
||||
print(f" N-gram TPS: {ngram['tps']:.1f} tok/s ({ngram['prompt_ms']:.0f}ms)")
|
||||
print(f" Speedup: {gain:+.1f}%")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+304
@@ -0,0 +1,304 @@
|
||||
#!/usr/bin/env bash
|
||||
# Benchmark n-gram speculative decoding vs no-spec for frank-large.
|
||||
#
|
||||
# This tests prompt-processing throughput at 64k context with two distinct
|
||||
# prompts: one code-heavy and one prose-heavy. Uses llama-server on scratch
|
||||
# ports with the OpenAI-compatible API to measure prompt_eval_duration_ms.
|
||||
#
|
||||
# Run on Dankclaw after stopping the production service:
|
||||
# sudo systemctl stop frank-large
|
||||
# ./scripts/bench-ngram-mod.sh
|
||||
# sudo systemctl start frank-large
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a; source "$ENV_FILE"; set +a
|
||||
fi
|
||||
|
||||
: "${LLAMA_BIN:=/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server}"
|
||||
: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX}"
|
||||
: "${MODEL_FILE:=Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf}"
|
||||
: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
|
||||
: "${CHAT_TEMPLATE_KWARGS:={}}"
|
||||
: "${BATCH_SIZE:=4096}"
|
||||
: "${UBATCH_SIZE:=4096}"
|
||||
: "${CTX_SIZE:=65536}"
|
||||
: "${NUM_THREADS:=16}"
|
||||
: "${THREAD_BATCH:=32}"
|
||||
: "${CACHE_TYPE_K:=f16}"
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
: "${FIT:=off}"
|
||||
: "${TEMPERATURE:=0.7}"
|
||||
: "${TOP_P:=0.8}"
|
||||
: "${TOP_K:=20}"
|
||||
: "${MIN_P:=0.0}"
|
||||
: "${REASONING:=on}"
|
||||
: "${REASONING_FORMAT:=deepseek}"
|
||||
|
||||
MODEL="$MODEL_DIR/$MODEL_FILE"
|
||||
HOSTNAME=$(hostname 2>/dev/null || echo "dankclaw")
|
||||
|
||||
# Prompt files (generated by generate-prompts.py or passed via args)
|
||||
PROMPT_CODE="${1:-}"
|
||||
PROMPT_PROSE="${2:-}"
|
||||
|
||||
if [[ -z "$PROMPT_CODE" || -z "$PROMPT_PROSE" ]]; then
|
||||
echo "ERROR: Need two prompt file paths."
|
||||
echo "Usage: $0 <prompt_code.txt> <prompt_prose.txt>"
|
||||
echo "Generate with: python3 scripts/generate-prompts.py"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -r "$MODEL" ]]; then
|
||||
echo "missing or unreadable model: $MODEL" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SERVER_DIR="$(cd -- "$(dirname -- "$LLAMA_BIN")" && pwd)"
|
||||
ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
|
||||
LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
||||
|
||||
# Scratch ports
|
||||
PORT_NGRAM=7771
|
||||
PORT_BASELINE=7772
|
||||
|
||||
cleanup() {
|
||||
echo "Cleaning up servers..."
|
||||
# Kill any leftover llama-server on our ports
|
||||
for port in $PORT_NGRAM $PORT_BASELINE; do
|
||||
fuser -k "$port"/tcp 2>/dev/null || true
|
||||
done
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
wait_for_server() {
|
||||
local port=$1
|
||||
local max_wait=120
|
||||
local elapsed=0
|
||||
while (( elapsed < max_wait )); do
|
||||
if curl -s "http://127.0.0.1:$port/health" | grep -q '"running"'; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
(( elapsed++ ))
|
||||
done
|
||||
echo "ERROR: Server on port $port did not start within ${max_wait}s" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
get_prompt_tokens() {
|
||||
local file=$1
|
||||
# Count tokens approximately (1 token ~ 4 chars for English/code)
|
||||
wc -c < "$file" | awk '{printf "%d", $1/4}'
|
||||
}
|
||||
|
||||
run_single_test() {
|
||||
local port=$1
|
||||
local spec_type="$2"
|
||||
local prompt_file=$3
|
||||
local label=$4
|
||||
local ngram_n_max="${5:-24}"
|
||||
local ngram_n_min="${6:-4}"
|
||||
|
||||
echo "=== $label ==="
|
||||
echo " spec-type: $spec_type"
|
||||
echo " ngram-mod n-max: $ngram_n_max (if enabled)"
|
||||
echo " ngram-mod n-min: $ngram_n_min (if enabled)"
|
||||
echo " prompt: $prompt_file ($(wc -c < "$prompt_file") bytes)"
|
||||
|
||||
# Build command
|
||||
local cmd=(
|
||||
/usr/bin/env
|
||||
ROCM_PATH="$ROCM_PATH"
|
||||
HIP_VISIBLE_DEVICES=0
|
||||
LD_LIBRARY_PATH="$LIBRARY_PATH"
|
||||
HSA_OVERRIDE_GFX_VERSION="${HSA_OVERRIDE_GFX_VERSION:-11.5.1}"
|
||||
"$LLAMA_BIN"
|
||||
--model "$MODEL"
|
||||
--chat-template-file "$CHAT_TEMPLATE_FILE"
|
||||
--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS"
|
||||
--host 127.0.0.1
|
||||
--port "$port"
|
||||
--jinja
|
||||
--device ROCm0
|
||||
--gpu-layers 999
|
||||
--parallel 1
|
||||
--ctx-size "$CTX_SIZE"
|
||||
--batch-size "$BATCH_SIZE"
|
||||
--ubatch-size "$UBATCH_SIZE"
|
||||
--fit "$FIT"
|
||||
--flash-attn on
|
||||
--cache-type-k "$CACHE_TYPE_K"
|
||||
--cache-type-v "$CACHE_TYPE_V"
|
||||
--threads "$NUM_THREADS"
|
||||
--threads-batch "$THREAD_BATCH"
|
||||
--cache-prompt
|
||||
--cache-idle-slots
|
||||
--metrics
|
||||
--temp "$TEMPERATURE"
|
||||
--top-p "$TOP_P"
|
||||
--top-k "$TOP_K"
|
||||
--min-p "$MIN_P"
|
||||
--reasoning "$REASONING"
|
||||
--reasoning-format "$REASONING_FORMAT"
|
||||
--reasoning-budget -1
|
||||
)
|
||||
|
||||
# Add spec flags
|
||||
if [[ "$spec_type" == "ngram-mod" ]]; then
|
||||
cmd+=(
|
||||
--spec-type "ngram-mod"
|
||||
--spec-ngram-mod-n-max "$ngram_n_max"
|
||||
--spec-ngram-mod-n-min "$ngram_n_min"
|
||||
)
|
||||
else
|
||||
cmd+=(
|
||||
--spec-type "none"
|
||||
)
|
||||
fi
|
||||
|
||||
# Start server with stderr to log
|
||||
local log_file="/tmp/ngram-bench-${label//[^a-zA-Z0-9]/-}.log"
|
||||
echo " Starting server (log: $log_file)..."
|
||||
|
||||
"${cmd[@]}" 2> "$log_file" &
|
||||
local server_pid=$!
|
||||
|
||||
if ! wait_for_server "$port"; then
|
||||
echo "ERROR: Server failed to start" >&2
|
||||
kill "$server_pid" 2>/dev/null || true
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo " Server ready."
|
||||
|
||||
# Send prompt via API
|
||||
echo " Sending prompt..."
|
||||
local prompt_text
|
||||
prompt_text=$(cat "$prompt_file")
|
||||
|
||||
local response
|
||||
response=$(curl -s -X POST "http://127.0.0.1:$port/v1/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"model\": \"frank-large\",
|
||||
\"prompt\": \"$prompt_text\",
|
||||
\"max_tokens\": 128,
|
||||
\"temperature\": 0,
|
||||
\"stream\": false
|
||||
}")
|
||||
|
||||
# Extract metrics
|
||||
local prompt_tokens prompt_eval_duration_ms total_duration_ms
|
||||
prompt_tokens=$(echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['prompt_eval_count'])" 2>/dev/null || echo "0")
|
||||
prompt_eval_duration_ms=$(echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['prompt_eval_duration_ms'])" 2>/dev/null || echo "0")
|
||||
total_duration_ms=$(echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['total_duration_ms'])" 2>/dev/null || echo "0")
|
||||
|
||||
# Calculate throughput
|
||||
local prompt_tps=0
|
||||
if [[ "$prompt_eval_duration_ms" -gt 0 ]]; then
|
||||
prompt_tps=$(python3 -c "print(f'{($prompt_tokens / $prompt_eval_duration_ms) * 1000:.1f}')")
|
||||
fi
|
||||
|
||||
echo " Prompt tokens: $prompt_tokens"
|
||||
echo " Prompt eval: ${prompt_eval_duration_ms}ms"
|
||||
echo " Prompt TPS: $prompt_tps tok/s"
|
||||
echo " Total duration: ${total_duration_ms}ms"
|
||||
|
||||
# Kill server
|
||||
kill "$server_pid" 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
echo " Done."
|
||||
echo "---"
|
||||
|
||||
# Output structured results for parsing
|
||||
echo "RESULT|$label|$prompt_file|$prompt_tokens|$prompt_eval_duration_ms|$total_duration_ms|$prompt_tps"
|
||||
}
|
||||
|
||||
echo "╔══════════════════════════════════════════════════════════╗"
|
||||
echo "║ N-gram Mod Benchmark — frank-large ║"
|
||||
echo "║ STRIX ROCmFP4 · 64k context · two prompts ║"
|
||||
echo "╚══════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
echo "Host: $HOSTNAME"
|
||||
echo "Model: $MODEL"
|
||||
echo "Build: $(basename "$LLAMA_BIN")"
|
||||
echo "Context: $CTX_SIZE"
|
||||
echo ""
|
||||
|
||||
# Run baseline (no spec)
|
||||
echo "=== BASELINE (no speculative decoding) ==="
|
||||
echo ""
|
||||
|
||||
BASELINE_CODE_RESULT=$(run_single_test $PORT_BASELINE "none" "$PROMPT_CODE" "Baseline-Code" 24 4)
|
||||
BASELINE_PROSE_RESULT=$(run_single_test $PORT_NGRAM "none" "$PROMPT_PROSE" "Baseline-Prose" 24 4)
|
||||
|
||||
echo ""
|
||||
echo "=== NGRAM-MOD (speculative decoding enabled) ==="
|
||||
echo ""
|
||||
|
||||
# Run n-gram mod
|
||||
NGRAM_CODE_RESULT=$(run_single_test $PORT_NGRAM "ngram-mod" "$PROMPT_CODE" "Ngram-Code" 24 4)
|
||||
NGRAM_PROSE_RESULT=$(run_single_test $PORT_BASELINE "ngram-mod" "$PROMPT_PROSE" "Ngram-Prose" 24 4)
|
||||
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════════════════════════╗"
|
||||
echo "║ RESULTS SUMMARY ║"
|
||||
echo "╚══════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
|
||||
# Parse and display results
|
||||
python3 << 'PYEOF'
|
||||
import sys
|
||||
|
||||
results = []
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line.startswith("RESULT|"):
|
||||
continue
|
||||
parts = line.split("|")
|
||||
if len(parts) != 8:
|
||||
continue
|
||||
label, prompt_file, prompt_tokens, prompt_eval_ms, total_ms, tps = parts[1], parts[2], int(parts[3]), int(parts[4]), int(parts[5]), float(parts[6])
|
||||
results.append({
|
||||
'label': label,
|
||||
'prompt_file': prompt_file,
|
||||
'prompt_tokens': prompt_tokens,
|
||||
'prompt_eval_ms': prompt_eval_ms,
|
||||
'total_ms': total_ms,
|
||||
'tps': tps
|
||||
})
|
||||
|
||||
# Compare baseline vs ngram for each prompt type
|
||||
for code_result in results:
|
||||
if not code_result['label'].startswith('Ngram-'):
|
||||
continue
|
||||
prompt_type = code_result['label'].replace('Ngram-', '').replace('Ngram', 'Baseline').replace('-', '-')
|
||||
baseline = next((r for r in results if r['label'] == 'Baseline-' + prompt_type), None)
|
||||
if not baseline:
|
||||
continue
|
||||
|
||||
spec = code_result['label'] == 'Ngram-' + prompt_type
|
||||
|
||||
print(f"Prompt: {prompt_type}")
|
||||
print(f" Tokens: {code_result['prompt_tokens']}")
|
||||
if spec:
|
||||
print(f" Spec: ngram-mod (n_max=24, n_min=4)")
|
||||
print(f" Prompt: {code_result['prompt_eval_ms']}ms | {code_result['tps']:>10.1f} tok/s")
|
||||
baseline_tps = baseline['tps']
|
||||
if baseline_tps > 0:
|
||||
improvement = ((code_result['tps'] - baseline_tps) / baseline_tps) * 100
|
||||
print(f" Speedup: {improvement:+.1f}% vs baseline ({baseline_tps:.1f} tok/s)")
|
||||
else:
|
||||
print(f" Spec: none (baseline)")
|
||||
print(f" Prompt: {baseline['prompt_eval_ms']}ms | {baseline['tps']:>10.1f} tok/s")
|
||||
print()
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
echo "Raw data lines above (RESULT|... format) can be parsed for automation."
|
||||
echo ""
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Benchmark decode throughput across speculative decoding configs.
|
||||
|
||||
Sweeps 4 configs: none, ngram-mod, draft-mtp, draft-mtp+ngram-mod.
|
||||
Each config generates 16k tokens from a minimal prompt at temp 0.
|
||||
|
||||
Runs ON the target host (Dankclaw). Service management uses local sudo.
|
||||
Usage:
|
||||
python3 bench-spec-decode.py [--max-tokens N] [--host H] [--port P]
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
SERVICE = "frank-large"
|
||||
ENV_FILE = "/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ALIAS = "frank-large"
|
||||
DEFAULT_HOST = "localhost"
|
||||
DEFAULT_PORT = 7770
|
||||
DEFAULT_MAX_TOKENS = 16384
|
||||
HEALTH_TIMEOUT = 1200 # 20 min for cold load
|
||||
HEALTH_INTERVAL = 5
|
||||
|
||||
PROMPT = (
|
||||
"Write a comprehensive, detailed essay about the complete history of "
|
||||
"computing, from the earliest mechanical calculators through modern AI. "
|
||||
"Cover every major era in detail, including key figures, machines, and "
|
||||
"breakthroughs. Be as thorough as possible."
|
||||
)
|
||||
|
||||
CONFIGS = [
|
||||
("none", "none"),
|
||||
("ngram-mod", "ngram-mod"),
|
||||
("draft-mtp", "draft-mtp"),
|
||||
("draft-mtp+ngram-mod", "draft-mtp,ngram-mod"),
|
||||
]
|
||||
|
||||
|
||||
def local(cmd):
|
||||
"""Run a command locally (the script runs on the target host)."""
|
||||
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=120)
|
||||
if r.returncode != 0:
|
||||
print(f" WARNING: {cmd} -> exit {r.returncode}: {r.stderr.strip()}",
|
||||
file=sys.stderr)
|
||||
return r
|
||||
|
||||
|
||||
def sed_spec_type(value):
|
||||
local(f"sudo sed -i 's/^SPEC_TYPE=.*/SPEC_TYPE={value}/' {ENV_FILE}")
|
||||
|
||||
|
||||
def wait_health(host, port, timeout=HEALTH_TIMEOUT):
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
try:
|
||||
resp = urllib.request.urlopen(
|
||||
f"http://{host}:{port}/health", timeout=10
|
||||
)
|
||||
if resp.status == 200:
|
||||
return True
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, ConnectionError,
|
||||
OSError):
|
||||
pass
|
||||
time.sleep(HEALTH_INTERVAL)
|
||||
return False
|
||||
|
||||
|
||||
def benchmark(host, port, alias, max_tokens):
|
||||
body = json.dumps({
|
||||
"model": alias,
|
||||
"messages": [{"role": "user", "content": PROMPT}],
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0,
|
||||
"stream": False,
|
||||
}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"http://{host}:{port}/v1/chat/completions",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
t0 = time.time()
|
||||
resp = urllib.request.urlopen(req, timeout=1200)
|
||||
wall = time.time() - t0
|
||||
data = json.loads(resp.read())
|
||||
return data, wall
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--host", default=DEFAULT_HOST)
|
||||
ap.add_argument("--port", type=int, default=DEFAULT_PORT)
|
||||
ap.add_argument("--max-tokens", type=int, default=DEFAULT_MAX_TOKENS)
|
||||
args = ap.parse_args()
|
||||
|
||||
# Read original SPEC_TYPE from the env file
|
||||
r = local(f"grep '^SPEC_TYPE=' {ENV_FILE}")
|
||||
original_spec = r.stdout.strip()
|
||||
orig_val = (original_spec.split("=", 1)[1]
|
||||
if "=" in original_spec else "draft-mtp,ngram-mod")
|
||||
print(f"Original: {original_spec}")
|
||||
print(f"Max tokens: {args.max_tokens}")
|
||||
print(f"Configs: {[c[0] for c in CONFIGS]}")
|
||||
print()
|
||||
|
||||
# Stop + mask the production service
|
||||
print("Stopping + masking production service...")
|
||||
local(f"sudo systemctl stop {SERVICE}")
|
||||
local(f"sudo systemctl mask {SERVICE}")
|
||||
time.sleep(2)
|
||||
|
||||
results = []
|
||||
for name, spec_type in CONFIGS:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Config: {name} (SPEC_TYPE={spec_type})")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Set SPEC_TYPE
|
||||
sed_spec_type(spec_type)
|
||||
print(f"Set SPEC_TYPE={spec_type}")
|
||||
|
||||
# Start the service
|
||||
local(f"sudo systemctl unmask {SERVICE}")
|
||||
local(f"sudo systemctl start {SERVICE}")
|
||||
print("Starting service...")
|
||||
|
||||
# Wait for health
|
||||
print(f"Waiting for health (up to {HEALTH_TIMEOUT}s)...")
|
||||
if not wait_health(args.host, args.port):
|
||||
print(f"FAILED: service did not become healthy for {name}")
|
||||
results.append({
|
||||
"config": name, "spec_type": spec_type,
|
||||
"error": "health timeout",
|
||||
})
|
||||
local(f"sudo systemctl stop {SERVICE}")
|
||||
local(f"sudo systemctl mask {SERVICE}")
|
||||
continue
|
||||
|
||||
print("Healthy. Running benchmark...")
|
||||
|
||||
# Run benchmark
|
||||
try:
|
||||
data, wall = benchmark(
|
||||
args.host, args.port, ALIAS, args.max_tokens
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"FAILED: {e}")
|
||||
results.append({
|
||||
"config": name, "spec_type": spec_type,
|
||||
"error": str(e),
|
||||
})
|
||||
local(f"sudo systemctl stop {SERVICE}")
|
||||
local(f"sudo systemctl mask {SERVICE}")
|
||||
continue
|
||||
|
||||
timings = data.get("timings", {})
|
||||
usage = data.get("usage", {})
|
||||
pred_n = timings.get("predicted_n", usage.get("completion_tokens", 0))
|
||||
pred_ms = timings.get("predicted_ms", 0)
|
||||
pred_ps = timings.get("predicted_per_second", 0)
|
||||
prompt_n = timings.get("prompt_n", usage.get("prompt_tokens", 0))
|
||||
prompt_ms = timings.get("prompt_ms", 0)
|
||||
|
||||
print(f" Generated: {pred_n} tokens")
|
||||
print(f" Wall time: {wall:.1f}s")
|
||||
print(f" Decode: {pred_ps:.2f} tok/s")
|
||||
print(f" Prefill: {prompt_n} tokens in {prompt_ms:.0f}ms"
|
||||
f" ({prompt_n / max(prompt_ms, 1) * 1000:.1f} tok/s)")
|
||||
|
||||
results.append({
|
||||
"config": name,
|
||||
"spec_type": spec_type,
|
||||
"predicted_tokens": pred_n,
|
||||
"predicted_ms": pred_ms,
|
||||
"predicted_per_second": pred_ps,
|
||||
"prompt_tokens": prompt_n,
|
||||
"prompt_ms": prompt_ms,
|
||||
"wall_time": wall,
|
||||
})
|
||||
|
||||
# Stop + mask between configs
|
||||
local(f"sudo systemctl stop {SERVICE}")
|
||||
local(f"sudo systemctl mask {SERVICE}")
|
||||
time.sleep(2)
|
||||
|
||||
# Restore original SPEC_TYPE
|
||||
sed_spec_type(orig_val)
|
||||
print(f"\nRestoring SPEC_TYPE={orig_val}")
|
||||
print("Unmasking + starting production service...")
|
||||
local(f"sudo systemctl unmask {SERVICE}")
|
||||
local(f"sudo systemctl start {SERVICE}")
|
||||
|
||||
# Health check
|
||||
if wait_health(args.host, args.port):
|
||||
print("Production service healthy.")
|
||||
else:
|
||||
print("WARNING: production service not healthy after restore!")
|
||||
|
||||
# Print results table
|
||||
print(f"\n{'='*70}")
|
||||
print("RESULTS")
|
||||
print(f"{'='*70}")
|
||||
print(f"{'Config':<30} {'Tokens':>8} {'Wall(s)':>10} {'Decode tok/s':>14} "
|
||||
f"{'Error':>20}")
|
||||
print(f"{'-'*30} {'-'*8} {'-'*10} {'-'*14} {'-'*20}")
|
||||
for r in results:
|
||||
if "error" in r:
|
||||
print(f"{r['config']:<30} {'':>8} {'':>10} {'':>14} "
|
||||
f"{r['error']:>20}")
|
||||
else:
|
||||
print(f"{r['config']:<30} {r['predicted_tokens']:>8} "
|
||||
f"{r['wall_time']:>10.1f} {r['predicted_per_second']:>14.2f} "
|
||||
f"{'':>20}")
|
||||
print(f"{'='*70}")
|
||||
|
||||
# Save JSON results
|
||||
ts = time.strftime("%Y%m%d-%H%M%S")
|
||||
out = f"/tmp/bench-spec-decode-{ts}.json"
|
||||
with open(out, "w") as f:
|
||||
json.dump({"results": results, "max_tokens": args.max_tokens,
|
||||
"timestamp": ts}, f, indent=2)
|
||||
print(f"Results saved: {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,453 @@
|
||||
{%- set template_version = "qwen3.8-froggeric-v22.4.0" %}
|
||||
{%- set _tool_format = tool_call_format if tool_call_format is defined else 'xml' %}
|
||||
{%- set image_count = namespace(value=0) %}
|
||||
{%- set video_count = namespace(value=0) %}
|
||||
{%- set add_vision_id = add_vision_id if add_vision_id is defined else false %}
|
||||
{%- set enable_thinking = enable_thinking if enable_thinking is defined else true %}
|
||||
{%- set auto_disable_thinking_with_tools = auto_disable_thinking_with_tools if auto_disable_thinking_with_tools is defined else false %}
|
||||
{%- if preserve_reasoning is defined and preserve_reasoning is not none %}
|
||||
{%- set _preserve_thinking = preserve_reasoning %}
|
||||
{%- elif preserve_thinking is defined and preserve_thinking is not none %}
|
||||
{%- set _preserve_thinking = preserve_thinking %}
|
||||
{%- else %}
|
||||
{%- set _preserve_thinking = true %}
|
||||
{%- endif %}
|
||||
{%- set max_tool_arg_chars = max_tool_arg_chars if max_tool_arg_chars is defined else 0 %}
|
||||
{%- set max_tool_response_chars = max_tool_response_chars if max_tool_response_chars is defined else 0 %}
|
||||
{%- set _default_reasoning_effort = 'medium' %}
|
||||
{%- set _has_tools = (tools is defined and tools and tools is iterable and tools is not mapping) %}
|
||||
{%- set _effort_raw = (reasoning_effort | string | lower) if reasoning_effort is defined and reasoning_effort is not none else _default_reasoning_effort %}
|
||||
{%- set _initial_thinking = enable_thinking %}
|
||||
{%- if _effort_raw in ('none', 'off') %}
|
||||
{%- set _initial_thinking = false %}
|
||||
{%- set _initial_effort = 'medium' %}
|
||||
{%- elif _effort_raw in ('minimal', 'low') %}
|
||||
{%- set _initial_effort = 'low' %}
|
||||
{%- elif _effort_raw in ('high', 'xhigh', 'max', 'ultracode', 'extreme') %}
|
||||
{%- set _initial_effort = 'xhigh' %}
|
||||
{%- else %}
|
||||
{%- set _initial_effort = 'medium' %}
|
||||
{%- endif %}
|
||||
{%- set ns_state = namespace(thinking=_initial_thinking, effort=_initial_effort) %}
|
||||
{%- if auto_disable_thinking_with_tools and _has_tools %}
|
||||
{%- set ns_state.thinking = false %}
|
||||
{%- endif %}
|
||||
{%- for msg in messages %}
|
||||
{%- if msg.role == 'system' or msg.role == 'developer' or msg.role == 'user' %}
|
||||
{%- if msg.content is string %}
|
||||
{%- if '<|think_off|>' in msg.content %}
|
||||
{%- set ns_state.thinking = false %}
|
||||
{%- elif '<|think_on|>' in msg.content %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- elif '<|think_xhigh|>' in msg.content or '<|think_high|>' in msg.content or '<|think_ultracode|>' in msg.content or '<|think_extreme|>' in msg.content or '<|think_max|>' in msg.content %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'xhigh' %}
|
||||
{%- elif '<|think_low|>' in msg.content or '<|think_minimal|>' in msg.content %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'low' %}
|
||||
{%- elif '<|think_medium|>' in msg.content %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'medium' %}
|
||||
{%- endif %}
|
||||
{%- elif msg.content is iterable and msg.content is not mapping %}
|
||||
{%- for item in msg.content %}
|
||||
{%- if item is string %}
|
||||
{%- set _item_text = item %}
|
||||
{%- elif item is mapping and 'text' in item and item.text is string %}
|
||||
{%- set _item_text = item.text %}
|
||||
{%- else %}
|
||||
{%- set _item_text = '' %}
|
||||
{%- endif %}
|
||||
{%- if _item_text %}
|
||||
{%- if '<|think_off|>' in _item_text %}
|
||||
{%- set ns_state.thinking = false %}
|
||||
{%- elif '<|think_on|>' in _item_text %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- elif '<|think_xhigh|>' in _item_text or '<|think_high|>' in _item_text or '<|think_ultracode|>' in _item_text or '<|think_extreme|>' in _item_text or '<|think_max|>' in _item_text %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'xhigh' %}
|
||||
{%- elif '<|think_low|>' in _item_text or '<|think_minimal|>' in _item_text %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'low' %}
|
||||
{%- elif '<|think_medium|>' in _item_text %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'medium' %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- set reasoning_instructions = '' %}
|
||||
{%- if ns_state.thinking %}
|
||||
{%- if ns_state.effort == 'xhigh' %}
|
||||
{%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}
|
||||
{%- elif ns_state.effort == 'low' %}
|
||||
{%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
|
||||
{%- if content is string %}
|
||||
{{- content }}
|
||||
{%- elif content is iterable and content is not mapping %}
|
||||
{%- for item in content %}
|
||||
{%- if item is mapping %}
|
||||
{%- if item.type == 'image' or 'image' in item or 'image_url' in item %}
|
||||
{%- if is_system_content %}
|
||||
{{- raise_exception('System message cannot contain images.') }}
|
||||
{%- endif %}
|
||||
{%- if do_vision_count %}
|
||||
{%- set image_count.value = image_count.value + 1 %}
|
||||
{%- endif %}
|
||||
{%- if add_vision_id %}
|
||||
{{- 'Picture ' ~ image_count.value ~ ': ' }}
|
||||
{%- endif %}
|
||||
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
|
||||
{%- elif item.type == 'video' or 'video' in item %}
|
||||
{%- if is_system_content %}
|
||||
{{- raise_exception('System message cannot contain videos.') }}
|
||||
{%- endif %}
|
||||
{%- if do_vision_count %}
|
||||
{%- set video_count.value = video_count.value + 1 %}
|
||||
{%- endif %}
|
||||
{%- if add_vision_id %}
|
||||
{{- 'Video ' ~ video_count.value ~ ': ' }}
|
||||
{%- endif %}
|
||||
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
|
||||
{%- elif 'text' in item %}
|
||||
{{- item.text }}
|
||||
{%- else %}
|
||||
{{- raise_exception('Unexpected item type in content.') }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- item | string }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- elif content is none or content is undefined %}
|
||||
{{- '' }}
|
||||
{%- else %}
|
||||
{{- raise_exception('Unexpected content type.') }}
|
||||
{%- endif %}
|
||||
{%- endmacro %}
|
||||
{%- if not messages %}
|
||||
{{- raise_exception('No messages provided.') }}
|
||||
{%- endif %}
|
||||
{%- set head = namespace(count=0, seen_non_system=false) %}
|
||||
{%- for message in messages %}
|
||||
{%- set _is_sys = (message.role == 'system' or message.role == 'developer') %}
|
||||
{%- if _is_sys and not head.seen_non_system %}
|
||||
{%- set head.count = head.count + 1 %}
|
||||
{%- else %}
|
||||
{%- set head.seen_non_system = true %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- set sys_state = namespace(content='') %}
|
||||
{%- for message in messages[:head.count] %}
|
||||
{%- set _part = render_content(message.content, false, true) | trim %}
|
||||
{%- if '<|think_off|>' in _part %}{%- set _part = _part.split('<|think_off|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_on|>' in _part %}{%- set _part = _part.split('<|think_on|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_xhigh|>' in _part %}{%- set _part = _part.split('<|think_xhigh|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_high|>' in _part %}{%- set _part = _part.split('<|think_high|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_ultracode|>' in _part %}{%- set _part = _part.split('<|think_ultracode|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_extreme|>' in _part %}{%- set _part = _part.split('<|think_extreme|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_max|>' in _part %}{%- set _part = _part.split('<|think_max|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_medium|>' in _part %}{%- set _part = _part.split('<|think_medium|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_low|>' in _part %}{%- set _part = _part.split('<|think_low|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_minimal|>' in _part %}{%- set _part = _part.split('<|think_minimal|>') | join('') | trim %}{%- endif %}
|
||||
{%- if _part %}
|
||||
{%- if sys_state.content %}
|
||||
{%- set sys_state.content = sys_state.content ~ '\n\n' ~ _part %}
|
||||
{%- else %}
|
||||
{%- set sys_state.content = _part %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- set _sc = sys_state.content %}
|
||||
{%- set _msgs = messages[head.count:] %}
|
||||
{%- if ns_state.thinking %}
|
||||
{%- set _terse_lead = 'Answer directly, after thinking. Lead with the answer, then only what it needs to be correct and usable.' %}
|
||||
{%- else %}
|
||||
{%- set _terse_lead = 'Answer directly and concisely. Give the answer with only what it needs to be correct and usable.' %}
|
||||
{%- endif %}
|
||||
{%- set _terse_core %}
|
||||
Never: open with preamble or pleasantries; restate the question; add filler transitions; hedge with niceties; or repeat a point you've already made.
|
||||
Always: keep essential steps, caveats, uncertainties, and specifics — never drop correctness or a needed warning for brevity. Keep the final answer lean. Use the least structure that conveys it (plain prose when short; lists or code only when they earn their place). If genuinely uncertain, say so and explain why — never omit uncertainty for the sake of brevity.
|
||||
If a user request is genuinely ambiguous, ask a sharp question, don't guess.
|
||||
{%- endset %}
|
||||
{%- set _terse = _terse_lead ~ '\n' ~ (_terse_core | trim) %}
|
||||
{#- `terse` chat-template kwarg: default true, so behaviour is unchanged unless a caller opts out.
|
||||
Pass {"terse": false} via chat_template_kwargs to serve the model with only its own system
|
||||
prompt -- useful for A/B-ing the terseness effect, or when a downstream prompt conflicts. #}
|
||||
{%- set _terse_on = terse if terse is defined else true %}
|
||||
{%- if _terse_on %}
|
||||
{%- if not _sc %}
|
||||
{%- set _sc = _terse | trim %}
|
||||
{%- else %}
|
||||
{%- set _sc = (_sc | trim) ~ '\n\n' ~ (_terse | trim) %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if _has_tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if reasoning_instructions %}
|
||||
{{- reasoning_instructions + '\n\n' }}
|
||||
{%- endif %}
|
||||
{{- '# Tools\n\nYou have access to the following functions:\n\n<tools>' }}
|
||||
{%- for tool in tools %}
|
||||
{{- '\n' }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- '\n</tools>' }}
|
||||
{%- if _tool_format == 'json' %}
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }}{%- if ns_state.thinking %}{{- '<think>\nBrief explanation of tool call\n</think>\n' }}{%- endif %}{{- '<tool_call>\n{"name": "example_function_name", "arguments": {"example_parameter_1": "value_1", "example_parameter_2": "This is the value for the second parameter"}}\n</tool_call>\n\n<IMPORTANT>\nReminder:\n' }}{%- if ns_state.thinking %}{{- '- You can use the <think></think> block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the <think></think> block.\n' }}{%- endif %}{{- '- Function calls MUST follow the specified format: a single JSON object with "name" and "arguments" keys inside <tool_call></tool_call> XML tags.\n- If you choose to call a tool, you MUST output the <tool_call> block IMMEDIATELY' }}{%- if ns_state.thinking %}{{- ' after thinking' }}{%- endif %}{{- ', with NO conversational text before it.\n- The <tool_call> tag MUST be at the very beginning of a new line, with NO spaces or indentation before it.\n- To call multiple functions, output a separate, completely closed <tool_call></tool_call> block for EACH function. Do NOT nest <tool_call> blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n</IMPORTANT>' }}
|
||||
{%- else %}
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }}{%- if ns_state.thinking %}{{- '<think>\nBrief explanation of tool call\n</think>\n' }}{%- endif %}{{- '<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n' }}{%- if ns_state.thinking %}{{- '- You can use the <think></think> block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the <think></think> block.\n' }}{%- endif %}{{- '- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags.\n- If you choose to call a tool, you MUST output the <tool_call> block IMMEDIATELY' }}{%- if ns_state.thinking %}{{- ' after thinking' }}{%- endif %}{{- ', with NO conversational text before it.\n- The <tool_call> and <function> tags MUST be at the very beginning of a new line, with NO spaces or indentation before them.\n- To call multiple functions, output a separate, completely closed <tool_call></tool_call> block for EACH function. Do NOT nest <tool_call> blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n</IMPORTANT>' }}
|
||||
{%- endif %}
|
||||
{%- if _sc %}
|
||||
{{- '\n\n' + _sc }}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if _sc %}
|
||||
{{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + _sc + '<|im_end|>\n' }}
|
||||
{%- elif reasoning_instructions %}
|
||||
{{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set _last_idx = _msgs | length - 1 %}
|
||||
{%- set ns = namespace(multi_step_tool=true, last_query_index=_last_idx) %}
|
||||
{%- for message in _msgs[::-1] %}
|
||||
{%- set index = (_msgs | length - 1) - loop.index0 %}
|
||||
{%- if ns.multi_step_tool and message.role == 'user' %}
|
||||
{%- set _rc = render_content(message.content, false) | trim %}
|
||||
{%- if not (_rc.startswith('<tool_response>') and _rc.endswith('</tool_response>')) %}
|
||||
{%- set ns.multi_step_tool = false %}
|
||||
{%- set ns.last_query_index = index %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if ns.multi_step_tool %}
|
||||
{%- if _last_idx > 50 %}
|
||||
{%- set ns.last_query_index = _last_idx %}
|
||||
{%- else %}
|
||||
{%- set ns.last_query_index = 0 %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set ns2 = namespace(prev_role='', consecutive_failures=0) %}
|
||||
{%- for message in _msgs %}
|
||||
{%- set is_system = (message.role == "system" or message.role == "developer") %}
|
||||
{%- set content = render_content(message.content, true, is_system) | trim %}
|
||||
{%- if is_system or message.role == 'user' %}
|
||||
{%- if '<|think_off|>' in content %}{%- set content = content.split('<|think_off|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_on|>' in content %}{%- set content = content.split('<|think_on|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_xhigh|>' in content %}{%- set content = content.split('<|think_xhigh|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_high|>' in content %}{%- set content = content.split('<|think_high|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_ultracode|>' in content %}{%- set content = content.split('<|think_ultracode|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_extreme|>' in content %}{%- set content = content.split('<|think_extreme|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_max|>' in content %}{%- set content = content.split('<|think_max|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_medium|>' in content %}{%- set content = content.split('<|think_medium|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_low|>' in content %}{%- set content = content.split('<|think_low|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_minimal|>' in content %}{%- set content = content.split('<|think_minimal|>') | join('') | trim %}{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if is_system %}
|
||||
{{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
|
||||
{%- elif message.role == 'user' %}
|
||||
{%- set ns2.consecutive_failures = 0 %}
|
||||
{{- '<|im_start|>user\n' + content + '<|im_end|>\n' }}
|
||||
{%- elif message.role == 'assistant' %}
|
||||
{%- set reasoning_content = '' %}
|
||||
{%- set _explicit_reasoning = '' %}
|
||||
{%- if message.reasoning_content is defined and message.reasoning_content is not none %}
|
||||
{%- if message.reasoning_content is string %}
|
||||
{%- set _explicit_reasoning = message.reasoning_content %}
|
||||
{%- else %}
|
||||
{%- set _explicit_reasoning = message.reasoning_content | string %}
|
||||
{%- endif %}
|
||||
{%- elif message.thinking is defined and message.thinking is not none %}
|
||||
{%- if message.thinking is string %}
|
||||
{%- set _explicit_reasoning = message.thinking %}
|
||||
{%- else %}
|
||||
{%- set _explicit_reasoning = message.thinking | string %}
|
||||
{%- endif %}
|
||||
{%- elif message.reasoning is defined and message.reasoning is not none %}
|
||||
{%- if message.reasoning is string %}
|
||||
{%- set _explicit_reasoning = message.reasoning %}
|
||||
{%- else %}
|
||||
{%- set _explicit_reasoning = message.reasoning | string %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if _explicit_reasoning %}
|
||||
{%- set _lead_end = '' %}
|
||||
{%- if content.startswith('<think>') and '</think>' in content %}
|
||||
{%- set _lead_end = '</think>' %}
|
||||
{%- elif content.startswith('<thinking>') and '</thinking>' in content %}
|
||||
{%- set _lead_end = '</thinking>' %}
|
||||
{%- elif content.startswith('</think>') %}
|
||||
{%- set _lead_end = '</think>' %}
|
||||
{%- elif content.startswith('</thinking>') %}
|
||||
{%- set _lead_end = '</thinking>' %}
|
||||
{%- endif %}
|
||||
{%- if _lead_end %}
|
||||
{%- set content = content.split(_lead_end)[-1].lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- set reasoning_content = _explicit_reasoning %}
|
||||
{%- else %}
|
||||
{%- set _think_end = '' %}
|
||||
{%- if content.startswith('</think>') %}
|
||||
{%- set _think_end = '</think>' %}
|
||||
{%- elif content.startswith('</thinking>') %}
|
||||
{%- set _think_end = '</thinking>' %}
|
||||
{%- elif '\n</think>' in content %}
|
||||
{%- set _think_end = '\n</think>' %}
|
||||
{%- elif '\n</thinking>' in content %}
|
||||
{%- set _think_end = '\n</thinking>' %}
|
||||
{%- elif '\n</ think>' in content %}
|
||||
{%- set _think_end = '\n</ think>' %}
|
||||
{%- elif '\n</think >' in content %}
|
||||
{%- set _think_end = '\n</think >' %}
|
||||
{%- elif content.startswith('<think>') and '</think>' in content %}
|
||||
{%- set _think_end = '</think>' %}
|
||||
{%- elif content.startswith('<thinking>') and '</thinking>' in content %}
|
||||
{%- set _think_end = '</thinking>' %}
|
||||
{%- endif %}
|
||||
{%- if _think_end %}
|
||||
{%- if 'thinking' in _think_end %}
|
||||
{%- set _think_start = '<thinking>' %}
|
||||
{%- else %}
|
||||
{%- set _think_start = '<think>' %}
|
||||
{%- endif %}
|
||||
{%- set reasoning_content = content.split(_think_end)[0].rstrip('\n') %}
|
||||
{%- if _think_start in reasoning_content %}
|
||||
{%- set reasoning_content = reasoning_content.split(_think_start)[-1].lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- set content = content.split(_think_end)[-1].lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set reasoning_content = reasoning_content | trim %}
|
||||
{%- if (_preserve_thinking or loop.index0 > ns.last_query_index) %}
|
||||
{{- '<|im_start|>assistant\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n' + content }}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls is defined and message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined and tool_call.function is not none %}
|
||||
{%- set tc = tool_call.function %}
|
||||
{%- else %}
|
||||
{%- set tc = tool_call %}
|
||||
{%- endif %}
|
||||
{%- set tc_name = tc.name if (tc.name is defined and tc.name is not none) else '' %}
|
||||
{%- if _tool_format == 'json' %}
|
||||
{%- if loop.first %}
|
||||
{%- if content | trim %}
|
||||
{{- '\n\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '\n' }}
|
||||
{%- endif %}
|
||||
{%- set _args = '{}' %}
|
||||
{%- if tc.arguments is defined and tc.arguments is not none %}
|
||||
{%- if tc.arguments is mapping %}
|
||||
{%- set _args = tc.arguments | tojson %}
|
||||
{%- elif tc.arguments is string %}
|
||||
{%- if tc.arguments %}
|
||||
{%- set _args = tc.arguments %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{%- set _args = tc.arguments | tojson %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n{"name": ' }}{{- tc_name | tojson }}{{- ', "arguments": ' }}{{- _args }}{{- '}\n</tool_call>' }}
|
||||
{%- else %}
|
||||
{%- if loop.first %}
|
||||
{%- if content | trim %}
|
||||
{{- '\n\n<tool_call>\n<function=' + tc_name + '>\n' }}
|
||||
{%- else %}
|
||||
{{- '<tool_call>\n<function=' + tc_name + '>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '\n<tool_call>\n<function=' + tc_name + '>\n' }}
|
||||
{%- endif %}
|
||||
{%- if tc.arguments is defined and tc.arguments is not none %}
|
||||
{%- if tc.arguments is mapping %}
|
||||
{%- for args_name, args_value in tc.arguments.items() %}
|
||||
{{- '<parameter=' + args_name + '>\n' }}
|
||||
{%- if args_value is string %}
|
||||
{%- set _av = args_value %}
|
||||
{%- else %}
|
||||
{%- set _av = args_value | tojson %}
|
||||
{%- endif %}
|
||||
{%- if max_tool_arg_chars > 0 and _av | length > max_tool_arg_chars %}
|
||||
{{- _av[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_av | length | string) ~ ' chars]' }}
|
||||
{%- else %}
|
||||
{{- _av }}
|
||||
{%- endif %}
|
||||
{{- '\n</parameter>\n' }}
|
||||
{%- endfor %}
|
||||
{%- else %}
|
||||
{%- if tc.arguments is string %}
|
||||
{%- set _raw_args = tc.arguments %}
|
||||
{%- else %}
|
||||
{%- set _raw_args = tc.arguments | tojson %}
|
||||
{%- endif %}
|
||||
{%- if _raw_args %}
|
||||
{%- if max_tool_arg_chars > 0 and _raw_args | length > max_tool_arg_chars %}
|
||||
{{- _raw_args[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_raw_args | length | string) ~ ' chars]' }}
|
||||
{%- else %}
|
||||
{{- _raw_args }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == 'tool' %}
|
||||
{%- set _content_lower = content | lower %}
|
||||
{%- set _content_head = _content_lower[:120] %}
|
||||
{%- set _is_code_or_grep = ('throw new ' in _content_lower or 'throw error' in _content_lower or 'console.error' in _content_lower or 'logger.error' in _content_lower or 'logging.error' in _content_lower or 'import ' in _content_head or 'def ' in _content_head or 'function ' in _content_head) %}
|
||||
{%- set _exit_code_zero = ('exit code: 0' in _content_head or 'process exited with code 0' in _content_head) %}
|
||||
{%- set _error_field_ok = ('"error": null' in _content_head or '"error":null' in _content_head or '"error": false' in _content_head or '"error":false' in _content_head or '"error": ""' in _content_head or '"error":""' in _content_head) %}
|
||||
{%- set _strong_error = (('"error":' in _content_head and not _error_field_ok) or '"status": "error"' in _content_head or '"status":"error"' in _content_head or 'traceback (most recent call last):' in _content_head or 'command not found' in _content_head or 'invalid syntax' in _content_head or 'fatal:' in _content_head or (('exit code: ' in _content_head or 'process exited with code' in _content_head) and not _exit_code_zero) or _content_head.startswith('exception:') or _content_head.startswith('failed to ')) %}
|
||||
{%- set _weak_error = ('error:' in _content_head or 'err!' in _content_head) %}
|
||||
{%- set _weak_suppressed = ('$ ' in _content_head or 'took ' in _content_head or content | length >= 600) %}
|
||||
{%- if not _is_code_or_grep and (_strong_error or (_weak_error and not _weak_suppressed)) %}
|
||||
{%- set ns2.consecutive_failures = ns2.consecutive_failures + 1 %}
|
||||
{%- else %}
|
||||
{%- set ns2.consecutive_failures = 0 %}
|
||||
{%- endif %}
|
||||
{%- if ns2.prev_role != 'tool' %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{%- if _tool_format != 'json' and max_tool_response_chars > 0 and content | length > max_tool_response_chars %}
|
||||
{%- set content = content[:max_tool_response_chars] + '\n[TRUNCATED - original length ' ~ (content | length | string) ~ ' chars]' %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' + content }}
|
||||
{%- if ns2.consecutive_failures >= 2 %}
|
||||
{{- '\n\n⚠️ SYSTEM WARNING: ' ~ ns2.consecutive_failures ~ ' consecutive tool errors detected. Your previous approach is incorrect. You MUST use a fundamentally different approach or corrected arguments.' }}
|
||||
{%- elif ns2.consecutive_failures == 1 %}
|
||||
{{- '\n\n⚠️ SYSTEM WARNING: The previous tool call returned an error. Diagnose the failure and retry with completely corrected arguments.' }}
|
||||
{%- endif %}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- set _next_role = _msgs[loop.index0 + 1].role %}
|
||||
{%- if _next_role != 'tool' %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>user\n[' + message.role + ']: ' + content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- set ns2.prev_role = message.role %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- if not ns_state.thinking %}
|
||||
{{- '<think>\n\n</think>\n\n' }}
|
||||
{%- else %}
|
||||
{{- '<think>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
@@ -0,0 +1,105 @@
|
||||
# Tunables for frank-large (mradermacher i1-Q4_K_M of
|
||||
# orcarouter/Qwen3.8-Flash-Next-Uncensored on ik_llama.cpp, Dankclaw,
|
||||
# RTX 3090 CUDA + Strix Halo CPU path, since 2026-09-15).
|
||||
#
|
||||
# Runtime: ik_llama.cpp master d85a1d7 (2026-09-15) with full qwen4exp
|
||||
# support (upstream PRs #2365/#2369/#2373 merged). Build recipe (CUDA for
|
||||
# the 3090 at sm_86, Zen5 AVX-512 IQK kernels for the CPU expert path):
|
||||
# cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON \
|
||||
# -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86 \
|
||||
# -DGGML_AVX512=ON -DGGML_AVX512_VBMI=ON -DGGML_AVX512_VNNI=ON \
|
||||
# -DGGML_AVX512_BF16=ON -DLLAMA_CURL=OFF
|
||||
# The startup banner must print HAVE_FANCY_SIMD is defined — without those
|
||||
# macros the quantized GEMM silently falls back to AVX2 (no warning).
|
||||
#
|
||||
# Model: mradermacher imatrix quant of the uncensored Flash-Next (qwen4exp
|
||||
# MoE: 512 experts / 10 used, 48 layers = 12 attention + 36 GDN/ssm, QSA
|
||||
# indexer, hyper-connections, PLE n-gram table ~320M rows). 111 GiB shipped
|
||||
# as 3 parts, concatenated on disk; each part's sha256 was verified against
|
||||
# its HF LFS oid at download time.
|
||||
#
|
||||
# Placement (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE):
|
||||
# - -ot "exps=CPU": ALL 512-expert tensors (121B elems, ~84 GB) live in
|
||||
# the Strix Halo's LPDDR5X via the plain CPU backend buffer — mmap-backed
|
||||
# page cache, ik's fastest known configuration for this shape (the
|
||||
# author's 2x3090 + Threadripper reference box measures ~50 t/s on the
|
||||
# same model family). The shared experts (shexp) keep their GPU-side
|
||||
# default: they run on every token.
|
||||
# - -ot "per_layer_token_embd=CPU": the ~32 GB PLE n-gram table follows —
|
||||
# only a tiny slice is touched per token, so it streams from page cache.
|
||||
# - NEVER use --cpu-moe on this host: it routes the CPU-overridden
|
||||
# weights through the CUDA_Host PINNED buffer (measured 2026-09-15:
|
||||
# 108.12 GiB page-locked, evicted every co-tenant into swap and starved
|
||||
# sshd into unreachability until a power cycle). -ot ...=CPU is the
|
||||
# mmap-backed path with identical DDR5 bandwidth for resident pages,
|
||||
# and the kernel degrades gracefully under pressure (page cache evict
|
||||
# + re-read) instead of stranding anyone.
|
||||
# - GGML_CUDA_NO_PINNED_WEIGHTS=1 (src/llama-load-tensors.cpp) keeps the
|
||||
# CPU-overridden weights mmapped and leaves only the small staging
|
||||
# buffers pinned. (GGML_CUDA_NO_PINNED would also unpin staging —
|
||||
# unnecessary here and slower for prefill transfers.)
|
||||
#
|
||||
# Sampling: greedy-by-default to preserve the halogen-era client contract
|
||||
# (a request without temperature decodes greedy; temp 0.0 = argmax).
|
||||
# Clients that send their own temperature get their own values.
|
||||
#
|
||||
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (see
|
||||
# the 2026-08-13 HIP incident in AGENTS.md). llama.cpp reads --host and
|
||||
# --temp as CLI flags, never as environment variables.
|
||||
|
||||
# ik_llama.cpp runtime binary (master d85a1d7 build).
|
||||
LLAMA_BIN=/home/dank/GitPackages/ik_llama.cpp/build/bin/llama-server
|
||||
|
||||
# Model package from
|
||||
# https://huggingface.co/mradermacher/Qwen3.8-Flash-Next-Uncensored-i1-GGUF
|
||||
# (parts concatenated). The mmproj is the matching vision tower from the
|
||||
# static repo mradermacher/Qwen3.8-Flash-Next-Uncensored-GGUF — ik_llama.cpp
|
||||
# links examples/mtmd, so image input survives the runtime swap.
|
||||
MODEL_DIR=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M
|
||||
MODEL_FILE=Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf
|
||||
MMPROJ_PATH=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M/Qwen3.8-Flash-Next-Uncensored.mmproj-f16.gguf
|
||||
|
||||
# Qwen-Sharp chat template (tracked in the repo, staged on the host). The
|
||||
# server emits reasoning in DeepSeek format so clients receive
|
||||
# reasoning_content separately instead of literal think tags in content.
|
||||
CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
|
||||
|
||||
# Alias + network.
|
||||
ALIAS=frank-large
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7770
|
||||
|
||||
# Serving: single 262144-token slot (the model's full context window),
|
||||
# batch 4096 / ubatch 512, 16+16 threads (the Halo CPU has 16 cores, no
|
||||
# SMT), f16 KV. The KV cache scales with the 12 full-attention layers only
|
||||
# (~6 GB f16 at 262k); the GDN/ssm layers keep constant-size state. If
|
||||
# VRAM headroom is needed, --cache-type-k/v q8_0 halves the KV footprint.
|
||||
CTX_SIZE=262144
|
||||
PARALLEL=1
|
||||
BATCH_SIZE=4096
|
||||
UBATCH_SIZE=512
|
||||
NUM_THREADS=16
|
||||
THREAD_BATCH=16
|
||||
CACHE_TYPE_K=f16
|
||||
CACHE_TYPE_V=f16
|
||||
|
||||
# GPU/MoE placement: everything on the 3090 except the experts and the PLE
|
||||
# table, which both live in host RAM via the CPU backend buffer (see the
|
||||
# placement notes above).
|
||||
NGL=999
|
||||
# ik_llama.cpp CUDA host-memory policy: CPU-overridden weights stay mmapped;
|
||||
# only the small staging buffers are pinned (see the placement notes above).
|
||||
GGML_CUDA_NO_PINNED_WEIGHTS=1
|
||||
|
||||
# Prompt cache (RAM budget in MiB) and slot state dir.
|
||||
CACHE_RAM=8192
|
||||
SLOT_SAVE_PATH=/home/dank/serverAppConfigs/scripts/slot-state
|
||||
|
||||
# Greedy-by-default (halogen contract). See sampling note above.
|
||||
TEMPERATURE=0.0
|
||||
|
||||
# Reasoning on: DeepSeek format projects thinking into the API's
|
||||
# reasoning_content field instead of literal think tags in content;
|
||||
# unlimited reasoning budget (the model decides).
|
||||
REASONING_FORMAT=deepseek
|
||||
REASONING_BUDGET=-1
|
||||
@@ -0,0 +1,21 @@
|
||||
[Unit]
|
||||
Description=ik_llama.cpp Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M 177B MoE (dankclaw, frank-large)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
# Staging buffers for the CUDA transfers are page-locked; the weights
|
||||
# themselves stay mmapped (GGML_CUDA_NO_PINNED_WEIGHTS=1), but the staging
|
||||
# budget still needs room. Parity with the halogen quadlet's
|
||||
# Ulimit=memlock=-1:-1.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve mradermacher/Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M as frank-large
|
||||
# via ik_llama.cpp on Dankclaw.
|
||||
#
|
||||
# Topology (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE):
|
||||
# the RTX 3090 (CUDA, sm_86) takes the dense/attention/output weights and
|
||||
# the KV cache (-ngl 999); ALL MoE expert tensors and the ~32 GB PLE
|
||||
# n-gram table stay in the Strix Halo's LPDDR5X via the plain CPU backend
|
||||
# buffer (-ot ...=CPU — mmap-backed page cache; only a tiny slice of PLE is
|
||||
# touched per token). NEVER use --cpu-moe on this host: it routes the
|
||||
# CPU-overridden weights through the CUDA_Host PINNED buffer (measured
|
||||
# 108.12 GiB page-locked on 2026-09-15, which evicted every co-tenant into
|
||||
# swap and starved sshd into unreachability until a power cycle). NEVER add
|
||||
# --no-mmap — the STRIX-era lesson transfers directly (the cgroup killer
|
||||
# takes the process with nothing in the server log).
|
||||
#
|
||||
# Runtime: ik_llama.cpp master d85a1d7 built with CUDA (sm_86) + Zen5
|
||||
# AVX-512 IQK kernels; the startup banner must show HAVE_FANCY_SIMD
|
||||
# is defined. Flag set verified against this binary's --help before this
|
||||
# script was written. ik has no --cache-prompt/--cache-idle-slots (prompt
|
||||
# caching is its default behavior) and no --alias: it is a single-model
|
||||
# server and reports the GGUF's general.name at /v1/models.
|
||||
#
|
||||
# Sampling: greedy-by-default (halogen client contract) — clients that
|
||||
# send their own temperature get their own values.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${LLAMA_BIN:=/home/dank/GitPackages/ik_llama.cpp/build/bin/llama-server}"
|
||||
: "${MODEL_DIR:=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M}"
|
||||
: "${MODEL_FILE:=Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf}"
|
||||
: "${MMPROJ_PATH:=$MODEL_DIR/Qwen3.8-Flash-Next-Uncensored.mmproj-f16.gguf}"
|
||||
: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7770}"
|
||||
: "${CTX_SIZE:=262144}"
|
||||
: "${PARALLEL:=1}"
|
||||
: "${BATCH_SIZE:=4096}"
|
||||
: "${UBATCH_SIZE:=512}"
|
||||
: "${NUM_THREADS:=16}"
|
||||
: "${THREAD_BATCH:=16}"
|
||||
: "${CACHE_TYPE_K:=f16}"
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
: "${NGL:=999}"
|
||||
: "${CPU_MOE:=on}"
|
||||
: "${FLASH_ATTN:=on}"
|
||||
: "${TEMPERATURE:=0.0}"
|
||||
: "${REASONING:=on}"
|
||||
: "${REASONING_FORMAT:=deepseek}"
|
||||
: "${REASONING_BUDGET:=-1}"
|
||||
: "${CACHE_RAM:=8192}"
|
||||
: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}"
|
||||
# Hard guard: on this host CUDA pinned host memory strands the co-tenants
|
||||
# (measured 2026-09-15: a default-policy start tried to page-lock 108.12 GiB,
|
||||
# evicted every co-tenant into swap and starved sshd into unreachability
|
||||
# until a power cycle). GGML_CUDA_NO_PINNED_WEIGHTS=1 keeps the
|
||||
# CPU-overridden weights mmapped (src/llama-load-tensors.cpp); the env file
|
||||
# sets it. Refuse to run without it.
|
||||
if [[ "${GGML_CUDA_NO_PINNED_WEIGHTS:-}" != "1" ]]; then
|
||||
echo "frank-large: refusing to start — GGML_CUDA_NO_PINNED_WEIGHTS=1 is mandatory on this host" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$SLOT_SAVE_PATH"
|
||||
|
||||
# The UVM device node is created lazily by nvidia-modprobe on the first
|
||||
# CUDA open after boot; make sure it exists before the server needs it.
|
||||
nvidia-modprobe -u -c=0 2>/dev/null || true
|
||||
|
||||
CPU_MOE_ARGS=()
|
||||
if [[ "$CPU_MOE" == "on" ]]; then
|
||||
# -ot ...=CPU (plain CPU backend buffer) is the mmap-backed path. NEVER
|
||||
# swap this for --cpu-moe: that routes the weights through the CUDA_Host
|
||||
# PINNED buffer (108.12 GiB measured) and strands the co-tenants.
|
||||
CPU_MOE_ARGS=(-ot "exps=CPU" -ot "per_layer_token_embd=CPU")
|
||||
fi
|
||||
|
||||
MMPROJ_ARGS=()
|
||||
if [[ -n "${MMPROJ_PATH:-}" && -f "$MMPROJ_PATH" ]]; then
|
||||
MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
|
||||
fi
|
||||
|
||||
TEMPLATE_ARGS=()
|
||||
if [[ -n "${CHAT_TEMPLATE_FILE:-}" && -f "$CHAT_TEMPLATE_FILE" ]]; then
|
||||
TEMPLATE_ARGS=(--chat-template-file "$CHAT_TEMPLATE_FILE")
|
||||
else
|
||||
TEMPLATE_ARGS=(--jinja)
|
||||
fi
|
||||
|
||||
exec "$LLAMA_BIN" \
|
||||
--model "$MODEL_DIR/$MODEL_FILE" \
|
||||
"${TEMPLATE_ARGS[@]}" \
|
||||
--host "$BIND_ADDR" \
|
||||
--port "$LISTEN_PORT" \
|
||||
--gpu-layers "$NGL" \
|
||||
"${CPU_MOE_ARGS[@]}" \
|
||||
--flash-attn "$FLASH_ATTN" \
|
||||
--parallel "$PARALLEL" \
|
||||
--ctx-size "$CTX_SIZE" \
|
||||
--batch-size "$BATCH_SIZE" \
|
||||
--ubatch-size "$UBATCH_SIZE" \
|
||||
--threads "$NUM_THREADS" \
|
||||
--threads-batch "$THREAD_BATCH" \
|
||||
--cache-type-k "$CACHE_TYPE_K" \
|
||||
--cache-type-v "$CACHE_TYPE_V" \
|
||||
--cache-ram "$CACHE_RAM" \
|
||||
--slot-save-path "$SLOT_SAVE_PATH" \
|
||||
--ctx-checkpoints 16 \
|
||||
--metrics \
|
||||
--temp "$TEMPERATURE" \
|
||||
--reasoning "$REASONING" \
|
||||
--reasoning-format "$REASONING_FORMAT" \
|
||||
--reasoning-budget "$REASONING_BUDGET" \
|
||||
"${MMPROJ_ARGS[@]}"
|
||||
Reference in New Issue
Block a user