feat: frank-default-large → llama.cpp Qwen3.6-27B Q8_K_P + DFlash draft + ngram-mod (bench pending)
This commit is contained in:
Frank
@@ -75,6 +75,16 @@ vllm serve <target-model> \
|
||||
|
||||
llama.cpp supports DFlash, DSpark, and MTP through `--spec-type`. When you pass `--spec-draft-hf`, llama.cpp auto-discovers the sidecar files and infers the type. The priority order is DSpark, DFlash, EAGLE-3, MTP.
|
||||
|
||||
`--spec-type` accepts a **comma-separated list** of speculative implementations. `common_speculative_init` (in `common/speculative.cpp`) builds a vector of implementations and chains drafting through all of them (`common_speculative_draft_params.drafting` is "used to chain the drafts through all the available implementations"). This means you can **layer ngram-mod on top of a DFlash draft model** in one server process:
|
||||
|
||||
```
|
||||
llama-server --hf-repo <target-GGUF> \
|
||||
--spec-draft-hf <dflash-draft-GGUF> \
|
||||
--spec-type draft-dflash,ngram-mod
|
||||
```
|
||||
|
||||
The DFlash drafter proposes, then ngram-mod runs as an additional implementation in the chain. Verified against upstream `common/speculative.cpp` (b10133+).
|
||||
|
||||
DFlash on llama.cpp:
|
||||
|
||||
```
|
||||
|
||||
Executable
+150
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env bash
|
||||
# bench-frank-large.sh — 64k-context benchmark for the frank-default-large stack.
|
||||
#
|
||||
# Measures generation throughput of Qwen3.6-27B Q8_K_P + DFlash + ngram-mod on
|
||||
# a single backend at 64k context. It starts llama-server on a scratch port,
|
||||
# drives one generation through the OpenAI-compatible API with a ~64k-token
|
||||
# prompt, and prints tokens/sec.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/bench-frank-large.sh <ROCm0|Vulkan0> [--generations N]
|
||||
#
|
||||
# Example:
|
||||
# ./scripts/bench-frank-large.sh ROCm0
|
||||
# ./scripts/bench-frank-large.sh Vulkan0 --generations 3
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DEVICE="${1:?usage: bench-frank-large.sh <ROCm0|Vulkan0> [--generations N]}"
|
||||
GENERATIONS=1
|
||||
if [ "${2:-}" = "--generations" ]; then
|
||||
GENERATIONS="${3:?--generations needs a count}"
|
||||
fi
|
||||
|
||||
MODEL_DIR=/home/dank/models
|
||||
MODEL_FILE=Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf
|
||||
DRAFT_FILE=Qwen3.6-27B-DFlash-Q8_0.gguf
|
||||
LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
|
||||
MODEL="$MODEL_DIR/$MODEL_FILE"
|
||||
DRAFT="$MODEL_DIR/$DRAFT_FILE"
|
||||
|
||||
PORT=7890
|
||||
CTX=65536
|
||||
PARALLEL=1
|
||||
BATCH=2048
|
||||
UBATCH=512
|
||||
OUT_TOKENS=256
|
||||
|
||||
for f in "$MODEL" "$DRAFT"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "missing required file: $f" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "=== 64k benchmark: device=$DEVICE model=$MODEL_FILE ==="
|
||||
|
||||
# --- start the server ----------------------------------------------------
|
||||
"$LLAMA_SERVER" \
|
||||
--model "$MODEL" \
|
||||
--spec-draft-model "$DRAFT" \
|
||||
--spec-type "draft-dflash,ngram-mod" \
|
||||
--spec-draft-n-max 8 \
|
||||
--spec-ngram-mod-n-min 48 \
|
||||
--spec-ngram-mod-n-max 64 \
|
||||
--spec-ngram-mod-n-match 24 \
|
||||
--device "$DEVICE" \
|
||||
--gpu-layers all \
|
||||
--cache-type-k q8_0 \
|
||||
--cache-type-v f16 \
|
||||
--ctx-size "$CTX" \
|
||||
--parallel "$PARALLEL" \
|
||||
--batch-size "$BATCH" \
|
||||
--ubatch-size "$UBATCH" \
|
||||
--alias bench \
|
||||
--host 127.0.0.1 \
|
||||
--port "$PORT" \
|
||||
> "/tmp/frankbench-$DEVICE.log" 2>&1 &
|
||||
SERVER_PID=$!
|
||||
echo "server pid $SERVER_PID on 127.0.0.1:$PORT"
|
||||
|
||||
trap 'kill $SERVER_PID 2>/dev/null; wait $SERVER_PID 2>/dev/null' EXIT
|
||||
|
||||
# --- wait for ready ------------------------------------------------------
|
||||
echo -n "waiting for server"
|
||||
ready=0
|
||||
for i in $(seq 1 400); do
|
||||
if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
|
||||
echo " [ready]"
|
||||
ready=1
|
||||
break
|
||||
fi
|
||||
echo -n "."
|
||||
sleep 2
|
||||
done
|
||||
if [ "$ready" != "1" ]; then
|
||||
echo ""
|
||||
echo "FAIL: server did not become ready"
|
||||
tail -25 "/tmp/frankbench-$DEVICE.log"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- drive N generations and measure --------------------------------------
|
||||
python3 - "$DEVICE" "$CTX" "$OUT_TOKENS" "$GENERATIONS" "$PORT" <<'PY'
|
||||
import sys, time, json, urllib.request, statistics
|
||||
|
||||
device = sys.argv[1]
|
||||
ctx = int(sys.argv[2])
|
||||
out_tokens = int(sys.argv[3])
|
||||
generations = int(sys.argv[4])
|
||||
port = sys.argv[5]
|
||||
|
||||
# ~64k-token prompt. One prose unit is ~110 chars ≈ 28 tokens for this model.
|
||||
# We aim for a prompt of ~60k tokens so generation stays within the 64k ctx.
|
||||
unit = ("The careful study of natural language reveals patterns of meaning "
|
||||
"that emerge from the deliberate arrangement of words and phrases. ")
|
||||
# approximate: unit is 160 chars, ~40 tokens; repeat to ~1500 units = ~60k tokens
|
||||
prompt = unit * 1500
|
||||
|
||||
def req(prompt, n):
|
||||
body = json.dumps({
|
||||
"prompt": prompt,
|
||||
"max_tokens": n,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
}).encode()
|
||||
r = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",
|
||||
data=body, headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(r, timeout=1800) as resp:
|
||||
return json.load(resp)
|
||||
|
||||
# prefill timing
|
||||
t0 = time.time()
|
||||
first = req(prompt, 0)
|
||||
t_pre = time.time() - t0
|
||||
used = first.get("usage", {})
|
||||
prompt_tokens = used.get("prompt_tokens", 0)
|
||||
print(f"prefill: {prompt_tokens} tokens in {t_pre:.2f}s "
|
||||
f"({prompt_tokens/t_pre:.1f} tok/s)")
|
||||
|
||||
# generation timing
|
||||
rates = []
|
||||
total_gentok = 0
|
||||
total_time = 0.0
|
||||
for i in range(generations):
|
||||
t0 = time.time()
|
||||
out = req(prompt, out_tokens)
|
||||
dt = time.time() - t0
|
||||
uit = out.get("usage", {})
|
||||
comp = uit.get("completion_tokens", 0)
|
||||
gt = dt - t_pre # approximate generation time = total - prefill
|
||||
r = comp / gt if gt > 0 else 0
|
||||
rates.append(r)
|
||||
total_gentok += comp
|
||||
total_time += gt
|
||||
print(f"gen {i+1}: {comp} tokens in {gt:.2f}s ({r:.1f} tok/s)")
|
||||
|
||||
avg = statistics.mean(rates) if rates else 0
|
||||
print(f"\nRESULT device={device} prompt_tokens={prompt_tokens} "
|
||||
f"avg_gen_tok_s={avg:.1f} over {generations} runs")
|
||||
PY
|
||||
@@ -0,0 +1,31 @@
|
||||
# Tunables for frank-default-large (HauhauCS Qwen3.6-27B Q8_K_P, DFlash+ngram-mod, dankclaw)
|
||||
# Runtime: llama.cpp llama-server; AMD Strix Halo (gfx1151) 96GB UMA; Ryzen AI MAX+ 395.
|
||||
MODEL_DIR=/home/dank/models
|
||||
ALIAS=frank-default-large
|
||||
HOST=0.0.0.0
|
||||
PORT=7770
|
||||
# 6 parallel slots x 65536 ctx each = 393216 total context pool.
|
||||
CTX_SIZE=393216
|
||||
PARALLEL=6
|
||||
# Spec: DFlash draft model layered with ngram-mod. --spec-type accepts a
|
||||
# comma-separated list; llama.cpp chains the implementations (verified in
|
||||
# common/speculative.cpp). --spec-draft-n-max must stay below the DFlash
|
||||
# block size (BF16 sidecar = 8 token block), so 8 is the ceiling.
|
||||
SPEC_TYPE=draft-dflash,ngram-mod
|
||||
SPEC_DRAFT_N_MAX=8
|
||||
SPEC_NGRAM_MOD_N_MIN=48
|
||||
SPEC_NGRAM_MOD_N_MAX=64
|
||||
SPEC_NGRAM_MOD_N_MATCH=24
|
||||
# Backend device for offload, from --list-devices. Pick the faster of:
|
||||
# ROCm0 (HIP / ROCm backend)
|
||||
# Vulkan0 (RADV Vulkan backend)
|
||||
DEVICE=ROCm0
|
||||
# Dense model (Qwen3.6-27B, not MoE): no MoE CPU offload. Full offload.
|
||||
N_GPU_LAYERS=all
|
||||
# KV on GPU, quantized (K q8_0, V f16); mirrors the small service profile.
|
||||
CACHE_TYPE_K=q8_0
|
||||
CACHE_TYPE_V=f16
|
||||
BATCH_SIZE=2048
|
||||
UBATCH_SIZE=512
|
||||
THREADS=8
|
||||
THREADS_HTTP=8
|
||||
@@ -1,16 +1,15 @@
|
||||
[Unit]
|
||||
Description=vLLM Ling-3.0-flash INT4 (CIRU gfx1151) -- frank-default-large
|
||||
Description=llama.cpp Qwen3.6-27B Q8_K_P DFlash+ngram-mod (frank-default-large)
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
WorkingDirectory=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native
|
||||
Environment="HOST=0.0.0.0"
|
||||
Environment="PORT=7770"
|
||||
Environment="SERVED_MODEL_NAME=frank-default-large"
|
||||
ExecStart=/home/dank/vllmScripts/scripts/serve-ling3-vllm.sh
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-large.env"
|
||||
ExecStart=/home/dank/vllmScripts/scripts/serve-llama-frank-large.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
|
||||
Executable
+74
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve HauhauCS Qwen3.6-27B-Uncensored-HauhauCS-Aggressive:Q8_K_P
|
||||
# with DFlash draft + ngram-mod speculative decoding on dankclaw
|
||||
# (AMD Strix Halo gfx1151, ROCm or Vulkan backend).
|
||||
# Runtime: llama.cpp llama-server.
|
||||
#
|
||||
# Tunables come from frank-default-large.env in this directory unless the
|
||||
# FRANK_LARGE_ENV_FILE environment variable overrides that path.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -----------------
|
||||
: "${MODEL_DIR:=/home/dank/models}"
|
||||
: "${MODEL_FILE:=Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf}"
|
||||
: "${DRAFT_FILE:=Qwen3.6-27B-DFlash-Q8_0.gguf}"
|
||||
: "${ALIAS:=frank-default-large}"
|
||||
: "${HOST:=0.0.0.0}"
|
||||
: "${PORT:=7770}"
|
||||
: "${CTX_SIZE:=393216}"
|
||||
: "${PARALLEL:=6}"
|
||||
: "${SPEC_TYPE:=draft-dflash,ngram-mod}"
|
||||
: "${SPEC_DRAFT_N_MAX:=8}"
|
||||
: "${SPEC_NGRAM_MOD_N_MIN:=48}"
|
||||
: "${SPEC_NGRAM_MOD_N_MAX:=64}"
|
||||
: "${SPEC_NGRAM_MOD_N_MATCH:=24}"
|
||||
: "${DEVICE:=ROCm0}"
|
||||
: "${N_GPU_LAYERS:=all}"
|
||||
: "${CACHE_TYPE_K:=q8_0}"
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
: "${BATCH_SIZE:=2048}"
|
||||
: "${UBATCH_SIZE:=512}"
|
||||
: "${THREADS:=8}"
|
||||
: "${THREADS_HTTP:=8}"
|
||||
|
||||
LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp/build/bin/llama-server}"
|
||||
MODEL="$MODEL_DIR/$MODEL_FILE"
|
||||
DRAFT="$MODEL_DIR/$DRAFT_FILE"
|
||||
|
||||
for f in "$MODEL" "$DRAFT"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "missing required file: $f" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
exec "$LLAMA_SERVER" \
|
||||
--model "$MODEL" \
|
||||
--spec-draft-model "$DRAFT" \
|
||||
--spec-type "$SPEC_TYPE" \
|
||||
--spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
|
||||
--spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN" \
|
||||
--spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX" \
|
||||
--spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH" \
|
||||
--device "$DEVICE" \
|
||||
--gpu-layers "$N_GPU_LAYERS" \
|
||||
--cache-type-k "$CACHE_TYPE_K" \
|
||||
--cache-type-v "$CACHE_TYPE_V" \
|
||||
--ctx-size "$CTX_SIZE" \
|
||||
--parallel "$PARALLEL" \
|
||||
--batch-size "$BATCH_SIZE" \
|
||||
--ubatch-size "$UBATCH_SIZE" \
|
||||
--threads "$THREADS" \
|
||||
--threads-http "$THREADS_HTTP" \
|
||||
--alias "$ALIAS" \
|
||||
--host "$HOST" \
|
||||
--port "$PORT"
|
||||
Reference in New Issue
Block a user