Frank
53ada02a63
Single full-context slot instead of two half-context slots. The 3090 takes ~33 of 66 layers (up from 26): the VRAM budget frees up because the f16 KV pool shrinks 401400 -> 262144. 33 layers is the max with a safety margin (34 -> 244 MiB headroom, 35 -> OOM). Verified math: /tmp/vram-math.py from live journal buffers.
81 lines
3.0 KiB
Bash
81 lines
3.0 KiB
Bash
# Head tunables — Qwen3.8-27B pipeline-parallel head (Dankclaw).
|
|
#
|
|
# The head is the LOCAL side of the split. It runs llama-server (b10433,
|
|
# HIP+Vulkan+RPC build in /home/dank/llama.cpp/build) with --rpc pointing at
|
|
# the worker on Dankserver. The model file lives on this machine only; the
|
|
# worker receives its tensor share over the network at load time.
|
|
#
|
|
# CRITICAL: this script must NOT pass --device. When --device is set,
|
|
# llama.cpp uses exactly that device list and silently drops the RPC device
|
|
# (src/llama.cpp llama_prepare_model_devices). Default device selection
|
|
# inserts RPC servers at the FRONT of the device list, so the split becomes
|
|
# [RPC0, iGPU] and --tensor-split index 0 is the worker.
|
|
#
|
|
# WARNING: do not name any variable here TEMP, HOST, or GPU_MAX_HEAP_SIZE
|
|
# (HIP runtime collision, see the 2026-08-13 TEMP incident).
|
|
|
|
MODEL_DIR=/home/dank/models
|
|
MODEL_FILE=Qwen3.8-27B-UD-Q8_K_XL.gguf
|
|
HEAD_BIN=/home/dank/llama.cpp/build/bin/llama-server
|
|
ALIAS=frank-default-large
|
|
BIND_ADDR=0.0.0.0
|
|
LISTEN_PORT=7770
|
|
|
|
# Comma-separated RPC endpoints (host:port). One worker here.
|
|
RPC_SERVERS=192.168.0.235:50052
|
|
|
|
# Model split fractions, one per device in order [RPC0, iGPU].
|
|
# 0.50,0.50 = ~33 of 66 layers on the 3090 - the maximum that fits its
|
|
# 24 GB card at the 262144 f16 KV pool (VRAM math 2026-08-16, from live
|
|
# journal buffers: 33 layers -> ~23.9 GB used, 714 MiB margin; 34 layers
|
|
# -> 244 MiB margin; 35 -> OOM). The 3090 is 2-3x faster per layer, so
|
|
# maxing its share minimizes the per-step device time in the pipeline.
|
|
TENSOR_SPLIT=0.50,0.50
|
|
|
|
# Context pool. 262144 total = 1 parallel slot x 262144 per slot
|
|
# (n_ctx_seq = n_ctx / n_seq_max, llama-context.cpp:293).
|
|
CTX_SIZE=262144
|
|
PARALLEL=1
|
|
|
|
# Speculative decoding. Matches production frank-default-large: MTP (bundled
|
|
# head, n_max 4 per the published tuning curve) with the ngram-mod fallback.
|
|
# Set SPEC_NONE=1 for a raw baseline with no speculation.
|
|
SPEC_NONE=0
|
|
SPEC_DEFAULT=1
|
|
MTP=1
|
|
SPEC_DRAFT_N_MAX=4
|
|
# Optional standalone MTP draft head (e.g. the Q4_0 ggml-org file). When set,
|
|
# it replaces the bundled in-model head via --spec-draft-model.
|
|
SPEC_DRAFT_MODEL=
|
|
SPEC_NGRAM_MOD_N_MIN=48
|
|
SPEC_NGRAM_MOD_N_MAX=64
|
|
SPEC_NGRAM_MOD_N_MATCH=24
|
|
|
|
# KV on GPU, unquantized (f16). 16 full-attention layers x 2 (K,V) x 4 heads
|
|
# x 256 dim x 2 bytes = 64 KiB/token -> 262144 tokens = 16.6 GiB total,
|
|
# held in proportion to the full-attention layers each device holds.
|
|
# Keep K == V so Flash Attention stays enabled.
|
|
CACHE_TYPE_K=f16
|
|
CACHE_TYPE_V=f16
|
|
|
|
# Batch sizing: production-verified values (2026-08-14).
|
|
BATCH_SIZE=8192
|
|
UBATCH_SIZE=256
|
|
|
|
# Sampling — Qwen3.8 thinking-mode defaults.
|
|
TEMPERATURE=1.0
|
|
TOP_K=20
|
|
TOP_P=0.95
|
|
MIN_P=0.0
|
|
|
|
THREADS=12
|
|
THREADS_HTTP=1
|
|
|
|
# 4 (info): clean journal for interactive use. Raise to 10 only when
|
|
# debugging the layer split (the per-layer assignment lines are debug-level).
|
|
VERBOSITY=4
|
|
|
|
# 1: --direct-io model load. Required in practice for large models loaded
|
|
# over RPC (load hangs otherwise; visorcraft Strix RPC finding).
|
|
DIRECT_IO=1
|