diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index e134c39..fe40f93 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -1,62 +1,78 @@ -# Tunables for frank-default-large (unsloth Qwen3.8-27B UD-Q8_K_XL, dankclaw) -# Runtime: llama.cpp llama-server (b10433+, rebuilt 2026-08-14 for bundled-MTP -# support); AMD Strix Halo (gfx1151) 96GB UMA; Ryzen AI MAX+ 395. +# Head tunables — Qwen3.8-27B pipeline-parallel head (Dankclaw). +# +# The head is the LOCAL side of the split. It runs llama-server (b10433, +# HIP+Vulkan+RPC build in /home/dank/llama.cpp/build) with --rpc pointing at +# the worker on Dankserver. The model file lives on this machine only; the +# worker receives its tensor share over the network at load time. +# +# CRITICAL: this script must NOT pass --device. When --device is set, +# llama.cpp uses exactly that device list and silently drops the RPC device +# (src/llama.cpp llama_prepare_model_devices). Default device selection +# inserts RPC servers at the FRONT of the device list, so the split becomes +# [RPC0, iGPU] and --tensor-split index 0 is the worker. +# +# WARNING: do not name any variable here TEMP, HOST, or GPU_MAX_HEAP_SIZE +# (HIP runtime collision, see the 2026-08-13 TEMP incident). + MODEL_DIR=/home/dank/models MODEL_FILE=Qwen3.8-27B-UD-Q8_K_XL.gguf -ALIAS=frank-default-large -HOST=0.0.0.0 -PORT=7770 -# Context pool. Qwen3.8 native context = 262144 per sequence. -# 524288 total / PARALLEL 2 = 262144 per slot, matching the pi client's -# contextWindow=262144 for frank-default-large. Model is 29GB; ROCm has -# ~70GB headroom, so 2x262k quantized KV fits. Verify at startup. -CTX_SIZE=524288 -# Parallel slots. 2026-08-15: DSpark multi-slot crashed the ROCm backend -# (illegal memory access, status 6/ABRT) under two concurrent 16k prose -# prompts, so DSpark was retired and --spec-default (ngram-mod) is used. -# ngram-mod is draftless and passed the strict parallel test historically. -# PARALLEL=2 is the current trial; pin to 1 if the 2x32k stream test fails. +HEAD_BIN=/home/dank/llama.cpp/build/bin/llama-server +ALIAS=frank-default +BIND_ADDR=0.0.0.0 +LISTEN_PORT=7770 + +# Comma-separated RPC endpoints (host:port). One worker here. +RPC_SERVERS=192.168.0.235:50052 + +# Model split fractions, one per device in order [RPC0, iGPU]. +# 0.40,0.60 = 40% of layers on the 3090 (~26 layers) - the maximum that +# fits its 24 GB card with 2x200700 f16 KV (see the KV math in +# docs/Pipeline Parallelism/Qwen3.8 PP Deployment.md). The 3090 is 2-3x +# faster per layer, so shifting weight to it is also a speed win. +TENSOR_SPLIT=0.40,0.60 + +# Context pool. 401400 total = 2 parallel slots x 200700 per slot +# (n_ctx_seq = n_ctx / n_seq_max, llama-context.cpp:293). +CTX_SIZE=401400 PARALLEL=2 -# Spec: llama.cpp --spec-default preset = ngram-mod (n_match 24, n_min 48, -# n_max 64). No draft model. Switched from DSpark 2026-08-15 after the -# DSpark multi-slot ROCm crash. MTP=1 prepends the bundled in-model MTP -# head (draft-mtp) before the ngram-mod fallback. See [[qwen3.8-spec-matrix]]. + +# Speculative decoding. Matches production frank-default-large: MTP (bundled +# head, n_max 4 per the published tuning curve) with the ngram-mod fallback. +# Set SPEC_NONE=1 for a raw baseline with no speculation. +SPEC_NONE=0 SPEC_DEFAULT=1 MTP=1 -SPEC_TYPE= -DRAFT_FILE= -SPEC_DRAFT_N_MAX= -SPEC_DRAFT_P_MIN= +SPEC_DRAFT_N_MAX=4 +# Optional standalone MTP draft head (e.g. the Q4_0 ggml-org file). When set, +# it replaces the bundled in-model head via --spec-draft-model. +SPEC_DRAFT_MODEL= SPEC_NGRAM_MOD_N_MIN=48 SPEC_NGRAM_MOD_N_MAX=64 SPEC_NGRAM_MOD_N_MATCH=24 -# Backend device from --list-devices. Vulkan0 selected 2026-08-15 for high -# context: at 262144 ctx (~200k prompt) Vulkan generates ~5.7 t/s vs ROCm -# ~3.8 t/s (48% faster decode), while ROCm prefills ~12% faster (119 vs 106 -# t/s). Generation is the interactive cost, so Vulkan0 wins at high ctx. -# See docs/bench-high-ctx results and [[qwen3.8-spec-matrix]]. -DEVICE=Vulkan0 -# Dense model: full offload. (Not MoE.) -N_GPU_LAYERS=all -# KV on GPU, quantized (K q8_0, V q8_0). Keep K == V for Flash Attention; -# mixed types silently disable FA (long-context decode collapses). -CACHE_TYPE_K=q8_0 -CACHE_TYPE_V=q8_0 -# Batch sizing (verified 2026-08-14 on Qwen3.8 UD-Q8_K_XL): -# ubatch 128 with batch>=2048 -> premature-EOS under concurrency (unsafe). -# ubatch 512 -> MTP acceptance collapses (~25%) -> gen ~9 t/s (slow). -# ubatch 256 -> ~30 t/s gen, concurrent-safe, ~330 t/s prefill. WINNER. -# batch 8192 keeps single-shot prefill flexibility; ubatch drives the graph. + +# KV on GPU, unquantized (f16). 16 full-attention layers x 2 (K,V) x 4 heads +# x 256 dim x 2 bytes = 64 KiB/token -> 401400 tokens = 24.5 GiB total. +# Keep K == V so Flash Attention stays enabled. +CACHE_TYPE_K=f16 +CACHE_TYPE_V=f16 + +# Batch sizing: production-verified values (2026-08-14). BATCH_SIZE=8192 UBATCH_SIZE=256 -# Sampling settings — per Qwen3.8-27B model card thinking mode. -# Thinking: temp=1.0, top_p=0.95, top_k=20, min_p=0. Clients may override. -# WARNING: do not name this variable TEMP. The ROCm HIP runtime reads the -# TEMP environment variable (JIT kernel-cache dir); TEMP=1.0 made the HIP -# runtime SEGV during model load on Aug 13 2026. + +# Sampling — Qwen3.8 thinking-mode defaults. TEMPERATURE=1.0 TOP_K=20 TOP_P=0.95 MIN_P=0.0 + THREADS=12 THREADS_HTTP=1 + +# 4 (info): clean journal for interactive use. Raise to 10 only when +# debugging the layer split (the per-layer assignment lines are debug-level). +VERBOSITY=4 + +# 1: --direct-io model load. Required in practice for large models loaded +# over RPC (load hangs otherwise; visorcraft Strix RPC finding). +DIRECT_IO=1 diff --git a/scripts/frank-default-large.service b/scripts/frank-default-large.service index 3951e4d..53fb69a 100644 --- a/scripts/frank-default-large.service +++ b/scripts/frank-default-large.service @@ -1,15 +1,14 @@ [Unit] -Description=llama.cpp Qwen3.8-27B UD-Q8_K_XL draft-mtp+ngram-mod (frank-default-large) -After=network.target +Description=llama.cpp Qwen3.8-27B pipeline-parallel head (Strix + RPC to Dankserver) (frank-default-large) +After=network-online.target +Wants=network-online.target [Service] Type=simple User=dank Group=dank -Environment="HOST=0.0.0.0" -Environment="PORT=7770" -Environment="FRANK_LARGE_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-large.env" -ExecStart=/home/dank/vllmScripts/scripts/serve-llama-frank-large.sh +Environment="FRANK_PP_HEAD_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-large.env" +ExecStart=/home/dank/vllmScripts/scripts/serve-pp-head.sh Restart=on-failure RestartSec=10 diff --git a/scripts/frank-pp-head.env b/scripts/frank-pp-head.env deleted file mode 100644 index fe40f93..0000000 --- a/scripts/frank-pp-head.env +++ /dev/null @@ -1,78 +0,0 @@ -# Head tunables — Qwen3.8-27B pipeline-parallel head (Dankclaw). -# -# The head is the LOCAL side of the split. It runs llama-server (b10433, -# HIP+Vulkan+RPC build in /home/dank/llama.cpp/build) with --rpc pointing at -# the worker on Dankserver. The model file lives on this machine only; the -# worker receives its tensor share over the network at load time. -# -# CRITICAL: this script must NOT pass --device. When --device is set, -# llama.cpp uses exactly that device list and silently drops the RPC device -# (src/llama.cpp llama_prepare_model_devices). Default device selection -# inserts RPC servers at the FRONT of the device list, so the split becomes -# [RPC0, iGPU] and --tensor-split index 0 is the worker. -# -# WARNING: do not name any variable here TEMP, HOST, or GPU_MAX_HEAP_SIZE -# (HIP runtime collision, see the 2026-08-13 TEMP incident). - -MODEL_DIR=/home/dank/models -MODEL_FILE=Qwen3.8-27B-UD-Q8_K_XL.gguf -HEAD_BIN=/home/dank/llama.cpp/build/bin/llama-server -ALIAS=frank-default -BIND_ADDR=0.0.0.0 -LISTEN_PORT=7770 - -# Comma-separated RPC endpoints (host:port). One worker here. -RPC_SERVERS=192.168.0.235:50052 - -# Model split fractions, one per device in order [RPC0, iGPU]. -# 0.40,0.60 = 40% of layers on the 3090 (~26 layers) - the maximum that -# fits its 24 GB card with 2x200700 f16 KV (see the KV math in -# docs/Pipeline Parallelism/Qwen3.8 PP Deployment.md). The 3090 is 2-3x -# faster per layer, so shifting weight to it is also a speed win. -TENSOR_SPLIT=0.40,0.60 - -# Context pool. 401400 total = 2 parallel slots x 200700 per slot -# (n_ctx_seq = n_ctx / n_seq_max, llama-context.cpp:293). -CTX_SIZE=401400 -PARALLEL=2 - -# Speculative decoding. Matches production frank-default-large: MTP (bundled -# head, n_max 4 per the published tuning curve) with the ngram-mod fallback. -# Set SPEC_NONE=1 for a raw baseline with no speculation. -SPEC_NONE=0 -SPEC_DEFAULT=1 -MTP=1 -SPEC_DRAFT_N_MAX=4 -# Optional standalone MTP draft head (e.g. the Q4_0 ggml-org file). When set, -# it replaces the bundled in-model head via --spec-draft-model. -SPEC_DRAFT_MODEL= -SPEC_NGRAM_MOD_N_MIN=48 -SPEC_NGRAM_MOD_N_MAX=64 -SPEC_NGRAM_MOD_N_MATCH=24 - -# KV on GPU, unquantized (f16). 16 full-attention layers x 2 (K,V) x 4 heads -# x 256 dim x 2 bytes = 64 KiB/token -> 401400 tokens = 24.5 GiB total. -# Keep K == V so Flash Attention stays enabled. -CACHE_TYPE_K=f16 -CACHE_TYPE_V=f16 - -# Batch sizing: production-verified values (2026-08-14). -BATCH_SIZE=8192 -UBATCH_SIZE=256 - -# Sampling — Qwen3.8 thinking-mode defaults. -TEMPERATURE=1.0 -TOP_K=20 -TOP_P=0.95 -MIN_P=0.0 - -THREADS=12 -THREADS_HTTP=1 - -# 4 (info): clean journal for interactive use. Raise to 10 only when -# debugging the layer split (the per-layer assignment lines are debug-level). -VERBOSITY=4 - -# 1: --direct-io model load. Required in practice for large models loaded -# over RPC (load hangs otherwise; visorcraft Strix RPC finding). -DIRECT_IO=1 diff --git a/scripts/frank-pp-head.service b/scripts/frank-pp-head.service deleted file mode 100644 index 411618f..0000000 --- a/scripts/frank-pp-head.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=llama.cpp Qwen3.8-27B pipeline-parallel head (Strix + RPC to Dankserver) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=dank -Group=dank -Environment="FRANK_PP_HEAD_ENV_FILE=/home/dank/vllmScripts/scripts/frank-pp-head.env" -ExecStart=/home/dank/vllmScripts/scripts/serve-pp-head.sh -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/scripts/serve-pp-head.sh b/scripts/serve-pp-head.sh index ebac11d..5d31b97 100755 --- a/scripts/serve-pp-head.sh +++ b/scripts/serve-pp-head.sh @@ -8,14 +8,14 @@ # # IMPORTANT: no --device flag. The RPC device only joins the model's device # list through default device selection; an explicit --device silently drops -# it. See the header of frank-pp-head.env. +# it. See the header of frank-default-large.env. # -# Tunables come from frank-pp-head.env in this directory unless the +# Tunables come from frank-default-large.env in this directory unless the # FRANK_PP_HEAD_ENV_FILE environment variable overrides that path. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ENV_FILE="${FRANK_PP_HEAD_ENV_FILE:-$SCRIPT_DIR/frank-pp-head.env}" +ENV_FILE="${FRANK_PP_HEAD_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}" if [[ -f "$ENV_FILE" ]]; then set -a # shellcheck disable=SC1090