From 57b0be037c22b1cffbb28876b4d1d5b5f887abda Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 14 Aug 2026 13:26:17 -0700 Subject: [PATCH] dankclaw: switch frank-default-large to unsloth Qwen3.8-27B UD-Q8_K_XL - rebuild llama.cpp stock to b10433 (9b05354ec) for bundled-MTP support - MODEL_FILE -> Qwen3.8-27B-UD-Q8_K_XL.gguf; DRAFT_FILE empty (MTP bundled) - SPEC: draft-mtp, n_max=10, p_min off (~30 t/s stable) - matrix winner - CTX 524288 / parallel 2 = 262144/slot (matches pi contextWindow) - serve script: bundled-MTP (no --spec-draft-model), optional --spec-draft-p-min --- scripts/frank-default-large.env | 68 +++++++++++++++--------------- scripts/serve-llama-frank-large.sh | 28 ++++++------ 2 files changed, 49 insertions(+), 47 deletions(-) diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 97adfce..b6b2273 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -1,51 +1,51 @@ -# Tunables for frank-default-large (HauhauCS Qwen3.6-27B Q8_K_P, DFlash+ngram-mod, dankclaw) -# Runtime: llama.cpp llama-server; AMD Strix Halo (gfx1151) 96GB UMA; Ryzen AI MAX+ 395. +# Tunables for frank-default-large (unsloth Qwen3.8-27B UD-Q8_K_XL, dankclaw) +# Runtime: llama.cpp llama-server (b10433+, rebuilt 2026-08-14 for bundled-MTP +# support); AMD Strix Halo (gfx1151) 96GB UMA; Ryzen AI MAX+ 395. MODEL_DIR=/home/dank/models +MODEL_FILE=Qwen3.8-27B-UD-Q8_K_XL.gguf ALIAS=frank-default-large HOST=0.0.0.0 PORT=7770 -# 3 parallel slots x 200700 ctx each = 602100 total context pool. -CTX_SIZE=602100 -PARALLEL=3 -# Spec: speculative decoding mode. -# draft-dflash - DFlash block-diffusion draft (benchmark winner pre-2026-08-14) -# BUT: premature EOS under parallel load when a new task -# reuses a cached prompt (shared draft ctx corruption). Retired. -# draft-mtp - MTP sidecar draft (a4lg/Qwen3.6-27B-MTP-ONLY-Q8_0.gguf) -# ngram-mod - draftless ngram matching (no draft model, no shared ctx) -# Benched 2026-08-14: see docs/frank-default-large-benchmark.md for the winner. +# Context pool. Qwen3.8 native context = 262144 per sequence. +# 524288 total / PARALLEL 2 = 262144 per slot, matching the pi client's +# contextWindow=262144 for frank-default-large. Model is 29GB; ROCm has +# ~70GB headroom, so 2x262k quantized KV fits. Verify at startup. +CTX_SIZE=524288 +PARALLEL=2 +# Spec: selected from the 2026-08-14 matrix (bench-spec-matrix.sh). +# WINNER: draft-mtp alone, n_max=10, p_min off -> ~30 t/s, STABLE, no +# premature stop. Layered draft-mtp,ngram-mod was faster on paper (median +# 133 t/s) but is non-deterministic (10-148 t/s) and shows a degenerate +# instant-EOS (tok=0) -> the same premature-stop class that retired DFlash. +# Rejected as production-unsafe. SPEC_TYPE=draft-mtp -# Draft model file (only used when SPEC_TYPE contains 'draft-'). -# DFlash: Qwen3.6-27B-DFlash-Q8_0.gguf ; MTP: Qwen3.6-27B-MTP-ONLY-Q8_0.gguf -DRAFT_FILE=Qwen3.6-27B-MTP-ONLY-Q8_0.gguf -# Draft length per spec type: dflash 8, mtp 2-3. -SPEC_DRAFT_N_MAX=3 +# Bundled MTP: Qwen3.8 keeps the MTP head in the target GGUF. llama.cpp +# auto-detects it (b10411+). DRAFT_FILE MUST stay EMPTY. +DRAFT_FILE= +# Draft length + sampling floor, from the spec matrix (n_max 2..10, p_min +# off/0.8; p_min 0.8 did not help draft-mtp and slowed it slightly). +SPEC_DRAFT_N_MAX=10 +SPEC_DRAFT_P_MIN= SPEC_NGRAM_MOD_N_MIN=48 SPEC_NGRAM_MOD_N_MAX=64 SPEC_NGRAM_MOD_N_MATCH=24 -# Backend device from --list-devices. ROCm wins on generation; Vulkan renders -# DFlash spec unusable (NONE-op abort) and gen is slower on ngram (6.9 t/s). +# Backend device from --list-devices. ROCm0 is the production backend. DEVICE=ROCm0 -# Dense model (Qwen3.6-27B, not MoE): no MoE CPU offload. Full offload. +# Dense model: full offload. (Not MoE.) N_GPU_LAYERS=all -# KV on GPU, quantized (K q8_0, V q8_0). V MUST match K: this llama.cpp -# build has GGML_CUDA_FA_ALL_QUANTS=OFF, and Flash Attention requires -# K type == V type. Mixed q8_0/f16 silently disables FA, collapsing -# long-context decode to ~1 t/s at 200k ctx. q8_0 V halves V memory too. +# KV on GPU, quantized (K q8_0, V q8_0). Keep K == V for Flash Attention; +# mixed types silently disable FA (long-context decode collapses). CACHE_TYPE_K=q8_0 CACHE_TYPE_V=q8_0 -# Batch sizing: empirically verified 2026-08-13 with caches dropped. -# batch 8192 / ubatch 2048 loads and serves (61 GiB VRAM peak, ctx 602100). -# ubatch is the compute-graph driver; larger ubatch = more VRAM, faster prefill. -# Re-verified 2026-08-14 during the spec swap (see benchmark doc). -BATCH_SIZE=8192 -UBATCH_SIZE=8192 -# Sampling settings — per HauhauCS Qwen3.6-27B model card. -# Thinking-general: temp=1.0, top_p=0.95, top_k=20, min_p=0. -# Clients may override per-request; these are the server-side defaults. +# Batch sizing (start conservative; tune after verifying memory): +# ubatch is the compute-graph driver; larger = more VRAM, faster prefill. +BATCH_SIZE=2048 +UBATCH_SIZE=512 +# Sampling settings — per Qwen3.8-27B model card thinking mode. +# Thinking: temp=1.0, top_p=0.95, top_k=20, min_p=0. Clients may override. # WARNING: do not name this variable TEMP. The ROCm HIP runtime reads the # TEMP environment variable (JIT kernel-cache dir); TEMP=1.0 made the HIP -# runtime SEGV (NULL unref) during model load on Aug 13 2026. +# runtime SEGV during model load on Aug 13 2026. TEMPERATURE=1.0 TOP_K=20 TOP_P=0.95 diff --git a/scripts/serve-llama-frank-large.sh b/scripts/serve-llama-frank-large.sh index 54bb437..e102789 100755 --- a/scripts/serve-llama-frank-large.sh +++ b/scripts/serve-llama-frank-large.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash -# Serve HauhauCS Qwen3.6-27B-Uncensored-HauhauCS-Aggressive:Q8_K_P -# with DFlash draft speculative decoding (draft-dflash) on dankclaw -# (AMD Strix Halo gfx1151, ROCm backend). -# Runtime: llama.cpp llama-server. +# Serve unsloth Qwen3.8-27B UD-Q8_K_XL on dankclaw with bundled-MTP +# speculative decoding (draft-mtp). ROCm backend. +# Runtime: llama.cpp llama-server (>= b10411). # # Tunables come from frank-default-large.env in this directory unless the # FRANK_LARGE_ENV_FILE environment variable overrides that path. @@ -19,15 +18,15 @@ fi # --- defaults (used when the env file does not set them) ----------------- : "${MODEL_DIR:=/home/dank/models}" -: "${MODEL_FILE:=Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf}" -: "${DRAFT_FILE:=Qwen3.6-27B-DFlash-Q8_0.gguf}" +: "${MODEL_FILE:=Qwen3.8-27B-UD-Q8_K_XL.gguf}" +: "${DRAFT_FILE:=}" # empty: Qwen3.8 bundles its MTP drafter in-model : "${ALIAS:=frank-default-large}" : "${HOST:=0.0.0.0}" : "${PORT:=7770}" -: "${CTX_SIZE:=602100}" -: "${PARALLEL:=3}" -: "${SPEC_TYPE:=draft-dflash}" -: "${SPEC_DRAFT_N_MAX:=8}" +: "${CTX_SIZE:=262144}" # Qwen3.8 native context (env may raise for parallel≥2) +: "${PARALLEL:=2}" +: "${SPEC_TYPE:=draft-mtp}" +: "${SPEC_DRAFT_N_MAX:=10}" : "${SPEC_NGRAM_MOD_N_MIN:=48}" : "${SPEC_NGRAM_MOD_N_MAX:=64}" : "${SPEC_NGRAM_MOD_N_MATCH:=24}" @@ -53,16 +52,19 @@ if [[ ! -f "$MODEL" ]]; then exit 1 fi -# Draft-model flags apply only to spec types that use a draft file -# (draft-dflash, draft-mtp, draft-eagle3). ngram-mod is draftless. +# Draft-model flag applies only when a separate draft file is provided. +# Qwen3.8 bundles MTP in-model; leave DRAFT_FILE empty to use bundled MTP. SPEC_ARGS=() -if [[ "$SPEC_TYPE" == *draft-* ]]; then +if [[ -n "$DRAFT_FILE" ]]; then if [[ ! -f "$DRAFT" ]]; then echo "missing required file: $DRAFT" >&2 exit 1 fi SPEC_ARGS+=(--spec-draft-model "$DRAFT") fi +if [[ -n "${SPEC_DRAFT_P_MIN:-}" ]]; then + SPEC_ARGS+=(--spec-draft-p-min "$SPEC_DRAFT_P_MIN") +fi exec "$LLAMA_SERVER" \ --model "$MODEL" \