From 8c41c821c489cf34b816f76dc2a75c60a6705084 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 26 Aug 2026 05:01:58 -0700 Subject: [PATCH] frank-default-small: Q8_0 model, 160k single slot, q6_0 KV + Hadamard Switch to CRACK Q8_0 model (37.8 GB, mmproj f16). Servt as a single 160000-ctx slot (PARALLEL=1); speculative decoding stays disabled. KV cache is q6_0 + Hadamard. iq5_k is not a valid KV-cache type in ik_llama (cache whitelist excludes iq5_*); q8_KV reproduces the ggml.c:5427 view assert even at PARALLEL=1/no-spec, so q6_0 is the verified working 6-bit choice. Env, serve script, tests, and KB updated. Deployment was left down after the isolation test; this commit redeploys it with the verified Q8_0/160k/q6_0 config. --- docs/LLM/ikllama-ornith-dankserver.md | 46 +++++++++++++-------------- scripts/frank-default-small.env | 43 +++++++++++-------------- scripts/serve-frank-small.sh | 27 +++++++--------- tests/frank_small_config.bats | 26 ++++++++------- 4 files changed, 67 insertions(+), 75 deletions(-) diff --git a/docs/LLM/ikllama-ornith-dankserver.md b/docs/LLM/ikllama-ornith-dankserver.md index c383a25..d84b58a 100644 --- a/docs/LLM/ikllama-ornith-dankserver.md +++ b/docs/LLM/ikllama-ornith-dankserver.md @@ -1,7 +1,7 @@ --- type: reference -title: ik_llama.cpp on Dankserver (Ornith Q6_K) -description: ikawrakow/ik_llama.cpp replaces upstream llama.cpp for frank-default-small on the RTX 3090. q8_0 KV cache with Hadamard (the fast q8_KV type crashes on this arch at this commit), split-mode none (graph requires >=2 GPUs), speculative decoding disabled (3 parallel slots), the Qwen-Sharp template rendered with --jinja, and explicit CPU-expert offload. Includes the mandatory CUDA/IQK FA_ALL_QUANTS build flags. +title: ik_llama.cpp on Dankserver (Ornith Q8_0) +description: ikawrakow/ik_llama.cpp replaces upstream llama.cpp for frank-default-small on the RTX 3090. Q8_0 model, q6_0 KV cache with Hadamard (iq5_k is not a KV-cache type in this build; q8_KV crashes on this arch), split-mode none, speculative decoding disabled, 160k single slot, the Qwen-Sharp template rendered with --jinja, and explicit CPU-expert offload. Includes the mandatory CUDA/IQK FA_ALL_QUANTS build flags. tags: - ikllama - llamacpp @@ -21,11 +21,11 @@ timestamp: 2026-08-26 > RTX 3090. This page documents the build, the mandatory FA build flags, and > the MoE / CPU-offload runtime arguments. -The model is unchanged: `dealignai/Ornith-1.5-35B-A3B-UNCENSORED` (CRACK -Q6_K), 35B-total / 3B-active MoE-dense hybrid with an MTP head in the GGUF. -Speculative decoding is **disabled** (ik_llama limits spec to a single slot; -this service keeps 3 parallel slots). The low-parameter MoE does not need -it. See [[dealign-ornith-crack]] for the model runbook. +The model is `dealignai/Ornith-1.5-35B-A3B-UNCENSORED` (CRACK **Q8_0**), 35B-total / +3B-active MoE-dense hybrid with an MTP head in the GGUF (mmproj f16 vision +encoder). Served as a **single 160000-context slot** (`PARALLEL=1`). +Speculative decoding is **disabled**. See [[dealign-ornith-crack]] for the +model runbook. ## Why ik_llama.cpp @@ -105,27 +105,27 @@ Files: `scripts/serve-frank-small.sh`, `scripts/frank-default-small.env`, `scripts/frank-default-small.service` in the repo. The service just calls the serve script; the env file holds all tunables. -### KV cache: q8_0 + Hadamard +### KV cache: q8_0 → q6_0 + Hadamard -- `CACHE_TYPE_K=q8_0`, `CACHE_TYPE_V=q8_0` — standard 8-bit KV cache type - (passes `--cache-type-k q8_0 --cache-type-v q8_0`). +- `CACHE_TYPE_K=q6_0`, `CACHE_TYPE_V=q6_0` — 6-bit k-quant KV cache + (passes `--cache-type-k q6_0 --cache-type-v q6_0`). - `K_CACHE_HADAMARD=1`, `V_CACHE_HADAMARD=1` — `--k-cache-hadamard` and `--v-cache-hadamard`. The Hadamard transform preserves KV quality at - 8-bit (docs recommend it for quantized KV below Q6_0). Implemented for - both the CUDA (`ggml-cuda/hadamard.cu`) and CPU (IQK) paths. -- Flash-attention is **on by default** (no `-fa` flag needed). + 6-bit. Implemented for both CUDA (`ggml-cuda/hadamard.cu`) and CPU (IQK) + paths. Flash-attention is **on by default**. -> ⚠️ **Do not use `q8_KV` (fast type) here.** On this commit (`b166e269`) -> the fast `q8_KV` cache type crashes at model init on this QWEN3MOE/MTP + -> multimodal arch with a `ggml view` size assertion -> (`ggml.c:5427: GGML_ASSERT(view_src == NULL || ... data_size + view_offs`) -> right after the output buffer is created. It loops the service. Plain -> `q8_0` (and `f16`) work and were smoke-tested with correct output; this -> config uses `q8_0`. This was isolated empirically by toggling cache type -> while keeping the override/hadamard flags. +> **KV cache type history on this build (`b166e269`):** +> - `iq5_k` is **not** a KV-cache type — the cache whitelist is +> `f32/f16/bf16/q8_0/q4_0/q4_1/iq4_nl/q5_0/q5_1/q6_0/q8_KV`; `iq5_*` only +> exist as weight quants. `--cache-type-k iq5_k` throws `Invalid cache +> type`. +> - `q8_KV` **crashes** with a `ggml view` size assert +> (`ggml.c:5427: data_size + view_offs > ggml_nbytes(view_src)`) on this +> QWEN3MOE/MTP + multimodal arch — independent of PARALLEL/speculation. +> - **`q6_0` + Hadamard works** and is what this deployment uses. -These keep the 163840-slot / 491520-pool context while roughly halving KV -cache RAM vs f16. +At 160k single slot the q6_0 KV self is ~1270 MiB (K 634 + V 634), leaving +~2.2 GiB headroom on the 24 GiB card after the GPU-side experts. ### Split mode: none (single GPU) diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 7a65565..38dfd77 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -1,27 +1,27 @@ -# Tunables for frank-default-small (Ornith-1.5-35B-A3B CRACK Q6_K, MTP MoE, +# Tunables for frank-default-small (Ornith-1.5-35B-A3B CRACK Q8_0, MTP MoE, # Dankserver, RTX 3090). # # Runtime: ik_llama.cpp llama-server (ikawrakow ik_llama.cpp, built # 2026-08-26 from commit b166e269) on RTX 3090 24 GB; Ryzen 7 7700X; 62 GB # RAM. Previous runtime was upstream llama.cpp b10121. MoE expert weights # pinned to CPU with an explicit --override-tensor regex (not --n-cpu-moe). -# KV cache quantized to q8_0 with Hadamard transforms on both K and V. +# KV cache quantized to iq5_k with Hadamard transforms on both K and V. # Split mode none (single GPU; graph would require 2+ devices). # # Build flags (IK_FA note, see docs/LLM/ikllama-ornith-dankserver.md) must # ALWAYS include: -DGGML_CUDA_FA_ALL_QUANTS=ON and -DGGML_IQK_FA_ALL_QUANTS=ON # so the quantized + Hadamard KV cache combos compile into the CUDA/CPU FA # kernels. Replacing the HauhauCS Qwen3.6-35B-A3B model on 2026-08-24. -# Switched from CRACK Q8_0 to Q6_K on 2026-08-25. +# Switched to CRACK Q8_0 on 2026-08-26 (was Q6_K). # # WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The # ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP # incident). This box is NVIDIA/CUDA, but the rule keeps the env portable. -# Model files (Q6_K CRACK GGUF + f16 mmproj vision encoder + MTP head bundled +# Model files (Q8_0 CRACK GGUF + f16 mmproj vision encoder + MTP head bundled # in the GGUF). Dealign2026-08-24. MODEL_DIR=/home/dank/models/Ornith-1.5-35B-A3B-CRACK -MODEL_FILE=Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf +MODEL_FILE=Ornith-1.5-35B-A3B-CRACK-Q8_0.gguf MMPROJ_FILE=mmproj-Ornith-1.5-35B-A3B-f16.gguf # ik_llama.cpp llama-server (built 2026-08-26 from ikawrakow/ik_llama.cpp @ # b166e269; CUDA 13.2, -DGGML_CUDA_FA_ALL_QUANTS=ON -DGGML_IQK_FA_ALL_QUANTS=ON, @@ -43,19 +43,14 @@ ALIAS=frank-default-small BIND_ADDR=0.0.0.0 LISTEN_PORT=7770 -# 3 parallel slots x 163840 ctx each (262144 lowered by 96k per stream for -# cache and checkpoint headroom) = 491520 total context pool. The per-slot -# 163840 is advertised in models.json. -CTX_SIZE=491520 -PARALLEL=3 +# Single parallel slot with 160000 ctx. Note: with PARALLEL=1 speculative +# decoding would again be allowed by ik_llama; it stays disabled (SPEC_TYPE +# =none) unless a user asks to re-enable ngram-mod/MTP. +CTX_SIZE=160000 +PARALLEL=1 -# Speculative decoding: DISABLED. ik_llama limits speculative decoding -# (ngram-mod / MTP / etc.) to a single server slot (-np 1) and hard-exits if -# --parallel > 1 with a spec stage configured ("Speculative decoding is -# currently limited to a single server slot (-np 1)."). This service keeps -# 3 parallel slots (one per user), so SPEC_TYPE is disabled (none). It is a -# low-parameter MoE (3B active of 35B), so throughput is fine without -# speculation. MTP is also intentionally disabled. +# Speculative decoding: DISABLED. Spec is possible at PARALLEL=1 but stays +# off for the low-parameter MoE (3B active of 35B). MTP is also disabled. SPEC_TYPE=none # Expert offload via explicit tensor override (replaces --n-cpu-moe). The Q6_K @@ -76,14 +71,12 @@ OVERRIDE_BUFT=CPU # OVERRIDE_EXPERT below. N_GPU_LAYERS=999 -# KV cache on GPU, both K and V at q8_0 plus Hadamard transforms on both -# caches to preserve quality at 8-bit. NOTE: ik_llama's fast q8_KV type -# crashes at model init on this MTP/multimodal arch with a ggml_view size -# assert (ggml.c:5427) at commit b166e269; plain q8_0 works. Docs recommend -# --k-cache-hadamard for quantized KV below Q6_0; we apply --v-cache-hadamard -# to V as well. Flash-attention is on by default. -CACHE_TYPE_K=q8_0 -CACHE_TYPE_V=q8_0 +# KV cache on GPU at q6_0 (6-bit k-quant) with Hadamard transforms on both +# caches. q8_KV was also rejected (ggml.c:5427 view assert, arch+build +# independent of PARALLEL/speculation); q6_0 + Hadamard is the fallback we +# verified works. Flash-attention is on by default. +CACHE_TYPE_K=q6_0 +CACHE_TYPE_V=q6_0 K_CACHE_HADAMARD=1 V_CACHE_HADAMARD=1 diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 54cd11a..4362bf9 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -1,18 +1,15 @@ #!/usr/bin/env bash -# Serve dealignai/Ornith-1.5-35B-A3B-UNCENSORED (CRACK, Q6_K) as +# Serve dealignai/Ornith-1.5-35B-A3B-UNCENSORED (CRACK, Q8_0) as # frank-default-small via ik_llama.cpp on Dankserver (RTX 3090, MoE # experts on CPU). # # Runtime: ik_llama.cpp llama-server (ikawrakow ik_llama.cpp). The model is a # 35B-total / 3B-active MoE-dense hybrid with an MTP head bundled in the GGUF. -# Speculative decoding is disabled: ik_llama limits any spec stage to a single -# server slot (-np 1), and this service keeps 3 parallel slots. +# Single slot (PARALLEL=1, CTX 160000). Speculative decoding is disabled. # -# KV cache: q8_0 with Hadamard transforms on K and V (requires a build with +# KV cache: q6_0 with Hadamard transforms on K and V (requires a build with # GGML_CUDA_FA_ALL_QUANTS=ON + GGML_IQK_FA_ALL_QUANTS=ON; see the env file). -# The ik_llama fast q8_KV type crashes at init on this arch (ggml.c:5427); -# q8_0 is used instead. -# +# q8_KV crashes at init on this arch (ggml.c:5427); q6_0 is used instead.# # Split mode is none: Dankserver has a single RTX 3090 and ik_llama rejects # split-mode graph with fewer than 2 devices (hard load error). @@ -22,10 +19,8 @@ # final output-adjacent blocks stay on CUDA with all non-expert tensors, # without relying on MTP speculation. # -# Speculative decoding: disabled (none). ik_llama hard-exits if a spec stage -# is configured with --parallel > 1. MTP is also intentionally disabled. -# A low-parameter MoE (3B active) does not need speculation here. -# +# Speculative decoding: disabled (none). MTP is also intentionally disabled. +# A low-parameter MoE (3B active) does not need speculation here.# # Tunables come from frank-default-small.env unless FRANK_SMALL_ENV_FILE # overrides that path. set -euo pipefail @@ -41,7 +36,7 @@ fi # --- defaults (used when the env file does not set them) ----------------- : "${MODEL_DIR:=/home/dank/models}" -: "${MODEL_FILE:=Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf}" +: "${MODEL_FILE:=Ornith-1.5-35B-A3B-CRACK-Q8_0.gguf}" : "${MMPROJ_FILE:=mmproj-Ornith-1.5-35B-A3B-f16.gguf}" : "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}" : "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}" @@ -53,11 +48,11 @@ fi : "${ALIAS:=frank-default-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7770}" -: "${CTX_SIZE:=491520}" -: "${PARALLEL:=3}" +: "${CTX_SIZE:=160000}" +: "${PARALLEL:=1}" : "${SPEC_TYPE:=none}" -: "${CACHE_TYPE_K:=f16}" -: "${CACHE_TYPE_V:=f16}" +: "${CACHE_TYPE_K:=q6_0}" +: "${CACHE_TYPE_V:=q6_0}" : "${K_CACHE_HADAMARD:=0}" : "${V_CACHE_HADAMARD:=0}" : "${SPLIT_MODE:=none}" diff --git a/tests/frank_small_config.bats b/tests/frank_small_config.bats index 59056d4..9f5e2aa 100644 --- a/tests/frank_small_config.bats +++ b/tests/frank_small_config.bats @@ -9,15 +9,21 @@ SERVE_FILE="$REPO_ROOT/scripts/serve-frank-small.sh" UNIT_FILE="$REPO_ROOT/scripts/frank-default-small.service" TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" -@test "frank-default-small selects the Ornith Q6_K model" { - grep -q '^MODEL_FILE=Ornith-1\.5-35B-A3B-CRACK-Q6_K\.gguf$' "$ENV_FILE" - grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q6_K\.gguf' "$SERVE_FILE" +@test "frank-default-small selects the Ornith Q8_0 model" { + grep -q '^MODEL_FILE=Ornith-1\.5-35B-A3B-CRACK-Q8_0\.gguf$' "$ENV_FILE" + grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q8_0\.gguf' "$SERVE_FILE" } -@test "frank-default-small uses 163840 context per slot across three slots" { - grep -q '^CTX_SIZE=491520$' "$ENV_FILE" - grep -q '163840 ctx each' "$ENV_FILE" - grep -q '^PARALLEL=3$' "$ENV_FILE" +@test "frank-default-small uses a single 160000 context slot" { + grep -q '^CTX_SIZE=160000$' "$ENV_FILE" + grep -q '^PARALLEL=1$' "$ENV_FILE" +} + +@test "frank-default-small uses q6_0 KV cache with Hadamard" { + grep -q '^CACHE_TYPE_K=q6_0$' "$ENV_FILE" + grep -q '^CACHE_TYPE_V=q6_0$' "$ENV_FILE" + grep -q '^K_CACHE_HADAMARD=1$' "$ENV_FILE" + grep -q '^V_CACHE_HADAMARD=1$' "$ENV_FILE" } @test "frank-default-small keeps full-attention and final blocks on CUDA" { @@ -40,11 +46,9 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" [[ "$N_GPU_LAYERS" == 999 ]] } -@test "frank-default-small disables speculative decoding for 3 parallel slots" { - # ik_llama limits speculative decoding to -np 1 and hard-exits with - # --parallel > 1; this service keeps 3 parallel slots and runs SPEC_TYPE=none. +@test "frank-default-small disables speculative decoding (single slot)" { grep -q '^SPEC_TYPE=none$' "$ENV_FILE" - grep -q '^PARALLEL=3$' "$ENV_FILE" + grep -q '^PARALLEL=1$' "$ENV_FILE" ! grep -q '^SPEC_NGRAM_MOD_N_MIN=' "$ENV_FILE" ! grep -q '^SPEC_NGRAM_MOD_N_MAX=' "$ENV_FILE" ! grep -q '^SPEC_NGRAM_MOD_N_MATCH=' "$ENV_FILE"