diff --git a/docs/LLM/ikllama-ornith-dankserver.md b/docs/LLM/ikllama-ornith-dankserver.md index 2de8590..61b47f1 100644 --- a/docs/LLM/ikllama-ornith-dankserver.md +++ b/docs/LLM/ikllama-ornith-dankserver.md @@ -1,7 +1,7 @@ --- type: reference title: ik_llama.cpp on Dankserver (Ornith Q8_0) -description: ikawrakow/ik_llama.cpp replaces upstream llama.cpp for frank-default-small on the RTX 3090. Q8_0 model, KV K q8_0 / V q5_0 with Hadamard (iq5_k is not a KV-cache type in this build; q8_KV crashes on this arch), split-mode none, speculative decoding disabled, 3 parallel slots x 160k ctx, the Qwen-Sharp template rendered with --jinja, and explicit CPU-expert offload. Includes the mandatory CUDA/IQK FA_ALL_QUANTS build flags. +description: ikawrakow/ik_llama.cpp replaces upstream llama.cpp for frank-default-small on the RTX 3090. Q8_0 model, KV K default f16 / V q6_0 with Hadamard on V only (iq5_k is not a KV-cache type in this build; q8_KV crashes on this arch), split-mode none, speculative decoding disabled, 3 parallel slots x 163840 ctx, the Qwen-Sharp template rendered with --jinja, and explicit CPU-expert offload. Includes the mandatory CUDA/IQK FA_ALL_QUANTS build flags. tags: - ikllama - llamacpp @@ -105,16 +105,17 @@ Files: `scripts/serve-frank-small.sh`, `scripts/frank-default-small.env`, `scripts/frank-default-small.service` in the repo. The service just calls the serve script; the env file holds all tunables. -### KV cache: K q8_0 / V q5_0 + Hadamard +### KV cache: K default f16 / V q6_0 + Hadamard (V only) -- `CACHE_TYPE_K=q8_0`, `CACHE_TYPE_V=q5_0` — asymmetric KV precision: the - K cache at 8-bit, the V cache at 5-bit (passes `--cache-type-k q8_0 - --cache-type-v q5_0`). Keeping K higher-precision than V is a common - asymmetric budget split. -- `K_CACHE_HADAMARD=1`, `V_CACHE_HADAMARD=1` — `--k-cache-hadamard` and - `--v-cache-hadamard`. The Hadamard transform preserves KV quality at - quantized precision. Implemented for both CUDA (`ggml-cuda/hadamard.cu`) - and CPU (IQK) paths. Flash-attention is **on by default**. +- `CACHE_TYPE_K=f16` — K cache at the default full-precision f16 + (passes `--cache-type-k f16`). No Hadamard on K. +- `CACHE_TYPE_V=q6_0` — V cache at 6-bit k-quant + (passes `--cache-type-v q6_0`). +- `V_CACHE_HADAMARD=1` — `--v-cache-hadamard` on V only. The Hadamard + transform preserves V quality at 6-bit; K is full-precision f16 and does + not need it (`K_CACHE_HADAMARD=0`). Implemented for both CUDA + (`ggml-cuda/hadamard.cu`) and CPU (IQK) paths. Flash-attention is **on + by default**. > **KV cache type history on this build (`b166e269`):** > - `iq5_k` is **not** a KV-cache type — the cache whitelist is @@ -124,12 +125,13 @@ the serve script; the env file holds all tunables. > - `q8_KV` **crashes** with a `ggml view` size assert > (`ggml.c:5427: data_size + view_offs > ggml_nbytes(view_src)`) on this > QWEN3MOE/MTP + multimodal arch — independent of PARALLEL/speculation. -> - `q6_0` (both caches) boots and served, but with `PARALLEL=3` the KV -> tripled, so the K cache went back up to `q8_0` and V down to `q5_0` for -> the chosen K/V budget split. +> - The deployment settled on K at default **f16** (full precision, no +> Hadamard) with V at **q6_0 + Hadamard**. -With 3 parallel slots at 160k context, the KV cache is ~3x a single slot's. -The GPU-side expert set is reduced to fit (see the expert section). +With 3 parallel slots at 163840 context the KV cache is large; keeping K at +full precision f16 while quantizing only V (with Hadamard) is the chosen +budget split. The GPU-side expert set stays reduced to fit (see the expert +section). ### Split mode: none (single GPU) diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 193e535..89c6aca 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -73,13 +73,12 @@ OVERRIDE_BUFT=CPU # OVERRIDE_EXPERT below. N_GPU_LAYERS=999 -# KV cache on GPU: K at q8_0 (8-bit) and V at q5_0 (5-bit), each with -# Hadamard transforms. The V cache at lower precision than K is a common -# asymmetric choice; q8_0 worked (plain q8_KV still crashes here, ggml.c -# :5427). Flash-attention is on by default. -CACHE_TYPE_K=q8_0 -CACHE_TYPE_V=q5_0 -K_CACHE_HADAMARD=1 +# KV cache on GPU: K back at the default f16 (no Hadamard on K; the K +# cache is kept full-precision so it does not need the transform), and V at +# q6_0 (6-bit k-quant) with Hadamard. Flash-attention is on by default. +CACHE_TYPE_K=f16 +CACHE_TYPE_V=q6_0 +K_CACHE_HADAMARD=0 V_CACHE_HADAMARD=1 # Split mode: none. Dankserver has a single RTX 3090 and ik_llama requires diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 4a4cc45..03cf37d 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -8,9 +8,10 @@ # Three parallel slots (PARALLEL=3), each with 160000 ctx. Speculative decoding # is disabled (ik_llama limits spec to a single slot). # -# KV cache: K q8_0 + V q5_0 with Hadamard transforms (requires a build -# with GGML_CUDA_FA_ALL_QUANTS=ON + GGML_IQK_FA_ALL_QUANTS=ON; see the env -# file). q8_KV crashes at init on this arch (ggml.c:5427); q8_0/q5_0 work.# +# KV cache: K at the default f16 (no Hadamard), V at q6_0 with Hadamard. +# (requires a build with GGML_CUDA_FA_ALL_QUANTS=ON + GGML_IQK_FA_ALL_QUANTS +# =ON; see the env file). q8_KV crashes at init on this arch (ggml.c:5427); +# V uses q6_0 instead.# # Split mode is none: Dankserver has a single RTX 3090 and ik_llama rejects # split-mode graph with fewer than 2 devices (hard load error). @@ -52,10 +53,10 @@ fi : "${CTX_SIZE:=491520}" : "${PARALLEL:=3}" : "${SPEC_TYPE:=none}" -: "${CACHE_TYPE_K:=q8_0}" -: "${CACHE_TYPE_V:=q5_0}" +: "${CACHE_TYPE_K:=f16}" +: "${CACHE_TYPE_V:=q6_0}" : "${K_CACHE_HADAMARD:=0}" -: "${V_CACHE_HADAMARD:=0}" +: "${V_CACHE_HADAMARD:=1}" : "${SPLIT_MODE:=none}" : "${VERBOSITY:=4}" # Expert offload: --override-tensor regex pinned to the CPU buft. Same pattern diff --git a/tests/frank_small_config.bats b/tests/frank_small_config.bats index 875661f..70be16f 100644 --- a/tests/frank_small_config.bats +++ b/tests/frank_small_config.bats @@ -20,10 +20,11 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" grep -q '163840 per slot' "$ENV_FILE" } -@test "frank-default-small uses K q8_0 + V q5_0 KV cache with Hadamard" { - grep -q '^CACHE_TYPE_K=q8_0$' "$ENV_FILE" - grep -q '^CACHE_TYPE_V=q5_0$' "$ENV_FILE" - grep -q '^K_CACHE_HADAMARD=1$' "$ENV_FILE" +@test "frank-default-small uses default f16 K and q6_0 V KV cache" { + grep -q '^CACHE_TYPE_K=f16$' "$ENV_FILE" + grep -q '^CACHE_TYPE_V=q6_0$' "$ENV_FILE" + # K at default f16 has no Hadamard; V at q6_0 keeps Hadamard. + grep -q '^K_CACHE_HADAMARD=0$' "$ENV_FILE" grep -q '^V_CACHE_HADAMARD=1$' "$ENV_FILE" }