diff --git a/docs/LLM/ikllama-ornith-dankserver.md b/docs/LLM/ikllama-ornith-dankserver.md index 5c99ce6..bca6a3c 100644 --- a/docs/LLM/ikllama-ornith-dankserver.md +++ b/docs/LLM/ikllama-ornith-dankserver.md @@ -1,7 +1,7 @@ --- type: reference title: ik_llama.cpp on Dankserver (Ornith Q6_K) -description: ikawrakow/ik_llama.cpp replaces upstream llama.cpp for frank-default-small on the RTX 3090. q8_0 KV cache with Hadamard (the fast q8_KV type crashes on this arch at this commit), split-mode none (graph requires >=2 GPUs), speculative decoding disabled (3 parallel slots), and explicit CPU-expert offload. Includes the mandatory CUDA/IQK FA_ALL_QUANTS build flags. +description: ikawrakow/ik_llama.cpp replaces upstream llama.cpp for frank-default-small on the RTX 3090. q8_0 KV cache with Hadamard (the fast q8_KV type crashes on this arch at this commit), split-mode none (graph requires >=2 GPUs), speculative decoding disabled (3 parallel slots), the Qwen-Sharp template rendered with --jinja, and explicit CPU-expert offload. Includes the mandatory CUDA/IQK FA_ALL_QUANTS build flags. tags: - ikllama - llamacpp @@ -144,17 +144,38 @@ cache RAM vs f16. > single-GPU host you must use `none` (or `layer`, which also needs >1). > Do not assume `--split-mode graph` degrades gracefully on one GPU. -### Expert offload (MoE on CPU) +### Expert offload (MoE: CPU with a partial GPU slice) -The expert-offload pattern is unchanged from upstream llama.cpp: the +The expert-offload pattern from upstream llama.cpp carries over: the `--override-tensor` regex pins the sparse expert (`ffn_*_exps`) tensors to the CPU buft, leaving every full-attention block plus blocks 35-40 on CUDA. +On 2026-08-26 we brought expert blocks **30, 32, 33, 34 back to CUDA** +(~2.5 GiB) to use the ~5.4 GiB GPU headroom for lower latency; the rest of +that CPU range stays on CPU. -- `OVERRIDE_EXPERT` — the layer-bounded regex (same as before the switch). +- `OVERRIDE_EXPERT` — the layer-bounded regex listing only the blocks whose + experts stay on CPU: `(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22| + 24|25|26|28|29)`. Blocks 30,32,33,34 (and the always-on full-attention + + 35-40 set) are left off the list so their experts load to CUDA. - `OVERRIDE_BUFT=CPU`. - `N_GPU_LAYERS=999` — ik_llama `-ngl` / `--gpu-layers` only accepts an integer (the literal `all` is rejected). 999 means "load as many tensor - layers as possible", while the expert override still pins the exps to CPU. + layers as possible", while the expert override still pins the listed exps + to CPU. + +To move more experts back to the GPU later, drop more block numbers from +`OVERRIDE_EXPERT` (each block is ~630 MiB; verify `nvidia-smi` headroom +first and keep ~2-3 GiB margin for KV/compute). + +### Qwen-Sharp chat template via --jinja + +`--chat-template-file` alone only stores the template text; ik_llama **does +not** render a custom template as real Jinja unless `--jinja` is also given +(`inputs.use_jinja ? common_chat_templates_apply_jinja : `). Without `--jinja` it warns it only supports "commonly used +templates" and silently uses built-in handlers, dropping the Qwen-Sharp +template's thinking / image / tool formatting. So the serve script passes +both `--chat-template-file` and `--jinja` (gated on `USE_JINJA`, set to `1`). ### Speculative decoding: disabled (3 parallel slots) diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index f982079..c5dba45 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -34,6 +34,10 @@ LLAMA_SERVER=/home/dank/ik_llama.cpp/build/bin/llama-server # staged on the host). # Source: https://huggingface.co/peculiar-ragdoll/Qwen-Sharp-Chat-Templates CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja +# Render the Qwen-Sharp template with the true Jinja engine (--jinja). Without +# it ik_llama falls back to built-in non-Jinja handlers and drops the custom +# template's thinking/image/tool formatting. +USE_JINJA=1 ALIAS=frank-default-small BIND_ADDR=0.0.0.0 @@ -60,7 +64,11 @@ SPEC_TYPE=none # 3,7,11,15,19,23,27,31,35,36,37,38,39,40. Move all other expert tensors to # CPU. All non-expert tensors stay on the 24 GB card. Buffer type "CPU" is the # CPU buft name on this host. -OVERRIDE_EXPERT='blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps' +# +# 2026-08-26: brought back expert blocks 30,32,33,34 to CUDA (~2.5 GiB) to +# use the ~5.4 GiB GPU headroom for lower latency; the remaining larger-CPU +# range below those stays on CPU. +OVERRIDE_EXPERT='blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29)\.ffn_(up|down|gate|gate_up)_(ch|)exps' OVERRIDE_BUFT=CPU # ik_llama --gpu-layers/-ngl does NOT accept the literal "all"; use 999 to # mean "load as many layers as possible". Expert exps are pinned to CPU by diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 942b8e6..54cd11a 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -45,6 +45,11 @@ fi : "${MMPROJ_FILE:=mmproj-Ornith-1.5-35B-A3B-f16.gguf}" : "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}" : "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}" +# Render the Qwen-Sharp template with the true Jinja engine. ik_llama only +# applies --chat-template-file as real Jinja when --jinja is passed; +# otherwise it falls back to built-in non-Jinja handlers and silently drops +# the custom template's thinking/image/tool formatting. +: "${USE_JINJA:=1}" : "${ALIAS:=frank-default-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7770}" @@ -92,6 +97,7 @@ exec "$LLAMA_SERVER" \ --model "$MODEL" \ --mmproj "$MMPROJ" \ --chat-template-file "$CHAT_TEMPLATE_FILE" \ + $([ "$USE_JINJA" = "1" ] && printf -- "--jinja") \ --spec-type "$SPEC_TYPE" \ --override-tensor "$OVERRIDE_EXPERT=$OVERRIDE_BUFT" \ --cache-type-k "$CACHE_TYPE_K" \ diff --git a/tests/frank_small_config.bats b/tests/frank_small_config.bats index 31022e3..18577a0 100644 --- a/tests/frank_small_config.bats +++ b/tests/frank_small_config.bats @@ -22,7 +22,7 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" @test "frank-default-small keeps full-attention and final blocks on CUDA" { source "$ENV_FILE" - [[ "$OVERRIDE_EXPERT" == 'blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps' ]] + [[ "$OVERRIDE_EXPERT" == 'blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29)\.ffn_(up|down|gate|gate_up)_(ch|)exps' ]] [[ "$OVERRIDE_EXPERT" != *'|3|'* ]] [[ "$OVERRIDE_EXPERT" != *'|7|'* ]] [[ "$OVERRIDE_EXPERT" != *'|11|'* ]] @@ -68,3 +68,10 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" grep -q 'qwen3\.8-froggeric-v22\.4\.0' "$TEMPLATE_FILE" grep -q 'Qwen-Sharp' "$UNIT_FILE" } + +@test "frank-default-small renders the custom template with --jinja" { + # Without --jinja, ik_llama only supports built-in non-Jinja handlers and + # silently drops the custom Qwen-Sharp template formatting. + grep -q '^USE_JINJA=1$' "$ENV_FILE" + grep -q -- '--jinja' "$SERVE_FILE" +}