From af6ffeeb643d5b03b9b214603aeb9e102de8da5a Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 25 Aug 2026 00:29:14 -0700 Subject: [PATCH] Switch frank-default-small to Ornith Q6_K --- docs/LLM/Dankserver System Specs.md | 6 +-- docs/LLM/dealign-ornith-crack.md | 62 ++++++++++++++++++----------- docs/LLM/deployment.md | 20 +++++----- docs/LLM/index.md | 4 +- scripts/frank-default-small.env | 32 ++++++++------- scripts/frank-default-small.service | 2 +- scripts/serve-frank-small.sh | 12 +++--- tests/frank_small_config.bats | 37 +++++++++++++++++ 8 files changed, 115 insertions(+), 60 deletions(-) create mode 100644 tests/frank_small_config.bats diff --git a/docs/LLM/Dankserver System Specs.md b/docs/LLM/Dankserver System Specs.md index 459890d..3cfeebb 100644 --- a/docs/LLM/Dankserver System Specs.md +++ b/docs/LLM/Dankserver System Specs.md @@ -60,12 +60,12 @@ Dankserver is an NVIDIA GPU server for vLLM inference. The host runs Ubuntu 26.0 ### Active Services -The active service on Dankserver is `frank-default-small.service`: the llama.cpp Ornith-1.5-35B-A3B CRACK (Q8_0) on port 7770, with MoE expert weights held on CPU via `--override-tensor`. The earlier PP worker (`frank-pp-worker.service`) was retired with the 2026-08-17 redo. See [[deployment]]. +The active service on Dankserver is `frank-default-small.service`: the llama.cpp Ornith-1.5-35B-A3B CRACK (Q6_K) on port 7770, with expert tensors from blocks 0 through 36 held on CPU via `--override-tensor`. The earlier PP worker (`frank-pp-worker.service`) was retired with the 2026-08-17 redo. See [[deployment]]. ## Hardware Relevance -- 24 GB VRAM is close to saturated with this config. At idle ~22.1 GB used, leaving ~2.0 GiB headroom at `--n-cpu-moe 36` (4 layers' experts on GPU) with `ubatch 3072` and a dual-f16 KV pool of 13680 MiB at 700416 cells (2026-08-14 refit; the same pool at 800768 cells OOM'd the load warmup). A llama-bench sweep ([[frank-default-small-benchmark]]) guided this state: prefill is the priority, so `ubatch 3072` stays (top prefill tier), and `ncmoe 36` was raised from 35 to fix a cuBLAS OOM-class crash during big prefills by freeing ~0.75 GiB peak VRAM at only -2% prefill. If it still crashes under load, add `GGML_CUDA_FORCE_MMQ=1`. If decode speed matters more, use `ncmoe 32 - ubatch 1024` (gen ~48.7 t/s). The tuning guidance applied to the retired `frank-default-small` service; the PP worker's allocations are fixed at load. -- 62 GB system RAM holds the ~29 GB of MoE expert weights with room to spare (~57 GB available after load). +- 24 GB VRAM hosts the Q6_K non-expert tensors and expert tensors from blocks 37 through 40. The new context pool contains 589824 f16 cells across three 196608-token slots. The configuration targets about 1.5 to 2 GiB of free VRAM for checkpoint overhead. Verify the actual headroom after each model restart with `nvidia-smi` and the service journal. +- 62 GB system RAM holds the ~22.76 GiB of CPU expert tensors with room to spare. - 127 GB swap is available for CPU KV cache paging if GPU KV offload is disabled. - Root disk has approximately 140 GB free. Store the GGUF models under `/home/dank/models` (about 32 GB total). diff --git a/docs/LLM/dealign-ornith-crack.md b/docs/LLM/dealign-ornith-crack.md index b3a30e3..75822b2 100644 --- a/docs/LLM/dealign-ornith-crack.md +++ b/docs/LLM/dealign-ornith-crack.md @@ -1,7 +1,7 @@ --- type: reference title: dealign Ornith-1.5-35B-A3B CRACK on Dankserver -description: dealignai/Ornith-1.5-35B-A3B-UNCENSORED (CRACK Q8_0) served by upstream llama.cpp on the RTX 3090. MoE model with the MTP head bundled; expert weights offloaded to CPU with an explicit --override-tensor. frank-default-small. +description: dealignai/Ornith-1.5-35B-A3B-UNCENSORED (CRACK Q6_K) served by upstream llama.cpp on the RTX 3090. MoE model with the MTP head bundled; selected expert weights offloaded to CPU with an explicit --override-tensor. frank-default-small. tags: - llamacpp - cuda @@ -14,12 +14,13 @@ timestamp: 2026-08-24 # dealign Ornith-1.5-35B-A3B CRACK on Dankserver -> **Active 2026-08-24.** `frank-default-small` serves this model. It -> replaced the HauhauCS Qwen3.6-35B-A3B Q6_K_P service on the same upstream -> llama.cpp binary (b10121). +> **Active 2026-08-25.** `frank-default-small` serves this model. It +> replaced the CRACK Q8_0 file on the same upstream llama.cpp binary (b10121). +> The Q6_K file was downloaded from the model repository and validated with +> `llama-gguf` before installation. `frank-default-small` (port 7770) serves -`Ornith-1.5-35B-A3B-CRACK-Q8_0.gguf` from `dealignai` on Hugging Face. It is +`Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf` from `dealignai` on Hugging Face. It is a **MoE** model (256 experts, 8 active) with a hybrid GatedDeltaNet (SSM) + attention architecture, abliterated (uncensored). The MTP speculative head is **bundled in the GGUF**, so `draft-mtp` auto-detects it from the target @@ -29,26 +30,36 @@ model. | File | Size | |------|------| -| `Ornith-1.5-35B-A3B-CRACK-Q8_0.gguf` | 37.2 GB | -| `mmproj-Ornith-1.5-35B-A3B-f16.gguf` (vision) | 0.90 GB | +| `Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf` | 27.2 GiB | +| `mmproj-Ornith-1.5-35B-A3B-f16.gguf` (vision) | 0.84 GiB | Store the files under `/home/dank/models/Ornith-1.5-35B-A3B-CRACK/`. -## Expert Offload — `--override-tensor`, not `--n-cpu-moe` +## Layer Audit and Expert Offload -The service pins the MoE expert weights to CPU with an **explicit tensor -override** rather than the layer-counter `--n-cpu-moe`. The override uses -the same internal regex that `--cpu-moe` / `--n-cpu-moe` expand: +The Q6_K file contains 41 transformer blocks and 753 tensors. Each block +contains 630 MiB of expert tensors. Blocks 3, 7, 11, 15, 19, 23, 27, 31, +35, and 39 contain full attention. Block 40 contains the bundled MTP head. + +The non-expert tensors total 2.02 GiB. The layer totals are 660.96 MiB for +ordinary SSM blocks, 655.81 MiB for full-attention blocks, and 662.40 MiB for +block 40. The expert portion is 630 MiB in every block. + +The service keeps all non-expert tensors on CUDA. It also keeps expert tensors +from blocks 37 through 40 on CUDA. Those blocks sit next to the output and +include the bundled MTP head. Expert tensors from blocks 0 through 36 move to +CPU through this explicit override: ``` -OVERRIDE_EXPERT=\.ffn_(up|down|gate|gate_up)_(ch|)exps +OVERRIDE_EXPERT='blk\.(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36)\.ffn_(up|down|gate|gate_up)_(ch|)exps' OVERRIDE_BUFT=CPU ``` -This is passed as `--override-tensor "$OVERRIDE_EXPERT=$OVERRIDE_BUFT"`. -Only the FFN expert tensors move to CPU; every other tensor stays on the -24 GB card up to `--gpu-layers`. To list the buffer type names on the host, -run a bogus override and read the error: +This places 3.08 GiB of expert weights on CUDA and 22.76 GiB on CPU. The +selection leaves the output-adjacent blocks and MTP path on CUDA while the +f16 context cache uses the remaining GPU budget. + +To list the buffer type names on the host, run a bogus override: ``` llama-server --model -c 64 -ot foo=BADTYPE @@ -72,22 +83,27 @@ proven best. `--spec-draft-n-max 4` is the knee. ``` llama-server \ - -m Ornith-1.5-35B-A3B-CRACK-Q8_0.gguf \ + -m Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf \ --mmproj mmproj-Ornith-1.5-35B-A3B-f16.gguf \ --chat-template-file chat-template.jinja \ --spec-type draft-mtp \ - --override-tensor "\.ffn_(up|down|gate|gate_up)_(ch|)exps=CPU" \ - -ngl all -c 688128 -np 3 \ + --override-tensor "blk\\.(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36)\\.ffn_(up|down|gate|gate_up)_(ch|)exps=CPU" \ + -ngl all -c 589824 -np 3 \ --cache-type-k f16 --cache-type-v f16 --no-mmap \ --temp 1.0 --top-p 0.95 --top-k 20 ``` ## Known Issues -1. The 37.2 GB Q8_0 file is large for the 62 GB RAM box; lower `PARALLEL` - or switch quant if VRAM or RAM is tight. -2. `--n-cpu-moe` remains supported; this service deliberately prefers the - explicit tensor override so only the expert tensors reach CPU. +1. The 27.2 GiB Q6_K file and 0.84 GiB vision projector fit the 62 GiB RAM + box with the selected expert offload. +2. The context pool contains 589824 cells. Three slots receive 196608 cells + each, which is 64K below the model context per stream. +3. `--n-cpu-moe` remains supported. This service prefers the explicit tensor + override because it preserves the final four expert blocks on CUDA. +4. The service uses the Qwen-Sharp template from + `peculiar-ragdoll/Qwen-Sharp-Chat-Templates` through + `scripts/chat-template.jinja`. ## See Also diff --git a/docs/LLM/deployment.md b/docs/LLM/deployment.md index 5a4c1de..783a6f2 100644 --- a/docs/LLM/deployment.md +++ b/docs/LLM/deployment.md @@ -17,7 +17,7 @@ repository to a running service. | Target | Host | Runtime | Service | Port | |--------|------|---------|---------|------| -| Dankserver | dank@dankserver | upstream llama.cpp (Ornith-1.5-35B-A3B-CRACK Q8_0, native MTP draft, MoE experts offloaded via --override-tensor) | frank-default-small | 7770 | +| Dankserver | dank@dankserver | upstream llama.cpp (Ornith-1.5-35B-A3B-CRACK Q6_K, native MTP draft, selected MoE experts offloaded via --override-tensor) | frank-default-small | 7770 | | Dankclaw | dank@dankclaw | CIRU ROCmFPX llama.cpp (Qwen3.8-27B-Uncensored-ROCmFP4 Q6 AGENT, standalone MTP Q4_0 head, 262K ctx / 1 slot) | frank-default-large | 7770 | ## Workflow @@ -122,17 +122,17 @@ curl -sf http://127.0.0.1:7770/health ### Dankserver — frank-default-small The active service on Dankserver is upstream llama.cpp (b10121) serving -`Ornith-1.5-35B-A3B-CRACK-Q8_0.gguf` under the alias `frank-default-small` +`Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf` under the alias `frank-default-small` on port 7770. It runs the stock `/home/dank/llama.cpp/build/bin/llama-server` -binary with the MoE expert weights pinned to CPU by an explicit -`--override-tensor` regex (the expert `\.ffn_(up|down|gate|gate_up)_(ch|)exps` -pattern, buffer type `CPU`) and 7 CPU threads, native `draft-mtp` -speculative decoding (bundled MoE MTP head, n_max 4), f16 KV, context -688128 (3 slots of ~229K each, 32k headroom per stream), batch 4096 / -ubatch 2048, `--no-mmap`. The -unit is `scripts/frank-default-small.service`. It launches through +binary with expert tensors from blocks 0 through 36 pinned to CPU by an +explicit `--override-tensor` regex. Expert tensors from blocks 37 through 40 +stay on CUDA. The service uses 7 CPU threads, native `draft-mtp` +speculative decoding, f16 KV, context 589824 (3 slots of 196608 each), batch +4096, ubatch 2048, and `--no-mmap`. The unit is +`scripts/frank-default-small.service`. It launches through `scripts/serve-frank-small.sh`, which reads tunables from -`scripts/frank-default-small.env`. See [[dealign-ornith-crack]]. +`scripts/frank-default-small.env`. The chat template comes from +`peculiar-ragdoll/Qwen-Sharp-Chat-Templates`. See [[dealign-ornith-crack]]. ### Dankclaw — frank-default-large diff --git a/docs/LLM/index.md b/docs/LLM/index.md index 84df901..30b64e5 100644 --- a/docs/LLM/index.md +++ b/docs/LLM/index.md @@ -19,8 +19,8 @@ graph LR A[RTX 3090 24GB + 62GB RAM] B[upstream llama.cpp b10121 frank-default-small, port 7770] - C[Ornith-1.5-35B-A3B-CRACK Q8_0 -native MTP, MoE experts offloaded via override, 688128 ctx] + C[Ornith-1.5-35B-A3B-CRACK Q6_K +native MTP, selected experts offloaded via override, 589824 ctx] A --> B --> C end diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index c8b3920..14a9c08 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -1,34 +1,37 @@ -# Tunables for frank-default-small (Ornith-1.5-35B-A3B CRACK Q8_0, MTP MoE, +# Tunables for frank-default-small (Ornith-1.5-35B-A3B CRACK Q6_K, MTP MoE, # Dankserver, RTX 3090). # # Runtime: upstream llama.cpp llama-server (b10121). RTX 3090 24 GB; Ryzen # 7 7700X; 62 GB RAM. MoE expert weights pinned to CPU with an explicit # --override-tensor regex (not --n-cpu-moe). Replaced the HauhauCS -# Qwen3.6-35B-A3B model on 2026-08-24. +# Qwen3.6-35B-A3B model on 2026-08-24. Switched from CRACK Q8_0 to Q6_K on +# 2026-08-25. # # WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The # ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP # incident). This box is NVIDIA/CUDA, but the rule keeps the env portable. -# Model files (Q8_0 CRACK GGUF + f16 mmproj vision encoder + MTP head bundled +# Model files (Q6_K CRACK GGUF + f16 mmproj vision encoder + MTP head bundled # in the GGUF). Dealign2026-08-24. MODEL_DIR=/home/dank/models/Ornith-1.5-35B-A3B-CRACK -MODEL_FILE=Ornith-1.5-35B-A3B-CRACK-Q8_0.gguf +MODEL_FILE=Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf MMPROJ_FILE=mmproj-Ornith-1.5-35B-A3B-f16.gguf # Upstream llama.cpp binary (b10121). LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server -# Qwen-Sharp chat template (tracked in the repo, staged on the host). +# Qwen-Sharp chat template from peculiar-ragdoll (tracked in the repo, +# staged on the host). +# Source: https://huggingface.co/peculiar-ragdoll/Qwen-Sharp-Chat-Templates CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja ALIAS=frank-default-small BIND_ADDR=0.0.0.0 LISTEN_PORT=7770 -# 3 parallel slots x 229376 ctx each (262144 lowered by 32k per stream for -# cache headroom) = 688128 total context pool. The per-slot 229376 matches -# the per-slot context advertised in models.json (229376). -CTX_SIZE=688128 +# 3 parallel slots x 196608 ctx each (262144 lowered by 64k per stream for +# cache and checkpoint headroom) = 589824 total context pool. The per-slot +# 196608 matches the per-slot context advertised in models.json (196608). +CTX_SIZE=589824 PARALLEL=3 # Spec: MoE model with a bundled MTP head -> native draft-mtp, n_max 4 @@ -38,11 +41,12 @@ SPEC_DRAFT_N_MAX=4 SPEC_DRAFT_N_MIN=0 SPEC_DRAFT_P_MIN=0.0 -# Expert offload via explicit tensor override (replaces --n-cpu-moe). Only -# the MoE FFN exps tensors move to CPU; everything else up to --gpu-layers -# stays on the 24 GB card. The regex is the same one the built-in --cpu-moe -# expands. Buffer type "CPU" is the CPU buft name on this host. -OVERRIDE_EXPERT='\.ffn_(up|down|gate|gate_up)_(ch|)exps' +# Expert offload via explicit tensor override (replaces --n-cpu-moe). The Q6_K +# layer audit found 630 MiB of expert tensors per block. Keep the four +# output-adjacent blocks 37-40 on CUDA, including the bundled MTP block 40. +# Move expert tensors from blocks 0-36 to CPU. All non-expert tensors stay on +# the 24 GB card. Buffer type "CPU" is the CPU buft name on this host. +OVERRIDE_EXPERT='blk\.(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36)\.ffn_(up|down|gate|gate_up)_(ch|)exps' OVERRIDE_BUFT=CPU N_GPU_LAYERS=all diff --git a/scripts/frank-default-small.service b/scripts/frank-default-small.service index ab74e48..064adf3 100644 --- a/scripts/frank-default-small.service +++ b/scripts/frank-default-small.service @@ -1,5 +1,5 @@ [Unit] -Description=llama.cpp Ornith-1.5-35B-A3B CRACK Q8_0 MTP + Sharp template (dankserver, frank-default-small) +Description=llama.cpp Ornith-1.5-35B-A3B CRACK Q6_K MTP + Qwen-Sharp template (dankserver, frank-default-small) After=network-online.target Wants=network-online.target diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 66b7b8d..bc65dad 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -1,16 +1,14 @@ #!/usr/bin/env bash -# Serve dealignai/Ornith-1.5-35B-A3B-UNCENSORED (CRACK, Q8_0) as +# Serve dealignai/Ornith-1.5-35B-A3B-UNCENSORED (CRACK, Q6_K) as # frank-default-small via upstream llama.cpp on Dankserver (RTX 3090, MoE # experts on CPU). # # Runtime: upstream llama.cpp llama-server (b10121). The model is a 35B-total / # 3B-active MoE-dense hybrid with the MTP speculative head bundled in the GGUF. # -# Expert offload: instead of the layer-counter --n-cpu-moe, the expert (FFN -# exps) tensors are pinned to CPU explicitly with --override-tensor. The -# pattern is the internal MoE exps regex (the same one --n-cpu-moe expands), -# so only the Mixture-of-Experts weight tensors move to CPU; every other -# tensor stays on the GPU up to --gpu-layers. +# Expert offload: the expert (FFN exps) tensors selected by the layer-bounded +# --override-tensor pattern are pinned to CPU. The final output-adjacent +# blocks stay on CUDA with all non-expert tensors, including the MTP head. # # Spec: MoE model with a bundled MTP head -> native draft-mtp (the head is # auto-detected from the target model; b10121 creates the MTP draft context @@ -32,7 +30,7 @@ fi # --- defaults (used when the env file does not set them) ----------------- : "${MODEL_DIR:=/home/dank/models}" -: "${MODEL_FILE:=Ornith-1.5-35B-A3B-CRACK-Q8_0.gguf}" +: "${MODEL_FILE:=Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf}" : "${MMPROJ_FILE:=mmproj-Ornith-1.5-35B-A3B-f16.gguf}" : "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}" : "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}" diff --git a/tests/frank_small_config.bats b/tests/frank_small_config.bats new file mode 100644 index 0000000..ec7aae4 --- /dev/null +++ b/tests/frank_small_config.bats @@ -0,0 +1,37 @@ +#!/usr/bin/env bats +# Seam: frank-default-small model, context, tensor placement, and template. +# This test covers only the Q6_K deployment contract. + +load helpers + +ENV_FILE="$REPO_ROOT/scripts/frank-default-small.env" +SERVE_FILE="$REPO_ROOT/scripts/serve-frank-small.sh" +UNIT_FILE="$REPO_ROOT/scripts/frank-default-small.service" +TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" + +@test "frank-default-small selects the Ornith Q6_K model" { + grep -q '^MODEL_FILE=Ornith-1\.5-35B-A3B-CRACK-Q6_K\.gguf$' "$ENV_FILE" + grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q6_K\.gguf' "$SERVE_FILE" +} + +@test "frank-default-small uses 196608 context per slot across three slots" { + grep -q '^CTX_SIZE=589824$' "$ENV_FILE" + grep -q '196608 ctx each' "$ENV_FILE" + grep -q '^PARALLEL=3$' "$ENV_FILE" +} + +@test "frank-default-small keeps final four expert blocks on CUDA" { + grep -q '^OVERRIDE_EXPERT=' "$ENV_FILE" + grep -Fq '35|36)\.ffn_' "$ENV_FILE" + grep -q '^OVERRIDE_BUFT=CPU$' "$ENV_FILE" + grep -q '^N_GPU_LAYERS=all$' "$ENV_FILE" +} + +@test "frank-default-small uses the tracked Qwen-Sharp template" { + grep -q '^CHAT_TEMPLATE_FILE=' "$ENV_FILE" + grep -q 'peculiar-ragdoll/Qwen-Sharp-Chat-Templates' "$ENV_FILE" + grep -q 'chat-template\.jinja' "$SERVE_FILE" + [ -s "$TEMPLATE_FILE" ] + grep -q 'qwen3\.8-froggeric-v22\.4\.0' "$TEMPLATE_FILE" + grep -q 'Qwen-Sharp' "$UNIT_FILE" +}