diff --git a/AGENTS.md b/AGENTS.md index 3e0a6a5..054bd07 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,7 @@ merge absorbed the former `vllmScripts` repo, history preserved). | Target | Service | Runtime | Model | Port | |--------|---------|---------|-------|------| -| Dankserver | `frank-primary` | unsloth llama.cpp b11030-mix prebuilt (ROCm gfx120X, bundled runtime, root systemd unit) | unsloth Qwen3.8-Flash-Next-GGUF Q_8 (175.30 GiB, 6 split parts) with the shared Q_8 MTP head (depth 2) chained behind `ngram-mod`, BF16 vision (mmproj), layer split `-ts 1,1` across both R9700s with the routed experts + 51B-param n-gram table computed in host RAM (`-ot exps=CPU,ngram_embeddings=CPU,per_layer_token_embd.weight=CPU` — the shared-head borrow path needs the last), 262144 ctx single stream (1 slot), mmap on, `--cache-ram 16384` | 7770 | +| Dankserver | `frank-primary` | unsloth llama.cpp b11030-mix prebuilt (ROCm gfx120X, bundled runtime, root systemd unit) | unsloth Qwen3.8-Flash-Next-GGUF Q_8 (175.30 GiB, 6 split parts) with the shared Q_8 MTP head (depth 2) chained behind `ngram-mod`, BF16 vision (mmproj), layer split `-ts 1,1` across both R9700s with the routed experts + 51B-param n-gram table computed in host RAM (`-ot exps=CPU,per_layer_token_embd.weight=CPU` — the shared-head borrow path needs the last), 262144 ctx single stream (1 slot), mmap on, `--cache-ram 16384` | 7770 | | Dankclaw | `frank-large` | unsloth llama.cpp b11030 prebuilt (ROCm gfx1151, root systemd unit) | unsloth Qwen3.8-27B-GGUF:UD-Q8_K_XL (27B dense, 31.5 GiB) with the z-lab DFlash2 Q4_K_M drafter (depth 4) on the gfx1151 iGPU (Strix Halo LPDDR, 114 GiB GTT), 262144 ctx single stream (1 slot), q8_0 KV, text only | 7770 | | Dankclaw | `frank-small` | mainline llama.cpp master 37b53fd (root systemd unit) | HauhauCS Gemma4-12B-QAT-Uncensored-Balanced Q4_K_M (BF16 mmproj + MTP assistant sidecar, `draft-mtp,ngram-mod`) on the RTX 3090, 1048576 pool / 4 slots of 262,144, q8_0 KV, the fleet's fastest vision endpoint | 7771 | @@ -355,7 +355,7 @@ old) on 2026-09-26, which had replaced radiance frank-medium the same day. **Placement: compute the experts in RAM, never stream them.** Layer split `-ts 1,1` with `-ngl 999`; the routed experts and the 51B-param n-gram (PLE) table are overridden to CPU (`-ot -exps=CPU,ngram_embeddings=CPU,per_layer_token_embd.weight=CPU`) and computed +exps=CPU,per_layer_token_embd.weight=CPU`) and computed from the mmap page cache — the x4-pair law (streaming over PCIe loses at 3.7 t/s; computing in RAM does not care). `per_layer_token_embd.weight=CPU` is REQUIRED by the shared MTP head's borrow path (a GPU-resident borrow diff --git a/scripts/dump-gguf.py b/scripts/dump-gguf.py new file mode 100755 index 0000000..7b8c500 --- /dev/null +++ b/scripts/dump-gguf.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Dump GGUF part headers: split metadata, tensor names/dims/types/bytes. + +Reads ONLY the header region (metadata + tensor-info table); tensor data is +never touched, so a 50 GiB part scans in milliseconds. Split GGUFs are valid +GGUFs per part; part 1 of this payload is metadata-only (ntensors=0) and +parts 2-6 carry the tensors. Confirmed layout for this payload: GGUF v3 — +magic, u32 version, u64 ntensors, u64 nkv, then kvs (v3: u64-length strings), +then tensor infos (v3: u64-length names). +""" +import struct, sys, traceback + +TYPE_SIZE = {0:4,1:2,2:1,3:1/2,4:4,5:8,6:2,7:1,8:0.4375,9:1.4375,10:1,11:0.5, + 12:2,13:4/3,14:2,15:2,16:4,17:2,18:2,19:1,20:2,21:1,22:2,23:2, + 24:2,25:1.0625,26:1,27:1.375,28:2,29:2,30:2,31:4,32:8,33:2.125, + 34:2.5,35:2,36:2.375,37:1.5625,38:2.3125,39:2.1875,40:0,41:1.75, + 42:1.75,43:1.8125,44:2,45:2.25,46:1.5,47:2.0625,48:2.625,49:1.0625, + 50:2.3125,51:2.875,52:2,53:2.3125,54:2.75,55:2.5,56:2.875,57:2.25, + 58:2,59:3.8125,60:2.8125,61:1.5625,62:1.5,63:0.5} + +class P: + def __init__(self, f): + self.f = f + self.ver = 2 + def rd(self, n): + b = self.f.read(n) + if len(b) != n: + raise EOFError(f"short read {len(b)}/{n}") + return b + def u8(self): return self.rd(1)[0] + def u32(self): return struct.unpack("= 3 else self.u32() + return self.rd(n).decode("utf-8", "replace") + def val(self, t): + # GGUF kv value types: 0 u8, 1 i8, 2 u16, 3 i16, 4 u32, 5 i32, + # 6 f32, 7 bool, 8 string, 9 array, 10 u64, 11 i64, 12 f64. + if t == 8: return self.s() + if t == 9: + et = self.u32(); n = self.u64() + return [self.val(et) for _ in range(n)] + if t == 0: return self.u8() + if t == 1: return struct.unpack("= pairs passed as repeated --override-tensor -# flags: -# * exps=CPU — the routed experts. ~100 GiB at Q8, read once per token; -# the CPU backend computes them from page cache. -# * the n-gram embedding block (the 51B-param PLE table) — sparse row -# lookups, tiny PCIe traffic from CPU RAM; NEVER pin with a GPU target -# (the 23 -> 13.4 t/s regression). -# * per_layer_token_embd.weight=CPU — REQUIRED by the shared MTP head's -# borrow path: GPU-resident, it crashes the draft load with -# "invalid vector subscript" (llama.cpp PR #28243). -# The exact tensor pattern for the n-gram block was discovered at load time -# on 2026-09-26 (see the runbook); this list is the deployed set. +# flags. Both overrides are pinned by the header dump of parts 2-3 +# (2026-09-26, /home/dank/models/dump-gguf.py — GGUF v3, 512 experts +# top-10, qwen4exp.ple.ngram_size=3, heads_per_ngram=8): +# * exps=CPU — the routed experts (blk.*.ffn_*_exps.weight, ~100 GiB at +# Q8_0), read once per token; the CPU backend computes them from page +# cache. +# * per_layer_token_embd.weight=CPU — the 51.2B-element PLE n-gram table +# (dims [160, 320001536], Q8_0, 54.4 GiB — the whole of part 3): +# sparse row lookups, tiny PCIe traffic from CPU RAM. It doubles as +# the shared MTP head's borrow tensor — GPU-resident, the draft load +# crashes with "invalid vector subscript" (llama.cpp PR #28243) — so +# this override is REQUIRED, not just a placement choice. NEVER give +# it a GPU target (the 23 -> 13.4 t/s regression) and never --no-mmap +# (the table must stay page-cache-backed; anonymous memory +# cgroup-kills silently). NGL=999 -OT_OVERRIDES=exps=CPU,ngram_embeddings=CPU,per_layer_token_embd.weight=CPU +OT_OVERRIDES=exps=CPU,per_layer_token_embd.weight=CPU TENSOR_SPLIT=1,1 # The drafter lives on the first R9700 (the card the load log shows holding diff --git a/scripts/serve-frank-primary.sh b/scripts/serve-frank-primary.sh index 58ecfb2..6fd1f4d 100755 --- a/scripts/serve-frank-primary.sh +++ b/scripts/serve-frank-primary.sh @@ -50,7 +50,7 @@ fi : "${PARALLEL_SLOTS:=1}" : "${SPEC_DRAFT_N_MAX:=2}" : "${NGL:=999}" -: "${OT_OVERRIDES:=exps=CPU,ngram_embeddings=CPU,per_layer_token_embd.weight=CPU}" +: "${OT_OVERRIDES:=exps=CPU,per_layer_token_embd.weight=CPU}" : "${TENSOR_SPLIT:=1,1}" : "${SPEC_DRAFT_DEVICE:=ROCm0}" : "${BATCH_SIZE:=8192}" diff --git a/serverAppConfigs-docs/LLM/frank-primary-llamacpp-qwen38-flash-next-q8.md b/serverAppConfigs-docs/LLM/frank-primary-llamacpp-qwen38-flash-next-q8.md index eaab636..6e6aed7 100644 --- a/serverAppConfigs-docs/LLM/frank-primary-llamacpp-qwen38-flash-next-q8.md +++ b/serverAppConfigs-docs/LLM/frank-primary-llamacpp-qwen38-flash-next-q8.md @@ -41,13 +41,21 @@ Lucebox DS4.1 quadlet that held the port for hours earlier the same day speed. - **The placement law, not a preference:** the routed experts and the 51B-param n-gram (PLE) embedding table are **computed in host RAM** - (`-ot exps=CPU,ngram_embeddings=CPU`) and never streamed over PCIe. The + (`-ot exps=CPU,per_layer_token_embd.weight=CPU`) and never streamed over PCIe. The x4-pair measurement on the same model shape: streaming loses (3.7 t/s), computing in RAM does not care (20+ t/s). The dense weights, attention, projector and MTP head sit on the two cards (~29 GiB); the ~100 GiB of expert pages live in mmap page cache — `--no-mmap` is forbidden (it forces the table into anonymous memory; the cgroup kill is silent) and pinning the PLE table with a GPU target cost 23 → 13.4 t/s when tried. + The tensor names are verified, not guessed: the GGUF v3 header dump + (`dump-gguf.py`, run on the fetched parts 2026-09-26) shows the PLE + table is one tensor — `per_layer_token_embd.weight`, dims + `[160, 320001536]`, Q8_0, 54.4 GiB, exactly the whole of part 3 — and + the model is `qwen4exp` with 512 experts top-10 + (`qwen4exp.ple.ngram_size=3`, `heads_per_ngram=8`). The same tensor is + the MTP head's borrow target, so the override is required even if the + placement were optional. - **Two cards, layer split** (`-ts 1,1`): both R9700s are on CPU root ports with full PCIe atomics (the ProArt B850-Creator swap, 2026-09-20), but this deployment uses llama.cpp's layer split — diff --git a/serverAppConfigs-docs/LLM/index.md b/serverAppConfigs-docs/LLM/index.md index 3bd55ee..950e130 100644 --- a/serverAppConfigs-docs/LLM/index.md +++ b/serverAppConfigs-docs/LLM/index.md @@ -62,7 +62,7 @@ frank-small, port 7771] unsloth llama.cpp (b11030-mix prebuilt, ROCm gfx120X, bundled runtime) serving unsloth/Qwen3.8-Flash-Next-GGUF at Q8_0 across both R9700s — routed experts and the 51B-param n-gram table computed in host RAM - (`-ot exps=CPU,ngram_embeddings=CPU`), the shared Q8_0 MTP head + (`-ot exps=CPU,per_layer_token_embd.weight=CPU`), the shared Q8_0 MTP head (depth 2) chained behind ngram-mod, BF16 vision, one full 262,144-token conversation, `reasoning_content`, the placement law, the cutover and rollback diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats index a19a446..3864756 100755 --- a/tests/frank_primary_unit.bats +++ b/tests/frank_primary_unit.bats @@ -59,12 +59,12 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" # computing them in RAM does not care (20+ tok/s on the same model # shape). exps=CPU plus the n-gram embedding table on CPU; the PLE # table must NEVER carry a GPU target (the 23 -> 13.4 t/s regression). - grep -q '^OT_OVERRIDES=exps=CPU,ngram_embeddings=CPU,per_layer_token_embd.weight=CPU$' "$ENV_FILE" grep -q -- '--override-tensor "$ot"' "$SERVE" # ...and mmap must stay on: --no-mmap forces the 51B-param table into # anonymous memory and the cgroup kill is silent. ! grep -q -- '--no-mmap' "$SERVE" ! grep -q '^MMPROJ_ON_GPU' "$ENV_FILE" + grep -q '^OT_OVERRIDES=exps=CPU,per_layer_token_embd.weight=CPU$' "$ENV_FILE" } @test "the projector is explicit, never autoloaded" {