frank-zealot: cutover to Ternary-Bonsai-2 PQ2_0 with q8_0 KV
- env: Bonsai2 payload pair, CACHE_TYPE_K/V=q8_0 (f16 OOMs at 2x200,704: 34 KiB vs 64 KiB/token over the 16 full-attention layers), measured boot ledger 22,664 used / 1,461 free incl. the 873 MiB mmproj that lands AFTER the engine fit check, q4_0 fallback rung, CRACK one-line MODEL_FILE swap - deployer: runtime/model/mmproj gates re-keyed to the PrismML vendor build (the only loader for GGML tensor type 142 + prism.hadamard.* metadata) - seam: payload trio pinned; KV argv assertions now derived from CACHE_TYPE_* in the env file — the wrong-var-name f16 fallback that produced the first failed trial window can no longer pass silently
This commit is contained in:
Frank
+19
-16
@@ -1222,23 +1222,26 @@ deploy_llm_dragoon() {
|
||||
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
# deploy_llm_zealot — frank-zealot is llama.cpp (master build, CUDA sm_86) on
|
||||
# the RTX 3090 as a ROOT unit: the runtime is a native binary tree under
|
||||
# ~/GitPackages/llama-master/build-cuda, so there is no container image and
|
||||
# no generated unit to avoid. relink_unit points
|
||||
# deploy_llm_zealot — frank-zealot is llama.cpp (PrismML fork build) on the
|
||||
# RTX 3090 as a ROOT unit: the runtime is a native binary tree under
|
||||
# ~/GitPackages/llama-prism (vendor release build 10754, commit 2459f68b5 —
|
||||
# the ONLY loader that registers the payload's custom tensor type 142 and
|
||||
# reads its prism.hadamard.* metadata), so there is no container image and no
|
||||
# generated unit to avoid. relink_unit points
|
||||
# /etc/systemd/system/frank-zealot.service at the repo file and boot
|
||||
# persistence is that unit's [Install] section.
|
||||
#
|
||||
# It is a SEPARATE endpoint on port 7771; frank-dragoon keeps 7770. The 3090 is
|
||||
# not shared: frank-dragoon's engine runs on the gfx1151 iGPU.
|
||||
LLM_ZEALOT_MODEL_FILE="${LLM_ZEALOT_MODEL_FILE:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf}"
|
||||
LLM_ZEALOT_RUNTIME="${LLM_ZEALOT_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
# The projector IS the vision capability here: the model is natively
|
||||
# multimodal and the Q6_K file carries no vision tensors — the publisher ships
|
||||
# its own BF16 projector in the same repo, so a deploy that starts without
|
||||
# this file silently ships a text-only endpoint. Checked unconditionally, like
|
||||
# frank-dragoon's.
|
||||
LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf}"
|
||||
LLM_ZEALOT_MODEL_FILE="${LLM_ZEALOT_MODEL_FILE:-/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-PQ2_0.gguf}"
|
||||
LLM_ZEALOT_RUNTIME="${LLM_ZEALOT_RUNTIME:-/home/dank/GitPackages/llama-prism/llama-server}"
|
||||
# The projector IS the vision capability here: the Bonsai-2 text file carries
|
||||
# no vision tensors, so the publisher's own mmproj (Qwen3-VL tower, 27 blocks,
|
||||
# 768 px, qwen3vl_merger, 629,246,976 B) is the whole image path — a deploy
|
||||
# that starts without it silently ships a text-only endpoint. Checked
|
||||
# unconditionally, like frank-dragoon's. Its 873.10 MiB worst-case VRAM lands
|
||||
# AFTER the engine's fit check, so it is not inside the q8_0 budget.
|
||||
LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-mmproj-Q8_0.gguf}"
|
||||
|
||||
deploy_llm_zealot() {
|
||||
# Refuse rather than restart-loop: a unit whose runtime or payload is
|
||||
@@ -1246,15 +1249,15 @@ deploy_llm_zealot() {
|
||||
# endpoint dark with nothing in the log to say why.
|
||||
if [[ ! -x "$LLM_ZEALOT_RUNTIME" ]]; then
|
||||
echo "ERROR: llama.cpp runtime absent at $LLM_ZEALOT_RUNTIME" >&2
|
||||
echo "frank-zealot: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2
|
||||
echo "frank-zealot: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2
|
||||
echo "frank-zealot: && cmake --build build-cuda --target llama-server) first." >&2
|
||||
echo "frank-zealot: fetch the PrismML fork release build (build 10754," >&2
|
||||
echo "frank-zealot: commit 2459f68b5) into ~/GitPackages/llama-prism first —" >&2
|
||||
echo "frank-zealot: stock llama.cpp refuses the payload's custom tensor type 142." >&2
|
||||
return 1
|
||||
fi
|
||||
for f in "$LLM_ZEALOT_MODEL_FILE" "$LLM_ZEALOT_MMPROJ"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: frank-zealot payload file absent at $f" >&2
|
||||
echo "frank-zealot: run scripts/download-frank-zealot.sh first;" >&2
|
||||
echo "frank-zealot: run scripts/download-frank-zealot-bonsai2.sh first;" >&2
|
||||
echo "frank-zealot: refusing to start a unit that cannot load." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
+53
-43
@@ -1,59 +1,69 @@
|
||||
# frank-zealot — llama.cpp (master, CUDA sm_86) serving
|
||||
# HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive at Q6_K on the RTX 3090,
|
||||
# :7771.
|
||||
# frank-zealot — llama.cpp (PrismML fork build, CUDA) serving
|
||||
# prism-ml/Ternary-Bonsai-2-27B-gguf at PQ2_0 on the RTX 3090, :7771.
|
||||
#
|
||||
# 2026-10-07 (Frank): swap from Gemma4-26B-A4B QAT Q4_K_M (one 200,704 slot)
|
||||
# to Qwen3.5-9B Q6_K at TWO 200,704-token conversations, EXACT (f16) KV,
|
||||
# MTP disabled (SPEC_TYPE=none), log verbosity 4.
|
||||
# 2026-10-17 (Frank): swap from Qwen3.5-9B-Uncensored Q6_K to the Bonsai-2-27B
|
||||
# ternary checkpoint at the SAME shape Frank asked for — TWO 200,704-token
|
||||
# conversations — with q8_0 K + q8_0 V (f16 is arithmetically impossible here,
|
||||
# see the KV law), MTP off (the file carries no nextn/mtp tensors), verbosity 4.
|
||||
#
|
||||
# Architecture — read from the GGUF's own header (HTTP range read, not the
|
||||
# model card): arch qwen35, 32 layers, dense 9B, full_attention_interval 4
|
||||
# ⇒ 8 full-attention layers (3,7,11,...,31) + 24 Gated DeltaNet linear layers,
|
||||
# 16 attention heads / 4 KV heads × 256 dim, ssm conv_kernel 4 / state_size
|
||||
# 128 / group_count 16 / time_step_rank 32 / inner_size 4096, native 262,144,
|
||||
# vocab 248,320, file_type 18 (Q6_K). Tensor-type histogram: Q6_K×250 +
|
||||
# F32×177 — standard types only, so the stock master build loads it (no
|
||||
# custom-quant lock-in like the ROCmFPX era). NO nextn/mtp tensors exist, so
|
||||
# in-file MTP is impossible here; SPEC_TYPE=none is forced by the payload.
|
||||
# THE RUNTIME IS A LOCK, NOT A PREFERENCE. The file is general.file_type 141
|
||||
# with 402 tensors of the CUSTOM GGML type 142 (pq2_0) plus prism.hadamard.*
|
||||
# metadata (normalized-Sylvester-Walsh transform, block 1024, 401 named
|
||||
# weights, inverse on token_embd, gdn_v_grouped). Stock llama.cpp refuses type
|
||||
# 142 at load; only the PrismML fork registers the type AND reads the Hadamard
|
||||
# metadata. The vendor prebuilt build 10754 (commit 2459f68b5) is what every
|
||||
# number in this file was measured on; sha256 12ae5baba033d8cd…
|
||||
LLAMA_BIN=/home/dank/GitPackages/llama-prism/llama-server
|
||||
MODEL_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-PQ2_0.gguf
|
||||
MMPROJ_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-mmproj-Q8_0.gguf
|
||||
# The CRACK re-export (dealignai/Bonsai-2-27B-Ternary-CRACK-GGUF, uncensored,
|
||||
# already on disk at /home/dank/models/Bonsai2-CRACK/) is a ONE-LINE swap: the
|
||||
# header census proves byte-identical geometry (851 tensors, same
|
||||
# {pq2_0:402, f32:353, bf16:96} histogram, 64 blocks, vocab 248,320, same
|
||||
# 8,952-char chat template) and the official file's sha256 is the Hub LFS oid.
|
||||
# Bonsai-2 is the BASE (censored) Qwen3.8 family; if a compaction or subagent
|
||||
# request ever comes back with a refusal, point MODEL_FILE at the CRACK file
|
||||
# and restart — the VRAM ledger below does not move.
|
||||
#
|
||||
# KV rate law: 8 full layers × 4 KV heads × 256 dim = 8,192 elements per
|
||||
# token per cache. f16 K + f16 V = 32,768 B/token = 32 KiB/token, so the
|
||||
# 401,408-token pool is 12.25 GiB. The 24 DeltaNet layers cost a fixed
|
||||
# recurrent state per SLOT (not per token) — that is why two 200k slots fit
|
||||
# where the 26B MoE fit only one.
|
||||
#
|
||||
# Feasibility comes from the predecessor's MEASURED boots, not a formula:
|
||||
# MiMo-V2.6-Distill-Qwen-9B (identical qwen35 geometry, 7,793,710,624 B) at
|
||||
# 2×200,704 f16 K + f16 V measured 21,276 used / 2,849 free at ubatch 1024
|
||||
# and 21,846 / 2,279 at ubatch 2048. This file is 415 MiB lighter in weights,
|
||||
# its BF16 mmproj is 3 MiB heavier ⇒ projected ~21,430 used / ~2,700 free.
|
||||
# The verbosity-4 boot ledger is the referee.
|
||||
#
|
||||
# Ubatch 2048 is the measured CUDA optimum (measured on the MiMo predecessor
|
||||
# at this exact arch + shape: +4.6 % prefill at 8.9k depth, decode flat, cost
|
||||
# 616 MiB). 1024 is the fallback rung if the boot lands under the 2 GiB floor.
|
||||
LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server
|
||||
MODEL_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf
|
||||
MMPROJ_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
|
||||
# Served model id — the harness/family name (Protoss rename completed to the
|
||||
# wire 2026-10-09: harness ids + homepage cards 2026-10-08, /v1/models aliases
|
||||
# this day). The SERVICE, unit, scripts and this env file keep the
|
||||
# frank-zealot name; only the alias on the wire changed. Consumers that
|
||||
# key on the served id: OpenWebUI (model row, access_grant, api_configs
|
||||
# model_ids) and the harness models.yml.
|
||||
# Served model id — the harness/family name. The SERVICE, unit, scripts and
|
||||
# this env file keep the frank-zealot name; only the alias on the wire is the
|
||||
# model id. Consumers that key on the served id: OpenWebUI (model row,
|
||||
# access_grant, api_configs model_ids) and the harness models.yml.
|
||||
ALIAS=frank-zealot
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7771
|
||||
# CTX_SIZE is the POOL llama.cpp divides across slots: 401,408 / 2 = 200,704
|
||||
# per conversation. MUST stay divisible by PARALLEL (the seam enforces it) and
|
||||
# the harness contextWindow quotes the PER-SLOT number — a longer prompt is
|
||||
# answered with a 400, never truncated.
|
||||
# answered with a 400, never truncated. The model's native window is 262,144;
|
||||
# 2 × 262,144 at q8_0 projects 25,758 MiB and does NOT fit this card (measured
|
||||
# f16 analogue: 41,118). One 262,144 slot fits with 5,995 MiB free but
|
||||
# serializes compaction, so the pair stays.
|
||||
CTX_SIZE=401408
|
||||
PARALLEL=2
|
||||
BATCH_SIZE=4096
|
||||
UBATCH_SIZE=2048
|
||||
CACHE_TYPE_K=f16
|
||||
CACHE_TYPE_V=f16
|
||||
# KV law (from the GGUF header, then confirmed by the boot ledger): 16
|
||||
# full-attention layers (full_attention_interval 4 over 64 blocks) × 4 KV heads
|
||||
# × 256 dim = 16,384 elements per token per cache. q8_0 bills 17,408 B per
|
||||
# cache ⇒ 34,816 B/token = 34 KiB, so the 401,408 pool is 13,328.00 MiB — the
|
||||
# ledger said exactly that. f16 would bill 25,088 MiB and the boot DIES:
|
||||
# `cudaMalloc failed: out of memory` on the 25,088.00 MiB request, engine fit
|
||||
# projection 33,198 vs 23,558 free. The 48 DeltaNet layers bill a fixed
|
||||
# recurrent state per SLOT — ~150 MiB each (context 13,627 − KV 13,328 = 299
|
||||
# for two slots), which is why pool arithmetic alone under-prices the shape.
|
||||
CACHE_TYPE_K=q8_0
|
||||
CACHE_TYPE_V=q8_0
|
||||
# Boot ledger (2026-10-17, this exact shape, nvidia-smi settled):
|
||||
# 22,664 MiB used / 1,461 MiB free of 24,123
|
||||
# fit projection 21,439 = 6,539 model + 13,627 context + 1,272 compute
|
||||
# the mtmd projector's 873.10 MiB worst-case is NOT inside that projection —
|
||||
# it lands after the fit check, which is the whole difference between
|
||||
# "2.6 GiB free on paper" and 1.4 GiB free in reality. Watch it.
|
||||
# Fallback rung if the margin is ever eaten (desktop, co-tenant): q4_0 KV
|
||||
# measured 16,392 used / 7,733 free at the same 2 × 200,704.
|
||||
# 1024 is the ubatch fallback (−616 MiB, measured on the 9B predecessor at
|
||||
# this card/arch; 2048 was the measured CUDA optimum, +4.6 % prefill).
|
||||
SPEC_TYPE=none
|
||||
NGL=99
|
||||
# Log verbosity threshold (llama.cpp -lv). 4 = debug: KV layout, buffer sizes
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
# Seam: the frank-zealot llama.cpp root unit on dankclaw.
|
||||
# frank-zealot is the RTX 3090 endpoint; since 2026-10-07 it serves
|
||||
# HauhauCS's Qwen3.5-9B-Uncensored-HauhauCS-Aggressive (Q6_K, qwen35 hybrid:
|
||||
# 32 layers, 8 full-attention + 24 Gated DeltaNet, no MTP tensors in the file)
|
||||
# via the CUDA (sm_86) build of llama.cpp master: TWO 200,704-token
|
||||
# conversations (--ctx-size 401408 --parallel 2), exact f16 K + f16 V under
|
||||
# flash attention, the publisher's own BF16 vision projector, and MTP off —
|
||||
# the file carries no nextn/mtp tensors, so there is no in-file head to draft
|
||||
# with. Log verbosity 4 ships so the VRAM-fit forensics (KV layout, buffer
|
||||
# sizes, graph capture) land in the journal.
|
||||
# frank-zealot is the RTX 3090 endpoint; since 2026-10-17 it serves
|
||||
# prism-ml/Ternary-Bonsai-2-27B-gguf at PQ2_0 — the ternary Bonsai-2-27B, a
|
||||
# qwen35 hybrid with 64 blocks (16 full-attention + 48 Gated DeltaNet, no
|
||||
# nextn/mtp tensors in the file) — via the PrismML fork build of llama.cpp
|
||||
# (vendor build 10754), the ONLY loader that registers the payload's custom
|
||||
# tensor type 142 and honours its prism.hadamard.* metadata: TWO 200,704-token
|
||||
# conversations (--ctx-size 401408 --parallel 2), q8_0 K + q8_0 V under flash
|
||||
# attention (f16 KV does not fit this card at this pool — the boot dies on a
|
||||
# 25,088 MiB cudaMalloc), the publisher's own Q8_0 vision projector, and MTP
|
||||
# off. Log verbosity 4 ships so the VRAM-fit forensics land in the journal.
|
||||
# This seam runs the serve script against a stub engine and asserts on the
|
||||
# argv it actually execs, so what is pinned is the command the operator gets.
|
||||
|
||||
@@ -95,10 +96,15 @@ run_stage() {
|
||||
grep -q 'FRANK_ZEALOT_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-zealot.env' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the shipped env points at the llama.cpp runtime and the Qwen3.5-9B Q6_K GGUF" {
|
||||
# The payload path IS the endpoint's identity.
|
||||
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE"
|
||||
grep -qE '^MODEL_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3\.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K\.gguf$' "$ENV_FILE"
|
||||
@test "the shipped env points at the fork runtime and the Bonsai-2 PQ2_0 pair" {
|
||||
# The payload path IS the endpoint's identity, and the runtime is part of
|
||||
# it here: the file's custom tensor type 142 + prism.hadamard.* metadata
|
||||
# make the PrismML build the only loader, so a retarget that moves the
|
||||
# model but leaves the master build behind ships an endpoint that cannot
|
||||
# load at all.
|
||||
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-prism/llama-server$' "$ENV_FILE"
|
||||
grep -qE '^MODEL_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-PQ2_0\.gguf$' "$ENV_FILE"
|
||||
grep -qE '^MMPROJ_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-mmproj-Q8_0\.gguf$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "the deployer gates on exactly the payload paths the env file ships" {
|
||||
@@ -135,19 +141,23 @@ run_stage() {
|
||||
[ $((pool / slots)) -eq 200704 ]
|
||||
}
|
||||
|
||||
@test "the KV contract is exact f16 K + f16 V, flash-attn on" {
|
||||
@test "the KV dtype pair follows the env file and flash-attn stays pinned on" {
|
||||
# Asserted against the env rather than a literal so a retune moves one
|
||||
# file, not two. The shipped pair is q8_0/q8_0: this payload bills
|
||||
# 16 full-attention layers × 4 KV heads × 256 dim = 16,384 elements per
|
||||
# token per cache, so f16/f16 would need 25,088 MiB of the 24,123 MiB card
|
||||
# (measured: cudaMalloc OOM, engine fit 33,198 vs 23,558 free) while q8_0
|
||||
# bills 34,816 B/token = 13,328.00 MiB at the 401,408 pool. The boot
|
||||
# ledger, not this file, is the referee. flash-attn is not decoration:
|
||||
# quantized V is only legal with it, and it must stay explicit so a
|
||||
# future quant swap cannot silently fall back.
|
||||
local env_k env_v
|
||||
env_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")"
|
||||
env_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")"
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
# Frank, 2026-10-07: the 9B rung funds unquantized KV (the 26B MoE era
|
||||
# shipped q8_0 V because the card was full). qwen35 bills the pool on the
|
||||
# 8 full-attention layers only (8 × 4 KV heads × 256 dim = 8,192
|
||||
# elements/token ⇒ 32 KiB/token at f16/f16, 12.25 GiB at the 401,408
|
||||
# pool); the 24 DeltaNet layers cost a fixed recurrent state per slot.
|
||||
# The boot ledger, not this file, is the referee. flash-attn stays pinned:
|
||||
# any quantized-V fallback (q8_0 = this build's FP8 class) requires it, and
|
||||
# the build carries GGML_CUDA_FA_ALL_QUANTS=ON.
|
||||
[ "$(argv_value --cache-type-k)" = "f16" ]
|
||||
[ "$(argv_value --cache-type-v)" = "f16" ]
|
||||
[ "$(argv_value --cache-type-k)" = "$env_k" ]
|
||||
[ "$(argv_value --cache-type-v)" = "$env_v" ]
|
||||
[ "$(argv_value --flash-attn)" = "on" ]
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user