diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 4fe482c..33a8b11 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1222,23 +1222,26 @@ deploy_llm_dragoon() { wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200 } -# deploy_llm_zealot — frank-zealot is llama.cpp (master build, CUDA sm_86) on -# the RTX 3090 as a ROOT unit: the runtime is a native binary tree under -# ~/GitPackages/llama-master/build-cuda, so there is no container image and -# no generated unit to avoid. relink_unit points +# deploy_llm_zealot — frank-zealot is llama.cpp (PrismML fork build) on the +# RTX 3090 as a ROOT unit: the runtime is a native binary tree under +# ~/GitPackages/llama-prism (vendor release build 10754, commit 2459f68b5 — +# the ONLY loader that registers the payload's custom tensor type 142 and +# reads its prism.hadamard.* metadata), so there is no container image and no +# generated unit to avoid. relink_unit points # /etc/systemd/system/frank-zealot.service at the repo file and boot # persistence is that unit's [Install] section. # # It is a SEPARATE endpoint on port 7771; frank-dragoon keeps 7770. The 3090 is # not shared: frank-dragoon's engine runs on the gfx1151 iGPU. -LLM_ZEALOT_MODEL_FILE="${LLM_ZEALOT_MODEL_FILE:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf}" -LLM_ZEALOT_RUNTIME="${LLM_ZEALOT_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" -# The projector IS the vision capability here: the model is natively -# multimodal and the Q6_K file carries no vision tensors — the publisher ships -# its own BF16 projector in the same repo, so a deploy that starts without -# this file silently ships a text-only endpoint. Checked unconditionally, like -# frank-dragoon's. -LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf}" +LLM_ZEALOT_MODEL_FILE="${LLM_ZEALOT_MODEL_FILE:-/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-PQ2_0.gguf}" +LLM_ZEALOT_RUNTIME="${LLM_ZEALOT_RUNTIME:-/home/dank/GitPackages/llama-prism/llama-server}" +# The projector IS the vision capability here: the Bonsai-2 text file carries +# no vision tensors, so the publisher's own mmproj (Qwen3-VL tower, 27 blocks, +# 768 px, qwen3vl_merger, 629,246,976 B) is the whole image path — a deploy +# that starts without it silently ships a text-only endpoint. Checked +# unconditionally, like frank-dragoon's. Its 873.10 MiB worst-case VRAM lands +# AFTER the engine's fit check, so it is not inside the q8_0 budget. +LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-mmproj-Q8_0.gguf}" deploy_llm_zealot() { # Refuse rather than restart-loop: a unit whose runtime or payload is @@ -1246,15 +1249,15 @@ deploy_llm_zealot() { # endpoint dark with nothing in the log to say why. if [[ ! -x "$LLM_ZEALOT_RUNTIME" ]]; then echo "ERROR: llama.cpp runtime absent at $LLM_ZEALOT_RUNTIME" >&2 - echo "frank-zealot: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2 - echo "frank-zealot: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2 - echo "frank-zealot: && cmake --build build-cuda --target llama-server) first." >&2 + echo "frank-zealot: fetch the PrismML fork release build (build 10754," >&2 + echo "frank-zealot: commit 2459f68b5) into ~/GitPackages/llama-prism first —" >&2 + echo "frank-zealot: stock llama.cpp refuses the payload's custom tensor type 142." >&2 return 1 fi for f in "$LLM_ZEALOT_MODEL_FILE" "$LLM_ZEALOT_MMPROJ"; do if [[ ! -f "$f" ]]; then echo "ERROR: frank-zealot payload file absent at $f" >&2 - echo "frank-zealot: run scripts/download-frank-zealot.sh first;" >&2 + echo "frank-zealot: run scripts/download-frank-zealot-bonsai2.sh first;" >&2 echo "frank-zealot: refusing to start a unit that cannot load." >&2 return 1 fi diff --git a/scripts/frank-zealot.env b/scripts/frank-zealot.env index 32e22db..11e816b 100644 --- a/scripts/frank-zealot.env +++ b/scripts/frank-zealot.env @@ -1,59 +1,69 @@ -# frank-zealot — llama.cpp (master, CUDA sm_86) serving -# HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive at Q6_K on the RTX 3090, -# :7771. +# frank-zealot — llama.cpp (PrismML fork build, CUDA) serving +# prism-ml/Ternary-Bonsai-2-27B-gguf at PQ2_0 on the RTX 3090, :7771. # -# 2026-10-07 (Frank): swap from Gemma4-26B-A4B QAT Q4_K_M (one 200,704 slot) -# to Qwen3.5-9B Q6_K at TWO 200,704-token conversations, EXACT (f16) KV, -# MTP disabled (SPEC_TYPE=none), log verbosity 4. +# 2026-10-17 (Frank): swap from Qwen3.5-9B-Uncensored Q6_K to the Bonsai-2-27B +# ternary checkpoint at the SAME shape Frank asked for — TWO 200,704-token +# conversations — with q8_0 K + q8_0 V (f16 is arithmetically impossible here, +# see the KV law), MTP off (the file carries no nextn/mtp tensors), verbosity 4. # -# Architecture — read from the GGUF's own header (HTTP range read, not the -# model card): arch qwen35, 32 layers, dense 9B, full_attention_interval 4 -# ⇒ 8 full-attention layers (3,7,11,...,31) + 24 Gated DeltaNet linear layers, -# 16 attention heads / 4 KV heads × 256 dim, ssm conv_kernel 4 / state_size -# 128 / group_count 16 / time_step_rank 32 / inner_size 4096, native 262,144, -# vocab 248,320, file_type 18 (Q6_K). Tensor-type histogram: Q6_K×250 + -# F32×177 — standard types only, so the stock master build loads it (no -# custom-quant lock-in like the ROCmFPX era). NO nextn/mtp tensors exist, so -# in-file MTP is impossible here; SPEC_TYPE=none is forced by the payload. +# THE RUNTIME IS A LOCK, NOT A PREFERENCE. The file is general.file_type 141 +# with 402 tensors of the CUSTOM GGML type 142 (pq2_0) plus prism.hadamard.* +# metadata (normalized-Sylvester-Walsh transform, block 1024, 401 named +# weights, inverse on token_embd, gdn_v_grouped). Stock llama.cpp refuses type +# 142 at load; only the PrismML fork registers the type AND reads the Hadamard +# metadata. The vendor prebuilt build 10754 (commit 2459f68b5) is what every +# number in this file was measured on; sha256 12ae5baba033d8cd… +LLAMA_BIN=/home/dank/GitPackages/llama-prism/llama-server +MODEL_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-PQ2_0.gguf +MMPROJ_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-mmproj-Q8_0.gguf +# The CRACK re-export (dealignai/Bonsai-2-27B-Ternary-CRACK-GGUF, uncensored, +# already on disk at /home/dank/models/Bonsai2-CRACK/) is a ONE-LINE swap: the +# header census proves byte-identical geometry (851 tensors, same +# {pq2_0:402, f32:353, bf16:96} histogram, 64 blocks, vocab 248,320, same +# 8,952-char chat template) and the official file's sha256 is the Hub LFS oid. +# Bonsai-2 is the BASE (censored) Qwen3.8 family; if a compaction or subagent +# request ever comes back with a refusal, point MODEL_FILE at the CRACK file +# and restart — the VRAM ledger below does not move. # -# KV rate law: 8 full layers × 4 KV heads × 256 dim = 8,192 elements per -# token per cache. f16 K + f16 V = 32,768 B/token = 32 KiB/token, so the -# 401,408-token pool is 12.25 GiB. The 24 DeltaNet layers cost a fixed -# recurrent state per SLOT (not per token) — that is why two 200k slots fit -# where the 26B MoE fit only one. -# -# Feasibility comes from the predecessor's MEASURED boots, not a formula: -# MiMo-V2.6-Distill-Qwen-9B (identical qwen35 geometry, 7,793,710,624 B) at -# 2×200,704 f16 K + f16 V measured 21,276 used / 2,849 free at ubatch 1024 -# and 21,846 / 2,279 at ubatch 2048. This file is 415 MiB lighter in weights, -# its BF16 mmproj is 3 MiB heavier ⇒ projected ~21,430 used / ~2,700 free. -# The verbosity-4 boot ledger is the referee. -# -# Ubatch 2048 is the measured CUDA optimum (measured on the MiMo predecessor -# at this exact arch + shape: +4.6 % prefill at 8.9k depth, decode flat, cost -# 616 MiB). 1024 is the fallback rung if the boot lands under the 2 GiB floor. -LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server -MODEL_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf -MMPROJ_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf -# Served model id — the harness/family name (Protoss rename completed to the -# wire 2026-10-09: harness ids + homepage cards 2026-10-08, /v1/models aliases -# this day). The SERVICE, unit, scripts and this env file keep the -# frank-zealot name; only the alias on the wire changed. Consumers that -# key on the served id: OpenWebUI (model row, access_grant, api_configs -# model_ids) and the harness models.yml. +# Served model id — the harness/family name. The SERVICE, unit, scripts and +# this env file keep the frank-zealot name; only the alias on the wire is the +# model id. Consumers that key on the served id: OpenWebUI (model row, +# access_grant, api_configs model_ids) and the harness models.yml. ALIAS=frank-zealot BIND_ADDR=0.0.0.0 LISTEN_PORT=7771 # CTX_SIZE is the POOL llama.cpp divides across slots: 401,408 / 2 = 200,704 # per conversation. MUST stay divisible by PARALLEL (the seam enforces it) and # the harness contextWindow quotes the PER-SLOT number — a longer prompt is -# answered with a 400, never truncated. +# answered with a 400, never truncated. The model's native window is 262,144; +# 2 × 262,144 at q8_0 projects 25,758 MiB and does NOT fit this card (measured +# f16 analogue: 41,118). One 262,144 slot fits with 5,995 MiB free but +# serializes compaction, so the pair stays. CTX_SIZE=401408 PARALLEL=2 BATCH_SIZE=4096 UBATCH_SIZE=2048 -CACHE_TYPE_K=f16 -CACHE_TYPE_V=f16 +# KV law (from the GGUF header, then confirmed by the boot ledger): 16 +# full-attention layers (full_attention_interval 4 over 64 blocks) × 4 KV heads +# × 256 dim = 16,384 elements per token per cache. q8_0 bills 17,408 B per +# cache ⇒ 34,816 B/token = 34 KiB, so the 401,408 pool is 13,328.00 MiB — the +# ledger said exactly that. f16 would bill 25,088 MiB and the boot DIES: +# `cudaMalloc failed: out of memory` on the 25,088.00 MiB request, engine fit +# projection 33,198 vs 23,558 free. The 48 DeltaNet layers bill a fixed +# recurrent state per SLOT — ~150 MiB each (context 13,627 − KV 13,328 = 299 +# for two slots), which is why pool arithmetic alone under-prices the shape. +CACHE_TYPE_K=q8_0 +CACHE_TYPE_V=q8_0 +# Boot ledger (2026-10-17, this exact shape, nvidia-smi settled): +# 22,664 MiB used / 1,461 MiB free of 24,123 +# fit projection 21,439 = 6,539 model + 13,627 context + 1,272 compute +# the mtmd projector's 873.10 MiB worst-case is NOT inside that projection — +# it lands after the fit check, which is the whole difference between +# "2.6 GiB free on paper" and 1.4 GiB free in reality. Watch it. +# Fallback rung if the margin is ever eaten (desktop, co-tenant): q4_0 KV +# measured 16,392 used / 7,733 free at the same 2 × 200,704. +# 1024 is the ubatch fallback (−616 MiB, measured on the 9B predecessor at +# this card/arch; 2048 was the measured CUDA optimum, +4.6 % prefill). SPEC_TYPE=none NGL=99 # Log verbosity threshold (llama.cpp -lv). 4 = debug: KV layout, buffer sizes diff --git a/tests/frank_zealot_unit.bats b/tests/frank_zealot_unit.bats index a5e28be..f1a1a17 100755 --- a/tests/frank_zealot_unit.bats +++ b/tests/frank_zealot_unit.bats @@ -1,13 +1,14 @@ # Seam: the frank-zealot llama.cpp root unit on dankclaw. -# frank-zealot is the RTX 3090 endpoint; since 2026-10-07 it serves -# HauhauCS's Qwen3.5-9B-Uncensored-HauhauCS-Aggressive (Q6_K, qwen35 hybrid: -# 32 layers, 8 full-attention + 24 Gated DeltaNet, no MTP tensors in the file) -# via the CUDA (sm_86) build of llama.cpp master: TWO 200,704-token -# conversations (--ctx-size 401408 --parallel 2), exact f16 K + f16 V under -# flash attention, the publisher's own BF16 vision projector, and MTP off — -# the file carries no nextn/mtp tensors, so there is no in-file head to draft -# with. Log verbosity 4 ships so the VRAM-fit forensics (KV layout, buffer -# sizes, graph capture) land in the journal. +# frank-zealot is the RTX 3090 endpoint; since 2026-10-17 it serves +# prism-ml/Ternary-Bonsai-2-27B-gguf at PQ2_0 — the ternary Bonsai-2-27B, a +# qwen35 hybrid with 64 blocks (16 full-attention + 48 Gated DeltaNet, no +# nextn/mtp tensors in the file) — via the PrismML fork build of llama.cpp +# (vendor build 10754), the ONLY loader that registers the payload's custom +# tensor type 142 and honours its prism.hadamard.* metadata: TWO 200,704-token +# conversations (--ctx-size 401408 --parallel 2), q8_0 K + q8_0 V under flash +# attention (f16 KV does not fit this card at this pool — the boot dies on a +# 25,088 MiB cudaMalloc), the publisher's own Q8_0 vision projector, and MTP +# off. Log verbosity 4 ships so the VRAM-fit forensics land in the journal. # This seam runs the serve script against a stub engine and asserts on the # argv it actually execs, so what is pinned is the command the operator gets. @@ -95,10 +96,15 @@ run_stage() { grep -q 'FRANK_ZEALOT_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-zealot.env' "$UNIT" } -@test "the shipped env points at the llama.cpp runtime and the Qwen3.5-9B Q6_K GGUF" { - # The payload path IS the endpoint's identity. - grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE" - grep -qE '^MODEL_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3\.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K\.gguf$' "$ENV_FILE" +@test "the shipped env points at the fork runtime and the Bonsai-2 PQ2_0 pair" { + # The payload path IS the endpoint's identity, and the runtime is part of + # it here: the file's custom tensor type 142 + prism.hadamard.* metadata + # make the PrismML build the only loader, so a retarget that moves the + # model but leaves the master build behind ships an endpoint that cannot + # load at all. + grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-prism/llama-server$' "$ENV_FILE" + grep -qE '^MODEL_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-PQ2_0\.gguf$' "$ENV_FILE" + grep -qE '^MMPROJ_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-mmproj-Q8_0\.gguf$' "$ENV_FILE" } @test "the deployer gates on exactly the payload paths the env file ships" { @@ -135,19 +141,23 @@ run_stage() { [ $((pool / slots)) -eq 200704 ] } -@test "the KV contract is exact f16 K + f16 V, flash-attn on" { +@test "the KV dtype pair follows the env file and flash-attn stays pinned on" { + # Asserted against the env rather than a literal so a retune moves one + # file, not two. The shipped pair is q8_0/q8_0: this payload bills + # 16 full-attention layers × 4 KV heads × 256 dim = 16,384 elements per + # token per cache, so f16/f16 would need 25,088 MiB of the 24,123 MiB card + # (measured: cudaMalloc OOM, engine fit 33,198 vs 23,558 free) while q8_0 + # bills 34,816 B/token = 13,328.00 MiB at the 401,408 pool. The boot + # ledger, not this file, is the referee. flash-attn is not decoration: + # quantized V is only legal with it, and it must stay explicit so a + # future quant swap cannot silently fall back. + local env_k env_v + env_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")" + env_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")" serve_argv [ "$status" -eq 0 ] - # Frank, 2026-10-07: the 9B rung funds unquantized KV (the 26B MoE era - # shipped q8_0 V because the card was full). qwen35 bills the pool on the - # 8 full-attention layers only (8 × 4 KV heads × 256 dim = 8,192 - # elements/token ⇒ 32 KiB/token at f16/f16, 12.25 GiB at the 401,408 - # pool); the 24 DeltaNet layers cost a fixed recurrent state per slot. - # The boot ledger, not this file, is the referee. flash-attn stays pinned: - # any quantized-V fallback (q8_0 = this build's FP8 class) requires it, and - # the build carries GGML_CUDA_FA_ALL_QUANTS=ON. - [ "$(argv_value --cache-type-k)" = "f16" ] - [ "$(argv_value --cache-type-v)" = "f16" ] + [ "$(argv_value --cache-type-k)" = "$env_k" ] + [ "$(argv_value --cache-type-v)" = "$env_v" ] [ "$(argv_value --flash-attn)" = "on" ] }