frank-zealot: cutover to Ternary-Bonsai-2 PQ2_0 with q8_0 KV
- env: Bonsai2 payload pair, CACHE_TYPE_K/V=q8_0 (f16 OOMs at 2x200,704: 34 KiB vs 64 KiB/token over the 16 full-attention layers), measured boot ledger 22,664 used / 1,461 free incl. the 873 MiB mmproj that lands AFTER the engine fit check, q4_0 fallback rung, CRACK one-line MODEL_FILE swap - deployer: runtime/model/mmproj gates re-keyed to the PrismML vendor build (the only loader for GGML tensor type 142 + prism.hadamard.* metadata) - seam: payload trio pinned; KV argv assertions now derived from CACHE_TYPE_* in the env file — the wrong-var-name f16 fallback that produced the first failed trial window can no longer pass silently
This commit is contained in:
@@ -1,13 +1,14 @@
|
||||
# Seam: the frank-zealot llama.cpp root unit on dankclaw.
|
||||
# frank-zealot is the RTX 3090 endpoint; since 2026-10-07 it serves
|
||||
# HauhauCS's Qwen3.5-9B-Uncensored-HauhauCS-Aggressive (Q6_K, qwen35 hybrid:
|
||||
# 32 layers, 8 full-attention + 24 Gated DeltaNet, no MTP tensors in the file)
|
||||
# via the CUDA (sm_86) build of llama.cpp master: TWO 200,704-token
|
||||
# conversations (--ctx-size 401408 --parallel 2), exact f16 K + f16 V under
|
||||
# flash attention, the publisher's own BF16 vision projector, and MTP off —
|
||||
# the file carries no nextn/mtp tensors, so there is no in-file head to draft
|
||||
# with. Log verbosity 4 ships so the VRAM-fit forensics (KV layout, buffer
|
||||
# sizes, graph capture) land in the journal.
|
||||
# frank-zealot is the RTX 3090 endpoint; since 2026-10-17 it serves
|
||||
# prism-ml/Ternary-Bonsai-2-27B-gguf at PQ2_0 — the ternary Bonsai-2-27B, a
|
||||
# qwen35 hybrid with 64 blocks (16 full-attention + 48 Gated DeltaNet, no
|
||||
# nextn/mtp tensors in the file) — via the PrismML fork build of llama.cpp
|
||||
# (vendor build 10754), the ONLY loader that registers the payload's custom
|
||||
# tensor type 142 and honours its prism.hadamard.* metadata: TWO 200,704-token
|
||||
# conversations (--ctx-size 401408 --parallel 2), q8_0 K + q8_0 V under flash
|
||||
# attention (f16 KV does not fit this card at this pool — the boot dies on a
|
||||
# 25,088 MiB cudaMalloc), the publisher's own Q8_0 vision projector, and MTP
|
||||
# off. Log verbosity 4 ships so the VRAM-fit forensics land in the journal.
|
||||
# This seam runs the serve script against a stub engine and asserts on the
|
||||
# argv it actually execs, so what is pinned is the command the operator gets.
|
||||
|
||||
@@ -95,10 +96,15 @@ run_stage() {
|
||||
grep -q 'FRANK_ZEALOT_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-zealot.env' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the shipped env points at the llama.cpp runtime and the Qwen3.5-9B Q6_K GGUF" {
|
||||
# The payload path IS the endpoint's identity.
|
||||
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE"
|
||||
grep -qE '^MODEL_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3\.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K\.gguf$' "$ENV_FILE"
|
||||
@test "the shipped env points at the fork runtime and the Bonsai-2 PQ2_0 pair" {
|
||||
# The payload path IS the endpoint's identity, and the runtime is part of
|
||||
# it here: the file's custom tensor type 142 + prism.hadamard.* metadata
|
||||
# make the PrismML build the only loader, so a retarget that moves the
|
||||
# model but leaves the master build behind ships an endpoint that cannot
|
||||
# load at all.
|
||||
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-prism/llama-server$' "$ENV_FILE"
|
||||
grep -qE '^MODEL_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-PQ2_0\.gguf$' "$ENV_FILE"
|
||||
grep -qE '^MMPROJ_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-mmproj-Q8_0\.gguf$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "the deployer gates on exactly the payload paths the env file ships" {
|
||||
@@ -135,19 +141,23 @@ run_stage() {
|
||||
[ $((pool / slots)) -eq 200704 ]
|
||||
}
|
||||
|
||||
@test "the KV contract is exact f16 K + f16 V, flash-attn on" {
|
||||
@test "the KV dtype pair follows the env file and flash-attn stays pinned on" {
|
||||
# Asserted against the env rather than a literal so a retune moves one
|
||||
# file, not two. The shipped pair is q8_0/q8_0: this payload bills
|
||||
# 16 full-attention layers × 4 KV heads × 256 dim = 16,384 elements per
|
||||
# token per cache, so f16/f16 would need 25,088 MiB of the 24,123 MiB card
|
||||
# (measured: cudaMalloc OOM, engine fit 33,198 vs 23,558 free) while q8_0
|
||||
# bills 34,816 B/token = 13,328.00 MiB at the 401,408 pool. The boot
|
||||
# ledger, not this file, is the referee. flash-attn is not decoration:
|
||||
# quantized V is only legal with it, and it must stay explicit so a
|
||||
# future quant swap cannot silently fall back.
|
||||
local env_k env_v
|
||||
env_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")"
|
||||
env_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")"
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
# Frank, 2026-10-07: the 9B rung funds unquantized KV (the 26B MoE era
|
||||
# shipped q8_0 V because the card was full). qwen35 bills the pool on the
|
||||
# 8 full-attention layers only (8 × 4 KV heads × 256 dim = 8,192
|
||||
# elements/token ⇒ 32 KiB/token at f16/f16, 12.25 GiB at the 401,408
|
||||
# pool); the 24 DeltaNet layers cost a fixed recurrent state per slot.
|
||||
# The boot ledger, not this file, is the referee. flash-attn stays pinned:
|
||||
# any quantized-V fallback (q8_0 = this build's FP8 class) requires it, and
|
||||
# the build carries GGML_CUDA_FA_ALL_QUANTS=ON.
|
||||
[ "$(argv_value --cache-type-k)" = "f16" ]
|
||||
[ "$(argv_value --cache-type-v)" = "f16" ]
|
||||
[ "$(argv_value --cache-type-k)" = "$env_k" ]
|
||||
[ "$(argv_value --cache-type-v)" = "$env_v" ]
|
||||
[ "$(argv_value --flash-attn)" = "on" ]
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user