diff --git a/scripts/frank-zealot.service b/scripts/frank-zealot.service index b74d7df..7841321 100644 --- a/scripts/frank-zealot.service +++ b/scripts/frank-zealot.service @@ -1,5 +1,5 @@ [Unit] -Description=llama.cpp Qwen3.5-9B-Uncensored-HauhauCS-Aggressive Q6_K GGUF (qwen35 hybrid, 2x200704 slots, f16 K+V, MTP off, mmproj vision) on the RTX 3090 (dankclaw, frank-zealot) +Description=llama.cpp (PrismML build 10754) Ternary-Bonsai-2-27B PQ2_0 GGUF (qwen35 hybrid, 2x200704 slots, q8_0 K+V, MTP off, mmproj vision) on the RTX 3090 (dankclaw, frank-zealot) After=network-online.target Wants=network-online.target diff --git a/scripts/serve-frank-zealot.sh b/scripts/serve-frank-zealot.sh index f186f4d..18024b4 100755 --- a/scripts/serve-frank-zealot.sh +++ b/scripts/serve-frank-zealot.sh @@ -1,28 +1,30 @@ #!/usr/bin/env bash -# Serve HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive (Q6_K) as -# frank-zealot on Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86). +# Serve prism-ml/Ternary-Bonsai-2-27B (PQ2_0 ternary quant) as frank-zealot on +# Dankclaw's RTX 3090 via llama.cpp (PrismML vendor build 10754, CUDA sm_86). # -# 2026-10-07 (Frank): Qwen3.5-9B swap at TWO 200,704-token conversations -# (--ctx-size 401408 --parallel 2) with EXACT f16 KV — the 9B rung frees the -# room the 26B MoE Gemma4 shape lacked, so quantized V retires. MTP stays off: -# this GGUF carries no MTP tensors at all (verified from its tensor-name -# list), so there is no in-file head to draft with. Verbosity 4 ships so the -# VRAM-fit forensics (KV layout, buffer sizes, graph capture) land in the journal. +# 2026-10-17 (Frank): Bonsai-2 PQ2_0 swap at TWO 200,704-token conversations +# (--ctx-size 401408 --parallel 2) with q8_0 K + q8_0 V — f16 KV does not fit +# this geometry: the engine's own fit projected 33,198 MiB against 23,558 free +# and cudaMalloc failed on the 25,088 MiB pool. MTP stays off: this GGUF +# carries no MTP tensors (verified from its tensor-name list). Verbosity 4 +# ships so the VRAM-fit forensics (KV layout, buffer sizes, graph capture) +# land in the journal. # -# Architecture: qwen35 — dense 9B, 32 layers with full_attention_interval 4 -# (8 full-attention layers at 3,7,...,31 + 24 Gated DeltaNet linear layers), -# 16 heads / 4 KV heads × 256 dim, native 262,144, vocab 248,320. The KV pool -# scales with the 8 full layers: 32 KiB/token at f16 K + f16 V (12.25 GiB at -# the 401,408 pool); the DeltaNet layers cost a fixed recurrent state per -# slot. Feasibility is priced from the MiMo-V2.6-Distill-Qwen-9B boots (same -# geometry, same card, same shape): 21,276/2,849 at ubatch 1024, 21,846/2,279 -# at ubatch 2048 — the boot ledger, not arithmetic, is the referee. -# Vision comes from the publisher's own BF16 projector (the model is natively -# multimodal), so --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is -# the documented text-only opt-out. +# Architecture: qwen35 — 27B, 64 blocks with full_attention_interval 4 (16 +# full-attention layers + 48 Gated DeltaNet linear layers), 24 heads / 4 KV +# heads × 256 dim, native 262,144, vocab 248,320. The KV pool scales with the +# 16 full layers: 34 KiB/token at q8_0 K + q8_0 V (13,328 MiB at the 401,408 +# pool); the DeltaNet layers cost a fixed recurrent state per SLOT (~150 MiB — +# boot-ledger context minus KV). Boot ledger: 22,664 used / 1,461 free, with +# the Q8_0 projector's 873 MiB landing AFTER the engine's fit projection. +# The runtime is a hard lock, not a preference: the file carries 402 tensors +# of custom GGML type 142 plus prism.hadamard.* metadata, which only the +# vendor build registers — stock llama.cpp refuses it at load. +# Vision comes from the publisher's Q8_0 projector, so --mmproj is what gives +# this endpoint eyes; MMPROJ_FILE="" is the documented text-only opt-out. # -# Sampling: llama.cpp takes it per request (the fleet harness sends the -# publisher recipe from models.yml: thinking temp 0.6 / top_p 0.95 / top_k 20). +# Sampling: llama.cpp takes it per request (the fleet harness sends the file's +# own template defaults from models.yml: thinking temp 1.0 / top_p 0.95 / top_k 20). # # Tunables come from frank-zealot.env unless FRANK_ZEALOT_ENV_FILE overrides. set -euo pipefail @@ -37,8 +39,8 @@ if [[ -f "$ENV_FILE" ]]; then fi # --- defaults (used when the env file does not set them) ------------------- -: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" -: "${MODEL_FILE:=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf}" +: "${LLAMA_BIN:=/home/dank/GitPackages/llama-prism/llama-server}" +: "${MODEL_FILE:=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-PQ2_0.gguf}" : "${ALIAS:=frank-zealot}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7771}" @@ -54,19 +56,20 @@ fi # throughput-neutral at a fixed ubatch. : "${BATCH_SIZE:=4096}" : "${UBATCH_SIZE:=2048}" -# Exact (unquantized) KV: f16 K + f16 V, restoring the MiMo-era contract that -# the 26B Gemma4 shape could not afford. 32 KiB/token at this geometry -# (8 full-attention layers × 4 KV heads × 256 dim); 12.25 GiB at the 401,408 -# pool. Explicit either way so the contract is visible in the argv, and -# --flash-attn stays required for any quantized-V fallback (q8_0 = llama.cpp's -# FP8 class; no true fp8 KV dtype exists in this build). -: "${CACHE_TYPE_K:=f16}" -: "${CACHE_TYPE_V:=f16}" -# Speculative decoding is OFF: this Q6_K file carries NO nextn/mtp tensors, so +# Quantized KV: q8_0 K + q8_0 V — the only pair that funds 2 × 200,704 on this +# card. 34 KiB/token at this geometry (16 full-attention layers × 4 KV heads × +# 256 dim); 13,328 MiB at the 401,408 pool. f16 measured OOM; q4_0 is the +# fallback rung (16,392 used / 7,733 free). Explicit either way so the contract +# is visible in the argv, and --flash-attn stays required for quantized V +# (no true fp8 KV dtype exists in this build). +: "${CACHE_TYPE_K:=q8_0}" +: "${CACHE_TYPE_V:=q8_0}" +# Speculative decoding is OFF: this PQ2_0 file carries NO nextn/mtp tensors, so # there is no in-file head (the law: MTP is a property of the file, not of the # flags). SPEC_TYPE=none strips BOTH speculative flags (flags move as a set); -# naming a drafter requires SPEC_DRAFT_FILE alongside SPEC_TYPE — the Gemma4 -# assistant on disk is arch gemma4-assistant and cannot drafter-match qwen35. +# an MTP-bearing CRACK re-export exists upstream (ManniX-ITA, +451 MB) if +# decode ever needs it — that is a new payload with its own VRAM bill, never a +# flag flip on this file. : "${SPEC_TYPE:=none}" : "${SPEC_DRAFT_N_MAX:=3}" : "${SPEC_DRAFT_FILE:=}" @@ -77,7 +80,7 @@ fi : "${VERBOSITY:=3}" # Vision projector. No colon on purpose: an explicitly empty value means # "text-only, on purpose" and must NOT be replaced by the default path. -: "${MMPROJ_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf}" +: "${MMPROJ_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-mmproj-Q8_0.gguf}" # Fail loudly rather than half-loading: a missing file is a download that # has not finished, and Restart=always would turn it into a crash loop that diff --git a/serverAppConfigs-docs/Dankclaw/OpenWebUI.md b/serverAppConfigs-docs/Dankclaw/OpenWebUI.md index 004e384..15ded01 100644 --- a/serverAppConfigs-docs/Dankclaw/OpenWebUI.md +++ b/serverAppConfigs-docs/Dankclaw/OpenWebUI.md @@ -43,8 +43,10 @@ As of 2026-10-07 the picker exposes **exactly one model: `frank-zealot`**. | 2 | `http://host.containers.internal:7771/v1` | yes | `frank-zealot` | llama.cpp CUDA (RTX 3090, 2×200,704) | Dankclaw | The `model` table carries exactly one decoration row, `frank-zealot` -(context length 200,704 = the per-slot window; description: Qwen3.5-9B -Q6_K + BF16 vision mmproj). +(context length 200,704 = the per-slot window; description since 2026-10-17: +Ternary-Bonsai-2-27B PQ2_0 + Q8_0 vision mmproj — the row's `description` is +live DB state, so a payload swap must re-key it in the database too, not only +in the repo). A third gate sits *after* the table, and it is the one that bites family accounts (2026-10-07): a model row is only visible to a non-admin user when