Merge pull request 'feat(frank-zealot): apply the refusal-direction adapter (rank-1 LoRA) at load' (#36) from feat/zealot-uncensored-lora into main
This commit was merged in pull request #36.
This commit is contained in:
@@ -27,7 +27,7 @@ merge absorbed the former `vllmScripts` repo, history preserved).
|
||||
|--------|---------|---------|---------|------|
|
||||
| Dankserver | `frank-archon` | radiance, digest-pinned, root unit, TP=2 | publisher `qwen3.8-next-flash-fp8-iq4r-moe.rad` | 7770 |
|
||||
| Dankclaw | `frank-dragoon` | halogen-flash-server 0.16.2, digest-pinned, root unit | `qwen38-flash-next-v2-abliterated.hgn` + n-gram + vision tower | 7770 |
|
||||
| Dankclaw | `frank-zealot` | llama.cpp PrismML fork build 10777, root unit — the only loader for this payload | Ternary-Bonsai-2-9B PQ2_0 + Q8_0 mmproj | 7771 |
|
||||
| Dankclaw | `frank-zealot` | llama.cpp PrismML fork build 10777, root unit — the only loader for this payload | Ternary-Bonsai-2-9B PQ2_0 + Q8_0 mmproj + rank-1 refusal adapter | 7771 |
|
||||
|
||||
Payload identity, shape, eras, laws and rollback live in ONE page each:
|
||||
[[frank-archon]], [[frank-dragoon]], [[frank-zealot]]. `scripts/<service>.env`
|
||||
@@ -440,7 +440,11 @@ disagrees with it is the bug.
|
||||
`prism.hadamard.*` dequant metadata — stock llama.cpp refuses it, and a
|
||||
loader ignoring that metadata dequantizes wrong); payload
|
||||
`Bonsai2-9B-MTP/…PQ2_0.gguf` + the publisher's **Q8_0** mmproj (the BF16
|
||||
tower can crash the server with concurrent image requests).
|
||||
tower can crash the server with concurrent image requests) + the rank-1
|
||||
refusal-direction adapter applied at load by `--lora-scaled …:1`. The
|
||||
adapter's rollback is `LORA_FILE=` (empty) in the env — one line, no rebuild
|
||||
— and build 10777 is also the gate that lets it load: older forks refuse it
|
||||
with `unsupported prism.hadamard.version: 2`.
|
||||
- Shape: `--ctx-size 602112 --parallel 3` = **three conversations at
|
||||
200,704 each**, `CACHE_TYPE_K=f16` + `CACHE_TYPE_V=q8_0` (asymmetric: 25,088
|
||||
B/token, 14,406 MiB of pool), batch 6144 / ubatch 1024,
|
||||
|
||||
@@ -1463,6 +1463,12 @@ LLM_ZEALOT_RUNTIME="${LLM_ZEALOT_RUNTIME:-/home/dank/GitPackages/llama-prism-b10
|
||||
# 868.32 MiB worst-case VRAM (595.29 model + 248.10 compute in the boot ledger)
|
||||
# lands AFTER the engine's fit check, so it is not inside the q8_0 budget.
|
||||
LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf}"
|
||||
# The refusal adapter is part of the payload identity now, so the deployer
|
||||
# checks it beside the other two files — but it has its OWN fetcher, so a
|
||||
# missing adapter must not send the operator to the bonsai9 script. Integrity
|
||||
# (bytes + sha256) is the fetcher's job at download time; this gate only asks
|
||||
# whether the file the env names exists, exactly like the two above.
|
||||
LLM_ZEALOT_LORA="${LLM_ZEALOT_LORA:-/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-Uncensored-LoRA.gguf}"
|
||||
|
||||
deploy_llm_zealot() {
|
||||
# Refuse rather than restart-loop: a unit whose runtime or payload is
|
||||
@@ -1484,6 +1490,13 @@ deploy_llm_zealot() {
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
if [[ ! -f "$LLM_ZEALOT_LORA" ]]; then
|
||||
echo "ERROR: frank-zealot refusal adapter absent at $LLM_ZEALOT_LORA" >&2
|
||||
echo "frank-zealot: run scripts/download-frank-zealot-uncensored-lora.sh first;" >&2
|
||||
echo "frank-zealot: the env names the adapter, so the unit would refuse to boot" >&2
|
||||
echo "frank-zealot: without it — starting the unit anyway is a crash loop." >&2
|
||||
return 1
|
||||
fi
|
||||
relink_unit frank-zealot
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-zealot.service
|
||||
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fetch the refusal-direction adapter that frank-zealot applies at load:
|
||||
# AtomicChat/Ternary-Bonsai-2-9B-Uncensored-LoRA-GGUF — a **3.2 MB rank-1 LoRA**
|
||||
# for prism-ml/Ternary-Bonsai-2-9B, not a checkpoint. Public repo: no auth
|
||||
# header, and never a token on the command line (a Bearer token is
|
||||
# world-readable in /proc/<pid>/cmdline).
|
||||
#
|
||||
# IT IS AN EDIT INSIDE THE COMPUTE GRAPH, NOT A SECOND MODEL. The card: B = -r
|
||||
# (the refusal direction, row 23 of its own probe matrix), A = P^T(s *
|
||||
# H(r^T W_fold)) built from the base pack's folded input basis, applied at every
|
||||
# transformer layer (`target_layers = all`, `sites = 64`), with the embedding
|
||||
# deliberately untouched because an embedding LoRA crashes this fork's graph.
|
||||
# `--lora-scaled <file>:<scale>` is the flag pair: scale 1 is the card's
|
||||
# "held-out refusals 0 %" point, scale 0 is the plain model again.
|
||||
#
|
||||
# THE RUNTIME IS A GATE, NOT A PREFERENCE. The card verified the adapter on
|
||||
# `prism-b10777-e3de715` — the exact build frank-zealot runs — and says older
|
||||
# forks (e.g. `prism-b10709`) refuse these packs with `unsupported
|
||||
# prism.hadamard.version: 2`. The fork is already pinned for the payload's
|
||||
# custom tensor type 142; this adapter adds a second reason it cannot move.
|
||||
#
|
||||
# PROVENANCE ASYMMETRY, STATED HONESTLY. The GGUF's own metadata names the pack
|
||||
# the direction was fitted against:
|
||||
# prism.abliterate.base_gguf = Ternary-Bonsai-2-9B-PTQ1_0.gguf
|
||||
# prism.abliterate.base_sha256 = 3561df7ac7d26a71ffa991ac2db740b10841c3a350aa2676ccd3ff5d428b4e8b
|
||||
# frank-zealot serves the PQ2_0 + MTP + Q8_0-embedding packing (its sha256 is
|
||||
# c79fcdfa…, see download-frank-zealot-bonsai9.sh), which is NOT that file and is
|
||||
# not on this host. The adapter still applies — same tensor names, and the
|
||||
# metadata is provenance rather than a checksum gate — but its strength was
|
||||
# calibrated on a sibling quantization. What proves the effect here is the
|
||||
# behaviour of the endpoint (a refusal probe before and after the swap), not the
|
||||
# metadata and not the card's table, which was measured on PTQ1_0.
|
||||
#
|
||||
# Fresh downloads, no resume; -f/--fail-with-body so an HTTP error is not a
|
||||
# success, then byte count and sha256 (this repo is NOT LFS — the file is a
|
||||
# plain git object, so the sha below is the content hash of the Hub's bytes as
|
||||
# fetched, not an LFS oid). Idempotent: a complete file is skipped.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-zealot-uncensored-lora.sh [DEST_DIR] [--verify-only]
|
||||
set -euo pipefail
|
||||
|
||||
DEST=""
|
||||
VERIFY_ONLY=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--verify-only) VERIFY_ONLY=1 ;;
|
||||
*) DEST="$arg" ;;
|
||||
esac
|
||||
done
|
||||
DEST="${DEST:-/home/dank/models/Bonsai2-9B-MTP}"
|
||||
|
||||
REPO="AtomicChat/Ternary-Bonsai-2-9B-Uncensored-LoRA-GGUF"
|
||||
# name | bytes | sha256 of the Hub's bytes
|
||||
FILES=(
|
||||
"Ternary-Bonsai-2-9B-Uncensored-LoRA.gguf|3155424|861b9c099fb5cd81438ba85bad30590847c777b1e18112586be7f4f9ae5fb07b"
|
||||
)
|
||||
|
||||
mkdir -p "$DEST"
|
||||
status=0
|
||||
for entry in "${FILES[@]}"; do
|
||||
IFS='|' read -r name size sha <<<"$entry"
|
||||
url="https://huggingface.co/${REPO}/resolve/main/${name}"
|
||||
f="$DEST/$name"
|
||||
have=0
|
||||
if [[ -f "$f" ]] && [[ "$(stat -c%s "$f")" == "$size" ]]; then
|
||||
if sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
have=1
|
||||
fi
|
||||
fi
|
||||
if (( have )); then
|
||||
echo "OK $name"
|
||||
continue
|
||||
fi
|
||||
if (( VERIFY_ONLY )); then
|
||||
echo "MISSING $name"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "FETCHING $name (from $REPO)"
|
||||
tmp="$f.part"
|
||||
curl -fsSL --fail-with-body -o "$tmp" "$url"
|
||||
mv "$tmp" "$f"
|
||||
if [[ "$(stat -c%s "$f")" != "$size" ]]; then
|
||||
echo "SIZE FAIL $name ($(stat -c%s "$f") != $size)"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
if ! sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
echo "SHA FAIL $name"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "VERIFIED $name"
|
||||
done
|
||||
if (( status == 0 )); then
|
||||
echo "DOWNLOAD_DONE $DEST"
|
||||
else
|
||||
echo "DOWNLOAD_FAILED $DEST" >&2
|
||||
fi
|
||||
exit "$status"
|
||||
@@ -1,5 +1,6 @@
|
||||
# frank-zealot — llama.cpp (PrismML fork build 10777, CUDA) on the RTX 3090,
|
||||
# :7771. Payload: Ternary-Bonsai-2-9B PQ2_0 with the MTP head in the file.
|
||||
# :7771. Payload: Ternary-Bonsai-2-9B PQ2_0 with the MTP head in the file, plus
|
||||
# the refusal-direction adapter applied at load.
|
||||
#
|
||||
# ONE OWNER: shape, payload identity, eras, laws and every measured number
|
||||
# live in serverAppConfigs-docs/LLM/frank-zealot.md. This file carries the
|
||||
@@ -13,6 +14,17 @@ MODEL_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-PQ2_0-MTP-Q8_0.g
|
||||
# Q8_0 IS MANDATORY: the publisher states the BF16 tower can crash llama-server
|
||||
# under concurrent image requests, which is exactly --parallel 3.
|
||||
MMPROJ_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf
|
||||
# Refusal-direction adapter (a rank-1 LoRA, 3.2 MB) applied at load; it edits
|
||||
# the compute graph, it is not a second model. Scale 1 is the card's "held-out
|
||||
# refusals 0 %" point. LORA_FILE="" is the documented opt-out — the plain pack
|
||||
# again, no rebuild — and LORA_SCALE=0 is the same result through the flag.
|
||||
# PROVENANCE ASYMMETRY: the adapter's own prism.abliterate.base_gguf names
|
||||
# Ternary-Bonsai-2-9B-PTQ1_0.gguf as the pack its direction was fitted against,
|
||||
# which is NOT this packing and is not on this host. The tensor names match, so
|
||||
# it applies; the strength was calibrated on a sibling quantization, so the
|
||||
# effect is proven by the endpoint's behaviour, never by the metadata.
|
||||
LORA_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-Uncensored-LoRA.gguf
|
||||
LORA_SCALE=1
|
||||
# Served id — the harness/family name, not a fact about the payload.
|
||||
# Id-keyed consumers: OpenWebUI (model row, access_grant, api_configs), models.yml.
|
||||
ALIAS=frank-zealot
|
||||
|
||||
@@ -33,6 +33,19 @@
|
||||
# what gives this endpoint eyes; MMPROJ_FILE="" is the documented text-only
|
||||
# opt-out.
|
||||
#
|
||||
# A refusal-direction adapter rides on top of the pack (Frank, 2026-10-19):
|
||||
# AtomicChat's rank-1 LoRA, applied at load by --lora-scaled. It is an edit
|
||||
# inside the compute graph, not a second model — B = -r (the refusal direction),
|
||||
# A built from the base pack's folded input basis, every transformer layer, the
|
||||
# embedding deliberately untouched because an embedding LoRA crashes this fork's
|
||||
# graph. The card verified it on prism-b10777-e3de715 — the build pinned here —
|
||||
# and says older forks refuse the pack outright (`unsupported
|
||||
# prism.hadamard.version: 2`), which makes the runtime lock doubly load-bearing.
|
||||
# Its own metadata names the PTQ1_0 packing as the pack the direction was fitted
|
||||
# against, so what proves the effect is a refusal probe against the live
|
||||
# endpoint, not the metadata and not the card's table. LORA_FILE="" drops the
|
||||
# flag and the endpoint is the plain pack again.
|
||||
#
|
||||
# Speculative decoding ships ON (Frank, 2026-10-19). The head is IN the file and
|
||||
# accepts well (0.798/0.617 per position, mean length 2.41, measured 2026-10-18);
|
||||
# what made it lose that evening was the bill, not the drafts — the draft context
|
||||
@@ -136,6 +149,13 @@ fi
|
||||
# Vision projector. No colon on purpose: an explicitly empty value means
|
||||
# "text-only, on purpose" and must NOT be replaced by the default path.
|
||||
: "${MMPROJ_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf}"
|
||||
# Refusal adapter. No colon on purpose, the same convention as MMPROJ_FILE: an
|
||||
# explicitly empty LORA_FILE is the documented opt-out (the plain pack again)
|
||||
# and must NOT be replaced by the default path. The scale travels INSIDE the
|
||||
# same argv token as the file, so LORA_SCALE only matters while LORA_FILE names
|
||||
# something; 0 is the card's "switch it off without touching the file" value.
|
||||
: "${LORA_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-Uncensored-LoRA.gguf}"
|
||||
: "${LORA_SCALE:=1}"
|
||||
|
||||
# Fail loudly rather than half-loading: a missing file is a download that
|
||||
# has not finished, and Restart=always would turn it into a crash loop that
|
||||
@@ -149,6 +169,17 @@ if [[ -n "$MMPROJ_FILE" ]]; then
|
||||
EXTRA_ARGS+=(--mmproj "$MMPROJ_FILE")
|
||||
fi
|
||||
|
||||
# The adapter reaches the engine as ONE token — --lora-scaled <file>:<scale> —
|
||||
# so a scale can never arrive without its file. It is applied once, at load;
|
||||
# nothing here hot-swaps it (--lora-init-without-apply plus POST /lora-adapters
|
||||
# is that shape, and a process holding three conversations is not the place to
|
||||
# re-parameterise weights mid-flight). A missing file is a fetch that never
|
||||
# finished, so refuse rather than let Restart=always loop on it.
|
||||
if [[ -n "$LORA_FILE" ]]; then
|
||||
[[ -e "$LORA_FILE" ]] || { echo "frank-zealot: missing $LORA_FILE" >&2; exit 1; }
|
||||
EXTRA_ARGS+=(--lora-scaled "${LORA_FILE}:${LORA_SCALE}")
|
||||
fi
|
||||
|
||||
# Prometheus scraping is opt-IN in llama.cpp: without --metrics the server
|
||||
# answers 501 on /metrics, and the fleet scrape then reports the endpoint DOWN.
|
||||
if [[ "$METRICS" = "1" ]]; then
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
type: reference
|
||||
title: "frank-zealot on Dankclaw — llama.cpp (PrismML fork 10777), Ternary-Bonsai-2-9B PQ2_0-MTP, 3 × 200704, f16 K + q8_0 V, MTP armed"
|
||||
title: "frank-zealot on Dankclaw — llama.cpp (PrismML fork 10777), Ternary-Bonsai-2-9B PQ2_0-MTP + refusal adapter, 3 × 200704, f16 K + q8_0 V, MTP armed"
|
||||
description: "The RTX 3090 endpoint: live shape exactly as the env ships it, every era with its rollback path, and its card laws."
|
||||
tags:
|
||||
- llm
|
||||
@@ -10,10 +10,10 @@ tags:
|
||||
- gguf
|
||||
- rtx3090
|
||||
- endpoint
|
||||
timestamp: 2026-10-18
|
||||
timestamp: 2026-10-19
|
||||
---
|
||||
|
||||
# frank-zealot — llama.cpp (PrismML) Bonsai-2-9B PQ2_0-MTP, 3 × 200,704, f16 K + q8_0 V, MTP armed
|
||||
# frank-zealot — llama.cpp (PrismML) Bonsai-2-9B PQ2_0-MTP + refusal adapter, 3 × 200,704, f16 K + q8_0 V, MTP armed
|
||||
|
||||
Dankclaw's RTX 3090 endpoint — `http://192.168.0.143:7771`, wire id `frank-zealot`, the fleet's **compaction hub**
|
||||
(three conversations at 200,704 each, the window the other two endpoints also cap at). One page
|
||||
@@ -29,11 +29,12 @@ is the bug.
|
||||
|---|---|
|
||||
| Checkpoint | `/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-PQ2_0-MTP-Q8_0.gguf` — 2,675,410,720 B, LFS sha256 `c79fcdfaf6b7bf47…` |
|
||||
| Projector | `…/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf` — 624,229,664 B, sha256 `e0cb6f6988cea323…` — **Q8_0 is mandatory**, not a preference (see Contract) |
|
||||
| Fetch | `scripts/download-frank-zealot-bonsai9.sh [DEST]` (`--verify-only`; `-f`, byte count, LFS sha256 per file) |
|
||||
| Adapter | `…/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-Uncensored-LoRA.gguf` — **3,155,424 B**, sha256 `861b9c099fb5cd81…` — a rank-1 refusal-direction LoRA (`adapter.type lora`, 128 tensors, every transformer layer, embedding untouched), applied at load by `--lora-scaled <file>:1`. Its metadata names `Ternary-Bonsai-2-9B-PTQ1_0.gguf` (`3561df7a…`) as the pack it was fitted against — **not this packing, and not on this host**; see Laws |
|
||||
| Fetch | `scripts/download-frank-zealot-bonsai9.sh [DEST]` (payload + projector) · `scripts/download-frank-zealot-uncensored-lora.sh [DEST]` (the adapter) — both `--verify-only`-able, `-f`, byte count and sha256 per file |
|
||||
| Runtime | `/home/dank/GitPackages/llama-prism-b10777/llama-server` — PrismML **fork**, `0.2.0-dev (build 10777, commit e3de71522)`; the only loader for this payload |
|
||||
| Trio | `scripts/frank-zealot.service` (root, relinked, `User=dank`, `Restart=on-failure`, `RestartSec=10`, `LimitMEMLOCK=infinity`, `TimeoutStopSec=180`) · `scripts/frank-zealot.env` · `scripts/serve-frank-zealot.sh` |
|
||||
| Address / alias | `BIND_ADDR=0.0.0.0`, `LISTEN_PORT=7771`, `ALIAS=frank-zealot` (a flag, not a fact) |
|
||||
| Flags | `--ctx-size 602112 --parallel 3` ⇒ 200,704 per slot · `--cache-type-k f16 --cache-type-v q8_0` (asymmetric, **14,406.00 MiB**) · `--flash-attn on` · `--batch-size 6144 --ubatch-size 1024` · `--n-gpu-layers 99` · `--jinja --reasoning auto --reasoning-format deepseek` · `--metrics` · `--mmproj` · `--log-verbosity 4` · `--presence-penalty 1.0` · **`--spec-type draft-mtp`** with `--spec-draft-n-max 2 --spec-draft-n-min 0 --spec-draft-p-min 0` |
|
||||
| Flags | `--ctx-size 602112 --parallel 3` ⇒ 200,704 per slot · `--cache-type-k f16 --cache-type-v q8_0` (asymmetric, **14,406.00 MiB**) · `--flash-attn on` · `--batch-size 6144 --ubatch-size 1024` · `--n-gpu-layers 99` · `--jinja --reasoning auto --reasoning-format deepseek` · `--metrics` · `--mmproj` · **`--lora-scaled Ternary-Bonsai-2-9B-Uncensored-LoRA.gguf:1`** · `--log-verbosity 4` · `--presence-penalty 1.0` · **`--spec-type draft-mtp`** with `--spec-draft-n-max 2 --spec-draft-n-min 0 --spec-draft-p-min 0` |
|
||||
|
||||
**Why this shape.** `general.architecture qwen35`: 33 blocks (32 trunk + `blk.32` = the MTP block)
|
||||
`full_attention_interval 4` ⇒ 8 full-attention + 24 Gated DeltaNet, 4 KV heads × 256 dim, `nextn_predict_layers 1`
|
||||
@@ -57,6 +58,17 @@ and `UBATCH_SIZE=2048` beside an armed drafter is still the shape that halved de
|
||||
`--spec-draft-model` — the draft rides the main model's weights, so a draft file would load the trunk twice.
|
||||
`SPEC_TYPE=none` is the rollback and strips the whole set.
|
||||
|
||||
**The adapter is behaviour, not shape.** It folds 3.2 MB of rank-1 tensors into the trunk at
|
||||
load, so it moves nothing in the budget equation above — no KV, no slot, no window. What it
|
||||
changes is what the endpoint answers, and the only instrument for that is a refusal probe:
|
||||
the same two prompts the plain pack answered `I cannot provide…` (captured on the live
|
||||
endpoint before the swap, `temperature 0.1`), re-asked after it. Budget **≥ 2 048 tokens** for
|
||||
that probe on this payload — the thinking channel swallows a 600-token request whole and
|
||||
`content` comes back empty with `finish_reason: length`, which reads as a refusal that never
|
||||
happened. The adapter's own card table is **not** evidence here: it was measured on the PTQ1_0
|
||||
packing. Rollback is `LORA_FILE=` (empty) in the env and one redeploy — no rebuild, no payload
|
||||
move; `LORA_SCALE=0` is the same result through the flag.
|
||||
|
||||
**Harness contract** (`~/.omp/agent/models.yml`, provider `llamacpp (dankclaw-3090)`, id `frank-zealot`):
|
||||
`contextWindow: 200704` — **the per-slot window**, because llama.cpp divides `--ctx-size` across slots and answers a
|
||||
longer prompt with a 400, never a truncation — plus `maxTokens: 16384`, `input: [text, image]`
|
||||
@@ -113,6 +125,7 @@ with 448 completion tokens (at `max_tokens: 64` the same request returns EMPTY c
|
||||
|
||||
| Window | Payload | Runtime | Shape | Replaced because | Rollback |
|
||||
|---|---|---|---|---|---|
|
||||
| 2026-10-19 → | the same 9B MTP file and projector, **plus** `Ternary-Bonsai-2-9B-Uncensored-LoRA.gguf` (3,155,424 B `861b9c09…`) applied by `--lora-scaled …:1` | PrismML fork 10777 (its verified build) | — (shape unchanged; the adapter is behaviour) | Frank ordered the refusal direction removed on this endpoint | one env line — `LORA_FILE=` (or `LORA_SCALE=0`) — and redeploy; the plain pack is the same file it already loads |
|
||||
| 2026-10-18 21:06 → 2026-10-19 | the same 9B MTP file and projector | PrismML fork 10777 | — | Frank ordered the third conversation back AND the drafter armed, and exact V funds neither: it bills 18,816 MiB at the pool that pays for the third slot, leaving nothing for the draft's 2,352 | one line each — `CTX_SIZE=524288 PARALLEL=2 CACHE_TYPE_V=f16 BATCH_SIZE=4096 UBATCH_SIZE=2048 SPEC_TYPE=none` — and redeploy; no payload move |
|
||||
| 2026-10-18 13:19 → same evening (the boot `68afc6d` replaced) | the same 9B MTP file and projector | PrismML fork 10777 | — | Frank ordered unquantized KV, and that pool cannot survive it: 24,576 MiB of f16 cache > 24,123 addressable | one env line — `CTX_SIZE=786432 PARALLEL=3 CACHE_TYPE_K/V=q8_0` — and redeploy; no payload move |
|
||||
| 2026-10-18 03:14 → 13:19 (arm under the unit, then the leftover drop-in) | `Bonsai2-PTQ1_0/Ternary-Bonsai-2-27B-PTQ1_0.gguf` 5,946,648,928 B + hardlinked Q8 mmproj | PrismML fork 10777 | arm: 2 × 200,704, batch/ubatch 3072/1536. The leftover `frank-zealot.service.d/ptq1.conf` then made the 13:16 cutover deploy boot THIS payload at 3 × 133,888 (`new slot, n_ctx = 133888` x3, journal read 2026-10-19) until the 13:19 boot | — | `/home/dank/models/Bonsai2-PTQ1_0/` + `scripts/download-frank-zealot-bonsai2-ptq1.sh` |
|
||||
@@ -169,13 +182,15 @@ with 448 completion tokens (at `max_tokens: 64` the same request returns EMPTY c
|
||||
- MTP is a property of the **file**: the plain PQ2_0 sibling could not draft at any flag value; only the `-MTP` export carries `blk.N.nextn.*`.
|
||||
- **The format is the runtime lock.** PQ2_0 is `general.file_type 141` over 202 tensors of custom GGML type **142** plus `prism.hadamard.*` metadata: stock llama.cpp refuses the type, and a loader ignoring the Hadamard transform would dequant **wrong, silently** (same lock class as ROCmFPX). Build ≥ b10776 is additionally the floor that pairs MTP with a quantized V cache.
|
||||
- Check arch support in the source tree: `grep -c` on the packed binary returns 0 for every arch, including the one it serves.
|
||||
- **Base alignment.** Bonsai-2 is the censored base lineage. `/home/dank/models/Bonsai2-CRACK/` matches the **27B** geometry (a one-line `MODEL_FILE` swap there, identical tensor census); no dealigned 9B exists, so for this payload the refusal escape hatch is a fetch, not a revert.
|
||||
- **Base alignment.** Bonsai-2 is the censored base lineage. `/home/dank/models/Bonsai2-CRACK/` matches the **27B** geometry (a one-line `MODEL_FILE` swap there, identical tensor census); no dealigned 9B exists, so for this payload the refusal escape hatch is a fetch, not a revert — which is exactly what was taken on 2026-10-19: the rank-1 adapter in the Adapter row above, behind `LORA_FILE`.
|
||||
- **A stale bench drop-in hijacks the production boot and no health check sees it.** An `Environment=FRANK_ZEALOT_ENV_FILE=/tmp/<arm>.env` in `frank-zealot.service.d/` booted a 27B PTQ1_0 payload while `/health` answered 200, `/v1/models` echoed the alias flag and the e2e gate passed 33/33 — none of them read the payload.
|
||||
- After any bench window: `systemctl show -p DropInPaths` → `tr '\0' ' ' </proc/$pid/cmdline` → ledger, in that order; drop-ins retire to `~/.AGENTTRASH/<date>-<topic>/`, and payload decisions land on this page the moment they are made, or an approved arm becomes indistinguishable from a rogue one a day later.
|
||||
- **An arm env must use the serve script's own variable names** (`CACHE_TYPE_K`/`CACHE_TYPE_V`, `BIND_ADDR`, `NGL`): an invented `KV_CACHE_TYPE` is swallowed by `: "${CACHE_TYPE_K:=f16}"` and the arm boots f16 — an OOM that reads like a model fault.
|
||||
- **Stop, shebang, GPU node.** llama.cpp rides out the default stop timeout (`TimeoutStopSec=180` is on the unit for that reason) and a hand-booted arm keeps answering 7771 behind a `failed` unit — kill it **by PID**, never `pkill -f llama-server`.
|
||||
- `serve-frank-zealot.sh` is a direct `ExecStart`, so a missing shebang is a `203/EXEC` boot loop at `Mem peak: 1.9M` with the GPU untouched: `head -1` is a deploy check.
|
||||
- `No CUDA GPUs are available` ⇒ `nvidia-modprobe -u -c=0`; KV spilled into unified memory halves decode silently, so re-read `nvidia-smi` after any shape change.
|
||||
- **An adapter's provenance is metadata, not a gate.** Its GGUF carries `prism.abliterate.base_gguf = Ternary-Bonsai-2-9B-PTQ1_0.gguf` and that pack's sha256; the endpoint applies the file to the PQ2_0+MTP packing anyway, because the loader matches **tensor names, not hashes**. So an adapter's effect is only ever proven behaviourally (a refusal probe against the live endpoint), never by the card, and an adapter whose rank landed on `token_embd` would be a crash rather than a degradation — the publisher leaves the embedding untouched for exactly that reason.
|
||||
- **The fork pin now has a third reason.** The payload's custom type 142, the b10776 floor that pairs MTP with a quantized V, and now this: older PrismML builds (e.g. `prism-b10709`) refuse the adapter outright with `unsupported prism.hadamard.version: 2`, so a runtime that can load the pack can still be unable to carry the adapter.
|
||||
|
||||
## Contract
|
||||
|
||||
@@ -210,12 +225,12 @@ with 448 completion tokens (at `max_tokens: 64` the same request returns EMPTY c
|
||||
boot ledger and the KV derivation all live on this page now; the header used to hold them, and two of its
|
||||
rung figures were wrong (`CACHE_TYPE_V=q8_0` and `CTX_SIZE=401408` both free **3,840 MiB** at the 524,288 pool,
|
||||
not 8,192 and not ~6.7 GiB — the Fallback rungs table above is the corrected statement).
|
||||
- `scripts/serve-frank-zealot.sh` — the exec line and its rationale comments; honors `FRANK_ZEALOT_ENV_FILE` for arm boots; exits 1 naming a missing binary, GGUF or projector rather than half-loading (a restart policy would turn an unfinished download into a crash loop that reads like a runtime fault).
|
||||
- `scripts/serve-frank-zealot.sh` — the exec line and its rationale comments; honors `FRANK_ZEALOT_ENV_FILE` for arm boots; exits 1 naming a missing binary, GGUF, projector or adapter rather than half-loading (a restart policy would turn an unfinished download into a crash loop that reads like a runtime fault).
|
||||
- `scripts/frank-zealot.service` — root unit, relinked into `/etc/systemd/system/`.
|
||||
- `scripts/download-frank-zealot-bonsai9.sh` (active) · `…-bonsai2.sh` · `…-bonsai2-ptq1.sh` ·
|
||||
- `scripts/download-frank-zealot-bonsai9.sh` (active payload + projector) · `scripts/download-frank-zealot-uncensored-lora.sh` (the refusal adapter; its header is the provenance record) · `…-bonsai2.sh` · `…-bonsai2-ptq1.sh` ·
|
||||
`scripts/download-frank-zealot.sh` (master-era GGUFs) · `old/download-frank-zealot-sglang-int8.sh` (SGLang).
|
||||
- `tests/frank_zealot_unit.bats` — stub-argv seam plus the VRAM fit gate.
|
||||
- `scripts/deploy-remote.sh` — `deploy_llm_zealot` (payload-path gates, MainPID-hardened health);
|
||||
- `scripts/deploy-remote.sh` — `deploy_llm_zealot` (payload-path gates including the adapter, MainPID-hardened health);
|
||||
- `shared/prometheus/prometheus.yml`, `shared/prometheus/rules/llm-fleet.yml`
|
||||
`shared/grafana/provisioning/dashboards/llm-fleet.json` — the scrape, the alert, the dashboard.
|
||||
- Live state, not repo files: the OpenWebUI picker row (back up `webui.db` before re-keying) and `~/.omp/agent/models.yml`.
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
# pool (4,096 B/position, 2,352.00 MiB here), and the 786,432 pool this shape
|
||||
# replaced cannot carry a drafter at any dtype inside the 24,123 MiB the driver
|
||||
# leaves addressable. The publisher's own Q8_0 vision projector rides along (the
|
||||
# BF16 one can crash on concurrent images). Log
|
||||
# BF16 one can crash on concurrent images). The payload also carries a
|
||||
# REFUSAL-DIRECTION ADAPTER: a rank-1 LoRA applied at load, so the seam pins the
|
||||
# --lora-scaled token (path and scale together), the scale knob, and the
|
||||
# empty-LORA_FILE opt-out.
|
||||
# This seam runs the serve script against a stub engine and asserts on the
|
||||
# argv it actually execs, so what is pinned is the command the operator gets.
|
||||
|
||||
@@ -39,8 +42,9 @@ STUB
|
||||
# (the env) instead of silently forking the test's filename constants.
|
||||
MODEL_STUB="$(basename "$(sed -nE 's/^MODEL_FILE=(.+)$/\1/p' "$ENV_FILE")")"
|
||||
MM_STUB="$(basename "$(sed -nE 's/^MMPROJ_FILE=(.+)$/\1/p' "$ENV_FILE")")"
|
||||
LORA_STUB="$(basename "$(sed -nE 's/^LORA_FILE=(.+)$/\1/p' "$ENV_FILE")")"
|
||||
DRAFT_STUB="draft-assistant.gguf"
|
||||
touch "$MODEL_DIR/$MODEL_STUB" "$MODEL_DIR/$MM_STUB" "$MODEL_DIR/$DRAFT_STUB"
|
||||
touch "$MODEL_DIR/$MODEL_STUB" "$MODEL_DIR/$MM_STUB" "$MODEL_DIR/$LORA_STUB" "$MODEL_DIR/$DRAFT_STUB"
|
||||
}
|
||||
|
||||
teardown() {
|
||||
@@ -69,6 +73,8 @@ SPEC_DRAFT_N_MAX=$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
VERBOSITY=$(sed -nE 's/^VERBOSITY=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
METRICS=$(sed -nE 's/^METRICS=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
NGL=$(sed -nE 's/^NGL=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
LORA_FILE=$MODEL_DIR/$LORA_STUB
|
||||
LORA_SCALE=$(sed -nE 's/^LORA_SCALE=(.+)$/\1/p' "$ENV_FILE")
|
||||
EOF
|
||||
run env FRANK_ZEALOT_ENV_FILE="$envf" nvidia-modprobe=true bash "$SERVE"
|
||||
}
|
||||
@@ -117,7 +123,7 @@ run_stage() {
|
||||
grep -q 'FRANK_ZEALOT_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-zealot.env' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the shipped env points at the fork runtime and the Bonsai-2 9B MTP pair" {
|
||||
@test "the shipped env points at the fork runtime, the MTP pair and the refusal adapter" {
|
||||
# The payload path IS the endpoint's identity, and the runtime is part of
|
||||
# it here: the file's custom tensor type 142 + prism.hadamard.* metadata
|
||||
# make the PrismML build the only loader, and this build must be >= b10776
|
||||
@@ -127,6 +133,7 @@ run_stage() {
|
||||
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-prism-b10777/llama-server$' "$ENV_FILE"
|
||||
grep -qE '^MODEL_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-PQ2_0-MTP-Q8_0\.gguf$' "$ENV_FILE"
|
||||
grep -qE '^MMPROJ_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0\.gguf$' "$ENV_FILE"
|
||||
grep -qE '^LORA_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-Uncensored-LoRA\.gguf$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "the deployer gates on exactly the payload paths the env file ships" {
|
||||
@@ -135,7 +142,7 @@ run_stage() {
|
||||
# would be gating on a file nobody reads — and a missing projector would
|
||||
# ship a blind endpoint.
|
||||
local env_v dep_v path pair
|
||||
for pair in "MODEL_FILE LLM_ZEALOT_MODEL_FILE" "MMPROJ_FILE LLM_ZEALOT_MMPROJ"; do
|
||||
for pair in "MODEL_FILE LLM_ZEALOT_MODEL_FILE" "MMPROJ_FILE LLM_ZEALOT_MMPROJ" "LORA_FILE LLM_ZEALOT_LORA"; do
|
||||
env_v="${pair%% *}"
|
||||
dep_v="${pair##* }"
|
||||
path="$(sed -nE "s/^${env_v}=(.+)$/\1/p" "$ENV_FILE")"
|
||||
@@ -367,6 +374,45 @@ run_stage() {
|
||||
! grep -qxF -- '--mmproj' <<<"$output"
|
||||
}
|
||||
|
||||
@test "the refusal adapter reaches the engine as path and scale in ONE token" {
|
||||
# --lora-scaled takes FNAME:SCALE as a single argument, so the seam pins the
|
||||
# whole token. An adapter that reaches the engine without its scale — or a
|
||||
# scale without its adapter — is a different payload than the one the env
|
||||
# file describes, and nothing else would notice.
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
local scale
|
||||
scale="$(sed -nE 's/^LORA_SCALE=(.+)$/\1/p' "$ENV_FILE")"
|
||||
[ "$(argv_value --lora-scaled)" = "$MODEL_DIR/$LORA_STUB:$scale" ]
|
||||
}
|
||||
|
||||
@test "the scale knob rides inside the token, including the card's off value" {
|
||||
# Distinctive values first: proof the flag is wired to the env and not
|
||||
# merely to the script's default. Scale 0 is the card's second mode — the
|
||||
# direction present but inert — so the endpoint can be neutered without
|
||||
# deleting a file or rebuilding anything.
|
||||
serve_argv
|
||||
{ echo 'LORA_SCALE=0'; } >> "$STAGE/test.env"
|
||||
run_stage
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --lora-scaled)" = "$MODEL_DIR/$LORA_STUB:0" ]
|
||||
|
||||
sed -i 's|^LORA_SCALE=.*|LORA_SCALE=0.75|' "$STAGE/test.env"
|
||||
run_stage
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --lora-scaled)" = "$MODEL_DIR/$LORA_STUB:0.75" ]
|
||||
}
|
||||
|
||||
@test "LORA_FILE empty opts the adapter out and leaves no trace in the argv" {
|
||||
# The rollback is ABSENCE of the flag: an empty argument would have the
|
||||
# engine open "" and die at load, which is the failure this seam forbids.
|
||||
serve_argv
|
||||
sed -i 's|^LORA_FILE=.*|LORA_FILE=|' "$STAGE/test.env"
|
||||
run_stage
|
||||
[ "$status" -eq 0 ]
|
||||
! grep -qxF -- '--lora-scaled' <<<"$output"
|
||||
}
|
||||
|
||||
@test "the card sampler's only non-metadata term reaches the argv" {
|
||||
# general.sampling.* (temperature / top_p / top_k / min_p / repeat_penalty)
|
||||
# rides in the checkpoint and the engine reads it itself; presence penalty
|
||||
@@ -406,23 +452,33 @@ run_stage() {
|
||||
grep -qxF -- '--jinja' <<<"$output"
|
||||
}
|
||||
|
||||
@test "the serve script refuses a missing runtime, GGUF or projector" {
|
||||
@test "the serve script refuses a missing runtime, GGUF, projector or adapter" {
|
||||
rm -f "$MODEL_DIR/$MM_STUB"
|
||||
serve_argv
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"frank-zealot: missing"* ]]
|
||||
|
||||
# The adapter is payload: a fetch that stopped short must not boot a
|
||||
# silently-plain endpoint, and Restart=always would loop on the missing
|
||||
# file until somebody reads the name in the log.
|
||||
touch "$MODEL_DIR/$MM_STUB"
|
||||
rm -f "$MODEL_DIR/$LORA_STUB"
|
||||
serve_argv
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"frank-zealot: missing $MODEL_DIR/$LORA_STUB"* ]]
|
||||
|
||||
rm -f "$RUNTIME_DIR/bin/llama-server"
|
||||
serve_argv
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"frank-zealot: missing"* ]]
|
||||
}
|
||||
|
||||
@test "the deploy handler gates on binary, GGUF and projector, then MainPID-hardens health" {
|
||||
@test "the deploy handler gates on binary, GGUF, projector and adapter, then MainPID-hardens health" {
|
||||
grep -q 'deploy_llm_zealot' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_ZEALOT_RUNTIME' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_ZEALOT_MODEL_FILE' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_ZEALOT_MMPROJ' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_ZEALOT_LORA' "$REMOTE_DEPLOY"
|
||||
grep -q 'relink_unit frank-zealot' "$REMOTE_DEPLOY"
|
||||
grep -q 'systemctl enable frank-zealot.service' "$REMOTE_DEPLOY"
|
||||
grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY"
|
||||
|
||||
Reference in New Issue
Block a user