From 68a525740d357ada0ba88aabb2d6e5a1f6a2fd3a Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 7 Oct 2026 13:48:54 -0700 Subject: [PATCH] frank-secondary: swap Swift-1.5 RCO-IQ3_XXS -> Gemma4-26B-A4B QAT Q4_K_M Payload: HauhauCS/Gemma4-26B-A4B-QAT-Uncensored Q4_K_M (16,796,015,520 B) + the publisher's BF16 mmproj, /home/dank/models/Gemma4-26B-A4B/. Shape: ONE 200,704-token slot (--parallel 1; 2x200,704 does not fit the 24 GiB card), f16 K + q8_0 V under flash attention, batch 4096 / ubatch 1024, --log-verbosity 4, MTP OFF (the gemma4-assistant file is on disk, not loaded: its draft KV is built against the target's full window, measured +1.9 GiB -> 23,806/319 free, no margin beside the f16-K raise). Measured boots (nvidia-smi): q8_0 K + q8_0 V / ubatch 2048 / MTP off = 21,890 used / 2,235 free; with the MTP assistant = 23,806 / 319. Engine facts recorded: gemma4 hybrid attention (25 sliding-window + 5 full-attention layers) so KV scales with 5 layers only, and attention_k_eq_v is honoured by build 5fc4f3c (pool bills K-only; the measured q8_0 pool = 2.05 GiB = the K-only figure). Serve script gained VERBOSITY (+ --log-verbosity in the argv), the SPEC_DRAFT_FILE drafter path, and defaults/lore retargeted to Gemma4. deploy-remote.sh payload gates follow the env. The frank-secondary bats seam is rewritten for the new contract (argv-based) and fixes a pre-existing vacuity: serve_argv regenerates the test env, so mutations made before it were silently undone - tests that mutate now exec through run_stage. frank-secondary is the fleet compaction hub (200,704 is the only window that fits a worst-case compaction input). --- AGENTS.md | 136 ++++++------- scripts/deploy-remote.sh | 8 +- scripts/frank-secondary.env | 24 ++- scripts/serve-frank-secondary.sh | 71 +++---- ...rank-secondary-llama-gemma4-q4km-200704.md | 92 +++++++++ serverAppConfigs-docs/LLM/index.md | 24 ++- serverAppConfigs-docs/index.md | 12 +- tests/frank_secondary_unit.bats | 190 ++++++++++-------- 8 files changed, 335 insertions(+), 222 deletions(-) create mode 100644 serverAppConfigs-docs/LLM/frank-secondary-llama-gemma4-q4km-200704.md diff --git a/AGENTS.md b/AGENTS.md index d2aac91..9ca1b8f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,7 +27,7 @@ merge absorbed the former `vllmScripts` repo, history preserved). |--------|---------|---------|-------|------| | Dankserver | `frank-primary` | **radiance** (StillDeadcode v1.0.7 line; image `stilldeadcode/radiance@sha256:4c3f484564…` digest-pinned INDEX digest, 287 MiB, `libr4d` gfx1201 kernels, plain `podman run`, root systemd unit) | `Qwen/Qwen3.8-Flash-Next-FP8` official (172.8 GB, 131 shards, sha256-verified) converted **no-recipe** to `qwen4exp-fp8-native.rad` (177,109 MiB measured = 173.0 GiB) — **native modelopt block-FP8, no requantisation** — on **BOTH R9700s TP=2** (`--tp-wire wht6`), experts in a **40 GiB/rank pinned host pool** + VRAM cache (`--placement expert_tiered`, `--expert-vs-cache-ratio 0.82` = the 99 %-hit point), **PLE 47.7 GiB pinned to host RAM** (`--ngram-placement ram`; root unit + `LimitMEMLOCK=infinity`), **fp8 KV 7.32 KiB/token**, **MTP spec decode depth 3**, *… | Dankclaw | `frank-large` | **halogen-flash-server 0.16.2** (`ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84…`, digest-pinned, plain `podman run` via `scripts/serve-frank-large.sh`, root systemd unit) | Quat3rnion/halogen-swift1.5-qwen3.8-flash-next-v2-abliterated **`qwen38-flash-next-v2-swift15-abliterated.hgn`** (66,687,678,464 B = compacted r2 of the card's 89,775,868,032 B original, ~62 GiB resident; the v2 `.hgn` format loads in NO other engine), **+ REQUIRED `qwen38-flash-next-ngram.hgn`** (51,200,246,144 B, peonist @5cc17cea — read beside the checkpoint) **+ vision tower** (897,916,416 B, @e053f488 — **vision ON**, `image_url` parts only, ≥256 px probes), **MTP spec decode ALWAYS-ON (`HALOGEN_SPEC_ADAPT=0` — the adaptive default disables the head per request and HALVES decode)**, MTP head INSIDE the checkpoint (`HALOGEN_MTP_HEAD` unset; no sidecar fetched), **2 slots × 200,192 ctx** (KV pool 400,384 positions ≈ 10.7 GiB at ~28 KiB/position; the 27B slots>1-disables-drafting law does NOT carry over — batching and speculation coexist, the card's quickstart ships 2 with the head), **server-side sampler 1.0/0.95/20** (card recipe; models.yml extraBody matches), coupled cap/timeout 65536/7200, prompt cache ON by default (warm 100k turn ≈ 2 s vs ≈ 88 s cold) | 7770 | -| Dankclaw | `frank-secondary` | llama.cpp master (0.5.0-dev, commit `5fc4f3c`), CUDA sm_86 build (root systemd unit, relinked) | ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF **RCO-IQ3_XXS -mtp** (27B dense qwen3_5 hybrid, 10,442,827,392 B; a per-tensor allocation mix of stock types; the MTP head in the file **drafted** — **spec ON** (draft-mtp depth 3) since 2026-10-06 evening, ngram skipped on the recurrent-rollback law) + the publisher's own **BF16 mmproj** for **vision**, on the RTX 3090, **2 × 114,688** (Frank's 112k cap each since the 2026-10-06 late reshape, cut from 131,072 to fund the **ubatch ladder 512 → 1024 → 2048** — **1,005 MiB free at boot, 763 settled after a vision probe** (encoder buffers resident)) with **q8_0 K + q8_0 V (FP8 class, 34 KiB/token)** under flash attention, batch 4096 / **ubatch 2048** (the ceiling for this shape — 4096 would not fit); **prefill 1,209 t/s @ 3,760 tokens; ~52–58 t/s recipe decode** (MTP acceptance 34–54 %, temp-1.0 noise; decode is ubatch-invariant). Renamed from `frank-small` 2026-10-06 (87721ab). Supersedes the same-morning IQ3_S-mtp payloads: the two-slot no-spec shape and the one-slot MTP shape (both 2026-10-06; payload on disk).|7771| +| Dankclaw | `frank-secondary` | llama.cpp master (0.5.0-dev, commit `5fc4f3c`), CUDA sm_86 build (root systemd unit, relinked) | HauhauCS/Gemma4-26B-A4B-QAT-Uncensored **Q4_K_M** (25.2 B-param MoE, ~4 B active, 16,780,192,888 B reported by the engine; 30 layers = 25 sliding-window + 5 full-attention) + the publisher's own **BF16 mmproj** for **vision**, on the RTX 3090, **ONE 200,704-token slot** (`--parallel 1`; two 200,704 slots do not fit 24 GiB), **f16 K + q8_0 V** under flash attention (the file declares `attention_k_eq_v` and this build honours it — the pool bills K-only), batch 4096 / **ubatch 1024**, `--log-verbosity 4`, **MTP OFF** (the assistant is a SEPARATE file and its draft KV is built against the target's full window — 23,806 / 319 MiB free with it loaded, no margin beside the f16-K raise). Measured q8_0-K boot: **21,890 used / 2,235 free**. Cutover 2026-10-07 from Swift-1.5 RCO-IQ3_XXS 2×114,688. All three endpoints route harness **compaction** here. | 7771 | Retired endpoints (`frank-npu`, and every prior runtime) are in the [runbook index](#runbook-index). The full swap history lives in @@ -308,9 +308,10 @@ image]`) — `frank-large` returned to vision with the flash-server v2 swap 27-layer Qwen-VL tower — patching, pad-token expansion and 3-part rotary are the engine core; `--mm-max-patches` available), and `frank-secondary` through -the publisher's own `mmproj-BF16.gguf` passed as `--mmproj` — the Swift +the publisher's own BF16 mmproj passed as `--mmproj` — the Gemma4 GGUF itself carries no vision tensors. `frank-secondary`'s `contextWindow` is -**114688** (cut from 131072 on 2026-10-06 late to fund the ubatch ladder to 2048), the +**200704** (the one-slot window since the 2026-10-07 Gemma4 swap; `--parallel 1`, +so the pool IS the conversation), the per-slot window, not the model's native 262144: the harness must compact before llama.cpp answers a longer prompt with a 400. Note also that @@ -417,89 +418,63 @@ that (`scripts/build-rocmfpx.sh`, pin `36e9acd`) is untouched. `/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server`, `GGML_CUDA_FA_ALL_QUANTS=ON`) as a **root systemd unit** (`scripts/frank-secondary.service`, `relink_unit`ed into -`/etc/systemd/system/`), serving `ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF` -at **RCO-IQ3_XXS -mtp** (10,442,827,392 B, sha256 `e13d3e74…`) **plus -the publisher's own `mmproj-BF16.gguf`** (931,145,888 B, sha256 -`19acd7fc…`), both `/home/dank/models/Swift15-UncIQ3XXS/`, on port 7771. -Cutover 2026-10-05 (evening) on Frank's explicit contract: **128k -context cap, KV at FP8, vision on, as many 128k contexts as fit**; -reshaped three times on 2026-10-06 — morning to two 131,072 slots -no-spec, evening to this UNCERSED RCO-IQ3_XXS payload with MTP back -on, late the RENAME to **frank-secondary** (87721ab) at two 114,688 -slots with the ubatch ladder climbing 512 → 1024 → 2048 (f2575a1, -d8da7a9). The -same-day payloads it supersedes: the LIVE MiMo-V2.6 Q6_K shape and the -Ornith-1.5-9B Q8_0 swap that was committed (77decb5..380aafa) but never -deployed (blocked at the docs gate; the docs debt is what this section -settles). +`/etc/systemd/system/`), serving +`HauhauCS/Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-MTP` at +**Q4_K_M** (16,796,015,520 B on disk; the engine reports +16,780,192,888 B, 25,233,142,046 params, vocab 262,144) **plus the +publisher's own BF16 mmproj** (1,194,827,776 B), both +`/home/dank/models/Gemma4-26B-A4B/`, on port 7771. Cutover 2026-10-07 +from the Swift-1.5 RCO-IQ3_XXS 2×114,688 shape. The MTP assistant +(`mtp-gemma-4-26B-A4B-it.gguf`, 251,937,728 B) is on disk, **not loaded**. -**Architecture: qwen3_5 hybrid, bigger** — block_count 65 (64 real layers + -the blk.64 MTP head), `full_attention_interval 4` ⇒ **16 full-attention + -48 Gated DeltaNet**, native 262,144 window, dense. KV exists only in the 16 -attention layers (4 KV heads × 256 dim): 64 KiB/token at f16, **34 KiB/token -at q8_0**. The DeltaNet state is a fixed per-slot buffer (~150 MiB/slot at -48 linear layers × 48 value heads). Header facts were read from the Hub -file by HTTP range + struct parse before anything was downloaded. +**Architecture: gemma4 MoE hybrid** — 30 layers, 128 experts (8 active), +n_embd 2816, native 262,144 window. Attention is hybrid: **25 +sliding-window layers (window 1024, a fixed ring each) + 5 full-attention +layers (5, 11, 17, 23, 29)**, 8 KV heads × 256 dim — so the KV pool scales +with the 5 full layers ONLY: ~10.6 KiB/token at q8_0, ~20.5 KiB/token at +f16. `attention_k_eq_v: True` — `gemma4.cpp` maps the absent `wv` to +`Vcur = Kcur`, and this build's KV cache honours it (no V slots allocated: +the q8_0 pool measured ~2.05 GiB at 200,704, the K-only figure). **Bill +K-only.** -**The shape (2026-10-06 midnight; Frank's tight-fit order, MTP back on -in the evening, per-slot cut + ubatch climbed to 2048):** `--ctx-size -229376 --parallel 2` = **TWO 114,688-token conversations** (Frank's -112k cap each; `n_ctx_slot = CTX_SIZE / PARALLEL`; cut from 131,072 to -fund the ubatch rungs), `--flash-attn on` pinned, **q8_0 K + q8_0 V** -(llama.cpp's FP8 class — no true fp8 KV dtype exists; 8.5 effective -bits; supersedes the same-day f16/f16 rule written for the Ornith 9B -fit), all layers on the GPU, `--mmproj` for vision, batch 4096 / -ubatch 2048. **Speculative decoding is ON** (draft-mtp, depth 3): the --mtp file's head is drafted against the main model's own weights; -ngram-mod skipped on purpose — on a qwen35 hybrid ngram disables -recurrent-state rollback, so a rejected draft corrupts the DeltaNet -state. +**The shape (2026-10-07):** `--ctx-size 200704 --parallel 1` = ONE +conversation (Frank's 200k cap; the harness `contextWindow` quotes +200704), `--flash-attn on`, **f16 K + q8_0 V** (exact K, FP8-class V), +batch 4096 / **ubatch 1024** (cut from 2048 to fund the f16-K raise), +`--log-verbosity 4` (debug — the KV/buffer forensics land in the journal), +`--mmproj` for vision, **speculative decoding OFF** (`SPEC_TYPE=none`). -**Measured at the midnight boot (2026-10-06, 2 × 114,688 / ubatch -2048): 23,120 used / 1,005 MiB free, settling 763 MiB after a vision -probe** (`nvidia-smi` free; the encoder buffers stay resident on this -boot) — the per-slot cut freed ~1.05 GiB of KV and both ubatch rungs -spent it back (~0.3 + ~0.75 GiB measured; the 1024 rung had landed -22,374/1,751, the 2048 rung cost 746 MiB, ~130 over the 616 the rung -measured on the MiMo shape). The MTP draft ctx costs **~2.4 GiB -measured** (23,636 − 22,728 + 1,561 at the 262,144 draft window) — ~6× -the ~0.4 GiB the one-slot era assumed, because the drafter's KV is -built against the target's FULL context, not the 3-token proposal -depth; the lighter RCO-IQ3_XXS quant (−1.56 GiB vs IQ3_S-mtp) funds it. -**Prefill 1,209 t/s @ 3,760 tokens; decode 52.0–58.2 tok/s (recipe -temp 1.0) / 66.9 (temp 0)**, acceptance 34.2–54.4 % — decode is -ubatch-invariant, the spread is temp-1.0 acceptance noise. -`reasoning_effort` probe unchanged: `xhigh` 200, `high` → 500 Jinja -raise (the models.yml fold stays mandatory). +**Why MTP is off**: Gemma4's MTP is a SEPARATE assistant file (arch +`gemma4-assistant`), not an in-file head, and the draft context is built +against the target's FULL window — measured cost **~1.9 GiB** (23,806 used +/ 319 free with the assistant loaded, vs 21,890 / 2,235 without). The +assistant also warns `n_ctx_seq (200704) > n_ctx_train (131072)` — the +draft runs 53 % past its trained window. The q8_0-K boot leaves 2,235 MiB, +which the f16-K raise (+2.05 GiB) spends; MTP and exact-K cannot coexist on +this card. -**Vision with the publisher's own projector**: ajgazin's repo SHIPS -`mmproj-BF16.gguf` in the same repo (the IQ3_S era's unsloth mmproj was -built from the SHARED BASE `Qwen/Qwen3.8-27B`, legal because Swift-1.5's -vision stack was hash-proven byte-identical to the base). The probe -traps carry over: ≥256 px probe images or the model answers "video"; -only the OpenAI `image_url` content part reaches the model. **New on -this payload: the uncensored model thinks more than the censored -Swift 1.5** — a 64-token budget is eaten entirely by -`reasoning_content` (empty `content`); 128 tokens clears the vision -probe. +**Measured boots** (nvidia-smi, the referee): q8_0 K + q8_0 V, ubatch 2048, +MTP off: **21,890 used / 2,235 free**; same shape with the MTP assistant: +**23,806 / 319**. The f16-K + ubatch-1024 boot: see the runbook's ledger +(projected ~23 GiB). Decode is expected ubatch-invariant (Swift-era 3090 +law; unmeasured on Gemma4). -The RCO-IQ3_XXS tier is a **per-tensor allocation mix** of stock quant -types (IQ3_S 97 / BF16 96 / IQ3_XXS 83 / IQ2_S 72 / IQ4_XS 33 / IQ2_XS 32 -/ IQ2_XXS 30 / Q2_K 28 / Q4_K 19 / Q6_K 8 / IQ1_S 4 / IQ1_M 4) — stock -llama.cpp loads it; the tensor-type histogram is the law (see Key -Lessons). Publisher KLD vs Swift BF16: mean 0.002631 / 90th 0.042136 / -max 34.49 — RCO is the publisher's "more experimental" family, and the -payload is the UNCECERSED Swift 1.5 (23/100 refusals vs Swift 1.5's -98/100, KL 0.0884). +**Vision**: the publisher's own BF16 mmproj via `--mmproj`. Probe traps +carry over from the Swift era: ≥256 px probe images or the model answers +"video", and only the OpenAI `image_url` content part reaches the model. + +**Compaction hub**: `models.yml` routes `compactionModel` for +`frank-primary`, `frank-large` and `frank-secondary` here — 200,704 is the +only window in the fleet that fits a worst-case compaction input (~133k +tokens). Consequence of `--parallel 1`: compaction requests serialize +behind live generation on this slot. Runbook: -`serverAppConfigs-docs/LLM/frank-secondary-llama-swift15-unc-iq3xxs-mtp-114688.md`. -Rollback: `/home/dank/models/Swift15-IQ3S/` (the IQ3_S-mtp payload live -until the 2026-10-06 evening swap — revert da36fa3 for the two-slot -no-spec shape) or `/home/dank/models/MiMo-Q6K/` (revert to the -2×200,704 f16 shape — [[frank-secondary-llama-mimo-q6k-2x200704]]) or -`/home/dank/models/Ornith-Q8/` (the committed-never-deployed Q8_0; -revert to 380aafa's tree). +`serverAppConfigs-docs/LLM/frank-secondary-llama-gemma4-q4km-200704.md`. +Rollback: revert the swap commit — the Swift payload +(`/home/dank/models/Swift15-UncIQ3XXS/`), the MiMo Q6_K payload +(`/home/dank/models/MiMo-Q6K/`) and the Ornith Q8_0 payload +(`/home/dank/models/Ornith-Q8/`) all remain on disk. ### frank-primary — Radiance (StillDeadcode), Qwen3.8-Flash-Next FP8 native, TP=2 (Dankserver, ACTIVE) @@ -1022,7 +997,7 @@ Active: | Service | Runbook | |---------|---------| | frank-large (flash-server 0.16.2, abliterated Swift1.5 v2 .hgn + MTP always-on, 2×200,192, vision ON, Dankclaw) | `serverAppConfigs-docs/LLM/frank-large-flash-server-v2-swift15.md` | -| frank-secondary (llama.cpp CUDA, Swift-1.5-27B Uncensored RCO IQ3_XXS-mtp, 2×114,688 draft-mtp q8_0 KV, Dankclaw) | `serverAppConfigs-docs/LLM/frank-secondary-llama-swift15-unc-iq3xxs-mtp-114688.md` | +| frank-secondary (llama.cpp CUDA, Gemma4-26B-A4B QAT Q4_K_M, 1×200,704 f16-K/q8_0-V MTP-off, Dankclaw) | `serverAppConfigs-docs/LLM/frank-secondary-llama-gemma4-q4km-200704.md` | | frank-primary (radiance StillDeadcode, Qwen3.8-Flash-Next-FP8 native, 2×R9700 TP=2, Dankserver) | `serverAppConfigs-docs/LLM/frank-primary-radiance-qwen38-native-fp8.md` | | Dankclaw kernel posture (`amd_iommu=off`) | `serverAppConfigs-docs/LLM/dankclaw-iommu-translation-off.md` | | Deployment flow + incidents | `serverAppConfigs-docs/LLM/deployment.md` | @@ -1037,6 +1012,7 @@ Retired (rollback targets and history): | FreeToken PR #587 ROCm, Qwen3.8-Flash-Next-Uncensored-FP8, ONE R9700 (`frank-primary`) | Dankserver | 2026-10-05 → 10-05 (rollback target — tag `frank-primary-freetoken-pr587-20261005`; trio + Dockerfile in the tag, image `localhost/frank-primary-freetoken:pr587-77a7201` and orcarouter payload on disk) | `frank-primary-freetoken-pr587.md` | | llama.cpp master MiMo-V2.6-Distill-Qwen-9B Q6_K 2×200,704 f16 KV (`frank-secondary`) | Dankclaw | 2026-10-03 → 10-05 (rollback target — payload on disk; owns the ubatch-2048 A/B; the Ornith swap of 10-05 was committed but never deployed, so this was the LIVE payload Swift replaced) | `frank-secondary-llama-mimo-q6k-2x200704.md` | | llama.cpp Ornith-1.5-9B Q8_0, ONE 262,144 f16 KV (`frank-secondary`) | Dankclaw | 2026-10-05 (committed 77decb5→380aafa, never deployed — superseded by Swift the same evening; payload on disk) | trail in `frank-secondary-llama-swift15-unc-iq3xxs-mtp-114688.md` | +| llama.cpp CUDA Swift-1.5-27B Uncensored RCO IQ3_XXS-mtp, 2×114,688 draft-mtp q8_0 KV (`frank-secondary`) | Dankclaw | 2026-10-06 → 2026-10-07 (rollback target — payload on disk; owns the 3090 ubatch ladder A/B and the 34 KiB/token qwen35 KV law) | `frank-secondary-llama-swift15-unc-iq3xxs-mtp-114688.md` | | ROCmFPX llama.cpp fork Qwen3.8-27B Q8_0_ROCMFPX_AGENT + MTP (`frank-large`) | Dankclaw | 2026-10-02 → 10-03 (rollback target, payload on disk) | `frank-large-rocmfpx-q8agent-mtp.md` | | gufo v0.1.1 Qwen3.8-27B UD-Q8_K_XL/UD-Q4_K_XL + DFlash2 (`frank-large`) | Dankclaw | 2026-09-28 → 10-01, again 10-01 → 10-02 (payloads removed from disk 2026-10-03; rollback = re-download) | `frank-large-gufo-qwen38-27b.md` | | vLLM TheRock MiMo-V2.6-Distill-Qwen-9B BF16 + cross-model MTP (`frank-large`) | Dankclaw | 2026-10-01 (same-day rollback to gufo) | `frank-large-vllm-mimo-v26-bf16.md` | diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 46c3a7f..7571b37 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1225,13 +1225,13 @@ deploy_llm_large() { # # It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is # not shared: frank-large's engine runs on the gfx1151 iGPU. -LLM_SECONDARY_MODEL_FILE="${LLM_SECONDARY_MODEL_FILE:-/home/dank/models/Swift15-UncIQ3XXS/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf}" +LLM_SECONDARY_MODEL_FILE="${LLM_SECONDARY_MODEL_FILE:-/home/dank/models/Gemma4-26B-A4B/Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf}" LLM_SECONDARY_RUNTIME="${LLM_SECONDARY_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" # The projector IS the vision capability here: the GGUF carries no vision -# tensors, and the publisher ships mmproj-BF16.gguf in the same repo, so a -# deploy that starts without this file silently ships a text-only +# tensors, and the publisher ships its own BF16 projector in the same repo, so +# a deploy that starts without this file silently ships a text-only # endpoint. Checked unconditionally, like frank-large's. -LLM_SECONDARY_MMPROJ="${LLM_SECONDARY_MMPROJ:-/home/dank/models/Swift15-UncIQ3XXS/mmproj-BF16.gguf}" +LLM_SECONDARY_MMPROJ="${LLM_SECONDARY_MMPROJ:-/home/dank/models/Gemma4-26B-A4B/mmproj-Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf}" deploy_llm_secondary() { # Refuse rather than restart-loop: a unit whose runtime or payload is diff --git a/scripts/frank-secondary.env b/scripts/frank-secondary.env index 8d13187..df0a6e8 100644 --- a/scripts/frank-secondary.env +++ b/scripts/frank-secondary.env @@ -4,7 +4,7 @@ # # 2026-10-07 (Frank): swap from Swift-1.5-Qwen3.8-27B RCO-IQ3_XXS to # Gemma4-26B-A4B Q4_K_M, ONE 200,704 context, q8_0 KV (FP8 class), -# MTP via separate assistant file (--spec-draft-model). +# MTP DISABLED (SPEC_TYPE=none). # # Architecture: gemma4 MoE — 30 layers, 128 experts (8 active), # hybrid attention: 25 sliding_window + 5 full_attention @@ -15,8 +15,11 @@ # KV: q8_0 K + q8_0 V (FP8 class, 1.0625 B/elem). The build has no # true fp8 KV dtype; q8_0 is the FP8 class. # -# MTP: separate assistant file (mtp-gemma-4-26B-A4B-it.gguf, 0.252 GiB), -# loaded via --spec-draft-model + --spec-type draft-mtp. +# MTP is OFF: SPEC_TYPE=none strips --spec-type AND --spec-draft-model +# (spec flags move as a set). The assistant file +# (mtp-gemma-4-26B-A4B-it.gguf, 0.252 GiB) stays on disk as the drafter +# payload if MTP is ever re-enabled. Measured VRAM: MTP ON booted +# 23,806 used / 319 free; MTP OFF boots 21,890 used / 2,235 free. LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server MODEL_FILE=/home/dank/models/Gemma4-26B-A4B/Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf MMPROJ_FILE=/home/dank/models/Gemma4-26B-A4B/mmproj-Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf @@ -26,10 +29,15 @@ LISTEN_PORT=7771 CTX_SIZE=200704 PARALLEL=1 BATCH_SIZE=4096 -UBATCH_SIZE=2048 -CACHE_TYPE_K=q8_0 +UBATCH_SIZE=1024 +CACHE_TYPE_K=f16 CACHE_TYPE_V=q8_0 -SPEC_TYPE=draft-mtp -SPEC_DRAFT_N_MAX=3 -SPEC_DRAFT_FILE=/home/dank/models/Gemma4-26B-A4B/mtp-gemma-4-26B-A4B-it.gguf +SPEC_TYPE=none NGL=99 + +# 2026-10-07 (Frank): verbosity 4 (debug — KV layout, buffers, graph capture), +# ubatch 1024, K raised to f16 (V stays q8_0). f16 K costs +2.05 GiB over q8_0 +# (5 full-attn layers x 200,704 tok x 8 heads x 256 dim x 2 B); the MTP-off +# boot had 2,235 MiB free, so this is a deliberate tight fit — read the boot +# ledger, not the arithmetic. +VERBOSITY=4 diff --git a/scripts/serve-frank-secondary.sh b/scripts/serve-frank-secondary.sh index 884fb15..88ccb55 100755 --- a/scripts/serve-frank-secondary.sh +++ b/scripts/serve-frank-secondary.sh @@ -1,23 +1,22 @@ #!/usr/bin/env bash -# Serve ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF -# (RCO-IQ3_XXS, **MTP head in the file**) as frank-secondary on Dankclaw's -# RTX 3090 via llama.cpp (master build, CUDA sm_86). +# Serve HauhauCS/Gemma4-26B-A4B-QAT-Uncensored (QAT **Q4_K_M**) as +# frank-secondary on Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86). # -# 2026-10-06 (Frank): TWO 131,072 conversations (262144 / 2) at the 128k -# cap, ubatch 512, and MTP BACK ON (draft-mtp, depth 3 — the payload is -# 1.56 GiB lighter than the IQ3_S-mtp it replaced, which is what funds -# the draft ctx plus the second slot; ngram stays skipped — on a qwen35 -# hybrid it disables recurrent-state rollback). The q8_0 KV contract -# (FP8 class) is unchanged; the per-tensor flags stay explicit either -# way. +# 2026-10-07 (Frank): Gemma4-26B-A4B swap with MTP OFF and ONE 200,704-token +# conversation (--parallel 1) — the 24 GiB card cannot fund a second slot at +# this weight, and the MTP assistant's KV (built against the target's full +# window, ~1.9 GiB measured) plus the f16-K raise do not coexist on it. +# Verbosity 4 ships so the VRAM-fit forensics land in the journal. # -# Architecture: qwen35 hybrid — 64 real layers, 16 full-attention + 48 -# Gated DeltaNet (full_attention_interval 4), native 262144 window, plus -# the blk.64 MTP head (nextn_predict_layers=1). KV lives only in the 16 -# attention layers (34 KiB/token at q8_0); the DeltaNet state is per-slot -# fixed size. +# Architecture: gemma4_26b_a4b — 30-layer MoE (active ~4B), hybrid attention: +# 25 sliding-window layers (fixed 1024-token ring each) + 5 full-attention +# layers (5, 11, 17, 23, 29), 8 KV heads x 256 dim, n_ctx_train 262,144. KV +# therefore scales ONLY with the 5 full layers: ~30.7 KiB/token at f16 K + +# q8_0 V (~6.2 GiB at 200,704). attention_k_eq_v is declared by the file and +# honoured by the build (bill K-side accounting; the measured boot is the +# referee: 21,890 used / 2,235 free at q8_0/q8_0, MTP off). # Vision comes from the projector file (the GGUF carries no vision -# tensors; the publisher SHIPS mmproj-BF16.gguf in the same repo), so +# tensors; the publisher ships its own BF16 mmproj in the same repo), so # --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the # documented text-only opt-out. # @@ -38,40 +37,43 @@ fi # --- defaults (used when the env file does not set them) ------------------- : "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" -: "${MODEL_FILE:=/home/dank/models/Swift15-UncIQ3XXS/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf}" +: "${MODEL_FILE:=/home/dank/models/Gemma4-26B-A4B/Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf}" : "${ALIAS:=frank-secondary}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7771}" -# ctx-size is the POOL llama.cpp divides across slots: 262144 / 2 gives -# TWO 131,072 conversations — Frank's 128k cap each. 2026-10-06 (Frank): -# drop MTP, drop ubatch to 512, take two slots; the fit is tight by his -# explicit order (frank-secondary.env carries the arithmetic and the floor -# violation). -: "${CTX_SIZE:=262144}" -: "${PARALLEL:=2}" +# ctx-size is the POOL llama.cpp divides across slots; with --parallel 1 the +# pool IS the conversation window. 200,704 is inside the model's trained +# 262,144; the harness contextWindow MUST quote this number — a longer prompt +# is answered with a 400, never truncated. +: "${CTX_SIZE:=200704}" +: "${PARALLEL:=1}" # Compute-buffer geometry. Ubatch is a prefill knob (decode never moves). -# Frank's 2026-10-06 tight-fit order ships 4096/512 (claws back ~0.3 GiB -# toward the second slot); ubatch 1024 is the first A/B back if a future -# boot lands with headroom (a miss fails loud at graph capture). The +# 2026-10-07 (Frank): 1024 — cut from 2048 to fund the f16-K raise. The # logical batch is VRAM- and throughput-neutral at a fixed ubatch. : "${BATCH_SIZE:=4096}" -: "${UBATCH_SIZE:=512}" +: "${UBATCH_SIZE:=1024}" # KV at FP8 class = q8_0 on llama.cpp (1.0625 B/elem => 8.5 effective bits; # there is no true fp8 KV dtype). Explicit either way so the contract is # visible in the argv. : "${CACHE_TYPE_K:=q8_0}" : "${CACHE_TYPE_V:=q8_0}" -# Speculative decoding is ON by default (Frank, 2026-10-06 evening): -# draft-mtp against the head that is IN the file. SPEC_TYPE none strips -# BOTH speculative flags (flags move as a set) — a no-spec baseline must -# not carry half a spec pair. -: "${SPEC_TYPE:=draft-mtp}" +# Speculative decoding is OFF by default since the Gemma4 swap (Frank, +# 2026-10-07): Gemma4's MTP is a SEPARATE assistant file (arch +# gemma4-assistant), not an in-file head, and its KV is built against the +# target's full window — ~1.9 GiB measured, which the f16-K raise cannot +# afford. SPEC_TYPE=none strips BOTH speculative flags (flags move as a +# set); naming a drafter requires SPEC_DRAFT_FILE alongside SPEC_TYPE. +: "${SPEC_TYPE:=none}" : "${SPEC_DRAFT_N_MAX:=3}" : "${SPEC_DRAFT_FILE:=}" : "${NGL:=99}" +# Log verbosity threshold (llama.cpp -lv). 3 is the engine default (info); +# 4 adds debug detail (KV layout, buffer sizes, graph capture) which is what +# the VRAM-fit forensics need. Levels above 4 flood the journal. +: "${VERBOSITY:=3}" # Vision projector. No colon on purpose: an explicitly empty value means # "text-only, on purpose" and must NOT be replaced by the default path. -: "${MMPROJ_FILE=/home/dank/models/Swift15-UncIQ3XXS/mmproj-BF16.gguf}" +: "${MMPROJ_FILE=/home/dank/models/Gemma4-26B-A4B/mmproj-Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf}" # Fail loudly rather than half-loading: a missing file is a download that # has not finished, and Restart=always would turn it into a crash loop that @@ -118,6 +120,7 @@ ARGS=( --flash-attn on --cache-type-k "$CACHE_TYPE_K" --cache-type-v "$CACHE_TYPE_V" + --log-verbosity "$VERBOSITY" --jinja --reasoning auto --reasoning-format deepseek diff --git a/serverAppConfigs-docs/LLM/frank-secondary-llama-gemma4-q4km-200704.md b/serverAppConfigs-docs/LLM/frank-secondary-llama-gemma4-q4km-200704.md new file mode 100644 index 0000000..32e0ca8 --- /dev/null +++ b/serverAppConfigs-docs/LLM/frank-secondary-llama-gemma4-q4km-200704.md @@ -0,0 +1,92 @@ +# frank-secondary — llama.cpp CUDA, Gemma4-26B-A4B Q4_K_M, 1×200,704 (Dankclaw) + +**ACTIVE since 2026-10-07.** RTX 3090, port 7771, root systemd unit +`frank-secondary.service`, served by `scripts/serve-frank-secondary.sh` + +`scripts/frank-secondary.env`. Supersedes +[[frank-secondary-llama-swift15-unc-iq3xxs-mtp-114688]] (Swift-1.5-Qwen3.8-27B +RCO-IQ3_XXS-mtp, 2×114,688, in-file MTP). + +## Payload + +|File|Path|Bytes| +|---|---|---| +|Main (Q4_K_M)|`/home/dank/models/Gemma4-26B-A4B/Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf`|16,780,192,888| +|Vision (mmproj BF16)|`/home/dank/models/Gemma4-26B-A4B/mmproj-Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf`|~1.195 GiB| +|MTP assistant (on disk, NOT loaded)|`/home/dank/models/Gemma4-26B-A4B/mtp-gemma-4-26B-A4B-it.gguf`|~0.252 GiB| + +Repo: `HauhauCS/Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-MTP` +(base `google/gemma-4-26B-A4B-it`). Downloaded 2026-10-07 by plain +`curl -fL` (no token needed, ungated). + +## Shape + +``` +--ctx-size 200704 --parallel 1 # ONE conversation, Frank's 200k cap +--flash-attn on +--cache-type-k f16 --cache-type-v q8_0 # exact K, FP8-class V (q8_0) +--batch-size 4096 --ubatch-size 1024 +--n-gpu-layers 99 +--log-verbosity 4 # debug: KV layout, buffers, graph capture +--spec-type none # MTP OFF (see below) +--mmproj +--jinja --reasoning auto --reasoning-format deepseek +``` + +Measured boot at **q8_0 K + q8_0 V, ubatch 2048, MTP off** (2026-10-07 +morning): **21,890 MiB used / 2,235 MiB free** (`nvidia-smi`). With MTP ON +(the assistant loaded) the same shape was **23,806 / 319** — the drafter's +context costs ~1.9 GiB because its KV is built against the target's full +window ([[Llama Cpp Serving Sizing]]). + +Same evening (Frank): **K raised to f16, ubatch cut 2048 → 1024, verbosity +4**. f16 K doubles the K side (20.5 KiB/token over the 5 full layers ≈ 4.1 +GiB at 200,704, +2.05 GiB over q8_0); the ubatch cut claws back part of it +(the Swift-era 3090 measurement put the 2048 rung at ~746 MiB of compute +buffers). Projected usage ~23 GiB — **the boot ledger is the referee, not +this arithmetic**: on a 24,125 MiB card a failed graph capture is the failure +mode. Decode is ubatch-invariant (Swift-era 3090 law; unmeasured on Gemma4 — +inference). + +## Architecture facts + +Gemma4 MoE hybrid: 30 layers, 128 experts (8 active), n_embd 2816, 262,144 +native window, n_vocab 262,144. Hybrid attention: **25 sliding-window layers +(window 1024) + 5 full-attention layers (5, 11, 17, 23, 29)** — the KV pool +scales with the 5 full layers only (8 KV heads × 256 dim): q8_0 ≈ 10.6 +KiB/token, f16 ≈ 20.5 KiB/token. A 200,704 slot therefore costs ~2.05 GiB at +q8_0 — far less than the Swift 16-layer shape (34 KiB/token). +`attention_k_eq_v: True` — in `gemma4.cpp` the absent `wv` tensor makes +`Vcur = Kcur` in the compute graph, and in this build (5fc4f3c) the KV cache +**honours it: no V slots are allocated**. Measured: the q8_0 pool landed at +~2.05 GiB, exactly the K-only figure. **Bill K-only.** The `--cache-type-v` +flag is kept explicit for contract visibility; with `k_eq_v` it is inert. + +The llama.cpp master build at commit `5fc4f3c` supports the arch +(`src/models/gemma4.cpp`, `LLM_ARCH_GEMMA4 = "gemma4"`). Note: `grep -c` +arch names on the packed binary returns 0 for EVERY arch (including the +`qwen3_5` it serves) — check the source tree, not the binary. + +## MTP law + +MTP for Gemma4 is a SEPARATE file (`mtp-gemma-4-26B-A4B-it.gguf`, arch +`gemma4-assistant`), loaded via `--spec-draft-model` beside +`--spec-type draft-mtp` — unlike Swift, whose MTP head ships inside the main +GGUF. The serve script gained `SPEC_DRAFT_FILE` for this. 2026-10-07 (Frank): +**MTP disabled** — `SPEC_TYPE=none` strips both spec flags (flags move as a +set). The assistant payload stays on disk; setting `SPEC_TYPE=draft-mtp` + +`SPEC_DRAFT_FILE` re-enables it at the cost of ~1.9 GiB (boots at 319 MiB +free). + +## Verification + +- `curl http://192.168.0.143:7771/health` → `{"status":"ok"}` +- `/v1/models` reports `n_ctx: 200704`, `ftype: Q4_K - Medium` +- Vision probe: solid-color PNG ≥ 256 px via OpenAI `image_url` part, + expect the exact color word. + +## Rollback + +Revert to the Swift trio: `MODEL_FILE=/home/dank/models/Swift15-UncIQ3XXS/…`, +`CTX_SIZE=229376`, `PARALLEL=2`, in-file `SPEC_TYPE=draft-mtp` — the Swift +payload remains on disk. See +[[frank-secondary-llama-swift15-unc-iq3xxs-mtp-114688]]. diff --git a/serverAppConfigs-docs/LLM/index.md b/serverAppConfigs-docs/LLM/index.md index 2f6a513..f80a24b 100644 --- a/serverAppConfigs-docs/LLM/index.md +++ b/serverAppConfigs-docs/LLM/index.md @@ -16,10 +16,9 @@ digest serving the Quat3rnion ABLITERATED Swift1.5 finetune of Qwen3.8-Flash-Next v2, a ~62 GiB v2 .hgn that loads in no other engine, MTP spec decode always-on (HALOGEN_SPEC_ADAPT=0), TWO slots at 200,192 ctx, vision ON)** and -**llama.cpp CUDA (frank-secondary, port 7771: ajgazin Swift-1.5-Qwen3.8-27B -Uncensored Dynamic-MTP RCO-IQ3_XXS GGUF — 27B (the MTP head in the file -IS drafted, draft-mtp depth 3), TWO 131,072 conversations at Frank's -128k cap (the tightest fit in the endpoint's history — 489 MiB free), +**llama.cpp CUDA (frank-secondary, port 7771: HauhauCS Gemma4-26B-A4B QAT +Uncensored Q4_K_M GGUF — 26B MoE hybrid (25 sliding-window + 5 full-attention +layers), ONE 200,704-token conversation, MTP disabled, q8_0 K + q8_0 V (FP8 class), vision via the publisher's own BF16 mmproj, on the RTX 3090)**. @@ -52,8 +51,8 @@ frank-large, port 7770] 2 x 200192 ctx, MTP always-on, image input] G[llama.cpp master CUDA sm_86 frank-secondary, port 7771] - H[Swift-1.5-27B Uncensored RCO-IQ3_XXS-mtp GGUF -2 x 131072 ctx, q8_0 K + q8_0 V, draft-mtp depth 3, mmproj vision] + H[Gemma4-26B-A4B QAT Uncensored Q4_K_M GGUF +1 x 200704 ctx, q8_0 K + q8_0 V, MTP off, mmproj vision] E2 --> E --> F E2 --> G --> H end @@ -103,8 +102,17 @@ frank-secondary, port 7771] (`slow`/`advisor` → frank-primary) carried forward. Superseded by the MODEL swap: the abliterated v2 .hgn loads only in flash-server, so the engine moved with the model -- [[frank-secondary-llama-swift15-unc-iq3xxs-mtp-114688]] — **ACTIVE since - 2026-10-05 (evening); UNCECERSED RCO-IQ3_XXS with MTP back on since +- [[frank-secondary-llama-gemma4-q4km-200704]] — **ACTIVE since 2026-10-07** + (root systemd unit, port 7771): llama.cpp master (CUDA sm_86) serving + HauhauCS/Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-MTP **Q4_K_M** + on the RTX 3090 — ONE 200,704-token conversation, q8_0 K + q8_0 V, + vision via the publisher's mmproj. MTP is OFF (the gemma4-assistant + drafter costs ~1.9 GiB: 23,806 used/319 free with it, 21,890/2,235 + without). Supersedes the Swift-1.5 era (below). +- [[frank-secondary-llama-swift15-unc-iq3xxs-mtp-114688]] — **RETIRED + 2026-10-07** (superseded by gemma4-q4km; payload on disk as rollback); + ACTIVE 2026-10-05 (evening) → 2026-10-07; UNCECERSED RCO-IQ3_XXS with + MTP back on since 2026-10-06 evening; RENAMED from frank-small and reshaped to 2 × 114,688 / ubatch ladder 512 → 1024 → **2048** late 2026-10-06** (root systemd unit, port diff --git a/serverAppConfigs-docs/index.md b/serverAppConfigs-docs/index.md index aea807a..9c8fd3f 100644 --- a/serverAppConfigs-docs/index.md +++ b/serverAppConfigs-docs/index.md @@ -372,8 +372,16 @@ The LLM inference fleet on the two hosts. Full index: - [[ciru-activefpx-promptforge]] — the RETIRED v3 CIRU PromptForge runtime on Dankclaw (superseded 2026-08-24) - [[kingjones-strix-mtp]] — the kingjones Qwen3.8-27B ROCmFP4 STRIX MTP model on Dankclaw (active) - [[dealign-ornith-crack]] — the retired Ornith-1.5-35B-A3B CRACK model on Dankserver (retired 2026-09-02) -- [[frank-secondary-llama-swift15-unc-iq3xxs-mtp-114688]] — the ACTIVE - frank-secondary runbook since 2026-10-05 (evening; **UNCECERSED RCO-IQ3_XXS +- [[frank-secondary-llama-gemma4-q4km-200704]] — the **ACTIVE** frank-secondary + runbook since 2026-10-07: llama.cpp master (CUDA sm_86) serving + HauhauCS Gemma4-26B-A4B QAT Uncensored **Q4_K_M** on the RTX 3090 — ONE + 200,704-token conversation, q8_0 K + q8_0 V, MTP **disabled** (the + `gemma4-assistant` drafter costs ~1.9 GiB: boot was 23,806 used/319 free + with it, 21,890/2,235 without), vision through the publisher's mmproj +- [[frank-secondary-llama-swift15-unc-iq3xxs-mtp-114688]] — RETIRED 2026-10-07 + (superseded by gemma4; payload on disk as the rollback): the + frank-secondary runbook 2026-10-05 (evening) → 2026-10-07 + (**UNCECERSED RCO-IQ3_XXS with MTP back on since 2026-10-06 evening; renamed from frank-small and reshaped to TWO 114,688-token conversations at Frank's 112k cap with the ubatch ladder climbed to 2048 late 2026-10-06**): llama.cpp master (CUDA diff --git a/tests/frank_secondary_unit.bats b/tests/frank_secondary_unit.bats index 4cbe403..2dc6583 100755 --- a/tests/frank_secondary_unit.bats +++ b/tests/frank_secondary_unit.bats @@ -1,15 +1,15 @@ # Seam: the frank-secondary llama.cpp root unit on dankclaw. -# frank-secondary is the RTX 3090 endpoint; since 2026-10-06 (evening) it -# serves ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF at -# RCO-IQ3_XXS (27B dense qwen3_5 hybrid) via a CUDA (sm_86) build of -# llama.cpp master: TWO 131,072-token conversations at Frank's 128k cap -# (pool 262144 / parallel 2), q8_0 K + q8_0 V (FP8 class), the -# publisher's own BF16 vision projector, and MTP back ON (draft-mtp -# depth 3 — the head is in the file; the lighter quant funds the draft -# ctx plus the second slot). The budget test pins the arithmetic; the -# boot log carries the measured free. This seam runs the serve script -# against a stub engine and asserts on the argv it actually execs, so -# what is pinned is the command the operator gets. +# frank-secondary is the RTX 3090 endpoint; since 2026-10-07 it serves +# HauhauCS's Gemma4-26B-A4B-QAT-Uncensored (QAT Q4_K_M, 30-layer MoE: 25 +# sliding-window + 5 full-attention layers) via the CUDA (sm_86) build of +# llama.cpp master: ONE 200,704-token conversation (--parallel 1), f16 K + +# q8_0 V under flash attention, the publisher's own BF16 vision projector, +# and MTP OFF — Gemma4's MTP is a separate assistant file whose KV is built +# against the target's full window (~1.9 GiB measured), which the f16-K raise +# cannot afford on a 24 GiB card. Log verbosity 4 ships so the VRAM-fit +# forensics (KV layout, buffer sizes, graph capture) land in the journal. +# This seam runs the serve script against a stub engine and asserts on the +# argv it actually execs, so what is pinned is the command the operator gets. load helpers @@ -18,9 +18,6 @@ SERVE="$REPO_ROOT/scripts/serve-frank-secondary.sh" ENV_FILE="$REPO_ROOT/scripts/frank-secondary.env" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" ENV_LARGE="$REPO_ROOT/scripts/frank-large.env" -TARGET_STUB="Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf" -# Same basename as the shipped projector: the test env repoints it into $STAGE. -MM_STUB="mmproj-BF16.gguf" setup() { STAGE="$(mktemp -d)" @@ -32,7 +29,13 @@ setup() { printf '%s\n' "$@" STUB chmod +x "$RUNTIME_DIR/bin/llama-server" "$RUNTIME_DIR/bin" 2>/dev/null || chmod +x "$RUNTIME_DIR/bin/llama-server" - touch "$MODEL_DIR/$TARGET_STUB" "$MODEL_DIR/$MM_STUB" + # The stub payload files carry the SAME basenames as the shipped env, so + # the seam exercises the real paths, and a payload rename moves one file + # (the env) instead of silently forking the test's filename constants. + MODEL_STUB="$(basename "$(sed -nE 's/^MODEL_FILE=(.+)$/\1/p' "$ENV_FILE")")" + MM_STUB="$(basename "$(sed -nE 's/^MMPROJ_FILE=(.+)$/\1/p' "$ENV_FILE")")" + DRAFT_STUB="mtp-gemma-4-26B-A4B-it.gguf" + touch "$MODEL_DIR/$MODEL_STUB" "$MODEL_DIR/$MM_STUB" "$MODEL_DIR/$DRAFT_STUB" } teardown() { @@ -43,12 +46,10 @@ teardown() { # argv, one token per line, in $output. serve_argv() { local envf="${1:-$STAGE/test.env}" - local mmproj - mmproj="$(sed -nE 's/^MMPROJ_FILE=(.+)$/\1/p' "$ENV_FILE")" cat > "$envf" < 1024 increase — pool 229376, parallel 2, per-slot - # 114688 (the cap). - [ "$(argv_value --parallel)" -eq 2 ] - ctx="$(argv_value --ctx-size)" - par="$(argv_value --parallel)" - [ $((ctx % par)) -eq 0 ] - [ $((ctx / par)) -eq 114688 ] + # llama.cpp divides --ctx-size across slots, so the observable contract + # is the per-slot window. The 24 GiB card cannot fund a second 200,704 + # slot at this weight + KV shape (measured: the MTP drafter alone costs + # ~1.9 GiB), so --parallel 1 is the shipped contract and the harness + # contextWindow must quote 200704, not the model's native 262144. + [ "$(argv_value --parallel)" -eq 1 ] + [ "$(argv_value --ctx-size)" -eq 200704 ] } -@test "the KV contract is q8_0 both halves (FP8 class), flash-attn on" { +@test "the KV contract is f16 K + q8_0 V, flash-attn on" { serve_argv [ "$status" -eq 0 ] - # Frank, 2026-10-05 evening: "k/v at FP8" for the Swift-1.5 swap — - # llama.cpp's FP8 class is q8_0 (1.0625 B/elem => 8.5 effective bits; - # there is no true fp8 KV dtype). This SUPERSEDES the same-day f16/f16 - # rule written for the Ornith 9B fit. The flag pair is what makes this - # observable — a build without the FA path silently falls back, so both - # must appear in the argv the supervisor runs. - [ "$(argv_value --cache-type-k)" = "q8_0" ] + # Frank, 2026-10-07: K raised to f16 (exact), V stays q8_0 — llama.cpp's + # FP8 class (1.0625 B/elem; no true fp8 KV dtype exists). KV scales ONLY + # with the 5 full-attention layers of 30 (the 25 sliding-window layers + # cost a fixed 1024-token ring each), so the pair costs ~6.2 GiB at + # 200,704 — the boot ledger, not this file, is the referee. flash-attn is + # REQUIRED for quantized V, and the build carries GGML_CUDA_FA_ALL_QUANTS=ON; + # without the flag the engine silently falls back and the contract is a lie. + [ "$(argv_value --cache-type-k)" = "f16" ] [ "$(argv_value --cache-type-v)" = "q8_0" ] [ "$(argv_value --flash-attn)" = "on" ] } -@test "the KV budget fits two 114688 slots at q8_0 and forbids a third" { - # Rate, from the RCO-IQ3_XXS GGUF header (same qwen35 architecture): - # SIXTEEN full-attention layers (block_count 65 = 64 real layers + the - # blk.64 MTP head, full_attention_interval 4) x 4 kv heads x 256 dim x - # 1.0625 B = 17 KiB/token for K and the same for V => 34 KiB/token at - # q8_0 (budgeted 37 with slack; f16 would be 64). The non-KV half: - # RCO-IQ3_XXS weights 10,148 MiB (10,442,827,392 B) — 1.56 GiB LIGHTER - # than the IQ3_S-mtp predecessor — + vision tower ~1.4 GiB (931,145,888 - # B BF16 mmproj at the measured ~1.45x cost law) + compute buffers at - # ubatch 2048 + CUDA graphs + TWO DeltaNet states (~150 MiB each) + the - # MTP draft ctx => 14 GiB budgeted. Two 114,688 slots are the - # 2026-10-06 late shape (the lighter per-slot funds the ubatch - # rungs); a THIRD slot must not fit. - # change CTX_SIZE and this bites before the deploy does; the boot's - # nvidia-smi free is the measured truth. - ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" - kv_gib=$((ctx * 37 / 1048576)) - third_slot_kv=$(( (ctx + 114688) * 37 / 1048576 )) - [ $((kv_gib + 14)) -le 24 ] - [ $((third_slot_kv + 14)) -gt 24 ] +@test "log verbosity follows the env file" { + serve_argv + [ "$status" -eq 0 ] + # The shipped 4 (debug) is what makes VRAM-fit forensics possible from + # the journal (KV layout, buffer sizes, graph capture). + [ "$(argv_value --log-verbosity)" = "4" ] + + # And an env that names no verbosity still ships the engine default + # explicitly — the script's written-down 3, never a blank. + sed -i '/^VERBOSITY=/d' "$STAGE/test.env" + run_stage + [ "$status" -eq 0 ] + [ "$(argv_value --log-verbosity)" = "3" ] } @test "the compute-buffer pair follows the env file and stays self-consistent" { - # This endpoint ran on llama.cpp's own defaults until 2026-10-03, so the - # contract is: whatever rung the repo carries is the rung the engine is - # exec'd with — the number a benchmark attributes a prefill rate to must be - # a repo value, never an engine default nobody wrote down. Asserted against - # the env rather than a literal so a retune moves one file, not two. + # The contract is: whatever rung the repo carries is the rung the engine + # is exec'd with — the number a benchmark attributes a prefill rate to + # must be a repo value, never an engine default nobody wrote down. + # Asserted against the env rather than a literal so a retune moves one + # file, not two. local env_batch env_ubatch env_batch="$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" env_ubatch="$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" @@ -180,31 +183,46 @@ argv_value() { # serve script's fallbacks are written-down values, not blanks the engine # fills in silently. sed -i '/^BATCH_SIZE=/d;/^UBATCH_SIZE=/d' "$STAGE/test.env" - serve_argv + run_stage [ "$status" -eq 0 ] argv_value --batch-size | grep -qE '^[0-9]+$' argv_value --ubatch-size | grep -qE '^[0-9]+$' } -@test "speculation is on: the full draft-mtp pair reaches the argv" { - # Frank, 2026-10-06 evening: MTP back on. The head is IN the file, so - # draft-mtp drafts against the main model's weights (no second model - # copy). ngram-mod stays skipped: on a qwen35 hybrid it disables - # recurrent-state rollback — a rejected draft corrupts the DeltaNet - # state. The flags move as a set: the pair is present together or - # absent together — assert both in the argv, and the env carries the - # same contract. The serve script's defaults ship the pair even for an - # env that names no drafter. +@test "speculation is off: no drafter flag reaches the argv" { + # Frank, 2026-10-07: MTP OFF on Gemma4. The assistant is a SEPARATE file + # and its draft KV is built against the target's full 200,704 window — + # ~1.9 GiB measured (23,806/319 free), a knife-edge once K is f16. The + # flags move as a set: SPEC_TYPE=none strips --spec-type, --spec-draft-n-max + # AND --spec-draft-model together, so a no-spec boot cannot carry half a + # spec pair. serve_argv [ "$status" -eq 0 ] + ! grep -qxF -- '--spec-type' <<<"$output" + ! grep -qxF -- '--spec-draft-n-max' <<<"$output" + ! grep -qxF -- '--spec-draft-model' <<<"$output" + grep -qE '^SPEC_TYPE=none$' "$ENV_FILE" +} + +@test "naming a drafter ships the full MTP trio" { + # The opt-back-in path: SPEC_TYPE + SPEC_DRAFT_FILE (the separate + # gemma4-assistant gguf) must reach the engine as a complete set — a + # draft-mtp flag without its model file boots a drafter against the main + # weights and drafts nothing on this architecture. + serve_argv + { + echo 'SPEC_TYPE=draft-mtp' + echo 'SPEC_DRAFT_N_MAX=3' + echo "SPEC_DRAFT_FILE=$MODEL_DIR/$DRAFT_STUB" + } >> "$STAGE/test.env" + run_stage [ "$(argv_value --spec-type)" = "draft-mtp" ] [ "$(argv_value --spec-draft-n-max)" = "3" ] - grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE" - grep -qE '^SPEC_DRAFT_N_MAX=3$' "$ENV_FILE" + [ "$(argv_value --spec-draft-model)" = "$MODEL_DIR/$DRAFT_STUB" ] } @test "vision ships by default and MMPROJ_FILE empty opts out of the image path" { - # The Swift GGUF carries no vision tensors, so the projector argument IS + # The Gemma4 GGUF carries no vision tensors, so the projector argument IS # the vision capability: no flag, no eyes. serve_argv [ "$status" -eq 0 ] @@ -218,7 +236,7 @@ argv_value() { ! grep -qxF -- '--mmproj' <<<"$output" } -@test "the fleet reasoning contract survives the reslice to two wide contexts" { +@test "the fleet reasoning contract reaches the argv" { serve_argv [ "$status" -eq 0 ] # deepseek format = thinking lands in reasoning_content, the field the @@ -253,8 +271,8 @@ argv_value() { } @test "the two endpoints keep separate ports and the large handler never pkills llama-server" { - # frank-large's gufo sweep must not take down frank-secondary, whose - # process is llama-server on the SAME host since this swap. + # frank-large's sweeps must not take down frank-secondary, whose + # process is llama-server on the SAME host. grep -q '^LISTEN_PORT=7771$' "$ENV_FILE" local large large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")"