On main: peer frank-small WIP held for frank-medium deploy

scripts/deploy-remote.sh

@@ -1084,12 +1084,12 @@ wait_for_llm_large_ready() {
 #
 # It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
 # not shared: frank-large's halogen engine runs on the gfx1151 iGPU.
-LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
+LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/Gemma4-12B-QAT/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf}"
 
 deploy_llm_small() {
     if [[ ! -f "$LLM_SMALL_MODEL" ]]; then
         echo "ERROR: model absent at $LLM_SMALL_MODEL" >&2
-        echo "frank-small: fetch ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF (the -mtp build plus the mmproj) first;" >&2
+        echo "frank-small: fetch HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced (Q4_K_M, the mmproj and the MTP sidecar) first;" >&2
         echo "frank-small: refusing to start a unit that cannot load." >&2
         return 1
     fi

scripts/frank-small.env

@@ -1,110 +1,90 @@
-# Tunables for frank-small (ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF, the IQ3_S
-# build with the MTP head, plus the BF16 vision projector) on mainline
-# llama.cpp, Dankclaw's RTX 3090, since 2026-09-16.
+# Tunables for frank-small (HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-
+# Balanced, Q4_K_M, plus the BF16 vision projector and the separate MTP
+# sidecar) on mainline llama.cpp, Dankclaw's RTX 3090, since 2026-09-17.
 #
-# Runtime: llama.cpp master 37b53fd (2026-09-16), CUDA build for sm_86:
+# Runtime: llama.cpp master 37b53fd, CUDA build for sm_86:
 #   cmake -B build-cuda -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON \
-#     -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86
+#     -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86 -DGGML_CUDA_FA_ALL_QUANTS=ON
 #   cmake --build build-cuda --target llama-server -j
-# GGML_CUDA_FA_ALL_QUANTS is deliberately OFF in that build: it is only needed
-# for QUANTIZED KV with flash attention, and this service runs f16 KV (below).
+# GGML_CUDA_FA_ALL_QUANTS is ON: this service runs q8_0 KV with flash
+# attention, and without the flag quantized-KV prefill collapses ~12x on
+# CUDA (see the CUDA lesson in AGENTS.md).
 #
-# Model: ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF, the `-mtp` build of IQ3_S.
-# GSQ-RCO assigns a separate quantization type to every tensor from a
-# gradient-based search under a total size budget, rather than applying one
-# type throughout; IQ3_S at 3.50 bpw is the release's task-lossless operating
-# point (AIME25 100.00 and LiveCodeBench v6 85.71 match BF16 exactly,
-# GPQA-Diamond 0.51 behind, at 11.8 GB against 53.8).
+# Model: HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced at Q4_K_M
+# (6.87 GiB). Architecture (GGUF arch gemma4): 48 layers, MHA with 16 kv
+# heads x 512 head_dim, rope freq_base 1e6, final logit softcapping 30,
+# per-layer token embeddings (PLE) inside the file, and a sliding-window
+# attention pattern (attention.sliding_window = 1024) -- most layers cache
+# only a 1024-position window, which is why the KV pool is a fraction of
+# the full-ctx x all-layers number.
 #
-# WHY THAT FILE: the `-mtp` build is ~0.35 GB larger because it carries the
-# model's Multi-Token Prediction head as GGUF block `blk.64.nextn.*`
-# (qwen35.nextn_predict_layers = 1). llama.cpp loads it from the SAME file --
-# `--spec-type draft-mtp` sets load_mtp, and the draft context runs on the main
-# model's weights (common/common.cpp passes shares_model when there is no
-# --model-draft; speculative.cpp gives it LLAMA_CONTEXT_TYPE_MTP), so there is
-# no second 12 GB copy in VRAM. MTP is therefore a property of the FILE, not of
-# the flags: the plain IQ3_S build has no MTP tensors to load.
+# MTP: the Multi-Token Prediction head ships as a SEPARATE file
+# (mtp-gemma-4-12B-it.gguf, arch gemma4-assistant, 4 nextn layers, 0.24
+# GiB). Unlike the Qwen3.8 -mtp build (head inside the main file), this one
+# is passed via --model-draft with --spec-type draft-mtp, so the draft
+# context loads the assistant model. The same --spec-type value chains the
+# draftless ngram-mod matcher, which attaches unconditionally; two
+# context-backed drafters cannot share one process, so draft-mtp,ngram-mod
+# is the only legal shape that carries both.
 #
-# Architecture (GGUF arch qwen35): 64 layers laid out as 16 x (3 x (Gated
-# DeltaNet -> FFN) + 1 x (Gated Attention -> FFN)) -- 48 linear-attention
-# layers with CONSTANT-SIZE per-sequence state and 16 full-attention layers,
-# 4 KV heads x 256 head_dim. Only the attention layers take KV, so a token
-# costs 2 (K,V) x 4 heads x 256 x 2 B = 64 KiB at f16 (34 KiB at q8_0, 18 KiB
-# at q4_0), and the DeltaNet layers add ~0.16 GiB per sequence in f32 at any
-# depth.
+# SIZING (RTX 3090, 24576 MiB): not modelled by hand -- CTX_SIZE is left
+# unset and --fit sizes the KV pool (q8_0 K and V) against what the 6.87
+# GiB of weights, the 0.16 GiB projector, the MTP sidecar's draft context
+# and the compute buffers leave on the card. The engine prints the chosen
+# context at startup; read it from the log, do not hard-code it back here.
 #
-# SIZING (RTX 3090, 24576 MiB) -- modelled, and the loader's own ledger
-# ("KV self size", "compute buffer size") outranks it:
-#   weights                            11.29 GiB (12,120,016,960 B)
-#   mmproj                              0.87 GiB (931,146,528 B, 27 blocks)
-#   MTP draft KV                        0.50 GiB (its context is n_ctx long
-#                                       over one nextn layer: 4 KiB/token)
-#   DeltaNet state                      0.16 GiB (one sequence)
-#   CUDA context + prefill/decode bufs  ~1.5 GiB
-#   KV at CTX_SIZE                      8.00 GiB (131,072 x 64 KiB)
-#                                      --------
-#                                      ~22.3 GiB of 24, ~2 GiB of slack
-# The binding number is the KV: ~9.5 GiB is left after the fixed terms, which
-# at 64 KiB/token is ~155k tokens across all slots. The deployed shape is one
-# slot at 131,072, full f16 precision, no KV quantization risk, no rebuild.
-# Alternatives inside the same budget: 2 x 65536 at f16; 4 x 65536 or
-# 1 x 262144 at q8_0, both of which need a rebuild with
-# -DGGML_CUDA_FA_ALL_QUANTS=ON (without it quantized-KV prefill collapses
-# ~12x on CUDA -- see the CUDA lesson in AGENTS.md).
-# --parallel N DIVIDES --ctx-size between slots; it does not multiply. One slot
-# means a second concurrent request queues behind the first.
+# Vision: --mmproj loads the BF16 projector (11 tensors, 52M params; it
+# also carries an audio encoder, unused by the fleet). The fleet's image
+# path lives here, which is why halogen no longer sets HALOGEN_VISION_TOWER.
 #
-# Sampling: greedy by default, which is the fleet contract (a request that
-# sends no temperature decodes greedy; a client that sends one gets its own).
-# Qwen's own agentic evaluations used temperature 1.0 / top_p 0.95, and the
-# harness applies exactly that per model in models.yml (extraBody), the same
-# way it did for the earlier frank-large runtime.
+# Sampling: greedy by default (the fleet contract); a client that sends a
+# temperature gets its own, and the harness applies per-model overrides
+# from models.yml.
 #
-# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (HIP reads
-# those bare vars on the ROCm hosts -- see the 2026-08-13 incident in
+# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (HIP
+# reads those bare vars on the ROCm hosts -- see the 2026-08-13 incident in
 # AGENTS.md). This box is CUDA, but the rule keeps env files portable.
 
-# llama.cpp runtime binary: mainline master 37b53fd, CUDA sm_86.
+# llama.cpp runtime binary: mainline master 37b53fd, CUDA sm_86,
+# FA_ALL_QUANTS=ON.
 LLAMA_BIN=/home/dank/GitPackages/llama.cpp-mainline/build-cuda/bin/llama-server
 
-# Model package. The -mtp file is required for speculative decoding.
-MODEL_DIR=/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp
-MODEL_FILE=Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf
-MMPROJ_PATH=/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp/mmproj-Qwen3.8-27B-BF16.gguf
+# Model package. The MTP sidecar file is required for speculative decoding.
+MODEL_DIR=/home/dank/models/Gemma4-12B-QAT
+MODEL_FILE=Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf
+MMPROJ_PATH=/home/dank/models/Gemma4-12B-QAT/mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf
+DRAFT_FILE=mtp-gemma-4-12B-it.gguf
 
-# Fleet identity and network. 7771 is this service's own endpoint on Dankclaw;
-# 7770 stays frank-large's. The alias is the id at /v1/models and what clients
-# ask for -- the harness entry in models.yml names it.
+# Fleet identity and network. 7771 is this service's own endpoint on
+# Dankclaw; 7770 stays frank-large's. The alias is the id at /v1/models and
+# what clients ask for -- the harness entry in models.yml names it.
 ALIAS=frank-small
 BIND_ADDR=0.0.0.0
 LISTEN_PORT=7771
 
-# One slot at 131,072 tokens with f16 KV (see the sizing note above).
-CTX_SIZE=131072
+# Context: intentionally unset -- --fit decides it (see the sizing note).
+# One slot; a second concurrent request queues behind the first.
 PARALLEL=1
-CACHE_TYPE_K=f16
-CACHE_TYPE_V=f16
+CACHE_TYPE_K=q8_0
+CACHE_TYPE_V=q8_0
 
-# Batch geometry: llama.cpp's own defaults (2048 logical / 512 physical). The
-# prefill wall clock at this model size is set by the card, not by ubatch, and
-# the small ubatch keeps the DeltaNet chunk buffers small. Raise UBATCH_SIZE
-# only after reading the compute buffer size the loader reports.
+# Batch geometry: llama.cpp's own defaults (2048 logical / 512 physical).
 BATCH_SIZE=2048
 UBATCH_SIZE=512
 
 # CPU threads: this service runs entirely on the GPU (NGL=999), so threads
-# cover tokenization, sampling and the HTTP layer only. Deliberately modest --
-# this host's CPU is shared with halogen's engine, which is CPU-hungry.
+# cover tokenization, sampling and the HTTP layer only. Deliberately modest
+# -- this host's CPU is shared with halogen's engine, which is CPU-hungry.
 NUM_THREADS=8
 THREAD_BATCH=8
 
-# Every layer on the 3090, with flash attention.
+# Every layer on the 3090, with flash attention (required for q8 KV).
 NGL=999
 FLASH_ATTN=on
 
-# Speculative decoding from the model's own MTP head (see the model note).
-# Draft depth stays llama.cpp's default of 3 tokens.
-SPEC_TYPE=draft-mtp
+# Speculative decoding: the MTP sidecar (draft context) chained with the
+# draftless ngram-mod matcher. Draft depth stays llama.cpp's default of 3.
+SPEC_TYPE=draft-mtp,ngram-mod
 
 # Greedy default (fleet contract); reasoning is extracted into
 # reasoning_content with an unlimited budget (the model decides).

scripts/serve-frank-small.sh

@@ -1,21 +1,27 @@
 #!/usr/bin/env bash
-# Serve ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp as frank-small on Dankclaw's
-# RTX 3090 via mainline llama.cpp.
+# Serve HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced (Q4_K_M) as
+# frank-small on Dankclaw's RTX 3090 via mainline llama.cpp.
 #
-# Topology: the whole model lives in VRAM (-ngl 999) -- 11.29 GiB of IQ3_S-mtp
-# weights, 0.87 GiB of BF16 vision projector and ~8 GiB of f16 KV inside the
-# card's 24 GiB. The host CPU only tokenizes and samples here; this box's CPU
+# Topology: the whole model lives in VRAM (-ngl 999) -- 6.87 GiB of Q4_K_M
+# weights plus the BF16 vision projector and q8_0 KV inside the card's
+# 24 GiB. The host CPU only tokenizes and samples here; this box's CPU
 # belongs to the halogen engine that serves frank-large.
 #
-# MTP: the `-mtp` build carries the NextN block (blk.64.nextn.*), and
-# `--spec-type draft-mtp` makes llama.cpp load it and run the draft context on
-# the SAME weights. There is no --model-draft here on purpose: a separate draft
-# path would load a second 12 GB copy of a 12 GB model. Draft depth is
-# llama.cpp's default (--spec-draft-n-max 3).
+# MTP: this model's Multi-Token Prediction head ships as a SEPARATE file
+# (mtp-gemma-4-12B-it.gguf, arch gemma4-assistant, 4 nextn layers). It is
+# passed via --model-draft with --spec-type draft-mtp: the draft context
+# loads the assistant model. The same --spec-type value chains the
+# draftless ngram-mod matcher, which attaches unconditionally; two
+# context-backed drafters cannot share one process, so this chain is the
+# only shape that carries both.
 #
-# Vision: --mmproj loads the BF16 projector, so image content parts work. The
-# fleet's image path moved here from frank-large on 2026-09-16, which is why
-# halogen no longer sets HALOGEN_VISION_TOWER.
+# Context: CTX_SIZE is left unset and --fit sizes the KV pool (q8_0 K and
+# V) against what the weights, projector, draft context and compute
+# buffers leave on the card; the chosen value lands in the startup log.
+#
+# Vision: --mmproj loads the BF16 projector (it also carries an audio
+# encoder; vision is the fleet contract). The fleet's image path lives
+# here, which is why halogen no longer sets HALOGEN_VISION_TOWER.
 #
 # Sampling: greedy by default (the fleet contract). Clients that send their own
 # temperature get their own values, and the harness overrides per model from
@@ -40,11 +46,12 @@ fi
 : "${LLAMA_BIN:=/home/dank/GitPackages/llama.cpp-mainline/build-cuda/bin/llama-server}"
 : "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp}"
 : "${MODEL_FILE:=Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
-: "${MMPROJ_PATH:=$MODEL_DIR/mmproj-Qwen3.8-27B-BF16.gguf}"
+: "${MMPROJ_PATH:=$MODEL_DIR/mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf}"
+: "${DRAFT_FILE:=}"
 : "${ALIAS:=frank-small}"
 : "${BIND_ADDR:=0.0.0.0}"
 : "${LISTEN_PORT:=7771}"
-: "${CTX_SIZE:=131072}"
+: "${CTX_SIZE:=}"
 : "${PARALLEL:=1}"
 : "${CACHE_TYPE_K:=f16}"
 : "${CACHE_TYPE_V:=f16}"
@@ -65,6 +72,9 @@ fi
 for path in "$LLAMA_BIN" "$MODEL_DIR/$MODEL_FILE" "$MMPROJ_PATH"; do
     [[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
 done
+if [[ -n "${DRAFT_FILE:-}" ]]; then
+    [[ -e "$MODEL_DIR/$DRAFT_FILE" ]] || { echo "frank-small: missing $MODEL_DIR/$DRAFT_FILE" >&2; exit 1; }
+fi
 
 # The UVM device node is created lazily by nvidia-modprobe on the first CUDA
 # open after boot; make sure it exists before the server needs it.
@@ -80,15 +90,30 @@ if [[ -n "${SPEC_TYPE:-}" && "$SPEC_TYPE" != "none" ]]; then
     SPEC_ARGS=(--spec-type "$SPEC_TYPE")
 fi
 
+DRAFT_ARGS=()
+if [[ -n "${DRAFT_FILE:-}" ]]; then
+    DRAFT_ARGS=(--model-draft "$MODEL_DIR/$DRAFT_FILE")
+fi
+
+# CTX_SIZE set -> pinned context. Unset -> --fit sizes the KV pool against
+# whatever the weights, projector, draft context and compute buffers leave
+# on the card; the engine prints the chosen value at startup.
+if [[ -n "${CTX_SIZE:-}" ]]; then
+    CTX_ARGS=(--ctx-size "$CTX_SIZE")
+else
+    CTX_ARGS=(--fit on)
+fi
+
 exec "$LLAMA_BIN" \
     --model "$MODEL_DIR/$MODEL_FILE" \
     "${MMPROJ_ARGS[@]}" \
+    "${DRAFT_ARGS[@]}" \
     --alias "$ALIAS" \
     --host "$BIND_ADDR" \
     --port "$LISTEN_PORT" \
     --gpu-layers "$NGL" \
     --flash-attn "$FLASH_ATTN" \
-    --ctx-size "$CTX_SIZE" \
+    "${CTX_ARGS[@]}" \
     --parallel "$PARALLEL" \
     --batch-size "$BATCH_SIZE" \
     --ubatch-size "$UBATCH_SIZE" \