diff --git a/scripts/frank-secondary.env b/scripts/frank-secondary.env index f755a32..8d13187 100644 --- a/scripts/frank-secondary.env +++ b/scripts/frank-secondary.env @@ -1,98 +1,35 @@ # frank-secondary — llama.cpp (master, CUDA sm_86) serving -# ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF:RCO-IQ3_XXS -# **with the MTP head in the file** on the RTX 3090, :7771. +# HauhauCS/Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-MTP +# (Q4_K_M) on the RTX 3090, :7771. # -# 2026-10-06 (Frank, evening): the payload moved to ajgazin's UNCELERSED -# Swift-1.5-Qwen3.8-27B at RCO-IQ3_XXS (10,442,827,392 B) — 1.56 GiB -# lighter than the IQ3_S-mtp it replaces — with MTP BACK ON (draft-mtp, -# depth 3), TWO 131,072 slots, and ubatch 512. The uncensored finetune -# changes behavior by design (publisher: 23/100 refusals vs Swift 1.5's -# 98/100, KL 0.0884 from Swift 1.5) — Frank's order. The RCO tier is the -# same per-tensor ALLOCATION recipe as the IQ3_S file (histogram: iq3_s -# 97 / bf16 96 / iq3_xxs 83 / iq2_s 72 / iq4_xs 33 / iq2_xs 32 / iq2_xxs -# 30 / q2_k 28 / q4_k 19 / q6_k 8 / iq1_s 4 / iq1_m 4 — all stock types, -# stock llama.cpp loads it); publisher KLD: Mean 0.002631, 90 % 0.0421 -# (kld_results.csv). RCO files are "more experimental" per the README — -# the quality floor is the IQ3_S rung, not a promise. +# 2026-10-07 (Frank): swap from Swift-1.5-Qwen3.8-27B RCO-IQ3_XXS to +# Gemma4-26B-A4B Q4_K_M, ONE 200,704 context, q8_0 KV (FP8 class), +# MTP via separate assistant file (--spec-draft-model). # -# The architecture is unchanged (same model family): qwen35 hybrid — -# block_count 65 (64 real layers + the blk.64 MTP head), -# full_attention_interval 4 => SIXTEEN full-attention layers, 24 Q heads -# / 4 KV heads, k/v length 256, native window 262144, dense, -# nextn_predict_layers=1. The MTP head ships in every main GGUF of this -# repo (README); --spec-type draft-mtp loads it against the main model's -# own weights. +# Architecture: gemma4 MoE — 30 layers, 128 experts (8 active), +# hybrid attention: 25 sliding_window + 5 full_attention +# (full at layers [5, 11, 17, 23, 29]), sliding_window 1024, +# 8 KV heads, head_dim 256, attention_k_eq_v (V = K), +# native 262,144 window. # -# THE KV CONTRACT STAYS q8_0 K + q8_0 V (Frank, 2026-10-05): 34 KiB/token -# (16 attn layers x 4 heads x 256 x 1.0625 B, K and V). +# KV: q8_0 K + q8_0 V (FP8 class, 1.0625 B/elem). The build has no +# true fp8 KV dtype; q8_0 is the FP8 class. # -# Why TWO slots + MTP fits now (it did not under IQ3_S): -# card 24,576 MiB. Weights 9,963 MiB (10,442,827,392 B; IQ3_S was -# 11,561). Projector ~1,290 MiB (the PUBLISHER's OWN mmproj-BF16.gguf, -# 931,145,888 B, at the measured ~1.45x card law — no substitution). -# Compute buffers at ubatch 512 ~1.3 GiB. CUDA ctx ~500 MiB. DeltaNet -# state ~150 MiB/slot x 2. MTP draft ctx back: ~0.4 GiB. Non-KV ~13.9 -# GiB. Two q8_0 slots = 8,704 MiB. Predicted free ~2.6-3.2 GiB — back -# over the old 2-3 GiB floor that the IQ3_S two-slot boot violated -# (measured there: 1,397 MiB free). MEASURE nvidia-smi free at boot -# (quote the free field) and let the runbook carry the number. -# -# Payload history (rollback trail): /home/dank/models/Swift15-IQ3S/ (the -# IQ3_S-mtp that was live until this swap), Ornith-Q8/, MiMo-Q6K/ all -# stay on disk; one MODEL_FILE + shape edit back each. +# MTP: separate assistant file (mtp-gemma-4-26B-A4B-it.gguf, 0.252 GiB), +# loaded via --spec-draft-model + --spec-type draft-mtp. LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server -MODEL_FILE=/home/dank/models/Swift15-UncIQ3XXS/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf - -# Vision: the publisher SHIPS the projector — mmproj-BF16.gguf -# (931,145,888 B) from the same repo, fetched by -# scripts/download-frank-secondary.sh. No substitution needed for this -# payload (the unsloth base-tower proof of 2026-10-05 remains the -# fallback argument for the IQ3_S rollback payload). -# Empty string = text-only on purpose. -MMPROJ_FILE=/home/dank/models/Swift15-UncIQ3XXS/mmproj-BF16.gguf - -# Fleet endpoint: same port and alias as every frank-secondary before it, so the -# harness models.yml id and the homepage probe do not move. +MODEL_FILE=/home/dank/models/Gemma4-26B-A4B/Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf +MMPROJ_FILE=/home/dank/models/Gemma4-26B-A4B/mmproj-Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf ALIAS=frank-secondary BIND_ADDR=0.0.0.0 LISTEN_PORT=7771 - -# TWO conversations at Frank's 112k cap (2026-10-06 evening): --ctx-size is -# the POOL llama.cpp divides across slots; 229376 / 2 = 114,688 per -# conversation. Lowering from 128k (131,072) frees ~1 GiB of KV, which -# funds the ubatch 512 -> 1024 increase (prefill throughput). The -# per-slot window is the product contract, so models.yml follows to -# 114,688. -CTX_SIZE=229376 -PARALLEL=2 - -# Compute-buffer geometry. Ubatch is a prefill knob (decode never moves); -# the 1024 rung was the first A/B back once the per-slot context dropped -# to 112k (2026-10-06 evening), and 2048 is the second (Frank, same night) -# — the rung measured +4.6 % prefill for 616 MiB on the 3090 (2026-10-05 -# boot pair), affordable inside the 1,751 MiB the 112k cut freed. The pair -# is a measurement, not a law (loud failure at graph capture if it does -# not fit). +CTX_SIZE=200704 +PARALLEL=1 BATCH_SIZE=4096 UBATCH_SIZE=2048 - -# The KV contract for this swap (Frank, 2026-10-05): q8_0 K + q8_0 V — -# llama.cpp's FP8 class. The flags stay explicit so the choice is visible in -# the argv either way, and --flash-attn stays pinned on in the serve script: -# it is required for quantized V (the build carries -# GGML_CUDA_FA_ALL_QUANTS=ON since 2026-09-17, so q8_0 KV is a real path, -# not a silent f16 fallback). CACHE_TYPE_K=q8_0 CACHE_TYPE_V=q8_0 - -# Speculative decoding is ON (Frank, 2026-10-06 evening): draft-mtp at -# depth 3 against the head IN the file (the -mtp variant ships the -# blk.64.nextn.* tensors). ngram-mod stays skipped — on a qwen35 hybrid -# it disables recurrent-state rollback, so a rejected draft would -# corrupt the DeltaNet state; MTP needs no such bypass (its draft ctx -# re-derives state from the target's own weights). The flags move as a -# set: the serve script emits BOTH only when SPEC_TYPE names a drafter. SPEC_TYPE=draft-mtp SPEC_DRAFT_N_MAX=3 - +SPEC_DRAFT_FILE=/home/dank/models/Gemma4-26B-A4B/mtp-gemma-4-26B-A4B-it.gguf NGL=99 diff --git a/scripts/serve-frank-secondary.sh b/scripts/serve-frank-secondary.sh index 9a3e182..884fb15 100755 --- a/scripts/serve-frank-secondary.sh +++ b/scripts/serve-frank-secondary.sh @@ -67,6 +67,7 @@ fi # not carry half a spec pair. : "${SPEC_TYPE:=draft-mtp}" : "${SPEC_DRAFT_N_MAX:=3}" +: "${SPEC_DRAFT_FILE:=}" : "${NGL:=99}" # Vision projector. No colon on purpose: an explicitly empty value means # "text-only, on purpose" and must NOT be replaced by the default path. @@ -89,6 +90,10 @@ fi # baseline must not carry half a spec pair. if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then EXTRA_ARGS+=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX") +if [[ -n "$SPEC_DRAFT_FILE" ]]; then + [[ -e "$SPEC_DRAFT_FILE" ]] || { echo "frank-secondary: missing $SPEC_DRAFT_FILE" >&2; exit 1; } + EXTRA_ARGS+=(--spec-draft-model "$SPEC_DRAFT_FILE") +fi fi # The UVM device node is created lazily by nvidia-modprobe on the first CUDA