From f70460b0023ce3a3632ffe8cc52a7638478c371c Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Fri, 14 Aug 2026 12:50:59 -0700
Subject: [PATCH] frank-default-small: KV K+V to f16, add verbosity tunable
 (F16 KV diag)

F16 K removes the flash-attn F16-conversion workspace (MMA_F16 prefill
kernel needs no copy when K is already f16). Trades static KV pool size
for the context-proportional workspace growth implicated in the
cublasSgemm_v2 crash class. VERBOSITY=4 run logs real allocation sizes;
refit CTX_SIZE if the pool does not fit on the 24 GiB 3090.
---
 scripts/frank-default-small.env    | 11 +++++++++--
 scripts/serve-llama-frank-small.sh |  4 +++-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env
index 73c5c43..13969e4 100644
--- a/scripts/frank-default-small.env
+++ b/scripts/frank-default-small.env
@@ -21,9 +21,16 @@ NGRAM_MOD_N_MATCH=24
 # decode. Fixes the cuBLAS OOM crash class during big prefills.
 N_CPU_MOE=36
 N_GPU_LAYERS=all
-# KV cache on GPU, K quantized to q8_0, V stays f16 (user-locked decision).
-CACHE_TYPE_K=q8_0
+# KV cache on GPU, both K and V at f16 (2026-08-14 diagnostic).
+# F16 K removes the flash-attention F16-conversion workspace (the MMA_F16
+# prefill kernel needs no copy when K is already f16), trading static pool
+# size for the context-proportional workspace growth that caused the
+# cublasSgemm_v2 crashes. Run with VERBOSITY=4 so the load log shows the
+# real allocation sizes; refit CTX_SIZE if it does not fit.
+CACHE_TYPE_K=f16
 CACHE_TYPE_V=f16
+# Diagnostic verbosity for this run; logs the KV/graph allocation sizes.
+VERBOSITY=4
 # Store SWA checkpoints in RAM to hold peak GPU memory down for the
 # larger batch/ubatch (leaves headroom for ctx-checkpoints operation).
 CTX_CHECKPOINTS=4
diff --git a/scripts/serve-llama-frank-small.sh b/scripts/serve-llama-frank-small.sh
index 4c9b847..a5b59b1 100755
--- a/scripts/serve-llama-frank-small.sh
+++ b/scripts/serve-llama-frank-small.sh
@@ -30,8 +30,9 @@ fi
 : "${NGRAM_MOD_N_MIN:=48}"
 : "${NGRAM_MOD_N_MAX:=64}"
 : "${NGRAM_MOD_N_MATCH:=24}"
-: "${CACHE_TYPE_K:=q8_0}"
+: "${CACHE_TYPE_K:=f16}"
 : "${CACHE_TYPE_V:=f16}"
+: "${VERBOSITY:=3}"
 : "${N_CPU_MOE:=36}"     # 36 of 40 layers: MoE expert weights stay on CPU; 4 on GPU
 : "${N_GPU_LAYERS:=all}" # every non-expert tensor goes to GPU
 : "${BATCH_SIZE:=2048}"
@@ -66,6 +67,7 @@ exec "$LLAMA_SERVER" \
     --spec-ngram-mod-n-match "$NGRAM_MOD_N_MATCH" \
     --cache-type-k "$CACHE_TYPE_K" \
     --cache-type-v "$CACHE_TYPE_V" \
+    --verbosity "$VERBOSITY" \
     --ctx-size "$CTX_SIZE" \
     --parallel "$PARALLEL" \
     --n-cpu-moe "$N_CPU_MOE" \