From 3cf6353050e42c7649f87aa0b72c42aa9dc18186 Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Tue, 25 Aug 2026 04:13:00 -0700
Subject: [PATCH] Reduce large model context to 64K

---
 docs/LLM/deployment.md          | 2 +-
 docs/LLM/eemin-rocmi4-mtp.md    | 2 +-
 scripts/frank-default-large.env | 4 ++--
 tests/frank_large_config.bats   | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/LLM/deployment.md b/docs/LLM/deployment.md
index 2058840..2711ef8 100644
--- a/docs/LLM/deployment.md
+++ b/docs/LLM/deployment.md
@@ -145,7 +145,7 @@ The active service on Dankclaw is the charlie12345/ROCmFPX llama.cpp runtime
 against system ROCm 7.2.4 / amdclang 22) with a **strict Qwen MTP** draft
 (`--spec-type draft-mtp --spec-mtp-strict-qwen`) running on ROCm0 with all
 layers (`--spec-draft-ngl all`) — no separate draft file, because the MTP head
-is embedded in the main GGUF. K/V are f16, 131072 context (128K below the
+is embedded in the main GGUF. K/V are f16, 65536 context (192K below the
 native 262144 maximum; 1 slot, concurrent requests queue, the server never
 preempts a running generation),
 `-t 16 -tb 32`, batch 512 / ubatch 256 (ubatch is swappable; sweep it on 8k
diff --git a/docs/LLM/eemin-rocmi4-mtp.md b/docs/LLM/eemin-rocmi4-mtp.md
index 3bd9f10..715b910 100644
--- a/docs/LLM/eemin-rocmi4-mtp.md
+++ b/docs/LLM/eemin-rocmi4-mtp.md
@@ -112,7 +112,7 @@ path's greedy decisions; it does not remove the W4A4 activation approximation.
 
 ### Serving settings
 
-- 131072 context (128K below the native 262144 maximum), 1 slot, f16 KV,
+- 65536 context (192K below the native 262144 maximum), 1 slot, f16 KV,
   context checkpoints, prompt cache. The lower limit leaves headroom for the
   model plus KV cache in the host's unified-memory budget. Concurrent requests
   queue; the server never preempts a running generation.
diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env
index 74b1602..51903b4 100644
--- a/scripts/frank-default-large.env
+++ b/scripts/frank-default-large.env
@@ -36,11 +36,11 @@ ALIAS=frank-default-large
 BIND_ADDR=0.0.0.0
 LISTEN_PORT=7770
 
-# Serving: 131072 context (128K below the model's 262144 maximum), 1 slot. f16
+# Serving: 65536 context (192K below the model's 262144 maximum), 1 slot. f16
 # KV, context checkpoints, prompt cache, strict-MTP embedded draft on ROCm0.
 # This leaves sufficient unified-memory headroom for the model plus KV cache.
 # Concurrent requests queue; the server never preempts a running generation.
-CTX_SIZE=131072
+CTX_SIZE=65536
 PARALLEL=1
 # Batch/ubatch follow the qualified ROCmI4 W4A4 profile (see the bench in
 # scripts/bench-ubatch-rocmi4.sh for the ubatch sweep on 8k contexts).
diff --git a/tests/frank_large_config.bats b/tests/frank_large_config.bats
index b961f4e..551778e 100644
--- a/tests/frank_large_config.bats
+++ b/tests/frank_large_config.bats
@@ -40,7 +40,7 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
 }
 
 @test "frank-default-large fits the model and KV cache in unified memory" {
-    grep -q '^CTX_SIZE=131072$' "$ENV_FILE"
+    grep -q '^CTX_SIZE=65536$' "$ENV_FILE"
     grep -q ': "${CTX_SIZE:=262144}"' "$SERVE_FILE"
 }