From b854ec78ad14e8df415f6ce7eda9f379a1c2a9cb Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Mon, 17 Aug 2026 18:25:28 -0700
Subject: [PATCH] frank-default-small: enable prefix caching (opt-in for hybrid
 models; big TTFT win for pi's repeated system prompt)

---
 scripts/frank-default-small.env | 8 ++++++++
 scripts/serve-frank-small.sh    | 7 +++++++
 2 files changed, 15 insertions(+)

diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env
index 9938da7..b18f6cb 100644
--- a/scripts/frank-default-small.env
+++ b/scripts/frank-default-small.env
@@ -39,6 +39,14 @@ MAX_NUM_BATCHED_TOKENS=8192
 # keeps headroom for the host.
 GPU_MEMORY_UTILIZATION=0.92
 
+# Prefix caching. This model is hybrid (GatedDeltaNet + full attention); vLLM
+# keeps prefix caching OPT-IN by default for hybrid models ("while the feature
+# matures"), so it is off unless we ask for it. Turning it on reuses the KV of
+# the shared prompt prefix (pi re-sends the same system prompt + tool schema
+# every turn), so follow-up requests skip re-prefilling thousands of tokens.
+# Big TTFT win for the agent loop. Set 0 to disable.
+ENABLE_PREFIX_CACHING=1
+
 # ngram speculative decoding (vLLM ngram proposer).
 # prompt_lookup_max = ngram window; num_speculative_tokens = draft depth.
 NGRAM_LOOKUP_MAX=5
diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh
index 1987341..1ec83be 100755
--- a/scripts/serve-frank-small.sh
+++ b/scripts/serve-frank-small.sh
@@ -35,6 +35,7 @@ fi
 : "${NUM_SPEC_TOKENS:=5}"
 : "${ENABLE_AUTO_TOOL_CHOICE:=1}"
 : "${TOOL_CALL_PARSER:=qwen3_xml}"
+: "${ENABLE_PREFIX_CACHING:=1}"
 # The Avesed tokenizer_config.json has no chat_template; vLLM would use its
 # built-in template which does not render the model's XML tool format. Use
 # the repo's chat_template.jinja (ships the <tool_call> instructions).
@@ -51,6 +52,11 @@ if [[ "${ENABLE_AUTO_TOOL_CHOICE}" == "1" ]]; then
     TOOL_ARGS+=(--tool-call-parser "${TOOL_CALL_PARSER}")
 fi
 
+PC_ARGS=()
+if [[ "${ENABLE_PREFIX_CACHING}" == "1" ]]; then
+    PC_ARGS+=(--enable-prefix-caching)
+fi
+
 exec "$VLLM_BIN" serve "$MODEL" \
     --host "$BIND_ADDR" \
     --port "$LISTEN_PORT" \
@@ -64,4 +70,5 @@ exec "$VLLM_BIN" serve "$MODEL" \
     --reasoning-parser qwen3 \
     --trust-remote-code \
     --chat-template "$CHAT_TEMPLATE" \
+    "${PC_ARGS[@]}" \
     "${TOOL_ARGS[@]}"