frank-default-small: enable prefix caching (opt-in for hybrid models; big TTFT win for pi's repeated system prompt)

scripts/frank-default-small.env

@@ -39,6 +39,14 @@ MAX_NUM_BATCHED_TOKENS=8192
 # keeps headroom for the host.
 GPU_MEMORY_UTILIZATION=0.92
 
+# Prefix caching. This model is hybrid (GatedDeltaNet + full attention); vLLM
+# keeps prefix caching OPT-IN by default for hybrid models ("while the feature
+# matures"), so it is off unless we ask for it. Turning it on reuses the KV of
+# the shared prompt prefix (pi re-sends the same system prompt + tool schema
+# every turn), so follow-up requests skip re-prefilling thousands of tokens.
+# Big TTFT win for the agent loop. Set 0 to disable.
+ENABLE_PREFIX_CACHING=1
+
 # ngram speculative decoding (vLLM ngram proposer).
 # prompt_lookup_max = ngram window; num_speculative_tokens = draft depth.
 NGRAM_LOOKUP_MAX=5

scripts/serve-frank-small.sh

@@ -35,6 +35,7 @@ fi
 : "${NUM_SPEC_TOKENS:=5}"
 : "${ENABLE_AUTO_TOOL_CHOICE:=1}"
 : "${TOOL_CALL_PARSER:=qwen3_xml}"
+: "${ENABLE_PREFIX_CACHING:=1}"
 # The Avesed tokenizer_config.json has no chat_template; vLLM would use its
 # built-in template which does not render the model's XML tool format. Use
 # the repo's chat_template.jinja (ships the <tool_call> instructions).
@@ -51,6 +52,11 @@ if [[ "${ENABLE_AUTO_TOOL_CHOICE}" == "1" ]]; then
     TOOL_ARGS+=(--tool-call-parser "${TOOL_CALL_PARSER}")
 fi
 
+PC_ARGS=()
+if [[ "${ENABLE_PREFIX_CACHING}" == "1" ]]; then
+    PC_ARGS+=(--enable-prefix-caching)
+fi
+
 exec "$VLLM_BIN" serve "$MODEL" \
     --host "$BIND_ADDR" \
     --port "$LISTEN_PORT" \
@@ -64,4 +70,5 @@ exec "$VLLM_BIN" serve "$MODEL" \
     --reasoning-parser qwen3 \
     --trust-remote-code \
     --chat-template "$CHAT_TEMPLATE" \
+    "${PC_ARGS[@]}" \
     "${TOOL_ARGS[@]}"