diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 5ad2769..fc4dd93 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -42,9 +42,12 @@ NUM_SPEC_TOKENS=5 # Tool calling. Clients (pi, OpenWebUI) send tool_choice=auto; without # --enable-auto-tool-choice vLLM rejects it with HTTP 400. The Avesed # chat template renders tools in the Qwen3 XML format, so the qwen3_xml -# parser is the matching one. +# parser is the matching one. CHAT_TEMPLATE must point at the repo's +# chat_template.jinja (the tokenizer_config.json ships none; vLLM's +# built-in template does not render the XML tool instructions). ENABLE_AUTO_TOOL_CHOICE=1 TOOL_CALL_PARSER=qwen3_xml +CHAT_TEMPLATE=/home/dank/models/avesed-qwen36-chat_template.jinja # RDNA flash-attention for the vision encoder (and general FA paths). # Without this, the mm encoder falls back to torch SDPA and the warmup diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 150e72f..a5a188a 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -35,6 +35,10 @@ fi : "${NUM_SPEC_TOKENS:=5}" : "${ENABLE_AUTO_TOOL_CHOICE:=1}" : "${TOOL_CALL_PARSER:=qwen3_xml}" +# The Avesed tokenizer_config.json has no chat_template; vLLM would use its +# built-in template which does not render the model's XML tool format. Use +# the repo's chat_template.jinja (ships the instructions). +: "${CHAT_TEMPLATE:=/home/dank/models/avesed-qwen36-chat_template.jinja}" if [[ ! -x "$VLLM_BIN" ]]; then echo "missing vllm binary: $VLLM_BIN" >&2 @@ -59,4 +63,5 @@ exec "$VLLM_BIN" serve "$MODEL" \ --speculative-config "{\"method\":\"ngram\",\"prompt_lookup_max\":$NGRAM_LOOKUP_MAX,\"prompt_lookup_min\":$NGRAM_LOOKUP_MIN,\"num_speculative_tokens\":$NUM_SPEC_TOKENS}" \ --reasoning-parser qwen3 \ --trust-remote-code \ + --chat-template "$CHAT_TEMPLATE" \ "${TOOL_ARGS[@]}"