From 1cebbf47f6ab16b0639a263b6be434f85a143418 Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Mon, 17 Aug 2026 17:49:27 -0700
Subject: [PATCH] frank-default-small: swap to nameistoken Quark W8A8 INT8,
 batch 8192

- Model: nameistoken/Qwen3.6-35B-A3B-Quark-W8A8-INT8 (AMD Quark W8A8,
  vLLM-native expert layout). The Avesed quant emitted malformed Qwen3
  XML tool calls; the Quark model follows the format cleanly.
- max-num-batched-tokens lowered 16384 -> 8192 (user request).
- Chat template: neutral path qwen36-chat_template-tools.jinja (same
  base template; adds a concrete worked tool-call example).
---
 scripts/frank-default-small.env | 14 ++++++++++----
 scripts/serve-frank-small.sh    |  6 +++---
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env
index 5e3edef..9938da7 100644
--- a/scripts/frank-default-small.env
+++ b/scripts/frank-default-small.env
@@ -12,7 +12,12 @@
 # amd-torch-device-gfx1151 + vllm rocm variant) from the AMD index.
 
 VLLM_BIN=/home/dank/vllm-venv/bin/vllm
-MODEL=Avesed/Qwen3.6-35B-A3B-INT8-W8A8
+# 2026-08-17 (evening): swapped from Avesed/Qwen3.6-35B-A3B-INT8-W8A8 to
+# nameistoken/Qwen3.6-35B-A3B-Quark-W8A8-INT8 (AMD Quark W8A8, same
+# architecture, vLLM-native expert layout). The Avesed quant's tool-call
+# XML fidelity was poor (malformed <function> tags); the Quark model
+# follows the Qwen3 XML tool format cleanly.
+MODEL=nameistoken/Qwen3.6-35B-A3B-Quark-W8A8-INT8
 ALIAS=frank-default-small
 BIND_ADDR=0.0.0.0
 LISTEN_PORT=52625
@@ -23,9 +28,10 @@ LISTEN_PORT=52625
 MAX_MODEL_LEN=262144
 KV_CACHE_DTYPE=auto
 
-# Concurrency and batching (user request).
+# Concurrency and batching (user request). max-num-batched-tokens lowered
+# to 8192 on 2026-08-17 (evening). vLLM has no separate ubatch knob.
 MAX_NUM_SEQS=4
-MAX_NUM_BATCHED_TOKENS=16384
+MAX_NUM_BATCHED_TOKENS=8192
 
 # vLLM has no separate ubatch knob. The scheduler issues one batch per step
 # with up to max-num-batched-tokens tokens; the KV cache is preallocated at
@@ -47,7 +53,7 @@ NUM_SPEC_TOKENS=5
 # built-in template does not render the XML tool instructions).
 ENABLE_AUTO_TOOL_CHOICE=1
 TOOL_CALL_PARSER=qwen3_xml
-CHAT_TEMPLATE=/home/dank/models/avesed-qwen36-chat_template-tools.jinja
+CHAT_TEMPLATE=/home/dank/models/qwen36-chat_template-tools.jinja
 
 # RDNA flash-attention for the vision encoder (and general FA paths).
 # Without this, the mm encoder falls back to torch SDPA and the warmup
diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh
index a5a188a..1987341 100755
--- a/scripts/serve-frank-small.sh
+++ b/scripts/serve-frank-small.sh
@@ -21,14 +21,14 @@ fi
 
 # --- defaults (used when the env file does not set them) -------------------
 : "${VLLM_BIN:=/home/dank/vllm-venv/bin/vllm}"
-: "${MODEL:=Avesed/Qwen3.6-35B-A3B-INT8-W8A8}"
+: "${MODEL:=nameistoken/Qwen3.6-35B-A3B-Quark-W8A8-INT8}"
 : "${ALIAS:=frank-default-small}"
 : "${BIND_ADDR:=0.0.0.0}"
 : "${LISTEN_PORT:=52625}"
 : "${MAX_MODEL_LEN:=262144}"
 : "${KV_CACHE_DTYPE:=auto}"
 : "${MAX_NUM_SEQS:=4}"
-: "${MAX_NUM_BATCHED_TOKENS:=16384}"
+: "${MAX_NUM_BATCHED_TOKENS:=8192}"
 : "${GPU_MEMORY_UTILIZATION:=0.92}"
 : "${NGRAM_LOOKUP_MAX:=5}"
 : "${NGRAM_LOOKUP_MIN:=1}"
@@ -38,7 +38,7 @@ fi
 # The Avesed tokenizer_config.json has no chat_template; vLLM would use its
 # built-in template which does not render the model's XML tool format. Use
 # the repo's chat_template.jinja (ships the <tool_call> instructions).
-: "${CHAT_TEMPLATE:=/home/dank/models/avesed-qwen36-chat_template.jinja}"
+: "${CHAT_TEMPLATE:=/home/dank/models/qwen36-chat_template-tools.jinja}"
 
 if [[ ! -x "$VLLM_BIN" ]]; then
     echo "missing vllm binary: $VLLM_BIN" >&2