frank-default-small: swap to nameistoken Quark W8A8 INT8, batch 8192
- Model: nameistoken/Qwen3.6-35B-A3B-Quark-W8A8-INT8 (AMD Quark W8A8, vLLM-native expert layout). The Avesed quant emitted malformed Qwen3 XML tool calls; the Quark model follows the format cleanly. - max-num-batched-tokens lowered 16384 -> 8192 (user request). - Chat template: neutral path qwen36-chat_template-tools.jinja (same base template; adds a concrete worked tool-call example).
This commit is contained in:
1 parent
a17e638a1f
commit
1cebbf47f6
2 files changed
+13
-7
No files matched your search
@@ -21,14 +21,14 @@ fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${VLLM_BIN:=/home/dank/vllm-venv/bin/vllm}"
|
||||
: "${MODEL:=Avesed/Qwen3.6-35B-A3B-INT8-W8A8}"
|
||||
: "${MODEL:=nameistoken/Qwen3.6-35B-A3B-Quark-W8A8-INT8}"
|
||||
: "${ALIAS:=frank-default-small}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=52625}"
|
||||
: "${MAX_MODEL_LEN:=262144}"
|
||||
: "${KV_CACHE_DTYPE:=auto}"
|
||||
: "${MAX_NUM_SEQS:=4}"
|
||||
: "${MAX_NUM_BATCHED_TOKENS:=16384}"
|
||||
: "${MAX_NUM_BATCHED_TOKENS:=8192}"
|
||||
: "${GPU_MEMORY_UTILIZATION:=0.92}"
|
||||
: "${NGRAM_LOOKUP_MAX:=5}"
|
||||
: "${NGRAM_LOOKUP_MIN:=1}"
|
||||
@@ -38,7 +38,7 @@ fi
|
||||
# The Avesed tokenizer_config.json has no chat_template; vLLM would use its
|
||||
# built-in template which does not render the model's XML tool format. Use
|
||||
# the repo's chat_template.jinja (ships the <tool_call> instructions).
|
||||
: "${CHAT_TEMPLATE:=/home/dank/models/avesed-qwen36-chat_template.jinja}"
|
||||
: "${CHAT_TEMPLATE:=/home/dank/models/qwen36-chat_template-tools.jinja}"
|
||||
|
||||
if [[ ! -x "$VLLM_BIN" ]]; then
|
||||
echo "missing vllm binary: $VLLM_BIN" >&2
|
||||
|
||||
Reference in new issue
Block a user