111 lines
6.2 KiB
Bash
111 lines
6.2 KiB
Bash
# frank-large.env — vLLM production config for frank-large (Dankclaw iGPU :7770).
|
|
#
|
|
# 2026-10-01 cutover from gufo (git tag frank-large-gufo-q4kxl-20261001).
|
|
# Target: XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B in FULL BF16 precision
|
|
# (17.53 GiB, 4 shards) — the same architecture frank-small serves as an
|
|
# INT8 W8A16 AutoRound conversion on the 3090, here unquantized. The engine
|
|
# is the kyuz0 TheRock vLLM build, the only fleet-carried runtime path for
|
|
# gfx1151 (official vLLM images are CUDA-only).
|
|
#
|
|
# MEASURED 2026-10-01 bench window on this host (identical client, one
|
|
# session, temperature 0, 28,647-token prefill / 512-token decode):
|
|
#
|
|
# arm prefill decode acceptance (per position)
|
|
# BF16 base 17,876 t/s 13.1 t/s —
|
|
# BF16 + DFlash 8 tok 7,976 t/s 17.4 t/s pos0 59% pos1 26%; 14.8% token rate
|
|
# BF16 + MTP 2 tok 7,380 t/s 19.8 t/s pos0 71% pos1 44%; 57.4% token rate
|
|
#
|
|
# Decode laws: 13.1 t/s is ~94% of the Strix Halo LPDDR5x roof for BF16
|
|
# weights (17.5 GiB/token against ~256 GB/s) — a quant, not a bug; the same
|
|
# INT8 payload decodes 83.9 t/s on the 3090 because it moves ~12 GiB
|
|
# through ~936 GB/s. Speculative arms are the ONLY way past the wall, and
|
|
# both drafters pay ~55-59% of prefill (draft passes share the compute
|
|
# queues). Frank's call enabled MTP: the thinking-heavy traffic on this
|
|
# endpoint spends its wall clock in decode.
|
|
#
|
|
# MTP IS CROSS-MODEL: the head comes from Qwen/Qwen3.5-9B (the distill's
|
|
# source model, 15 mtp.* tensors, identical 4096-hidden/248320-vocab
|
|
# geometry) — MiMo itself ships ZERO mtp tensors, which is why frank-small's
|
|
# SGLang NEXTN is a 0.00-acceptance ghost (SGLang reads the head from the
|
|
# TARGET checkpoint; vLLM reads the draft from a separate path). The 71%/44%
|
|
# per-position acceptance is the distillation preserving Qwen3.5 hidden-state
|
|
# structure; it is measured, not assumed.
|
|
#
|
|
# NUMERICS NOTE (measured): with a spec drafter enabled, temperature-0 text
|
|
# DIFFERS from the no-spec run (greedy sha 1318aeb6 base vs 0208b45d dflash
|
|
# vs 20c41c38 mtp): batched verification changes the bf16 reduction order,
|
|
# so argmax near-ties diverge and cascade. Spec decode on vLLM is
|
|
# distribution-preserving, not bitwise-greedy-preserving. The gufo
|
|
# greedy==AR law does not carry over.
|
|
#
|
|
# HIP reads bare env vars TEMP, HOST, GPU_MAX_HEAP_SIZE — every key here is
|
|
# FRANK_LARGE_-prefixed so none can leak.
|
|
|
|
# --- runtime ----------------------------------------------------------------
|
|
# Digest-pinned (the :latest tag of this fork moves with no notice):
|
|
# image ID 2d24a8c5a3fd, vLLM 0.30.0 / torch 2.11.0 / ROCm 10.0.0 (TheRock),
|
|
# verified to ship vllm/v1/spec_decode/dflash.py, the qwen3_5 model registry
|
|
# entry (Qwen3_5ForConditionalGeneration), and the Triton GDN backend.
|
|
FRANK_LARGE_IMAGE=docker.io/kyuz0/vllm-therock-gfx1151@sha256:98c1f3ac9833e7fa45e9a82fb7a4315e6d95daaf2548a211c8da3a871b17533a
|
|
|
|
# --- payload ----------------------------------------------------------------
|
|
FRANK_LARGE_MODEL_DIR=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-BF16
|
|
FRANK_LARGE_MODEL_SHARDS=4
|
|
|
|
# --- identity ---------------------------------------------------------------
|
|
# Same port and served alias as every frank-large predecessor; the harness
|
|
# models.yml id is unchanged. The bench-window container (vllm-mimo) is
|
|
# swept before every start.
|
|
FRANK_LARGE_CONTAINER=frank-large-mimo
|
|
FRANK_LARGE_LEGACY_CONTAINERS="vllm-mimo"
|
|
FRANK_LARGE_SERVED_NAME=frank-large
|
|
FRANK_LARGE_PUBLISH=7770:7770
|
|
|
|
# --- engine shape (2026-10-01 window) ----------------------------------------
|
|
# Full native window 262,144 at UTIL 0.45 booted clean on gfx1151 in all
|
|
# three arms (~200-240 s warm start). KV grows only on the 8 full-attention
|
|
# layers (32 KiB/token bf16: 8 x 2 x 4 heads x 256 dim x 2 B); the 24 GDN
|
|
# layers keep a FIXED ~25 MiB state per running sequence, so the pool is
|
|
# generous and sessions are cheap. SEQS 4 preserves the four-session
|
|
# contract the gufo shape ran; vLLM shares the pool across them by
|
|
# scheduler, unlike gufo's per-session carve.
|
|
FRANK_LARGE_MAXLEN=262144
|
|
FRANK_LARGE_SEQS=4
|
|
FRANK_LARGE_UTIL=0.45
|
|
# First-deploy boot (2026-10-01): GPU KV cache size 819,332 tokens,
|
|
# Maximum concurrency for 262,144 tokens per request 3.13x; healthy at 230 s.
|
|
# Change MAXLEN/SEQS/UTIL or the image and this line moves — re-read it.
|
|
|
|
# --- speculative decode: cross-model MTP (measured +51% decode) ---------------
|
|
# mtp beats dflash here (19.8 vs 17.4 t/s) and needs 2 draft tokens, not 8:
|
|
# acceptance past position 1 is 44% and falling (DFlash's own curve decays
|
|
# 59/26/14/7). num_speculative_tokens beyond 2 is UNMEASURED — do not raise
|
|
# it without a fresh A/B. To flip drafters, set FRANK_LARGE_SPEC=dflash and
|
|
# repoint SPEC_DIR (the DFlash drafter stays on disk).
|
|
# The gufo-era law "spec decode is absent" was a frank-primary finding; this
|
|
# endpoint's traffic is decode-dominated thinking, which is the regime where
|
|
# spec wins (prefill costs ~55% in every arm).
|
|
FRANK_LARGE_SPEC=
|
|
FRANK_LARGE_SPEC_DIR=/home/dank/models/Qwen3.5-9B
|
|
FRANK_LARGE_SPEC_TOKENS=2
|
|
|
|
# --- AUTORESEARCH arm: llama.cpp runtime -------------------------------------
|
|
# master build (qwen35 GDN kernels, HIP gfx1151) on Q6_K GGUF: 7.79 GiB
|
|
# weights = 0.51x the BF16 bytes, so the LPDDR roof doubles. Cross-model
|
|
# DFlash draft (qwen3.5-9b, Q8_0). --reasoning-format deepseek emits
|
|
# reasoning_content, NOT vLLM's reasoning.
|
|
FRANK_LARGE_RUNTIME=llama
|
|
FRANK_LARGE_LLAMA_BIN=/home/dank/GitPackages/llama-master/build-hip/bin/llama-server
|
|
FRANK_LARGE_GGUF=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf
|
|
FRANK_LARGE_DRAFT_GGUF=/home/dank/models/DFlash-Qwen35/qwen3.5-9b-dflash-Q8_0.gguf
|
|
FRANK_LARGE_LLAMA_CTX=1048576
|
|
# --- client contract ----------------------------------------------------------
|
|
# vLLM emits the thinking text in `reasoning` (NOT llama.cpp/gufo's
|
|
# `reasoning_content`) — the harness models.yml compat block for this
|
|
# provider must set reasoningContentField accordingly. The qwen3 parser is
|
|
# what keeps thinking out of `content`. Tools ride the qwen3_coder parser
|
|
# with auto tool choice. --language-model-only drops the vision tower (the
|
|
# endpoint stays text-only; the tower can come back by dropping the flag).
|
|
FRANK_LARGE_REASONING_PARSER=qwen3
|
|
FRANK_LARGE_TOOL_PARSER=qwen3_coder
|