frank-small: revert vLLM cutover — back to llama.cpp Gemma4-12B Q4_K_M
The vLLM detour (a2b39b1, same day) is retired: the Q4_K_M llama.cpp setup
is the requested shape and the vLLM KV-quant options on sm_86 all measured
dead (fp8 hardware-impossible, int8 page-size unification, turboquant
mm-prefix). Root unit restored, vLLM quadlet/Dockerfile/fetch script moved
to .AGENTTRASH, runbook listed as same-day-reverted. Rollback target:
frank-small-vllm-gemma4.
This commit is contained in:
Frank committed
1 parent
bab8f7f839
commit
374041cd7a
15 files changed
+495
-637
No files matched your search
+113
-58
@@ -1,30 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve google/gemma-4-12B-it-qat-w4a16-ct as frank-small on Dankclaw's RTX
|
||||
# 3090 via vLLM (since 2026-09-18), with the google bf16 MTP assistant
|
||||
# for speculative decoding.
|
||||
# Serve HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced (Q4_K_M) as
|
||||
# frank-small on Dankclaw's RTX 3090 via mainline llama.cpp.
|
||||
#
|
||||
# This script is the CONTAINER ENTRYPOINT of the frank-small quadlet: the
|
||||
# vllm/vllm-openai image's own entrypoint is `vllm serve`, and the quadlet
|
||||
# replaces it with this file (mounted read-only out of the repo checkout) so
|
||||
# the flag set stays in git. The image pins the gemma4-unified vLLM build,
|
||||
# which is what loads gemma4_unified natively and runs the Gemma 4 MTP path.
|
||||
# Topology: the whole model lives in VRAM (-ngl 999) -- 6.87 GiB of Q4_K_M
|
||||
# weights plus the BF16 vision projector and q8_0 KV inside the card's
|
||||
# 24 GiB. The host CPU only tokenizes and samples here; this box's CPU
|
||||
# belongs to the halogen engine that serves frank-large.
|
||||
#
|
||||
# MTP: the drafter is a separate 4-layer assistant checkpoint in bf16
|
||||
# (Google's original gemma-4-12B-it-assistant). It goes in
|
||||
# --speculative-config as method "mtp" — vLLM's Gemma 4 MTP path, not a
|
||||
# generic draft model. The w4a16-ct re-quant of the assistant was tried
|
||||
# first and drafts at acceptance 1.00 (the drafter path does not apply the
|
||||
# Marlin kernel to its linears); the bf16 assistant accepts at 2.80.
|
||||
# MTP: this model's Multi-Token Prediction head ships as a SEPARATE file
|
||||
# (mtp-gemma-4-12B-it.gguf, arch gemma4-assistant, 4 nextn layers). It is
|
||||
# passed via --model-draft with --spec-type draft-mtp: the draft context
|
||||
# loads the assistant model. The same --spec-type value chains the
|
||||
# draftless ngram-mod matcher, which attaches unconditionally; two
|
||||
# context-backed drafters cannot share one process, so this chain is the
|
||||
# only shape that carries both.
|
||||
#
|
||||
# Sampling: greedy by default (fleet contract) via
|
||||
# --override-generation-config; clients that send their own temperature get
|
||||
# their own, and the harness overrides per model from models.yml.
|
||||
# Context: CTX_SIZE is set explicitly to PARALLEL x 262144 so every slot
|
||||
# gets the model's FULL window; --fit (the CTX_SIZE-unset branch below)
|
||||
# maximises the TOTAL pool instead, which would divide a smaller window
|
||||
# across slots. The 4-slot layout is measured to fit the card (see the
|
||||
# SIZING note in frank-small.env); more slots would fail the KV allocation.
|
||||
#
|
||||
# Tunables come from frank-small.env (mounted beside this file by the
|
||||
# quadlet) unless FRANK_SMALL_ENV_FILE overrides the path.
|
||||
# Vision + audio: --mmproj loads the BF16 projector, which also carries
|
||||
# an audio encoder — input_audio content parts transcribe verbatim
|
||||
# (verified 2026-09-17). The fleet's image path lives here, which is why
|
||||
# halogen no longer sets HALOGEN_VISION_TOWER.
|
||||
#
|
||||
# Sampling: greedy by default (the fleet contract). Clients that send their own
|
||||
# temperature get their own values, and the harness overrides per model from
|
||||
# models.yml. --jinja is pinned even though this build defaults it on: the
|
||||
# model's own template supplies reasoning_effort, preserve_thinking and the
|
||||
# tool-call grammar, and a future build that flips that default must not
|
||||
# silently take tools away.
|
||||
#
|
||||
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides it.
|
||||
set -euo pipefail
|
||||
|
||||
ENV_FILE="${FRANK_SMALL_ENV_FILE:-/scripts/frank-small.env}"
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
@@ -33,51 +46,93 @@ if [[ -f "$ENV_FILE" ]]; then
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${MODEL_DIR:=/models/main}"
|
||||
: "${DRAFT_DIR:=/models/assistant}"
|
||||
: "${LLAMA_BIN:=/home/dank/GitPackages/llama.cpp-mainline/build-cuda/bin/llama-server}"
|
||||
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp}"
|
||||
: "${MODEL_FILE:=Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
|
||||
: "${MMPROJ_PATH:=$MODEL_DIR/mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf}"
|
||||
: "${DRAFT_FILE:=}"
|
||||
: "${ALIAS:=frank-small}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=8000}"
|
||||
: "${MAX_MODEL_LEN:=262144}"
|
||||
: "${GPU_MEM_UTIL:=0.95}"
|
||||
: "${KV_CACHE_DTYPE:=auto}"
|
||||
: "${MAX_NUM_SEQS:=16}"
|
||||
: "${MAX_NUM_BATCHED_TOKENS:=16384}"
|
||||
: "${SPEC_NUM_TOKENS:=4}"
|
||||
: "${ENFORCE_EAGER:=0}"
|
||||
: "${LISTEN_PORT:=7771}"
|
||||
: "${CTX_SIZE:=}"
|
||||
: "${PARALLEL:=1}"
|
||||
: "${CACHE_TYPE_K:=f16}"
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
: "${BATCH_SIZE:=2048}"
|
||||
: "${UBATCH_SIZE:=512}"
|
||||
# Decode-shape knobs: the MTP drafter's per-round proposal length (build
|
||||
# default 3; the Gemma4 sidecar carries 4 nextn layers) and the engine's
|
||||
# busy-wait level (build default 50).
|
||||
: "${SPEC_DRAFT_N_MAX:=3}"
|
||||
: "${POLL_LEVEL:=50}"
|
||||
: "${NUM_THREADS:=8}"
|
||||
: "${THREAD_BATCH:=8}"
|
||||
: "${NGL:=999}"
|
||||
: "${FLASH_ATTN:=on}"
|
||||
: "${SPEC_TYPE:=draft-mtp}"
|
||||
: "${TEMPERATURE:=0.0}"
|
||||
: "${REASONING_FORMAT:=deepseek}"
|
||||
: "${REASONING_BUDGET:=-1}"
|
||||
|
||||
# Fail loudly rather than half-loading a model into VRAM: a missing file here
|
||||
# is a download that has not finished, and the unit's restart policy would
|
||||
# otherwise turn that into a crash loop that reads like a runtime fault.
|
||||
for path in "$MODEL_DIR/model.safetensors" "$MODEL_DIR/config.json" \
|
||||
"$DRAFT_DIR/model.safetensors" "$DRAFT_DIR/config.json"; do
|
||||
for path in "$LLAMA_BIN" "$MODEL_DIR/$MODEL_FILE" "$MMPROJ_PATH"; do
|
||||
[[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
|
||||
done
|
||||
|
||||
SPEC_CONFIG="{\"method\":\"mtp\",\"model\":\"${DRAFT_DIR}\",\"num_speculative_tokens\":${SPEC_NUM_TOKENS}}"
|
||||
|
||||
VLLM_FLAGS=(
|
||||
serve "$MODEL_DIR"
|
||||
--served-model-name "$ALIAS"
|
||||
--host "$BIND_ADDR"
|
||||
--port "$LISTEN_PORT"
|
||||
--max-model-len "$MAX_MODEL_LEN"
|
||||
--gpu-memory-utilization "$GPU_MEM_UTIL"
|
||||
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
||||
--max-num-seqs "$MAX_NUM_SEQS"
|
||||
--max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS"
|
||||
--speculative-config "$SPEC_CONFIG"
|
||||
--reasoning-parser gemma4
|
||||
--enable-auto-tool-choice
|
||||
--tool-call-parser gemma4
|
||||
--override-generation-config '{"temperature": 0}'
|
||||
)
|
||||
# ENFORCE_EAGER=1 skips Inductor/Triton compilation and CUDA graph capture
|
||||
# (slower steady-state decode, faster and more robust boot). The 3090 is
|
||||
# sm_86 (Ampere) — no native fp8 — so KV stays bf16 and the default
|
||||
# graphed path is the right one; the knob is a boot-debugging lever.
|
||||
if [[ "$ENFORCE_EAGER" == "1" ]]; then
|
||||
VLLM_FLAGS+=(--enforce-eager)
|
||||
if [[ -n "${DRAFT_FILE:-}" ]]; then
|
||||
[[ -e "$MODEL_DIR/$DRAFT_FILE" ]] || { echo "frank-small: missing $MODEL_DIR/$DRAFT_FILE" >&2; exit 1; }
|
||||
fi
|
||||
|
||||
exec vllm "${VLLM_FLAGS[@]}"
|
||||
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
|
||||
# open after boot; make sure it exists before the server needs it.
|
||||
nvidia-modprobe -u -c=0 2>/dev/null || true
|
||||
|
||||
MMPROJ_ARGS=()
|
||||
if [[ -n "${MMPROJ_PATH:-}" && -f "$MMPROJ_PATH" ]]; then
|
||||
MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
|
||||
fi
|
||||
|
||||
SPEC_ARGS=()
|
||||
if [[ -n "${SPEC_TYPE:-}" && "$SPEC_TYPE" != "none" ]]; then
|
||||
SPEC_ARGS=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
||||
fi
|
||||
|
||||
DRAFT_ARGS=()
|
||||
if [[ -n "${DRAFT_FILE:-}" ]]; then
|
||||
DRAFT_ARGS=(--model-draft "$MODEL_DIR/$DRAFT_FILE")
|
||||
fi
|
||||
|
||||
# CTX_SIZE set -> pinned context. Unset -> --fit sizes the KV pool against
|
||||
# whatever the weights, projector, draft context and compute buffers leave
|
||||
# on the card; the engine prints the chosen value at startup.
|
||||
if [[ -n "${CTX_SIZE:-}" ]]; then
|
||||
CTX_ARGS=(--ctx-size "$CTX_SIZE")
|
||||
else
|
||||
CTX_ARGS=(--fit on)
|
||||
fi
|
||||
|
||||
exec "$LLAMA_BIN" \
|
||||
--model "$MODEL_DIR/$MODEL_FILE" \
|
||||
"${MMPROJ_ARGS[@]}" \
|
||||
"${DRAFT_ARGS[@]}" \
|
||||
--alias "$ALIAS" \
|
||||
--host "$BIND_ADDR" \
|
||||
--port "$LISTEN_PORT" \
|
||||
--gpu-layers "$NGL" \
|
||||
--flash-attn "$FLASH_ATTN" \
|
||||
"${CTX_ARGS[@]}" \
|
||||
--parallel "$PARALLEL" \
|
||||
--batch-size "$BATCH_SIZE" \
|
||||
--ubatch-size "$UBATCH_SIZE" \
|
||||
--threads "$NUM_THREADS" \
|
||||
--threads-batch "$THREAD_BATCH" \
|
||||
--poll "$POLL_LEVEL" \
|
||||
--cache-type-k "$CACHE_TYPE_K" \
|
||||
--cache-type-v "$CACHE_TYPE_V" \
|
||||
"${SPEC_ARGS[@]}" \
|
||||
--jinja \
|
||||
--metrics \
|
||||
--temp "$TEMPERATURE" \
|
||||
--reasoning-format "$REASONING_FORMAT" \
|
||||
--reasoning-budget "$REASONING_BUDGET"
|
||||
Reference in new issue
Block a user