frank-small: llama.cpp -> vLLM (gemma-4-12B-it-qat-w4a16-ct, w4a16 Marlin)
Runtime swap on the RTX 3090 (port 7771): - vLLM gemma-x86_64-cu129 dev build, digest-pinned, audio extra pip-installed into localhost/vllm/vllm-openai:frank-small (Dockerfile, no COPY lines) - google/gemma-4-12B-it-qat-w4a16-ct (10.26 GiB int4) + Google's bf16 MTP assistant (845 MB) — the w4a16-ct assistant cannot draft in vLLM (drafter path skips Marlin; acceptance 1.00), bf16 measures 2.80-3.56 - root systemd unit -> rootless Quadlet (dankclaw/frank-small/quadlet/); manual CUDA device+lib mounts (no nvidia-container-toolkit on host) - bf16 KV (fp8 hardware-dead on sm_86), 289,647-token pool, 2 concurrent full-context sessions, CUDA graphs on - native text+image+audio: the fleet's multimodal endpoint - measured: pp@67k 1293 t/s, pp@213k 608 t/s, tg@213k ~13.6 t/s, tg@85k ~17.6 t/s - host driver 610.57.04 (Ubuntu 26.04); sm_86 is correct, no arch override - planner/scoping: frank-small now selected by dankclaw/frank-small/ + env/serve/download trio; old unit retired to scripts/.AGENTTRASH/ - docs: new runbook frank-small-vllm-gemma4.md; AGENTS.md, LLM index, top docs index updated; old runbook moved to retired
This commit is contained in:
Frank
+13
-5
@@ -319,6 +319,12 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
# its unit and fleet settings live in one .container file.
|
||||
_add(services, "frank-large")
|
||||
return services
|
||||
if len(parts) >= 2 and parts[1] == "frank-small":
|
||||
# The service is a rootless Quadlet container since 2026-09-18
|
||||
# (vLLM); its unit lives in one .container file. The old root
|
||||
# unit's trio is in scripts/ and maps there.
|
||||
_add(services, "frank-small")
|
||||
return services
|
||||
if len(parts) >= 2 and parts[1] == "observability":
|
||||
# The five observability containers became Quadlet units on
|
||||
# 2026-09-11. A unit change selects exactly its own service;
|
||||
@@ -402,11 +408,13 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
# change to it redeploying alertmanager is the only path that
|
||||
# reaches it on the host.
|
||||
_add(services, "alertmanager")
|
||||
if name in {"frank-small.env", "frank-small.service", "serve-frank-small.sh"}:
|
||||
# llama.cpp frank-small (2026-09-16) keeps its unit, env and serve
|
||||
# files here; each one selects frank-small only. The ik_llama.cpp
|
||||
# trio that lived here for two days is in scripts/.AGENTTRASH/,
|
||||
# which is archive material and plans nothing.
|
||||
if name in {"frank-small.env", "serve-frank-small.sh",
|
||||
"download-frank-small-vllm.sh"}:
|
||||
# vLLM frank-small (2026-09-18) keeps its env, entrypoint and
|
||||
# fetch script here; each one selects frank-small only. The
|
||||
# unit is the quadlet under dankclaw/frank-small/, and the old
|
||||
# root unit file is in scripts/.AGENTTRASH/, which is archive
|
||||
# material and plans nothing.
|
||||
_add(services, "frank-small")
|
||||
if name in {"frank-medium.env", "frank-medium.service", "serve-frank-medium.sh"}:
|
||||
# llama.cpp frank-medium (2026-09-16) keeps its unit, env and serve
|
||||
|
||||
+76
-14
@@ -1139,28 +1139,90 @@ wait_for_llm_large_ready() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# deploy_llm_small — frank-small is mainline llama.cpp on the RTX 3090, as a
|
||||
# ROOT unit: the runtime is a native CUDA binary, so there is no container
|
||||
# image to install and no generated unit to avoid. relink_unit points
|
||||
# /etc/systemd/system/frank-small.service at the repo file and boot persistence
|
||||
# is that unit's [Install] section.
|
||||
# deploy_llm_small — frank-small is a rootless Quadlet running vLLM on the
|
||||
# RTX 3090 since 2026-09-18: the runtime is the digest-pinned
|
||||
# vllm/vllm-openai gemma4-unified image, and the unit is generated from
|
||||
# dankclaw/frank-small/quadlet/. It replaces the mainline-llama.cpp
|
||||
# deployment, which ran as a ROOT unit.
|
||||
#
|
||||
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
|
||||
# not shared: frank-large's halogen engine runs on the gfx1151 iGPU.
|
||||
LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/Gemma4-12B-QAT/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf}"
|
||||
# TWO MANAGERS, ONE UNIT NAME. The llama.cpp incarnation ran as a ROOT unit
|
||||
# whose file is a symlink into this checkout; the vLLM unit is a USER
|
||||
# (quadlet) unit. Both must never exist at once — they would race for port
|
||||
# 7771 — so the root unit is retired first: stop, disable, move the /etc
|
||||
# symlink into the home trash (never `rm`), reload the root manager. This
|
||||
# host's `systemctl disable` has been observed to delete the unit file
|
||||
# itself (2026-08-26 incident), so the move tolerates a file that is
|
||||
# already gone.
|
||||
#
|
||||
# The payloads are fetched and sha256-verified by
|
||||
# scripts/download-frank-small-vllm.sh before first start; the handler
|
||||
# refuses to launch a unit that cannot load its model or its drafter.
|
||||
LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/gemma4-12b-qat-w4a16-ct/model.safetensors}"
|
||||
LLM_SMALL_DRAFT="${LLM_SMALL_DRAFT:-/home/dank/models/gemma4-12b-assistant-bf16/model.safetensors}"
|
||||
# Base image: the gemma-x86_64-cu129 build (vLLM 0.22.1rc1.dev357+
|
||||
# g74b5964f0) — the image that boots this checkpoint. The June 3
|
||||
# gemma4-unified build predates it and fails at weight load (missing
|
||||
# vision_embedder.patch_dense); the Sep 15 cu129-nightly fails at import
|
||||
# (torchvision ABI). The unit runs a LOCAL BUILD on top of this base —
|
||||
# dankclaw/frank-small/Dockerfile installs the vllm[audio] extra, which
|
||||
# the base ships without.
|
||||
VLLM_SMALL_BASE_IMAGE="docker.io/vllm/vllm-openai@sha256:e24d3d8e367ebc2a6ccfc8cf3cecee858a30a6c66b5b1642d85363cd745e001e"
|
||||
VLLM_SMALL_BASE_ALIAS="localhost/vllm/vllm-openai:gemma4-unified"
|
||||
VLLM_SMALL_IMAGE="localhost/vllm/vllm-openai:frank-small"
|
||||
|
||||
retire_root_llm_small() {
|
||||
local trash="$HOME/.AGENTTRASH/20260918-vllm-cutover/host-units"
|
||||
run_privileged systemctl stop frank-small.service 2>/dev/null || true
|
||||
run_privileged systemctl disable frank-small.service 2>/dev/null || true
|
||||
if [[ -e /etc/systemd/system/frank-small.service || -L /etc/systemd/system/frank-small.service ]]; then
|
||||
mkdir -p "$trash"
|
||||
run_privileged mv -- /etc/systemd/system/frank-small.service \
|
||||
"$trash/frank-small.service"
|
||||
echo "retired root unit frank-small.service -> $trash"
|
||||
else
|
||||
# This host's `systemctl disable` is allowed to delete the unit file
|
||||
# outright (2026-08-26 incident), so there is often nothing left to
|
||||
# move. Say so: previously no message meant no evidence either way.
|
||||
echo "root unit frank-small.service was already removed by disable"
|
||||
fi
|
||||
run_privileged systemctl daemon-reload
|
||||
}
|
||||
|
||||
deploy_llm_small() {
|
||||
if [[ ! -f "$LLM_SMALL_MODEL" ]]; then
|
||||
echo "ERROR: model absent at $LLM_SMALL_MODEL" >&2
|
||||
echo "frank-small: fetch HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced (Q4_K_M, the mmproj and the MTP sidecar) first;" >&2
|
||||
echo "frank-small: run scripts/download-frank-small-vllm.sh first (main model + MTP assistant, sha256-verified);" >&2
|
||||
echo "frank-small: refusing to start a unit that cannot load." >&2
|
||||
return 1
|
||||
fi
|
||||
relink_unit frank-small
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-small.service
|
||||
run_privileged systemctl restart frank-small.service
|
||||
wait_for_health frank-small http://127.0.0.1:7771/health 900
|
||||
if [[ ! -f "$LLM_SMALL_DRAFT" ]]; then
|
||||
echo "ERROR: MTP assistant absent at $LLM_SMALL_DRAFT" >&2
|
||||
echo "frank-small: run scripts/download-frank-small-vllm.sh first;" >&2
|
||||
echo "frank-small: refusing to start a unit that cannot draft." >&2
|
||||
return 1
|
||||
fi
|
||||
retire_root_llm_small
|
||||
# The predecessor can also be THIS quadlet -- a redeploy of a running
|
||||
# frank-small -- so stop the user unit before waiting on the port. On a
|
||||
# first cutover it is not running and this is a no-op.
|
||||
systemctl --user stop frank-small.service 2>/dev/null || true
|
||||
# Pull the base by digest, re-tag it to the FROM ref the Dockerfile
|
||||
# expects, then build the derived image the quadlet runs: the quadlet
|
||||
# path cannot resolve registry-prefixed refs from local storage at
|
||||
# create time (same workaround as halogen), and Pull=never keeps boots
|
||||
# off the registry. The build needs network for pip (the vllm[audio]
|
||||
# extra); podman build has it on by default.
|
||||
podman pull "$VLLM_SMALL_BASE_IMAGE"
|
||||
podman tag "$VLLM_SMALL_BASE_IMAGE" "$VLLM_SMALL_BASE_ALIAS"
|
||||
podman build --pull=never -t "$VLLM_SMALL_IMAGE" \
|
||||
-f dankclaw/frank-small/Dockerfile dankclaw/frank-small
|
||||
# Nothing may start until the predecessor has released the port.
|
||||
wait_port_free 7771 || return 1
|
||||
install_quadlet frank-small dankclaw/frank-small/quadlet frank-small
|
||||
# No enable: generated units cannot be enabled on this host (v1 quadlet
|
||||
# generator); boot persistence comes from [Install] in the unit file.
|
||||
systemctl --user restart frank-small.service
|
||||
wait_for_health frank-small http://127.0.0.1:7771/health 1200
|
||||
}
|
||||
|
||||
# deploy_llm_medium — frank-medium is mainline llama.cpp on the two R9700s as a
|
||||
|
||||
Executable
+80
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fetch the frank-small vLLM payloads on dankclaw (RTX 3090, since 2026-09-18):
|
||||
#
|
||||
# google/gemma-4-12B-it-qat-w4a16-ct
|
||||
# compressed-tensors w4a16 (int4, group 32) QAT checkpoint — the model
|
||||
# vLLM's gemma4_unified implementation loads natively. One 9.56 GiB
|
||||
# safetensors plus tokenizer/config.
|
||||
#
|
||||
# google/gemma-4-12B-it-assistant
|
||||
# the MTP draft/assistant (gemma4_unified_assistant, 4 layers) in
|
||||
# bf16 — Google's original. 806 MiB. The w4a16-ct re-quant
|
||||
# (kunhunjon) that was used first drafts at acceptance 1.00: vLLM's
|
||||
# MTP drafter path does not apply the Marlin w4a16 kernel to the
|
||||
# assistant's linears (no "Using MarlinLinearKernel" after the
|
||||
# drafter load), so its packed int4 weights are misread. The bf16
|
||||
# assistant accepts at 2.80 (measured 2026-09-19).
|
||||
#
|
||||
# Both repos are public and ungated; no token is needed. Every fetch is -f
|
||||
# (an HTTP error is a failure, not a 153-byte "artifact"), resumable, and the
|
||||
# two LFS payloads are sha256-verified against the Hub's declared oid — the
|
||||
# discipline from scripts/download-frank-medium-gguf.sh.
|
||||
#
|
||||
# The serve script refuses to start when either payload is absent, so a
|
||||
# half-finished run of this script reads as a missing file, not a runtime
|
||||
# fault.
|
||||
set -euo pipefail
|
||||
|
||||
MODELS_DIR="${MODELS_DIR:-/home/dank/models}"
|
||||
LOG_DIR="$MODELS_DIR/logs"
|
||||
mkdir -p "$LOG_DIR"
|
||||
LOG="$LOG_DIR/$(basename "$0" .sh).log"
|
||||
|
||||
log() { echo "$(date -Is) $*" | tee -a "$LOG"; }
|
||||
|
||||
# fetch <repo> <dir> <file> [sha256] [size]
|
||||
fetch() {
|
||||
local repo="$1" dir="$2" file="$3" sha="${4:-}" size="${5:-}"
|
||||
mkdir -p "$dir"
|
||||
if [[ -n "$size" && -f "$dir/$file" ]] && [[ "$(stat -c%s "$dir/$file")" == "$size" ]]; then
|
||||
log "skip $dir/$file (size $size matches)"
|
||||
else
|
||||
log "fetch $repo/$file"
|
||||
curl -fsSL -C - "https://huggingface.co/$repo/resolve/main/$file" -o "$dir/$file"
|
||||
log "fetched $dir/$file ($(stat -c%s "$dir/$file") bytes)"
|
||||
fi
|
||||
if [[ -n "$sha" ]]; then
|
||||
local actual
|
||||
actual="$(sha256sum "$dir/$file" | cut -d' ' -f1)"
|
||||
if [[ "$actual" != "$sha" ]]; then
|
||||
log "SHA256 MISMATCH $dir/$file: $actual != $sha"
|
||||
return 1
|
||||
fi
|
||||
log "sha256 ok $dir/$file"
|
||||
fi
|
||||
}
|
||||
|
||||
MAIN_REPO=google/gemma-4-12B-it-qat-w4a16-ct
|
||||
ASSISTANT_REPO=google/gemma-4-12B-it-assistant
|
||||
MAIN_DIR="$MODELS_DIR/gemma4-12b-qat-w4a16-ct"
|
||||
ASSISTANT_DIR="$MODELS_DIR/gemma4-12b-assistant-bf16"
|
||||
|
||||
# Hub-declared LFS oids (sha256), 2026-09-18.
|
||||
fetch "$MAIN_REPO" "$MAIN_DIR" model.safetensors \
|
||||
60b6e3989502969d8ae04185d72ecbbc7db63978d5af747a493d53895aa6bfa3 10264229896
|
||||
fetch "$MAIN_REPO" "$MAIN_DIR" tokenizer.json \
|
||||
cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f 32169626
|
||||
for f in config.json generation_config.json chat_template.jinja \
|
||||
processor_config.json tokenizer_config.json recipe.yaml; do
|
||||
fetch "$MAIN_REPO" "$MAIN_DIR" "$f"
|
||||
done
|
||||
|
||||
fetch "$ASSISTANT_REPO" "$ASSISTANT_DIR" model.safetensors \
|
||||
3279c173daddd7186e79d652ad94022415736d3a1370625696c898429b06d6df 845719296
|
||||
fetch "$ASSISTANT_REPO" "$ASSISTANT_DIR" tokenizer.json \
|
||||
c001d9ada50af662c94f5ab17ec7e09f6438d1bec8246c47fee6510693d8de35 32169884
|
||||
for f in config.json generation_config.json tokenizer_config.json; do
|
||||
fetch "$ASSISTANT_REPO" "$ASSISTANT_DIR" "$f"
|
||||
done
|
||||
|
||||
log "ALL FETCHED OK"
|
||||
+69
-100
@@ -1,112 +1,81 @@
|
||||
# Tunables for frank-small (HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-
|
||||
# Balanced, Q4_K_M, plus the BF16 vision projector and the separate MTP
|
||||
# sidecar) on mainline llama.cpp, Dankclaw's RTX 3090, since 2026-09-17.
|
||||
# Tunables for frank-small (since 2026-09-18): vLLM serving
|
||||
# google/gemma-4-12B-it-qat-w4a16-ct (compressed-tensors w4a16 — the format
|
||||
# Google publishes "for native, optimized inference with vLLM") with the
|
||||
# google bf16 MTP assistant, on Dankclaw's RTX 3090.
|
||||
#
|
||||
# Runtime: llama.cpp master 37b53fd, CUDA build for sm_86:
|
||||
# cmake -B build-cuda -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON \
|
||||
# -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86 -DGGML_CUDA_FA_ALL_QUANTS=ON
|
||||
# cmake --build build-cuda --target llama-server -j
|
||||
# GGML_CUDA_FA_ALL_QUANTS is ON: this service runs q8_0 KV with flash
|
||||
# attention, and without the flag quantized-KV prefill collapses ~12x on
|
||||
# CUDA (see the CUDA lesson in AGENTS.md).
|
||||
# Runtime: the vllm/vllm-openai gemma-x86_64-cu129 build (vLLM
|
||||
# 0.22.1rc1.dev357+g74b5964f0; base digest in scripts/deploy-remote.sh),
|
||||
# with the vllm[audio] extra installed on top (dankclaw/frank-small/
|
||||
# Dockerfile) so the model's native audio path works. That image carries
|
||||
# the native gemma4_unified implementation, the Gemma4MTPModel drafter,
|
||||
# the gemma4 reasoning and tool-call parsers, and CUDA 12.9 for the Ada
|
||||
# card. The older gemma4-unified build (June 3) predates this checkpoint
|
||||
# and fails at weight load; the cu129-nightly build (Sep 15) fails at
|
||||
# import (torchvision ABI); this one is the image that boots.
|
||||
#
|
||||
# Model: HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced at Q4_K_M
|
||||
# (6.87 GiB). Architecture (GGUF arch gemma4): 48 layers, MHA with 16 kv
|
||||
# heads x 512 head_dim, rope freq_base 1e6, final logit softcapping 30,
|
||||
# per-layer token embeddings (PLE) inside the file, and a sliding-window
|
||||
# attention pattern (attention.sliding_window = 1024) -- most layers cache
|
||||
# only a 1024-position window, which is why the KV pool is a fraction of
|
||||
# the full-ctx x all-layers number.
|
||||
# Model: 11.95B "unified" (encoder-free) — 48 layers of 5 sliding-window
|
||||
# (1024-token) + 1 full-attention, int4 group-32 weights with bf16
|
||||
# embeddings, vision and audio. The fleet's image path lives here: the model
|
||||
# ingests image (and audio) content natively, so no projector file exists
|
||||
# any more and nothing external has to load.
|
||||
#
|
||||
# MTP: the Multi-Token Prediction head ships as a SEPARATE file
|
||||
# (mtp-gemma-4-12B-it.gguf, arch gemma4-assistant, 4 nextn layers, 0.24
|
||||
# GiB). Unlike the Qwen3.8 -mtp build (head inside the main file), this one
|
||||
# is passed via --model-draft with --spec-type draft-mtp, so the draft
|
||||
# context loads the assistant model. The same --spec-type value chains the
|
||||
# draftless ngram-mod matcher, which attaches unconditionally; two
|
||||
# context-backed drafters cannot share one process, so draft-mtp,ngram-mod
|
||||
# is the only legal shape that carries both.
|
||||
# MTP: the draft model is a SEPARATE 4-layer assistant checkpoint in the
|
||||
# same int4 w4a16 compressed-tensors format — the card's compatibility rule
|
||||
# is that a QAT target must draft with a QAT drafter at the same precision.
|
||||
# It is passed in --speculative-config with method "mtp", vLLM's Gemma 4
|
||||
# MTP path, not a generic draft model.
|
||||
#
|
||||
# SIZING (RTX 3090, 24576 MiB): CTX_SIZE = PARALLEL x 262144 -- four
|
||||
# FULL-window slots, measured to fit: the 4-slot boot idles at 22,346
|
||||
# MiB and each extra full-context slot costs 3,450 MiB (measured 1-slot
|
||||
# 11,996 -> 4-slot 22,346). A fifth slot projects to ~25.8 GiB, past the
|
||||
# card, and the KV pool allocates upfront -- so 4 is the max. Explicit
|
||||
# CTX_SIZE bypasses --fit on purpose: --fit maximises the TOTAL pool,
|
||||
# which would shrink each slot below the full 262144 window once
|
||||
# parallel > 1.
|
||||
# SIZING (RTX 3090, 24576 MiB): the goal is the model's FULL 262144 window
|
||||
# for as many concurrent sessions as the card holds.
|
||||
# MAX_MODEL_LEN=262144 the model's max_position_embeddings
|
||||
# GPU_MEM_UTIL=0.95 the card is dedicated to this service
|
||||
# KV_CACHE_DTYPE=auto bf16 KV. fp8_e4m3 would halve the KV charge
|
||||
# (289,756 -> 623,666-token pool, one -> two
|
||||
# full-context sessions), but the 3090 is
|
||||
# sm_86 (Ampere) and has no native fp8: the
|
||||
# TRITON_ATTN backend Gemma4 forces emits
|
||||
# cvt.e4m3x2 instructions that exist only from
|
||||
# sm_89 (Ada), so fp8 KV cannot boot on this
|
||||
# card (measured 2026-09-19; the "sm_89
|
||||
# patch" experiment is in .AGENTTRASH).
|
||||
# MAX_NUM_SEQS=16 the batch ceiling; the KV pool, not this,
|
||||
# caps full-context concurrency
|
||||
# MAX_NUM_BATCHED_TOKENS=16384 the prefill chunk; larger chunks prefill
|
||||
# faster but grow the activation arena that
|
||||
# competes with the KV pool for the card
|
||||
# The boot log prints the KV pool size and "Maximum concurrency for N tokens
|
||||
# per request" — for a hybrid SWA model that line is the NAIVE ratio (pool /
|
||||
# max_model_len) and undercounts: the 42 sliding-window layers only retain a
|
||||
# 1024-token window per request, so the true full-context session count is
|
||||
# higher. Verify it behaviourally (concurrent long requests) on every
|
||||
# restart, not from these comments.
|
||||
#
|
||||
# Vision + audio: --mmproj loads the BF16 projector (11 tensors, 52M
|
||||
# params; it carries an audio encoder too). The fleet's image path lives
|
||||
# here, which is why halogen no longer sets HALOGEN_VISION_TOWER. Audio
|
||||
# WORKS: input_audio content parts transcribe verbatim (verified
|
||||
# 2026-09-17 with a 16k mono WAV).
|
||||
# Sampling: greedy by default (fleet contract) via
|
||||
# --override-generation-config; a client that sends its own temperature gets
|
||||
# its own, and the harness applies per-model overrides from models.yml.
|
||||
#
|
||||
# Sampling: greedy by default (the fleet contract); a client that sends a
|
||||
# temperature gets its own, and the harness applies per-model overrides
|
||||
# from models.yml.
|
||||
#
|
||||
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (HIP
|
||||
# reads those bare vars on the ROCm hosts -- see the 2026-08-13 incident in
|
||||
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (the
|
||||
# ROCm hosts read those bare vars — see the 2026-08-13 incident in
|
||||
# AGENTS.md). This box is CUDA, but the rule keeps env files portable.
|
||||
|
||||
# llama.cpp runtime binary: mainline master 37b53fd, CUDA sm_86,
|
||||
# FA_ALL_QUANTS=ON.
|
||||
LLAMA_BIN=/home/dank/GitPackages/llama.cpp-mainline/build-cuda/bin/llama-server
|
||||
# Container paths, bind-mounted by the quadlet. The host paths are declared
|
||||
# in the quadlet and the deploy handler, not here: this file is read inside
|
||||
# the container.
|
||||
MODEL_DIR=/models/main
|
||||
DRAFT_DIR=/models/assistant
|
||||
|
||||
# Model package. The MTP sidecar file is required for speculative decoding.
|
||||
MODEL_DIR=/home/dank/models/Gemma4-12B-QAT
|
||||
MODEL_FILE=Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf
|
||||
MMPROJ_PATH=/home/dank/models/Gemma4-12B-QAT/mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf
|
||||
DRAFT_FILE=mtp-gemma-4-12B-it.gguf
|
||||
|
||||
# Fleet identity and network. 7771 is this service's own endpoint on
|
||||
# Dankclaw; 7770 stays frank-large's. The alias is the id at /v1/models and
|
||||
# what clients ask for -- the harness entry in models.yml names it.
|
||||
# Fleet identity. 8000 is the in-container port; the quadlet publishes it as
|
||||
# host 7771, the endpoint Hermes, the harness and the homepage probe have
|
||||
# always used. The alias is the id at /v1/models and what clients ask for.
|
||||
ALIAS=frank-small
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7771
|
||||
LISTEN_PORT=8000
|
||||
|
||||
# Four FULL 262144-token contexts, one per slot (CTX_SIZE is the total
|
||||
# pool; --parallel divides it). See the SIZING note above for the fit
|
||||
# arithmetic; a fifth slot does not fit.
|
||||
CTX_SIZE=1048576
|
||||
PARALLEL=4
|
||||
CACHE_TYPE_K=q8_0
|
||||
CACHE_TYPE_V=q8_0
|
||||
|
||||
# Batch geometry: 8192 logical / 1024 physical. Measured on this card at
|
||||
# max length (254k-token prefill / 253k-ctx decode, 4 slots, q8 KV):
|
||||
# ubatch 512 -> pp 915 t/s, tg@250k 90-204; ubatch 1024 -> pp 962,
|
||||
# tg 257-302. Larger ubatches (2048) OOM the MTP draft context's pp
|
||||
# compute buffer (~3.3 GiB against 2.2 GiB headroom) and must not be
|
||||
# tried. Idle VRAM is 23,300 MiB at this geometry.
|
||||
BATCH_SIZE=8192
|
||||
UBATCH_SIZE=1024
|
||||
|
||||
# CPU threads: this service runs entirely on the GPU (NGL=999), so threads
|
||||
# cover tokenization, sampling and the HTTP layer only. Deliberately modest
|
||||
# -- this host's CPU is shared with halogen's engine, which is CPU-hungry.
|
||||
NUM_THREADS=8
|
||||
THREAD_BATCH=8
|
||||
|
||||
# Every layer on the 3090, with flash attention (required for q8 KV).
|
||||
NGL=999
|
||||
FLASH_ATTN=on
|
||||
|
||||
# Speculative decoding: the MTP sidecar (draft context) chained with the
|
||||
# draftless ngram-mod matcher. Draft depth and busy-wait level are made
|
||||
# explicit here (both were the implicit build defaults: 3 and 50).
|
||||
# Measured at 250k ctx: depth 4 + poll 100 DECODED WORSE (230-257 t/s)
|
||||
# than depth 3 + poll 50 (257-302 t/s) — the longer verification round
|
||||
# costs more attention re-read at long context than the extra accepted
|
||||
# token earns. Keep 3/50.
|
||||
SPEC_TYPE=draft-mtp,ngram-mod
|
||||
SPEC_DRAFT_N_MAX=3
|
||||
POLL_LEVEL=50
|
||||
|
||||
# Greedy default (fleet contract); reasoning is extracted into
|
||||
# reasoning_content with an unlimited budget (the model decides).
|
||||
TEMPERATURE=0.0
|
||||
REASONING_FORMAT=deepseek
|
||||
REASONING_BUDGET=-1
|
||||
# Full window, dedicated card, bf16 KV (see above), roomy batch ceiling,
|
||||
# 16k prefill chunk, 4-token MTP draft (the assistant carries 4 layers).
|
||||
MAX_MODEL_LEN=262144
|
||||
GPU_MEM_UTIL=0.95
|
||||
KV_CACHE_DTYPE=auto
|
||||
MAX_NUM_SEQS=16
|
||||
MAX_NUM_BATCHED_TOKENS=16384
|
||||
SPEC_NUM_TOKENS=4
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
[Unit]
|
||||
Description=llama.cpp Qwen3.8-27B GSQ-RCO IQ3_S MTP+vision (dankclaw, frank-small)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
# CUDA pins its staging buffers, so the process needs the memlock budget even
|
||||
# though the weights themselves are copied into VRAM. Parity with the halogen
|
||||
# quadlet's Ulimit=memlock=-1:-1 and the retired ik unit.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
+58
-113
@@ -1,43 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced (Q4_K_M) as
|
||||
# frank-small on Dankclaw's RTX 3090 via mainline llama.cpp.
|
||||
# Serve google/gemma-4-12B-it-qat-w4a16-ct as frank-small on Dankclaw's RTX
|
||||
# 3090 via vLLM (since 2026-09-18), with the google bf16 MTP assistant
|
||||
# for speculative decoding.
|
||||
#
|
||||
# Topology: the whole model lives in VRAM (-ngl 999) -- 6.87 GiB of Q4_K_M
|
||||
# weights plus the BF16 vision projector and q8_0 KV inside the card's
|
||||
# 24 GiB. The host CPU only tokenizes and samples here; this box's CPU
|
||||
# belongs to the halogen engine that serves frank-large.
|
||||
# This script is the CONTAINER ENTRYPOINT of the frank-small quadlet: the
|
||||
# vllm/vllm-openai image's own entrypoint is `vllm serve`, and the quadlet
|
||||
# replaces it with this file (mounted read-only out of the repo checkout) so
|
||||
# the flag set stays in git. The image pins the gemma4-unified vLLM build,
|
||||
# which is what loads gemma4_unified natively and runs the Gemma 4 MTP path.
|
||||
#
|
||||
# MTP: this model's Multi-Token Prediction head ships as a SEPARATE file
|
||||
# (mtp-gemma-4-12B-it.gguf, arch gemma4-assistant, 4 nextn layers). It is
|
||||
# passed via --model-draft with --spec-type draft-mtp: the draft context
|
||||
# loads the assistant model. The same --spec-type value chains the
|
||||
# draftless ngram-mod matcher, which attaches unconditionally; two
|
||||
# context-backed drafters cannot share one process, so this chain is the
|
||||
# only shape that carries both.
|
||||
# MTP: the drafter is a separate 4-layer assistant checkpoint in bf16
|
||||
# (Google's original gemma-4-12B-it-assistant). It goes in
|
||||
# --speculative-config as method "mtp" — vLLM's Gemma 4 MTP path, not a
|
||||
# generic draft model. The w4a16-ct re-quant of the assistant was tried
|
||||
# first and drafts at acceptance 1.00 (the drafter path does not apply the
|
||||
# Marlin kernel to its linears); the bf16 assistant accepts at 2.80.
|
||||
#
|
||||
# Context: CTX_SIZE is set explicitly to PARALLEL x 262144 so every slot
|
||||
# gets the model's FULL window; --fit (the CTX_SIZE-unset branch below)
|
||||
# maximises the TOTAL pool instead, which would divide a smaller window
|
||||
# across slots. The 4-slot layout is measured to fit the card (see the
|
||||
# SIZING note in frank-small.env); more slots would fail the KV allocation.
|
||||
# Sampling: greedy by default (fleet contract) via
|
||||
# --override-generation-config; clients that send their own temperature get
|
||||
# their own, and the harness overrides per model from models.yml.
|
||||
#
|
||||
# Vision + audio: --mmproj loads the BF16 projector, which also carries
|
||||
# an audio encoder — input_audio content parts transcribe verbatim
|
||||
# (verified 2026-09-17). The fleet's image path lives here, which is why
|
||||
# halogen no longer sets HALOGEN_VISION_TOWER.
|
||||
#
|
||||
# Sampling: greedy by default (the fleet contract). Clients that send their own
|
||||
# temperature get their own values, and the harness overrides per model from
|
||||
# models.yml. --jinja is pinned even though this build defaults it on: the
|
||||
# model's own template supplies reasoning_effort, preserve_thinking and the
|
||||
# tool-call grammar, and a future build that flips that default must not
|
||||
# silently take tools away.
|
||||
#
|
||||
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides it.
|
||||
# Tunables come from frank-small.env (mounted beside this file by the
|
||||
# quadlet) unless FRANK_SMALL_ENV_FILE overrides the path.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
|
||||
ENV_FILE="${FRANK_SMALL_ENV_FILE:-/scripts/frank-small.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
@@ -46,93 +33,51 @@ if [[ -f "$ENV_FILE" ]]; then
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${LLAMA_BIN:=/home/dank/GitPackages/llama.cpp-mainline/build-cuda/bin/llama-server}"
|
||||
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp}"
|
||||
: "${MODEL_FILE:=Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
|
||||
: "${MMPROJ_PATH:=$MODEL_DIR/mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf}"
|
||||
: "${DRAFT_FILE:=}"
|
||||
: "${MODEL_DIR:=/models/main}"
|
||||
: "${DRAFT_DIR:=/models/assistant}"
|
||||
: "${ALIAS:=frank-small}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7771}"
|
||||
: "${CTX_SIZE:=}"
|
||||
: "${PARALLEL:=1}"
|
||||
: "${CACHE_TYPE_K:=f16}"
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
: "${BATCH_SIZE:=2048}"
|
||||
: "${UBATCH_SIZE:=512}"
|
||||
# Decode-shape knobs: the MTP drafter's per-round proposal length (build
|
||||
# default 3; the Gemma4 sidecar carries 4 nextn layers) and the engine's
|
||||
# busy-wait level (build default 50).
|
||||
: "${SPEC_DRAFT_N_MAX:=3}"
|
||||
: "${POLL_LEVEL:=50}"
|
||||
: "${NUM_THREADS:=8}"
|
||||
: "${THREAD_BATCH:=8}"
|
||||
: "${NGL:=999}"
|
||||
: "${FLASH_ATTN:=on}"
|
||||
: "${SPEC_TYPE:=draft-mtp}"
|
||||
: "${TEMPERATURE:=0.0}"
|
||||
: "${REASONING_FORMAT:=deepseek}"
|
||||
: "${REASONING_BUDGET:=-1}"
|
||||
: "${LISTEN_PORT:=8000}"
|
||||
: "${MAX_MODEL_LEN:=262144}"
|
||||
: "${GPU_MEM_UTIL:=0.95}"
|
||||
: "${KV_CACHE_DTYPE:=auto}"
|
||||
: "${MAX_NUM_SEQS:=16}"
|
||||
: "${MAX_NUM_BATCHED_TOKENS:=16384}"
|
||||
: "${SPEC_NUM_TOKENS:=4}"
|
||||
: "${ENFORCE_EAGER:=0}"
|
||||
|
||||
# Fail loudly rather than half-loading a model into VRAM: a missing file here
|
||||
# is a download that has not finished, and the unit's restart policy would
|
||||
# otherwise turn that into a crash loop that reads like a runtime fault.
|
||||
for path in "$LLAMA_BIN" "$MODEL_DIR/$MODEL_FILE" "$MMPROJ_PATH"; do
|
||||
for path in "$MODEL_DIR/model.safetensors" "$MODEL_DIR/config.json" \
|
||||
"$DRAFT_DIR/model.safetensors" "$DRAFT_DIR/config.json"; do
|
||||
[[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
|
||||
done
|
||||
if [[ -n "${DRAFT_FILE:-}" ]]; then
|
||||
[[ -e "$MODEL_DIR/$DRAFT_FILE" ]] || { echo "frank-small: missing $MODEL_DIR/$DRAFT_FILE" >&2; exit 1; }
|
||||
|
||||
SPEC_CONFIG="{\"method\":\"mtp\",\"model\":\"${DRAFT_DIR}\",\"num_speculative_tokens\":${SPEC_NUM_TOKENS}}"
|
||||
|
||||
VLLM_FLAGS=(
|
||||
serve "$MODEL_DIR"
|
||||
--served-model-name "$ALIAS"
|
||||
--host "$BIND_ADDR"
|
||||
--port "$LISTEN_PORT"
|
||||
--max-model-len "$MAX_MODEL_LEN"
|
||||
--gpu-memory-utilization "$GPU_MEM_UTIL"
|
||||
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
||||
--max-num-seqs "$MAX_NUM_SEQS"
|
||||
--max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS"
|
||||
--speculative-config "$SPEC_CONFIG"
|
||||
--reasoning-parser gemma4
|
||||
--enable-auto-tool-choice
|
||||
--tool-call-parser gemma4
|
||||
--override-generation-config '{"temperature": 0}'
|
||||
)
|
||||
# ENFORCE_EAGER=1 skips Inductor/Triton compilation and CUDA graph capture
|
||||
# (slower steady-state decode, faster and more robust boot). The 3090 is
|
||||
# sm_86 (Ampere) — no native fp8 — so KV stays bf16 and the default
|
||||
# graphed path is the right one; the knob is a boot-debugging lever.
|
||||
if [[ "$ENFORCE_EAGER" == "1" ]]; then
|
||||
VLLM_FLAGS+=(--enforce-eager)
|
||||
fi
|
||||
|
||||
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
|
||||
# open after boot; make sure it exists before the server needs it.
|
||||
nvidia-modprobe -u -c=0 2>/dev/null || true
|
||||
|
||||
MMPROJ_ARGS=()
|
||||
if [[ -n "${MMPROJ_PATH:-}" && -f "$MMPROJ_PATH" ]]; then
|
||||
MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
|
||||
fi
|
||||
|
||||
SPEC_ARGS=()
|
||||
if [[ -n "${SPEC_TYPE:-}" && "$SPEC_TYPE" != "none" ]]; then
|
||||
SPEC_ARGS=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
||||
fi
|
||||
|
||||
DRAFT_ARGS=()
|
||||
if [[ -n "${DRAFT_FILE:-}" ]]; then
|
||||
DRAFT_ARGS=(--model-draft "$MODEL_DIR/$DRAFT_FILE")
|
||||
fi
|
||||
|
||||
# CTX_SIZE set -> pinned context. Unset -> --fit sizes the KV pool against
|
||||
# whatever the weights, projector, draft context and compute buffers leave
|
||||
# on the card; the engine prints the chosen value at startup.
|
||||
if [[ -n "${CTX_SIZE:-}" ]]; then
|
||||
CTX_ARGS=(--ctx-size "$CTX_SIZE")
|
||||
else
|
||||
CTX_ARGS=(--fit on)
|
||||
fi
|
||||
|
||||
exec "$LLAMA_BIN" \
|
||||
--model "$MODEL_DIR/$MODEL_FILE" \
|
||||
"${MMPROJ_ARGS[@]}" \
|
||||
"${DRAFT_ARGS[@]}" \
|
||||
--alias "$ALIAS" \
|
||||
--host "$BIND_ADDR" \
|
||||
--port "$LISTEN_PORT" \
|
||||
--gpu-layers "$NGL" \
|
||||
--flash-attn "$FLASH_ATTN" \
|
||||
"${CTX_ARGS[@]}" \
|
||||
--parallel "$PARALLEL" \
|
||||
--batch-size "$BATCH_SIZE" \
|
||||
--ubatch-size "$UBATCH_SIZE" \
|
||||
--threads "$NUM_THREADS" \
|
||||
--threads-batch "$THREAD_BATCH" \
|
||||
--poll "$POLL_LEVEL" \
|
||||
--cache-type-k "$CACHE_TYPE_K" \
|
||||
--cache-type-v "$CACHE_TYPE_V" \
|
||||
"${SPEC_ARGS[@]}" \
|
||||
--jinja \
|
||||
--metrics \
|
||||
--temp "$TEMPERATURE" \
|
||||
--reasoning-format "$REASONING_FORMAT" \
|
||||
--reasoning-budget "$REASONING_BUDGET"
|
||||
exec vllm "${VLLM_FLAGS[@]}"
|
||||
|
||||
Reference in New Issue
Block a user