dank/serverAppConfigs
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-small: llama.cpp -> vLLM (gemma-4-12B-it-qat-w4a16-ct, w4a16 Marlin)

Browse Source 
Runtime swap on the RTX 3090 (port 7771):
- vLLM gemma-x86_64-cu129 dev build, digest-pinned, audio extra pip-installed
  into localhost/vllm/vllm-openai:frank-small (Dockerfile, no COPY lines)
- google/gemma-4-12B-it-qat-w4a16-ct (10.26 GiB int4) + Google's bf16 MTP
  assistant (845 MB) — the w4a16-ct assistant cannot draft in vLLM (drafter
  path skips Marlin; acceptance 1.00), bf16 measures 2.80-3.56
- root systemd unit -> rootless Quadlet (dankclaw/frank-small/quadlet/);
  manual CUDA device+lib mounts (no nvidia-container-toolkit on host)
- bf16 KV (fp8 hardware-dead on sm_86), 289,647-token pool, 2 concurrent
  full-context sessions, CUDA graphs on
- native text+image+audio: the fleet's multimodal endpoint
- measured: pp@67k 1293 t/s, pp@213k 608 t/s, tg@213k ~13.6 t/s,
  tg@85k ~17.6 t/s
- host driver 610.57.04 (Ubuntu 26.04); sm_86 is correct, no arch override
- planner/scoping: frank-small now selected by dankclaw/frank-small/ +
  env/serve/download trio; old unit retired to scripts/.AGENTTRASH/
- docs: new runbook frank-small-vllm-gemma4.md; AGENTS.md, LLM index,
  top docs index updated; old runbook moved to retired
This commit is contained in:
Frank
2026-09-18 21:06:46 -07:00
parent cf06c4eab8
commit a2b39b179a
15 changed files with 821 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/deploy-remote.sh
+76 -14
View File
@@ -1139,28 +1139,90 @@ wait_for_llm_large_ready() {
return 0
}
# deploy_llm_small — frank-small is mainline llama.cpp on the RTX 3090, as a
# ROOT unit: the runtime is a native CUDA binary, so there is no container
# image to install and no generated unit to avoid. relink_unit points
# /etc/systemd/system/frank-small.service at the repo file and boot persistence
# is that unit's [Install] section.
# deploy_llm_small — frank-small is a rootless Quadlet running vLLM on the
# RTX 3090 since 2026-09-18: the runtime is the digest-pinned
# vllm/vllm-openai gemma4-unified image, and the unit is generated from
# dankclaw/frank-small/quadlet/. It replaces the mainline-llama.cpp
# deployment, which ran as a ROOT unit.
#
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
# not shared: frank-large's halogen engine runs on the gfx1151 iGPU.
LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/Gemma4-12B-QAT/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf}"
# TWO MANAGERS, ONE UNIT NAME. The llama.cpp incarnation ran as a ROOT unit
# whose file is a symlink into this checkout; the vLLM unit is a USER
# (quadlet) unit. Both must never exist at once — they would race for port
# 7771 — so the root unit is retired first: stop, disable, move the /etc
# symlink into the home trash (never `rm`), reload the root manager. This
# host's `systemctl disable` has been observed to delete the unit file
# itself (2026-08-26 incident), so the move tolerates a file that is
# already gone.
#
# The payloads are fetched and sha256-verified by
# scripts/download-frank-small-vllm.sh before first start; the handler
# refuses to launch a unit that cannot load its model or its drafter.
LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/gemma4-12b-qat-w4a16-ct/model.safetensors}"
LLM_SMALL_DRAFT="${LLM_SMALL_DRAFT:-/home/dank/models/gemma4-12b-assistant-bf16/model.safetensors}"
# Base image: the gemma-x86_64-cu129 build (vLLM 0.22.1rc1.dev357+
# g74b5964f0) — the image that boots this checkpoint. The June 3
# gemma4-unified build predates it and fails at weight load (missing
# vision_embedder.patch_dense); the Sep 15 cu129-nightly fails at import
# (torchvision ABI). The unit runs a LOCAL BUILD on top of this base —
# dankclaw/frank-small/Dockerfile installs the vllm[audio] extra, which
# the base ships without.
VLLM_SMALL_BASE_IMAGE="docker.io/vllm/vllm-openai@sha256:e24d3d8e367ebc2a6ccfc8cf3cecee858a30a6c66b5b1642d85363cd745e001e"
VLLM_SMALL_BASE_ALIAS="localhost/vllm/vllm-openai:gemma4-unified"
VLLM_SMALL_IMAGE="localhost/vllm/vllm-openai:frank-small"
retire_root_llm_small() {
local trash="$HOME/.AGENTTRASH/20260918-vllm-cutover/host-units"
run_privileged systemctl stop frank-small.service 2>/dev/null || true
run_privileged systemctl disable frank-small.service 2>/dev/null || true
if [[ -e /etc/systemd/system/frank-small.service || -L /etc/systemd/system/frank-small.service ]]; then
mkdir -p "$trash"
run_privileged mv -- /etc/systemd/system/frank-small.service \
"$trash/frank-small.service"
echo "retired root unit frank-small.service -> $trash"
else
# This host's `systemctl disable` is allowed to delete the unit file
# outright (2026-08-26 incident), so there is often nothing left to
# move. Say so: previously no message meant no evidence either way.
echo "root unit frank-small.service was already removed by disable"
fi
run_privileged systemctl daemon-reload
}
deploy_llm_small() {
if [[ ! -f "$LLM_SMALL_MODEL" ]]; then
echo "ERROR: model absent at $LLM_SMALL_MODEL" >&2
echo "frank-small: fetch HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced (Q4_K_M, the mmproj and the MTP sidecar) first;" >&2
echo "frank-small: run scripts/download-frank-small-vllm.sh first (main model + MTP assistant, sha256-verified);" >&2
echo "frank-small: refusing to start a unit that cannot load." >&2
return 1
fi
relink_unit frank-small
run_privileged systemctl daemon-reload
run_privileged systemctl enable frank-small.service
run_privileged systemctl restart frank-small.service
wait_for_health frank-small http://127.0.0.1:7771/health 900
if [[ ! -f "$LLM_SMALL_DRAFT" ]]; then
echo "ERROR: MTP assistant absent at $LLM_SMALL_DRAFT" >&2
echo "frank-small: run scripts/download-frank-small-vllm.sh first;" >&2
echo "frank-small: refusing to start a unit that cannot draft." >&2
return 1
fi
retire_root_llm_small
# The predecessor can also be THIS quadlet -- a redeploy of a running
# frank-small -- so stop the user unit before waiting on the port. On a
# first cutover it is not running and this is a no-op.
systemctl --user stop frank-small.service 2>/dev/null || true
# Pull the base by digest, re-tag it to the FROM ref the Dockerfile
# expects, then build the derived image the quadlet runs: the quadlet
# path cannot resolve registry-prefixed refs from local storage at
# create time (same workaround as halogen), and Pull=never keeps boots
# off the registry. The build needs network for pip (the vllm[audio]
# extra); podman build has it on by default.
podman pull "$VLLM_SMALL_BASE_IMAGE"
podman tag "$VLLM_SMALL_BASE_IMAGE" "$VLLM_SMALL_BASE_ALIAS"
podman build --pull=never -t "$VLLM_SMALL_IMAGE" \
-f dankclaw/frank-small/Dockerfile dankclaw/frank-small
# Nothing may start until the predecessor has released the port.
wait_port_free 7771 || return 1
install_quadlet frank-small dankclaw/frank-small/quadlet frank-small
# No enable: generated units cannot be enabled on this host (v1 quadlet
# generator); boot persistence comes from [Install] in the unit file.
systemctl --user restart frank-small.service
wait_for_health frank-small http://127.0.0.1:7771/health 1200
}
# deploy_llm_medium — frank-medium is mainline llama.cpp on the two R9700s as a