frank-small: revert vLLM cutover — back to llama.cpp Gemma4-12B Q4_K_M
The vLLM detour (a2b39b1, same day) is retired: the Q4_K_M llama.cpp setup
is the requested shape and the vLLM KV-quant options on sm_86 all measured
dead (fp8 hardware-impossible, int8 page-size unification, turboquant
mm-prefix). Root unit restored, vLLM quadlet/Dockerfile/fetch script moved
to .AGENTTRASH, runbook listed as same-day-reverted. Rollback target:
frank-small-vllm-gemma4.
This commit is contained in:
Frank
+14
-76
@@ -1139,90 +1139,28 @@ wait_for_llm_large_ready() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# deploy_llm_small — frank-small is a rootless Quadlet running vLLM on the
|
||||
# RTX 3090 since 2026-09-18: the runtime is the digest-pinned
|
||||
# vllm/vllm-openai gemma4-unified image, and the unit is generated from
|
||||
# dankclaw/frank-small/quadlet/. It replaces the mainline-llama.cpp
|
||||
# deployment, which ran as a ROOT unit.
|
||||
# deploy_llm_small — frank-small is mainline llama.cpp on the RTX 3090, as a
|
||||
# ROOT unit: the runtime is a native CUDA binary, so there is no container
|
||||
# image to install and no generated unit to avoid. relink_unit points
|
||||
# /etc/systemd/system/frank-small.service at the repo file and boot persistence
|
||||
# is that unit's [Install] section.
|
||||
#
|
||||
# TWO MANAGERS, ONE UNIT NAME. The llama.cpp incarnation ran as a ROOT unit
|
||||
# whose file is a symlink into this checkout; the vLLM unit is a USER
|
||||
# (quadlet) unit. Both must never exist at once — they would race for port
|
||||
# 7771 — so the root unit is retired first: stop, disable, move the /etc
|
||||
# symlink into the home trash (never `rm`), reload the root manager. This
|
||||
# host's `systemctl disable` has been observed to delete the unit file
|
||||
# itself (2026-08-26 incident), so the move tolerates a file that is
|
||||
# already gone.
|
||||
#
|
||||
# The payloads are fetched and sha256-verified by
|
||||
# scripts/download-frank-small-vllm.sh before first start; the handler
|
||||
# refuses to launch a unit that cannot load its model or its drafter.
|
||||
LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/gemma4-12b-qat-w4a16-ct/model.safetensors}"
|
||||
LLM_SMALL_DRAFT="${LLM_SMALL_DRAFT:-/home/dank/models/gemma4-12b-assistant-bf16/model.safetensors}"
|
||||
# Base image: the gemma-x86_64-cu129 build (vLLM 0.22.1rc1.dev357+
|
||||
# g74b5964f0) — the image that boots this checkpoint. The June 3
|
||||
# gemma4-unified build predates it and fails at weight load (missing
|
||||
# vision_embedder.patch_dense); the Sep 15 cu129-nightly fails at import
|
||||
# (torchvision ABI). The unit runs a LOCAL BUILD on top of this base —
|
||||
# dankclaw/frank-small/Dockerfile installs the vllm[audio] extra, which
|
||||
# the base ships without.
|
||||
VLLM_SMALL_BASE_IMAGE="docker.io/vllm/vllm-openai@sha256:e24d3d8e367ebc2a6ccfc8cf3cecee858a30a6c66b5b1642d85363cd745e001e"
|
||||
VLLM_SMALL_BASE_ALIAS="localhost/vllm/vllm-openai:gemma4-unified"
|
||||
VLLM_SMALL_IMAGE="localhost/vllm/vllm-openai:frank-small"
|
||||
|
||||
retire_root_llm_small() {
|
||||
local trash="$HOME/.AGENTTRASH/20260918-vllm-cutover/host-units"
|
||||
run_privileged systemctl stop frank-small.service 2>/dev/null || true
|
||||
run_privileged systemctl disable frank-small.service 2>/dev/null || true
|
||||
if [[ -e /etc/systemd/system/frank-small.service || -L /etc/systemd/system/frank-small.service ]]; then
|
||||
mkdir -p "$trash"
|
||||
run_privileged mv -- /etc/systemd/system/frank-small.service \
|
||||
"$trash/frank-small.service"
|
||||
echo "retired root unit frank-small.service -> $trash"
|
||||
else
|
||||
# This host's `systemctl disable` is allowed to delete the unit file
|
||||
# outright (2026-08-26 incident), so there is often nothing left to
|
||||
# move. Say so: previously no message meant no evidence either way.
|
||||
echo "root unit frank-small.service was already removed by disable"
|
||||
fi
|
||||
run_privileged systemctl daemon-reload
|
||||
}
|
||||
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
|
||||
# not shared: frank-large's halogen engine runs on the gfx1151 iGPU.
|
||||
LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/Gemma4-12B-QAT/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf}"
|
||||
|
||||
deploy_llm_small() {
|
||||
if [[ ! -f "$LLM_SMALL_MODEL" ]]; then
|
||||
echo "ERROR: model absent at $LLM_SMALL_MODEL" >&2
|
||||
echo "frank-small: run scripts/download-frank-small-vllm.sh first (main model + MTP assistant, sha256-verified);" >&2
|
||||
echo "frank-small: fetch HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced (Q4_K_M, the mmproj and the MTP sidecar) first;" >&2
|
||||
echo "frank-small: refusing to start a unit that cannot load." >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$LLM_SMALL_DRAFT" ]]; then
|
||||
echo "ERROR: MTP assistant absent at $LLM_SMALL_DRAFT" >&2
|
||||
echo "frank-small: run scripts/download-frank-small-vllm.sh first;" >&2
|
||||
echo "frank-small: refusing to start a unit that cannot draft." >&2
|
||||
return 1
|
||||
fi
|
||||
retire_root_llm_small
|
||||
# The predecessor can also be THIS quadlet -- a redeploy of a running
|
||||
# frank-small -- so stop the user unit before waiting on the port. On a
|
||||
# first cutover it is not running and this is a no-op.
|
||||
systemctl --user stop frank-small.service 2>/dev/null || true
|
||||
# Pull the base by digest, re-tag it to the FROM ref the Dockerfile
|
||||
# expects, then build the derived image the quadlet runs: the quadlet
|
||||
# path cannot resolve registry-prefixed refs from local storage at
|
||||
# create time (same workaround as halogen), and Pull=never keeps boots
|
||||
# off the registry. The build needs network for pip (the vllm[audio]
|
||||
# extra); podman build has it on by default.
|
||||
podman pull "$VLLM_SMALL_BASE_IMAGE"
|
||||
podman tag "$VLLM_SMALL_BASE_IMAGE" "$VLLM_SMALL_BASE_ALIAS"
|
||||
podman build --pull=never -t "$VLLM_SMALL_IMAGE" \
|
||||
-f dankclaw/frank-small/Dockerfile dankclaw/frank-small
|
||||
# Nothing may start until the predecessor has released the port.
|
||||
wait_port_free 7771 || return 1
|
||||
install_quadlet frank-small dankclaw/frank-small/quadlet frank-small
|
||||
# No enable: generated units cannot be enabled on this host (v1 quadlet
|
||||
# generator); boot persistence comes from [Install] in the unit file.
|
||||
systemctl --user restart frank-small.service
|
||||
wait_for_health frank-small http://127.0.0.1:7771/health 1200
|
||||
relink_unit frank-small
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-small.service
|
||||
run_privileged systemctl restart frank-small.service
|
||||
wait_for_health frank-small http://127.0.0.1:7771/health 900
|
||||
}
|
||||
|
||||
# deploy_llm_medium — frank-medium is mainline llama.cpp on the two R9700s as a
|
||||
|
||||
Reference in New Issue
Block a user