Frank
f55189e880
The podman 5.x v1 quadlet generator on dankclaw rejects ContainerEntrypoint with 'unsupported key in group Container', so the unit was never generated and the deploy died at restart with 'Unit frank-small.service not found'. Entry= is the v1 spelling.
101 lines
5.0 KiB
INI
101 lines
5.0 KiB
INI
# Quadlet unit: frank-small (rootless, boot-persistent).
|
|
#
|
|
# Install path on dankclaw: ~/.config/containers/systemd/frank-small.container
|
|
# Unit name: frank-small.service (container name: frank-small)
|
|
#
|
|
# The small inference service. vLLM (since 2026-09-18) serving
|
|
# google/gemma-4-12B-it-qat-w4a16-ct (compressed-tensors w4a16) with the
|
|
# google bf16 MTP assistant for speculative decoding, on the RTX
|
|
# 3090. It replaced the mainline-llama.cpp ROOT unit (the old
|
|
# scripts/frank-small.service is retired into scripts/.AGENTTRASH/).
|
|
#
|
|
# SHAPE. The quadlet replaces the image's `vllm serve` entrypoint with
|
|
# /scripts/serve-frank-small.sh (mounted read-only out of the repo checkout)
|
|
# so the flag set stays in git. The model and drafter directories are bound
|
|
# read-only: the payloads are fetched and sha256-verified by
|
|
# scripts/download-frank-small-vllm.sh before first start, and the deploy
|
|
# handler refuses to start the unit when either is absent.
|
|
#
|
|
# CUDA WITHOUT the nvidia-container-toolkit. This host has never run a CUDA
|
|
# container (the 3090 served frank-small as a native binary, the iGPU runs
|
|
# ROCm through /dev/kfd), so there is no nvidia hook to lean on. The recipe
|
|
# that works (measured 2026-09-18, torch sees the 3090 at 25.3 GB):
|
|
# - the four /dev/nvidia* nodes via AddDevice (they are 0666 on this host,
|
|
# so the rootless user namespace can open them);
|
|
# - the HOST driver's libcuda.so.1 and libnvidia-ml.so.1 mounted read-only
|
|
# — the image carries its own CUDA 12.9 runtime, but vLLM's platform
|
|
# probe needs the driver's NVML stub, and without it `vllm serve` dies at
|
|
# parser construction with "Failed to infer device type";
|
|
# - NVIDIA_VISIBLE_DEVICES=all.
|
|
# memlock must be unlimited: CUDA pins its staging buffers.
|
|
#
|
|
# SIZING. The tunables (full 262144 window, 0.95 of a dedicated 24 GiB card,
|
|
# bf16 KV, 16-seq batch ceiling, 16k prefill chunk, 4-token MTP draft) live
|
|
# in scripts/frank-small.env. The boot log's "Maximum concurrency for N
|
|
# tokens per request" line is the naive pool ratio for a hybrid SWA model
|
|
# and undercounts; verify the session count behaviourally on restart.
|
|
#
|
|
# IMAGE. The unit runs the localhost/-aliased tag: this host's podman quadlet
|
|
# path cannot resolve registry-prefixed refs from local storage at create
|
|
# time. The tag is a LOCAL BUILD, not a pull: deploy-remote.sh pulls the
|
|
# digest-pinned docker.io base (the gemma-x86_64-cu129 build, vLLM
|
|
# 0.22.1rc1.dev357+g74b5964f0 — the image that boots this checkpoint) and
|
|
# re-tags it, then builds dankclaw/frank-small/Dockerfile on top, which
|
|
# installs the vllm[audio] extra (av, scipy, soundfile,
|
|
# mistral_common[audio]) so the model's native audio path works. Pull=never
|
|
# keeps boots off the registry.
|
|
[Unit]
|
|
Description=frank-small — vLLM gemma4_unified w4a16 + MTP on RTX 3090
|
|
After=network-online.target
|
|
|
|
[Container]
|
|
ContainerName=frank-small
|
|
Image=localhost/vllm/vllm-openai:frank-small
|
|
# Published port stays 7771: Hermes, the harness and the homepage probe all
|
|
# point here. 8000 (vLLM's in-container port) is never reachable directly.
|
|
PublishPort=7771:8000
|
|
# CUDA device nodes (0666 on this host — rootless can open them).
|
|
AddDevice=/dev/nvidia0
|
|
AddDevice=/dev/nvidiactl
|
|
AddDevice=/dev/nvidia-uvm
|
|
AddDevice=/dev/nvidia-uvm-tools
|
|
# Host driver stubs the image's CUDA runtime and vLLM's NVML probe need.
|
|
# The image's LD_LIBRARY_PATH leads with /usr/local/nvidia/lib64 — the
|
|
# nvidia-container-toolkit's conventional mount point, which this host does
|
|
# not have — so the driver libs go there; the /usr/lib mounts cover torch
|
|
# Inductor's own lookup.
|
|
Volume=/usr/lib/x86_64-linux-gnu/libcuda.so.1:/usr/lib/x86_64-linux-gnu/libcuda.so.1:ro
|
|
Volume=/usr/lib/x86_64-linux-gnu/libcuda.so:/usr/lib/x86_64-linux-gnu/libcuda.so:ro
|
|
Volume=/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1:ro
|
|
Volume=/usr/lib/x86_64-linux-gnu/libcuda.so.1:/usr/local/nvidia/lib64/libcuda.so.1:ro
|
|
Volume=/usr/lib/x86_64-linux-gnu/libcuda.so:/usr/local/nvidia/lib64/libcuda.so:ro
|
|
Volume=/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1:/usr/local/nvidia/lib64/libnvidia-ml.so.1:ro
|
|
Environment=NVIDIA_VISIBLE_DEVICES=all
|
|
# Payloads, read-only: fetched and sha256-verified by
|
|
# scripts/download-frank-small-vllm.sh before first start.
|
|
Volume=/home/dank/models/gemma4-12b-qat-w4a16-ct:/models/main:ro
|
|
Volume=/home/dank/models/gemma4-12b-assistant-bf16:/models/assistant:ro
|
|
# Serve script and env, read-only out of the repo checkout.
|
|
Volume=/home/dank/serverAppConfigs/scripts:/scripts:ro
|
|
Entry=/scripts/serve-frank-small.sh
|
|
Environment=FRANK_SMALL_ENV_FILE=/scripts/frank-small.env
|
|
Ulimit=memlock=-1:-1
|
|
# /health answers 200 once the engine is up; vLLM 503s it while the model is
|
|
# still loading, so a green check means ready-to-serve, not process-alive.
|
|
# curl ships in the image.
|
|
HealthCmd=curl -sf http://127.0.0.1:8000/health
|
|
HealthInterval=30s
|
|
HealthTimeout=10s
|
|
HealthRetries=3
|
|
HealthStartPeriod=600s
|
|
Pull=never
|
|
|
|
[Service]
|
|
# First start reads ~10 GiB of weights and compiles the CUDA graphs.
|
|
TimeoutStartSec=900
|
|
Restart=on-failure
|
|
RestartSec=10
|
|
|
|
[Install]
|
|
WantedBy=default.target
|