Frank
ae137317ab
podman 5.7.0's v1 quadlet generator rejects both ContainerEntrypoint and Entry; the key is Entrypoint= (pkg/systemd/quadlet/quadlet.go KeyEntrypoint, mapped to --entrypoint).
101 lines
5.0 KiB
INI
101 lines
5.0 KiB
INI
# Quadlet unit: frank-small (rootless, boot-persistent).
|
|
#
|
|
# Install path on dankclaw: ~/.config/containers/systemd/frank-small.container
|
|
# Unit name: frank-small.service (container name: frank-small)
|
|
#
|
|
# The small inference service. vLLM (since 2026-09-18) serving
|
|
# google/gemma-4-12B-it-qat-w4a16-ct (compressed-tensors w4a16) with the
|
|
# google bf16 MTP assistant for speculative decoding, on the RTX
|
|
# 3090. It replaced the mainline-llama.cpp ROOT unit (the old
|
|
# scripts/frank-small.service is retired into scripts/.AGENTTRASH/).
|
|
#
|
|
# SHAPE. The quadlet replaces the image's `vllm serve` entrypoint with
|
|
# /scripts/serve-frank-small.sh (mounted read-only out of the repo checkout)
|
|
# so the flag set stays in git. The model and drafter directories are bound
|
|
# read-only: the payloads are fetched and sha256-verified by
|
|
# scripts/download-frank-small-vllm.sh before first start, and the deploy
|
|
# handler refuses to start the unit when either is absent.
|
|
#
|
|
# CUDA WITHOUT the nvidia-container-toolkit. This host has never run a CUDA
|
|
# container (the 3090 served frank-small as a native binary, the iGPU runs
|
|
# ROCm through /dev/kfd), so there is no nvidia hook to lean on. The recipe
|
|
# that works (measured 2026-09-18, torch sees the 3090 at 25.3 GB):
|
|
# - the four /dev/nvidia* nodes via AddDevice (they are 0666 on this host,
|
|
# so the rootless user namespace can open them);
|
|
# - the HOST driver's libcuda.so.1 and libnvidia-ml.so.1 mounted read-only
|
|
# — the image carries its own CUDA 12.9 runtime, but vLLM's platform
|
|
# probe needs the driver's NVML stub, and without it `vllm serve` dies at
|
|
# parser construction with "Failed to infer device type";
|
|
# - NVIDIA_VISIBLE_DEVICES=all.
|
|
# memlock must be unlimited: CUDA pins its staging buffers.
|
|
#
|
|
# SIZING. The tunables (full 262144 window, 0.95 of a dedicated 24 GiB card,
|
|
# bf16 KV, 16-seq batch ceiling, 16k prefill chunk, 4-token MTP draft) live
|
|
# in scripts/frank-small.env. The boot log's "Maximum concurrency for N
|
|
# tokens per request" line is the naive pool ratio for a hybrid SWA model
|
|
# and undercounts; verify the session count behaviourally on restart.
|
|
#
|
|
# IMAGE. The unit runs the localhost/-aliased tag: this host's podman quadlet
|
|
# path cannot resolve registry-prefixed refs from local storage at create
|
|
# time. The tag is a LOCAL BUILD, not a pull: deploy-remote.sh pulls the
|
|
# digest-pinned docker.io base (the gemma-x86_64-cu129 build, vLLM
|
|
# 0.22.1rc1.dev357+g74b5964f0 — the image that boots this checkpoint) and
|
|
# re-tags it, then builds dankclaw/frank-small/Dockerfile on top, which
|
|
# installs the vllm[audio] extra (av, scipy, soundfile,
|
|
# mistral_common[audio]) so the model's native audio path works. Pull=never
|
|
# keeps boots off the registry.
|
|
[Unit]
|
|
Description=frank-small — vLLM gemma4_unified w4a16 + MTP on RTX 3090
|
|
After=network-online.target
|
|
|
|
[Container]
|
|
ContainerName=frank-small
|
|
Image=localhost/vllm/vllm-openai:frank-small
|
|
# Published port stays 7771: Hermes, the harness and the homepage probe all
|
|
# point here. 8000 (vLLM's in-container port) is never reachable directly.
|
|
PublishPort=7771:8000
|
|
# CUDA device nodes (0666 on this host — rootless can open them).
|
|
AddDevice=/dev/nvidia0
|
|
AddDevice=/dev/nvidiactl
|
|
AddDevice=/dev/nvidia-uvm
|
|
AddDevice=/dev/nvidia-uvm-tools
|
|
# Host driver stubs the image's CUDA runtime and vLLM's NVML probe need.
|
|
# The image's LD_LIBRARY_PATH leads with /usr/local/nvidia/lib64 — the
|
|
# nvidia-container-toolkit's conventional mount point, which this host does
|
|
# not have — so the driver libs go there; the /usr/lib mounts cover torch
|
|
# Inductor's own lookup.
|
|
Volume=/usr/lib/x86_64-linux-gnu/libcuda.so.1:/usr/lib/x86_64-linux-gnu/libcuda.so.1:ro
|
|
Volume=/usr/lib/x86_64-linux-gnu/libcuda.so:/usr/lib/x86_64-linux-gnu/libcuda.so:ro
|
|
Volume=/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1:ro
|
|
Volume=/usr/lib/x86_64-linux-gnu/libcuda.so.1:/usr/local/nvidia/lib64/libcuda.so.1:ro
|
|
Volume=/usr/lib/x86_64-linux-gnu/libcuda.so:/usr/local/nvidia/lib64/libcuda.so:ro
|
|
Volume=/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1:/usr/local/nvidia/lib64/libnvidia-ml.so.1:ro
|
|
Environment=NVIDIA_VISIBLE_DEVICES=all
|
|
# Payloads, read-only: fetched and sha256-verified by
|
|
# scripts/download-frank-small-vllm.sh before first start.
|
|
Volume=/home/dank/models/gemma4-12b-qat-w4a16-ct:/models/main:ro
|
|
Volume=/home/dank/models/gemma4-12b-assistant-bf16:/models/assistant:ro
|
|
# Serve script and env, read-only out of the repo checkout.
|
|
Volume=/home/dank/serverAppConfigs/scripts:/scripts:ro
|
|
Entrypoint=/scripts/serve-frank-small.sh
|
|
Environment=FRANK_SMALL_ENV_FILE=/scripts/frank-small.env
|
|
Ulimit=memlock=-1:-1
|
|
# /health answers 200 once the engine is up; vLLM 503s it while the model is
|
|
# still loading, so a green check means ready-to-serve, not process-alive.
|
|
# curl ships in the image.
|
|
HealthCmd=curl -sf http://127.0.0.1:8000/health
|
|
HealthInterval=30s
|
|
HealthTimeout=10s
|
|
HealthRetries=3
|
|
HealthStartPeriod=600s
|
|
Pull=never
|
|
|
|
[Service]
|
|
# First start reads ~10 GiB of weights and compiles the CUDA graphs.
|
|
TimeoutStartSec=900
|
|
Restart=on-failure
|
|
RestartSec=10
|
|
|
|
[Install]
|
|
WantedBy=default.target
|