# Quadlet unit: frank-small (rootless, boot-persistent).
#
# Install path on dankclaw: ~/.config/containers/systemd/frank-small.container
# Unit name: frank-small.service (container name: frank-small)
#
# The small inference service. vLLM (since 2026-09-18) serving
# google/gemma-4-12B-it-qat-w4a16-ct (compressed-tensors w4a16) with the
# google bf16 MTP assistant for speculative decoding, on the RTX
# 3090. It replaced the mainline-llama.cpp ROOT unit (the old
# scripts/frank-small.service is retired into scripts/.AGENTTRASH/).
#
# SHAPE. The quadlet replaces the image's `vllm serve` entrypoint with
# /scripts/serve-frank-small.sh (mounted read-only out of the repo checkout)
# so the flag set stays in git. The model and drafter directories are bound
# read-only: the payloads are fetched and sha256-verified by
# scripts/download-frank-small-vllm.sh before first start, and the deploy
# handler refuses to start the unit when either is absent.
#
# CUDA WITHOUT the nvidia-container-toolkit. This host has never run a CUDA
# container (the 3090 served frank-small as a native binary, the iGPU runs
# ROCm through /dev/kfd), so there is no nvidia hook to lean on. The recipe
# that works (measured 2026-09-18, torch sees the 3090 at 25.3 GB):
#   - the four /dev/nvidia* nodes via AddDevice (they are 0666 on this host,
#     so the rootless user namespace can open them);
#   - the HOST driver's libcuda.so.1 and libnvidia-ml.so.1 mounted read-only
#     — the image carries its own CUDA 12.9 runtime, but vLLM's platform
#     probe needs the driver's NVML stub, and without it `vllm serve` dies at
#     parser construction with "Failed to infer device type";
#   - NVIDIA_VISIBLE_DEVICES=all.
# memlock must be unlimited: CUDA pins its staging buffers.
#
# SIZING. The tunables (full 262144 window, 0.95 of a dedicated 24 GiB card,
# bf16 KV, 16-seq batch ceiling, 16k prefill chunk, 4-token MTP draft) live
# in scripts/frank-small.env. The boot log's "Maximum concurrency for N
# tokens per request" line is the naive pool ratio for a hybrid SWA model
# and undercounts; verify the session count behaviourally on restart.
#
# IMAGE. The unit runs the localhost/-aliased tag: this host's podman quadlet
# path cannot resolve registry-prefixed refs from local storage at create
# time. The tag is a LOCAL BUILD, not a pull: deploy-remote.sh pulls the
# digest-pinned docker.io base (the gemma-x86_64-cu129 build, vLLM
# 0.22.1rc1.dev357+g74b5964f0 — the image that boots this checkpoint) and
# re-tags it, then builds dankclaw/frank-small/Dockerfile on top, which
# installs the vllm[audio] extra (av, scipy, soundfile,
# mistral_common[audio]) so the model's native audio path works. Pull=never
# keeps boots off the registry.
[Unit]
Description=frank-small — vLLM gemma4_unified w4a16 + MTP on RTX 3090
After=network-online.target

[Container]
ContainerName=frank-small
Image=localhost/vllm/vllm-openai:frank-small
# Published port stays 7771: Hermes, the harness and the homepage probe all
# point here. 8000 (vLLM's in-container port) is never reachable directly.
PublishPort=7771:8000
# CUDA device nodes (0666 on this host — rootless can open them).
AddDevice=/dev/nvidia0
AddDevice=/dev/nvidiactl
AddDevice=/dev/nvidia-uvm
AddDevice=/dev/nvidia-uvm-tools
# Host driver stubs the image's CUDA runtime and vLLM's NVML probe need.
# The image's LD_LIBRARY_PATH leads with /usr/local/nvidia/lib64 — the
# nvidia-container-toolkit's conventional mount point, which this host does
# not have — so the driver libs go there; the /usr/lib mounts cover torch
# Inductor's own lookup.
Volume=/usr/lib/x86_64-linux-gnu/libcuda.so.1:/usr/lib/x86_64-linux-gnu/libcuda.so.1:ro
Volume=/usr/lib/x86_64-linux-gnu/libcuda.so:/usr/lib/x86_64-linux-gnu/libcuda.so:ro
Volume=/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1:ro
Volume=/usr/lib/x86_64-linux-gnu/libcuda.so.1:/usr/local/nvidia/lib64/libcuda.so.1:ro
Volume=/usr/lib/x86_64-linux-gnu/libcuda.so:/usr/local/nvidia/lib64/libcuda.so:ro
Volume=/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1:/usr/local/nvidia/lib64/libnvidia-ml.so.1:ro
Environment=NVIDIA_VISIBLE_DEVICES=all
# Payloads, read-only: fetched and sha256-verified by
# scripts/download-frank-small-vllm.sh before first start.
Volume=/home/dank/models/gemma4-12b-qat-w4a16-ct:/models/main:ro
Volume=/home/dank/models/gemma4-12b-assistant-bf16:/models/assistant:ro
# Serve script and env, read-only out of the repo checkout.
Volume=/home/dank/serverAppConfigs/scripts:/scripts:ro
ContainerEntrypoint=/scripts/serve-frank-small.sh
Environment=FRANK_SMALL_ENV_FILE=/scripts/frank-small.env
Ulimit=memlock=-1:-1
# /health answers 200 once the engine is up; vLLM 503s it while the model is
# still loading, so a green check means ready-to-serve, not process-alive.
# curl ships in the image.
HealthCmd=curl -sf http://127.0.0.1:8000/health
HealthInterval=30s
HealthTimeout=10s
HealthRetries=3
HealthStartPeriod=600s
Pull=never

[Service]
# First start reads ~10 GiB of weights and compiles the CUDA graphs.
TimeoutStartSec=900
Restart=on-failure
RestartSec=10

[Install]
WantedBy=default.target