# Quadlet unit: frank-medium (rootless, boot-persistent).
#
# Install path on dankserver: ~/.config/containers/systemd/frank-medium.container
# Unit name: frank-medium.service (container name: frank-medium)
#
# The dankserver LLM endpoint (:7770, alias frank-medium): the vllm-radiance
# stack (the 0.13.0-e060895 image, PR #51) serving
# hugypufy/Swift-Qwen3.8-27B-PARO-MXFP6 (ParoQuant MXFP6, W4A8, 25.16 GiB)
# on the 2x Radeon AI PRO R9700 (gfx1201) as ONE tensor-parallel process
# (TP=2, R4D one-shot all-reduce) with the z-lab DFlash2 external drafter
# (SPEC=7) and UNQUANTIZED bf16 KV (KV_DTYPE=auto).
# The entrypoint is QUANTIZATION-AWARE: it reads the model's weight
# quantization from config.json and serves the plain FP8 path (no MXFP4
# patches, --quantization=fp8, ROCM_AITER unified attention), the MXFP4
# patch-and-serve path, or the ParoQuant MXFP6 stack (PR #51's launch6_p
# kernel). A model swap is now: change SNAP (+ SPEC_METHOD/SPEC), restart.
# The MXFP4 checkpoint (just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq, the
# 2026-09-20 to 2026-09-21 serving shape) is the on-disk rollback target.
#
# SHAPE. The launcher the upstream repo ships binds the server with
# --network=host (vLLM takes --host 0.0.0.0 --port 7770 directly), which is
# what this unit does too — no PublishPort. The fleet contract is unchanged:
# :7770, served model name frank-medium (SERVED_NAMES), OpenAI /v1.
#
# TP=2 IS INSTALLABLE HERE NOW, and it won on measurement. Until 2026-09-20
# this unit was archived un-installed because the second card sat behind B650
# chipset bridges advertising `AtomicOpsCap: Routing-`, which stopped RCCL
# kernel dispatch. The board swap (ProArt B850-Creator WiFi NEO) put both
# cards on CPU GPP root ports — Gen5 x8 each, every bridge Routing+, both
# endpoints ReqEn+ — and the collective-free R4D path runs at TP=2 without
# complaint. The conversation battery (scripts/bench-conversation.py, salted,
# same 10 prompts, 2026-09-20) measured this shape against the incumbent:
# decode geomean 95.8 t/s vs 45.7, mean TTFT 0.21 s vs 0.72 s, and 259 t/s
# aggregate against 4 concurrent streams vs 71.7. Evidence and the llama-side
# caveats: serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md and
# Dual-R9700-Cooperation.md.
#
# DEVICES. /dev/kfd + /dev/dri through the amdgpu/KFD stack; HIP/ROCR device
# indices 0,1 (the two R9700s). The 2 GiB Raphael iGPU enumerates as index 2
# and is excluded both by the index pin and by the upstream MIN_GPU_MIB=8192
# logic this unit replaces. --privileged --ipc=host --security-opt
# seccomp=unconfined --cap-add SYS_PTRACE is the upstream launcher's exact
# podman posture.
#
# MOUNTS. /models (rw checkpoints), the HF cache (tokenizer files),
# /cache (Triton/inductor/aiter/vLLM compile cache — first start compiles for
# several minutes, later starts reuse), /patches (the pinned radiance clone —
# its patches are applied at EVERY container start by the entrypoint), /r4d
# (the patched libr4d build; the image's shipped r4d.so NaNs the
# gated-delta-net and must never be used), and the entrypoint + healthcheck
# scripts and the chat template from this checkout (mounted read-only
# straight out of /home/dank/serverAppConfigs — deploy.sh git-pulls the
# checkout before dispatching).
#
# IMAGE. The unit runs the localhost/-aliased tag: this host's podman quadlet
# path cannot resolve registry-prefixed refs from local storage at create
# time (same as frank-large on dankclaw). deploy-remote.sh pulls the
# docker.io ref and re-tags it before the restart. Pull=never keeps boots off
# the registry.
#
# KEEP IN SYNC: the radiance clone is pinned (e060895...); if it is updated,
# re-diff dankserver/frank-medium/radiance-entrypoint.sh against the new
# serve-mxfp4.sh container command and bump the pin deliberately.
#
# DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the
# HIP runtime reads bare TEMP for its JIT cache path (2026-08-13 incident).
[Unit]
Description=frank-medium — vllm-radiance ParoQuant MXFP6 Swift Qwen3.8-27B on 2x R9700 (gfx1201)
After=network-online.target

[Container]
ContainerName=frank-medium
Image=localhost/radiance-vllm:0.13.0-e060895
# vLLM binds 0.0.0.0:7770 on the host network directly; no PublishPort.
AddDevice=/dev/kfd
AddDevice=/dev/dri
GroupAdd=keep-groups
# The upstream launcher's exact podman posture (rootless; podman-only flags
# like --replace are not needed under systemd management).
PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add SYS_PTRACE --network=host
# The two R9700s. The Raphael iGPU (index 2, 2 GiB) stays out: shared system
# memory is not a compute target.
Environment=ROCR_VISIBLE_DEVICES=0,1
Environment=HIP_VISIBLE_DEVICES=0,1
# Drafting: the external z-lab DFlash2-FP8 drafter (SPEC_METHOD=dflash). The
# plain-FP8 checkpoint (rollback) carries its own single-layer MTP head, but
# DFlash2 beat MTP self-speculation on decode, so the drafter serves either.
# Chat template: the Qwen-Sharp variant (peculiar-ragdoll v22.5.0) — the terse
# system prompt (opt out per-request with chat_template_kwargs {"terse":
# false}), message.reasoning support (vLLM's thinking field), and the
# tightened tool-call rendering. The predecessor (qwen-sharp-v22.4.0.jinja,
# mounted beside it) remains as the rollback target: flip CHAT_TEMPLATE.
Environment=SNAP=/models/Qwen3.8-27B-PARO-MXFP6
Environment=DRAFTER=/models/Qwen3.8-27B-DFlash2-FP8
Environment=CHAT_TEMPLATE=/templates/qwen-sharp-v22.5.0.jinja
# Fleet identity: the id at /v1/models and in every response body. Every
# client (Hermes, the harness, services.json) points at frank-medium.
Environment=SERVED_NAMES=frank-medium
Environment=PORT=7770
# Fixed 2-card tensor parallel: the tested configuration of the upstream
# repo; the R4D one-shot all-reduce kernel serves it natively.
Environment=TP=2
# Measured production batch shape (upstream README): MAXSEQS 8 peaks
# aggregate throughput at 573 t/s. MAXLEN is the served window (see KV_DTYPE).
Environment=MAXSEQS=8
Environment=CHUNK=8192
Environment=MAXLEN=262144
# MAXLEN is the whole pool shared across the MAXSEQS slots: 8 concurrent
# sessions split 262,144 tokens between them, so a full-window request
# excludes the others until it finishes (vLLM preempts rather than refuses).
# The llama deployment it replaced advertised 327,680 across 2 slots.
#
# VISION WORKS (verified 2026-09-21: a solid red 48x48 PNG is named "Red").
# config.json sets language_model_only=true and there is no separate mmproj,
# but the Qwen3_5ForConditionalGeneration architecture carries the vision tower
# through the Gemma4 merge path and it is not MXFP4-quantized, so vLLM serves
# image input on either checkpoint. harness models.yml keeps frank-medium
# input: [text, image].
# 0.97, down from 0.98: the startup free-memory check (free >= util x 31.86
# GiB) sat ~15 MiB inside the threshold on the 2026-09-26 boot (free 31.21
# GiB vs required 31.22 GiB) — the worker's own HIP context plus the
# amdgpu per-boot baseline is ~690 MiB at check time and that drifts a few
# tens of MiB between boots, so 0.98 crash-looped on every restart. 0.97
# buys ~160 MiB of headroom at the cost of ~5k KV tokens on the
# auto-sized pool.
Environment=GPU_UTIL=0.97
# Speculative decoding: DFlash2 external drafter at depth 7. A/B tested
# against MTP self-speculation (depth 8) on this FP8 checkpoint: DFlash2
# wins on decode (100.0 vs 85.9 t/s geomean, 287.1 vs 179.7 t/s burst-4)
# despite a lower accept length (4.24 vs 5.66) — the small separate drafter
# steps faster than the target's own MTP layer applied autoregressively.
Environment=SPEC_METHOD=dflash
Environment=SPEC=7
# KV cache: 0 = profile the shape (the documented plain-FP8 path's default;
# the upstream docker-compose leaves the pin commented out). The measured pin
# for this shape is 15989735424 (14.89 GiB, upstream, 2x R9700 32GB); after
# the first healthy start, recalibrate with calibrate-kv.sh to use the extra
# VRAM on the 34.2 GiB cards.
Environment=KV_MEM=0
# KV cache dtype: auto = UNQUANTIZED bf16 KV. The experiment: fp8 KV stacked on
# MXFP4's already-quantized W4A8 weights compounds attention precision loss, so
# clean KV should recover output quality. MXFP4's freed weights (19.37 vs 30.9
# GB) buy the pool back — the full 262,144 native window still fits unquantized
# (~290k-token f16 pool > 262,144). Flip to fp8 for the measured 943k-token pool.
Environment=KV_DTYPE=auto
# Checkpoints (rw: nothing downloads at runtime, but vLLM mmaps the shards;
# keep parity with the launcher's rw mount).
Volume=/home/dank/models:/models
# Tokenizer files ride the HF cache; HF_HUB_OFFLINE=1 is set in the
# entrypoint.
Volume=/home/dank/.cache/huggingface:/root/.cache/huggingface
# Compile cache. Keyed on model + stack version + every graph-changing knob;
# never share it across configurations (upstream rule).
Volume=/home/dank/.radiance-cache-mxfp6-013-pq-nqft-fp8s-gnq:/cache
# The pinned radiance clone: patches applied at every container start.
Volume=/home/dank/GitPackages/radiance-vllm-mxfp4:/patches:z
# The patched libr4d (pin b9e42ab + repo extras, key b9e42ab-rx6). The
# image's shipped r4d.so predates the GDN overflow fix and NaNs this model.
Volume=/home/dank/.cache/radiance-libr4d/b9e42ab-rx6:/r4d:z
# Entrypoint + healthcheck from this checkout, mounted read-only.
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/radiance-entrypoint.sh:/usr/local/bin/frank-medium-entrypoint.sh:ro
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/frank-medium-healthcheck.py:/usr/local/bin/frank-medium-healthcheck.py:ro
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/qwen-sharp-v22.4.0.jinja:/templates/qwen-sharp-v22.4.0.jinja:ro
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/qwen-sharp-v22.5.0.jinja:/templates/qwen-sharp-v22.5.0.jinja:ro
# The entrypoint must be the container command; the image's own entrypoint
# (which the patched script execs at the end) must not run unpatched.
PodmanArgs=--entrypoint /usr/local/bin/frank-medium-entrypoint.sh
# 200 only once the engine serves; during first-start compilation the
# container is legitimately unhealthy. HealthCmd is split on spaces into an
# argv array by the podman 5.x v1 generator, hence the file-based probe.
HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py
HealthInterval=60s
HealthTimeout=30s
HealthRetries=3
HealthStartPeriod=1800s
Pull=never

[Service]
# First start compiles Triton/inductor kernels (several minutes) before the
# engine loads ~9.5 GiB of weights per card.
TimeoutStartSec=1800
Restart=on-failure
RestartSec=10

[Install]
WantedBy=default.target