Frank
acd14ab7df
Replaces frank-medium (vllm-radiance TP=2) on :7770. The 377.2 GiB 43-layer MoE (256 routed experts, top-6) is mmapped: dense work and the MLA KV cache on the R9700, routed experts computed on CPU from the page cache (246 GiB RAM). The unit bypasses the image entrypoint (Qwen/DFlash-only flag set) and execs luce_server with the DS4 launch shape; --ds4-fused-decode's monolithic-load fallback IS the serving mode here. deploy_llm_primary gates on image + model byte count, stops/masks the radiance unit, waits for :7770, installs and starts. Runbook: frank-primary-lucebox-ds4.md.
86 lines
4.1 KiB
INI
86 lines
4.1 KiB
INI
# Quadlet unit: frank-primary (rootless, boot-persistent).
|
|
#
|
|
# Install path on dankserver: ~/.config/containers/systemd/frank-primary.container
|
|
# Unit name: frank-primary.service (container name: frank-primary)
|
|
#
|
|
# The dankserver LLM endpoint (:7770, alias frank-primary): the Lucebox
|
|
# runtime (github.com/Luce-Org/lucebox, local image localhost/lucebox-rocm:
|
|
# gfx1201 built from Dockerfile.rocm with LUCE_HIP_ARCHES=gfx1201 and
|
|
# ROCM_VERSION=7.14.1-full to match the host driver's ROCm 7.x line)
|
|
# serving DeepSeek-V4.1-Flash-ROCMFP23 (377.2 GiB GGUF, 43-layer MoE,
|
|
# 256 routed experts + 1 shared per layer, top-6 routing) on a single
|
|
# Radeon AI PRO R9700 (gfx1201). The model does not fit VRAM: dense work
|
|
# and the MLA KV cache sit on the R9700, the routed experts are computed
|
|
# on CPU from the mmapped page cache (246 GiB RAM holds the working set;
|
|
# the rest streams from NVMe). Lazy mmap page-in is the engine default —
|
|
# no flag.
|
|
#
|
|
# The image's entrypoint only forwards a fixed Qwen/DFlash flag set and
|
|
# cannot express the DS4 flags, so the unit bypasses it (--entrypoint) and
|
|
# runs the repo's frank-primary-entrypoint.sh, which execs the native
|
|
# luce_server with the DS4 launch shape (see that script's header).
|
|
#
|
|
# SHAPE. Host network, :7770, served model name frank-primary, OpenAI /v1
|
|
# (Chat Completions with tool metadata, Responses, Anthropic Messages,
|
|
# Open WebUI metadata) — the fleet contract is unchanged from
|
|
# frank-medium.
|
|
#
|
|
# DEVICES. /dev/kfd + /dev/dri through the amdgpu/KFD stack. HIP_VISIBLE_
|
|
# DEVICES=0 pins the first R9700 (the Raphael iGPU enumerates as index 2
|
|
# and stays out). The second R9700 is not used: the in-process
|
|
# heterogeneous expert-parallel path (LUCE_DS4_MOE_TP_INPROC) is qualified
|
|
# for gfx1151+gfx1201 pairs, not gfx1201+gfx1201.
|
|
#
|
|
# MOUNTS. /models (ro — the GGUF is mmapped, nothing writes), and the
|
|
# entrypoint + healthcheck scripts straight out of this checkout
|
|
# (deploy.sh git-pulls the checkout before dispatching).
|
|
#
|
|
# IMAGE. localhost/-aliased tag: this host's podman quadlet path cannot
|
|
# resolve registry-prefixed refs from local storage at create time (same
|
|
# as frank-medium). Pull=never keeps boots off the registry.
|
|
#
|
|
# CUTOVER. This unit REPLACES frank-medium on :7770. The deploy handler
|
|
# (deploy_llm_primary in scripts/deploy-remote.sh) stops and masks the
|
|
# radiance unit before starting this one; the radiance unit is retired to
|
|
# the home trash after frank-primary is verified (trash policy, never rm).
|
|
#
|
|
# DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the
|
|
# HIP runtime reads bare TEMP for its JIT cache path (2026-08-13 incident).
|
|
[Unit]
|
|
Description=frank-primary — Lucebox DeepSeek-V4.1-Flash-ROCMFP23 on R9700 (gfx1201), experts in RAM
|
|
After=network-online.target
|
|
|
|
[Container]
|
|
ContainerName=frank-primary
|
|
Image=localhost/lucebox-rocm:gfx1201
|
|
# Host network: luce_server binds 0.0.0.0:7770 directly; no PublishPort.
|
|
AddDevice=/dev/kfd
|
|
AddDevice=/dev/dri
|
|
GroupAdd=keep-groups
|
|
PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add SYS_PTRACE --network=host --entrypoint /usr/local/bin/frank-primary-entrypoint.sh
|
|
# First R9700 only (index pin excludes the Raphael iGPU at index 2).
|
|
Environment=HIP_VISIBLE_DEVICES=0
|
|
Environment=ROCR_VISIBLE_DEVICES=0
|
|
# The 377 GiB GGUF, mmapped; pages pull from disk/page cache on demand.
|
|
Volume=/home/dank/models:/models:ro
|
|
# Entrypoint + healthcheck from this checkout, mounted read-only.
|
|
Volume=/home/dank/serverAppConfigs/dankserver/frank-primary/frank-primary-entrypoint.sh:/usr/local/bin/frank-primary-entrypoint.sh:ro
|
|
Volume=/home/dank/serverAppConfigs/dankserver/frank-primary/frank-primary-healthcheck.py:/usr/local/bin/frank-primary-healthcheck.py:ro
|
|
# 200 only once the engine serves; first start page-ins the dense weights
|
|
# and builds the graphs, which takes a while. HealthCmd is split on spaces
|
|
# into an argv array by the podman 5.x v1 generator, hence the file probe.
|
|
HealthCmd=python3 /usr/local/bin/frank-primary-healthcheck.py
|
|
HealthInterval=60s
|
|
HealthTimeout=30s
|
|
HealthRetries=3
|
|
HealthStartPeriod=1800s
|
|
Pull=never
|
|
|
|
[Service]
|
|
TimeoutStartSec=1800
|
|
Restart=on-failure
|
|
RestartSec=10
|
|
|
|
[Install]
|
|
WantedBy=default.target
|