From 6b647c1558e5885291aaf3e2bdc5f1d2d6c7c1d4 Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 15 Sep 2026 20:59:55 -0700 Subject: [PATCH] frank-medium: swap ROCmFPX llama.cpp runtime for vllm-radiance MXFP4 Rootless Quadlet on dankserver serving just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq (the radiance stack's designated alternate target; ships its fp8 MTP head, no rewrite step) with the DFlash2-FP8 drafter at SPEC=7 on the 2x R9700s (TP=2, host network, :7770, alias frank-medium unchanged). - dankserver/frank-medium/: quadlet unit, container entrypoint (fork of serve-mxfp4.sh's patch block at pinned clone commit 97353293), healthcheck - dispatcher retires the root llama.cpp unit, re-tags the registry image to the localhost alias, installs the quadlet, waits on /health - planner maps dankserver/frank-medium/* to frank-medium; scripts-era mappings removed - ROCmFPX-era env/serve/unit/download scripts staged in scripts/.AGENTTRASH with .rocmfpx suffix; seams updated (frank_medium_quadlet.bats new) --- .../frank-medium/frank-medium-healthcheck.py | 17 ++ .../quadlet/frank-medium.container | 138 +++++++++++ .../frank-medium/radiance-entrypoint.sh | 225 ++++++++++++++++++ .../download-q8agent.sh.rocmfpx} | 0 .../frank-medium.env.rocmfpx} | 0 .../frank-medium.service.rocmfpx} | 0 .../serve-frank-medium.sh.rocmfpx} | 0 scripts/deploy-plan.py | 17 +- scripts/deploy-remote.sh | 50 ++-- scripts/test-changed.sh | 11 +- tests/changed_tests.bats | 10 +- tests/deployment_scoping.bats | 18 +- tests/frank_medium_quadlet.bats | 127 ++++++++++ 13 files changed, 566 insertions(+), 47 deletions(-) create mode 100755 dankserver/frank-medium/frank-medium-healthcheck.py create mode 100644 dankserver/frank-medium/quadlet/frank-medium.container create mode 100755 dankserver/frank-medium/radiance-entrypoint.sh rename scripts/{download-q8agent.sh => .AGENTTRASH/download-q8agent.sh.rocmfpx} (100%) rename scripts/{frank-medium.env => .AGENTTRASH/frank-medium.env.rocmfpx} (100%) rename scripts/{frank-medium.service => .AGENTTRASH/frank-medium.service.rocmfpx} (100%) rename scripts/{serve-frank-medium.sh => .AGENTTRASH/serve-frank-medium.sh.rocmfpx} (100%) create mode 100755 tests/frank_medium_quadlet.bats diff --git a/dankserver/frank-medium/frank-medium-healthcheck.py b/dankserver/frank-medium/frank-medium-healthcheck.py new file mode 100755 index 0000000..fb0455a --- /dev/null +++ b/dankserver/frank-medium/frank-medium-healthcheck.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +"""frank-medium container healthcheck: vLLM /health must answer 200. + +Runs INSIDE the container. The vLLM /health endpoint only answers once the +engine is up (weights loaded, cudagraphs captured), so a healthy container is +a serving container, not merely a live process. The HealthCmd in the quadlet +splits on spaces into an argv array (podman 5.x v1 generator), which is why +this lives in a file rather than an inline python one-liner. +""" +import sys +import urllib.request + +try: + with urllib.request.urlopen("http://127.0.0.1:7770/health", timeout=5) as r: + sys.exit(0 if r.status == 200 else 1) +except Exception: + sys.exit(1) diff --git a/dankserver/frank-medium/quadlet/frank-medium.container b/dankserver/frank-medium/quadlet/frank-medium.container new file mode 100644 index 0000000..0e9e6a4 --- /dev/null +++ b/dankserver/frank-medium/quadlet/frank-medium.container @@ -0,0 +1,138 @@ +# Quadlet unit: frank-medium (rootless, boot-persistent). +# +# Install path on dankserver: ~/.config/containers/systemd/frank-medium.container +# Unit name: frank-medium.service (container name: frank-medium) +# +# The dankserver LLM endpoint (:7770, alias frank-medium), since 2026-09-15 +# running the vllm-radiance MXFP4 stack (stilldeadcode/vllm-radiance:0.9.3 +# image + the ggz14/radiance-vllm-mxfp4 patches) serving +# just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq on the 2x Radeon AI PRO +# R9700 (gfx1201) cards. It replaced the charlie12345/ROCmFPX llama.cpp +# deployment, which served the same alias from the Q8_0_ROCMFPX_AGENT GGUF; +# the MXFP4 stack is tensor-parallel (TP=2, R4D one-shot all-reduce) with the +# DFlash2-FP8 block-diffusion drafter (SPEC=7) — the measured production +# configuration of the upstream repo, on our exact hardware signature +# (2x7551-32624, see the KV pin below). +# +# SHAPE. The launcher the upstream repo ships binds the server with +# --network=host (vLLM takes --host 0.0.0.0 --port 7770 directly), which is +# what this unit does too — no PublishPort. The fleet contract is unchanged: +# :7770, served model name frank-medium (SERVED_NAMES), OpenAI /v1. +# +# DEVICES. /dev/kfd + /dev/dri through the amdgpu/KFD stack; HIP/ROCR device +# indices 0,1 (the two R9700s). The 2 GiB Raphael iGPU enumerates as index 2 +# and is excluded both by the index pin and by the upstream MIN_GPU_MIB=8192 +# logic this unit replaces. --privileged --ipc=host --security-opt +# seccomp=unconfined --cap-add SYS_PTRACE is the upstream launcher's exact +# podman posture. +# +# MOUNTS. /models (rw checkpoints), the HF cache (tokenizer files), +# /cache (Triton/inductor/aiter/vLLM compile cache — first start compiles for +# several minutes, later starts reuse), /patches (the pinned radiance clone — +# its patches are applied at EVERY container start by the entrypoint), /r4d +# (the patched libr4d build; the image's shipped r4d.so NaNs the +# gated-delta-net and must never be used), and the entrypoint + healthcheck +# scripts from this checkout (mounted read-only straight out of +# /home/dank/serverAppConfigs — deploy.sh git-pulls the checkout before +# dispatching). +# +# IMAGE. The unit runs the localhost/-aliased tag: this host's podman quadlet +# path cannot resolve registry-prefixed refs from local storage at create +# time (same as frank-large on dankclaw). deploy-remote.sh pulls the +# docker.io ref and re-tags it before the restart. Pull=never keeps boots off +# the registry. +# +# KEEP IN SYNC: the radiance clone is pinned (97353293...); if it is updated, +# re-diff dankserver/frank-medium/radiance-entrypoint.sh against the new +# serve-mxfp4.sh container command and bump the pin deliberately. +# +# DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the +# HIP runtime reads bare TEMP for its JIT cache path (2026-08-13 incident). +[Unit] +Description=frank-medium — vllm-radiance MXFP4 Qwen3.8-27B-Uncensored on 2x R9700 (gfx1201) +After=network-online.target + +[Container] +ContainerName=frank-medium +Image=localhost/radiance-vllm:0.9.3 +# vLLM binds 0.0.0.0:7770 on the host network directly; no PublishPort. +AddDevice=/dev/kfd +AddDevice=/dev/dri +GroupAdd=keep-groups +# The upstream launcher's exact podman posture (rootless; podman-only flags +# like --replace are not needed under systemd management). +PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add SYS_PTRACE --network=host +# The two R9700s. The Raphael iGPU (index 2, 2 GiB) stays out: shared system +# memory is not a compute target. +Environment=ROCR_VISIBLE_DEVICES=0,1 +Environment=HIP_VISIBLE_DEVICES=0,1 +# The uncensored MXFP4 checkpoint ships its fp8 MTP head already, so no +# fp8_mtp.py rewrite step exists or is needed (unlike amd/Qwen3.8-27B-Quark-AWQ-MXFP4). +Environment=SNAP=/models/Qwen3.8-27B-Uncensored-MXFP4-awq +Environment=DRAFTER=/models/Qwen3.8-27B-DFlash2-FP8 +Environment=CHAT_TEMPLATE=/patches/qwen-fixed-v22.3.jinja +# Fleet identity: the id at /v1/models and in every response body. Every +# client (Hermes, the harness, services.json) points at frank-medium. +Environment=SERVED_NAMES=frank-medium +Environment=PORT=7770 +# Fixed 2-card tensor parallel: the tested configuration of the upstream +# repo; the R4D one-shot all-reduce kernel serves it natively. +Environment=TP=2 +# Measured production batch shape (upstream README): MAXSEQS 8 peaks +# aggregate throughput at 573 t/s; MAXLEN is the model's native context. +Environment=MAXSEQS=8 +Environment=CHUNK=8192 +Environment=MAXLEN=262144 +Environment=GPU_UTIL=0.98 +# Speculative decoding: the DFlash2 drafter at depth 7 is the configuration +# every published number (224 t/s single-stream decode, 4749-4810 t/s +# prefill) was measured with. +Environment=SPEC_METHOD=dflash +Environment=SPEC=7 +# KV cache pin, bytes. From the upstream repo's shipped kv-profiles.tsv row +# sig=2x7551-32624 maxseqs=8 chunk=8192 maxlen=262144 spec=dflash +# which is EXACTLY this host (2x R9700 DID 0x7551, 32624 MiB each). Pinned +# explicitly because this unit does not run the launcher's lookup; valid +# only at this batch shape (change MAXSEQS/CHUNK/SPEC_METHOD => recalibrate +# with calibrate-kv.sh or set KV_MEM=0 to let vLLM profile). Worth +# 943,581 KV tokens vs ~892,799 profiled (~5.7%). +Environment=KV_MEM=18563072000 +# Checkpoints (rw: nothing downloads at runtime, but vLLM mmaps the shards; +# keep parity with the launcher's rw mount). +Volume=/home/dank/models:/models +# Tokenizer files ride the HF cache; HF_HUB_OFFLINE=1 is set in the +# entrypoint. +Volume=/home/dank/.cache/huggingface:/root/.cache/huggingface +# Compile cache. Keyed on model + stack version + every graph-changing knob; +# never share it across configurations (upstream rule). +Volume=/home/dank/.radiance-cache-w4a8-093-gdnm-nqft-fp8s-gnq:/cache +# The pinned radiance clone: patches applied at every container start. +Volume=/home/dank/GitPackages/radiance-vllm-mxfp4:/patches:z +# The patched libr4d (pin b9e42ab + repo extras, key b9e42ab-rx6). The +# image's shipped r4d.so predates the GDN overflow fix and NaNs this model. +Volume=/home/dank/.cache/radiance-libr4d/b9e42ab-rx6:/r4d:z +# Entrypoint + healthcheck from this checkout, mounted read-only. +Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/radiance-entrypoint.sh:/usr/local/bin/frank-medium-entrypoint.sh:ro +Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/frank-medium-healthcheck.py:/usr/local/bin/frank-medium-healthcheck.py:ro +# The entrypoint must be the container command; the image's own entrypoint +# (which the patched script execs at the end) must not run unpatched. +PodmanArgs=--entrypoint /usr/local/bin/frank-medium-entrypoint.sh +# 200 only once the engine serves; during first-start compilation the +# container is legitimately unhealthy. HealthCmd is split on spaces into an +# argv array by the podman 5.x v1 generator, hence the file-based probe. +HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py +HealthInterval=60s +HealthTimeout=30s +HealthRetries=3 +HealthStartPeriod=1800s +Pull=never + +[Service] +# First start compiles Triton/inductor kernels (several minutes) before the +# engine loads ~9.5 GiB of weights per card. +TimeoutStartSec=1800 +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=default.target diff --git a/dankserver/frank-medium/radiance-entrypoint.sh b/dankserver/frank-medium/radiance-entrypoint.sh new file mode 100755 index 0000000..997ece8 --- /dev/null +++ b/dankserver/frank-medium/radiance-entrypoint.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# frank-medium container entrypoint: vllm-radiance MXFP4 on 2x Radeon AI PRO +# R9700 (gfx1201), serving just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq. +# +# This is a fork of the patch-and-serve block inside radiance-vllm-mxfp4's +# serve-mxfp4.sh (clone at /home/dank/GitPackages/radiance-vllm-mxfp4, pinned +# commit 9735329348ae1c9319f01cbe5f7004ec5f4dba63). The published image alone +# is NOT sufficient for this model: the shipped libr4d predates the +# gated-delta-net overflow fix and NaNs the output, and the MXFP4 gates need +# relaxing, so the repo's patches are applied at every container start from +# the /patches mount. It runs INSIDE the container (executed as the image +# command by the frank-medium quadlet); the launcher's host-side GPU/TP/KV +# detection is not needed here because this fleet is fixed 2x gfx1201 and the +# KV pin is pinned explicitly in the quadlet. +# +# KEEP IN SYNC: if the radiance clone is updated, re-diff this file against +# the new serve-mxfp4.sh container command and against the DRY_RUN output. +# +# Env contract: every knob below is ${VAR:-default}; the quadlet sets the +# fleet-specific ones (SNAP, PORT, SERVED_NAMES, TP, KV_MEM, ...). +set -euo pipefail + +# ---------------------------------------------------------------- serving shape +: "${SNAP:=/models/Qwen3.8-27B-Uncensored-MXFP4-awq}" +: "${DRAFTER:=/models/Qwen3.8-27B-DFlash2-FP8}" +: "${CHAT_TEMPLATE:=/patches/qwen-fixed-v22.3.jinja}" +: "${PORT:=7770}" +: "${SERVED_NAMES:=frank-medium}" +: "${MAXLEN:=262144}" +: "${MAXSEQS:=8}" +: "${CHUNK:=8192}" +: "${GPU_UTIL:=0.98}" +: "${TP:=2}" +: "${KV_MEM:=18563072000}" # shipped kv-profiles.tsv row 2x7551-32624 @ 8/8192/262144/dflash +: "${SPEC_METHOD:=dflash}" +if [ "$SPEC_METHOD" = dflash ]; then : "${SPEC:=7}"; else : "${SPEC:=4}"; fi +: "${MIN_M:=0}" # 0 = the W4A8 kernel serves every M (MIN_M=1 still routes M=1 to aiter) +: "${R4D_ATTN:=1}" +: "${ASYNC:=0}" + +if [ "$R4D_ATTN" = "1" ]; then ATTN=R4D; else ATTN=ROCM_AITER_UNIFIED_ATTN; fi +if [ "$ASYNC" = 1 ]; then ASYNC_FLAG="--async-scheduling"; UNPAD=false; else + ASYNC_FLAG="--no-async-scheduling"; UNPAD=true; fi + +# Drafting config. dflash: the DFlash2-FP8 block-diffusion drafter (its +# attention backend must support full cuda graphs -> TRITON_ATTN). The +# disable_padded_drafter_batch unpad is the single-stream lever (~+50%). +if [ "$SPEC_METHOD" = dflash ]; then + if [ ! -f "$DRAFTER/config.json" ]; then + echo "frank-medium: no dflash drafter at $DRAFTER" >&2 + exit 1 + fi + DRAFT_SAMPLE=${DRAFT_SAMPLE:-greedy} + SPEC_CFG="{\"method\":\"dflash\",\"model\":\"$DRAFTER\",\"num_speculative_tokens\":$SPEC,\"attention_backend\":\"TRITON_ATTN\",\"disable_padded_drafter_batch\":$UNPAD,\"draft_sample_method\":\"$DRAFT_SAMPLE\"}" + RADIANCE_DRAFT_RERANK=${RADIANCE_DRAFT_RERANK:-80} + RADIANCE_VERIFY_HEAD=${RADIANCE_VERIFY_HEAD:-1} +else + SPEC_CFG="{\"method\":\"mtp\",\"num_speculative_tokens\":$SPEC,\"attention_backend\":\"$ATTN\",\"disable_padded_drafter_batch\":$UNPAD}" + RADIANCE_DRAFT_RERANK=${RADIANCE_DRAFT_RERANK:-32} + RADIANCE_VERIFY_HEAD=${RADIANCE_VERIFY_HEAD:-0} +fi + +# The AR size gate compares raw bf16 bytes: CHUNK x hidden(5120) x 2. Derived +# so changing CHUNK cannot silently drop prefill back onto RCCL. +export RADIANCE_AR_MAX_KB=$(( (CHUNK * 5120 * 2) / 1024 + 4096 )) + +# ---------------------------------------------------------------- runtime env +export HF_HUB_OFFLINE=1 +export VLLM_LOGGING_LEVEL=${VLLM_LOGGING_LEVEL:-INFO} +export VLLM_NO_USAGE_STATS=${VLLM_NO_USAGE_STATS:-1} +export VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION=1 +export VLLM_ROCM_USE_AITER_MHA=0 VLLM_ROCM_USE_AITER_MLA=0 VLLM_ROCM_USE_AITER_MOE=0 +export VLLM_ROCM_USE_AITER_LINEAR=0 VLLM_ROCM_USE_AITER_FP8BMM=0 +export VLLM_ROCM_USE_AITER_FP4BMM=0 VLLM_ROCM_USE_AITER_RMSNORM=0 +export NCCL_PROTO=Simple +export RADIANCE_USE_R4D=${RADIANCE_USE_R4D:-1} RADIANCE_USE_R4D_AR=${RADIANCE_USE_R4D_AR:-1} +export RADIANCE_DFLASH_SELECTOR_TOPK="${RADIANCE_DFLASH_SELECTOR_TOPK:-}" +export RADIANCE_R4D_REPORT=1 +export RADIANCE_PRESHUFFLE=${RADIANCE_PRESHUFFLE:-1} +export RADIANCE_MXFP4_SANITIZE="${RADIANCE_MXFP4_SANITIZE:-0}" +export RADIANCE_MXFP4=1 RADIANCE_MXFP4_W4A8=1 RADIANCE_MXFP4_W4A8_MIN_M="$MIN_M" +export RADIANCE_FAST_DRAFT=${RADIANCE_FAST_DRAFT:-1} +export RADIANCE_DRAFT_TAU=${RADIANCE_DRAFT_TAU:-0.20} +export RADIANCE_DRAFT_RERANK +export RADIANCE_VERIFY_HEAD +export RADIANCE_VERIFY_HEAD_MAX_M=${RADIANCE_VERIFY_HEAD_MAX_M:-32} +export RADIANCE_MXFP4_DEBUG=${RADIANCE_MXFP4_DEBUG:-0} +export RADIANCE_MXFP4_PUREQUANT=${RADIANCE_MXFP4_PUREQUANT:-0} +export RADIANCE_MXFP4_SYNC=${RADIANCE_MXFP4_SYNC:-0} +export RADIANCE_MXFP4_CLONE=${RADIANCE_MXFP4_CLONE:-0} +export RADIANCE_MXFP4_CHECKX=${RADIANCE_MXFP4_CHECKX:-0} +export RADIANCE_MXFP4_PADOUT=${RADIANCE_MXFP4_PADOUT:-0} +export RADIANCE_MXFP4_TN4_MIN_M=${RADIANCE_MXFP4_TN4_MIN_M:-2048} +# The decode-kernel band must cover MAXSEQS x (SPEC+1) rows: 64 covers 8x8. +if [ "${MAXSEQS}" -gt 8 ]; then + export RADIANCE_MXFP4_DECODE_MAX_M=${RADIANCE_MXFP4_DECODE_MAX_M:-128} +fi +export RADIANCE_MXFP4_DECODE_NT=${RADIANCE_MXFP4_DECODE_NT:-1} +export RADIANCE_MXFP4_A_TILED_MIN_M=${RADIANCE_MXFP4_A_TILED_MIN_M:-513} +export RADIANCE_MXFP4_WPERM=${RADIANCE_MXFP4_WPERM:-1} +export RADIANCE_TP_PAD=${RADIANCE_TP_PAD:-0} # TP=2 here; 3 is the only non-zero value upstream supports +export RADIANCE_TP_PAD_DRAFTER=${RADIANCE_TP_PAD_DRAFTER:-1} +export RADIANCE_TP_PAD_STRICT=${RADIANCE_TP_PAD_STRICT:-1} +export RADIANCE_GDN_MERGE_INPROJ=${RADIANCE_GDN_MERGE_INPROJ:-1} +export RADIANCE_GDN_NORM_QUANT=${RADIANCE_GDN_NORM_QUANT:-1} +export RADIANCE_GDN_STRIDED_GATES=${RADIANCE_GDN_STRIDED_GATES:-0} +export RADIANCE_GDN_EMPTY_OUT=${RADIANCE_GDN_EMPTY_OUT:-0} +export R4D_ATTN_FP8=${R4D_ATTN_FP8:-3} +export RADIANCE_DFLASH_CALIB="${RADIANCE_DFLASH_CALIB:-}" +export RADIANCE_DFLASH_CALIB_TOKENS="${RADIANCE_DFLASH_CALIB_TOKENS:-200000}" +export RADIANCE_MXFP4_SHADOW="${RADIANCE_MXFP4_SHADOW:-}" +export RADIANCE_MXFP4_KERNEL_N="${RADIANCE_MXFP4_KERNEL_N:-}" +export RADIANCE_MXFP4_KERNEL_NK="${RADIANCE_MXFP4_KERNEL_NK:-}" +export RADIANCE_MXFP4_CHECKALL="${RADIANCE_MXFP4_CHECKALL:-}" +export RADIANCE_MXFP4_DECODE_KS="${RADIANCE_MXFP4_DECODE_KS:-}" +export RADIANCE_MXFP4_DECODE_BK="${RADIANCE_MXFP4_DECODE_BK:-}" +export RADIANCE_MXFP4_PERBLOCK_NK="${RADIANCE_MXFP4_PERBLOCK_NK:-}" +export RADIANCE_MXFP4_REFLINEAR="${RADIANCE_MXFP4_REFLINEAR:-0}" +export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-}" +export RADIANCE_MXFP4_EPIFAST="${RADIANCE_MXFP4_EPIFAST:-1}" +export RADIANCE_AR_OVERLAP=${RADIANCE_AR_OVERLAP:-0} +export RADIANCE_GDN_FUSED_UPDATE=${RADIANCE_GDN_FUSED_UPDATE:-1} +export RADIANCE_DYNAMIC_WIDTH=${RADIANCE_DYNAMIC_WIDTH:-1} +export RADIANCE_DYNW_ALPHA=${RADIANCE_DYNW_ALPHA:-0.35} +export RADIANCE_DYNW_MARGIN=${RADIANCE_DYNW_MARGIN:-2} +export RADIANCE_DYNW_MIN=${RADIANCE_DYNW_MIN:-2} +export RADIANCE_DYNW_MIN_BATCH=${RADIANCE_DYNW_MIN_BATCH:-3} +export RADIANCE_AR_QNB=${RADIANCE_AR_QNB:-96} +export RADIANCE_AR_QNT=${RADIANCE_AR_QNT:-1024} +export MXFP4_CUMODE=${MXFP4_CUMODE:-0} +export RADIANCE_STEP_TRACE=${RADIANCE_STEP_TRACE:-0} +export RADIANCE_AR_OVERLAP_MIN_M=${RADIANCE_AR_OVERLAP_MIN_M:-2048} +export RADIANCE_AR_OVERLAP_SLICES=${RADIANCE_AR_OVERLAP_SLICES:-4} +export RADIANCE_MXFP4_EPIFAST=${RADIANCE_MXFP4_EPIFAST:-1} +export RADIANCE_MXFP4_R4D_DECODE_MAX_M=${RADIANCE_MXFP4_R4D_DECODE_MAX_M:-0} +export RADIANCE_TOPK_TRITON_MIN_ROWS=${RADIANCE_TOPK_TRITON_MIN_ROWS:-1} +export RADIANCE_SKINNY_GEMM=${RADIANCE_SKINNY_GEMM:-1} +export RADIANCE_MXFP4_HOIST_QUANT=${RADIANCE_MXFP4_HOIST_QUANT:-1} +export RADIANCE_MXFP4_TRACED_QUANT=${RADIANCE_MXFP4_TRACED_QUANT:-1} +export RADIANCE_FP8_STREAM=${RADIANCE_FP8_STREAM:-1} +export RADIANCE_RMS_QUANT_FUSION=${RADIANCE_RMS_QUANT_FUSION:-1} +export RADIANCE_NVFP4_MXFP4=${RADIANCE_NVFP4_MXFP4:-0} +export RADIANCE_NVFP4_EXP=${RADIANCE_NVFP4_EXP:-mse} +export RADIANCE_NVFP4_FP8_LAYERS=${RADIANCE_NVFP4_FP8_LAYERS:-mxfp4} +export RADIANCE_NVFP4_BF16_LAYERS=${RADIANCE_NVFP4_BF16_LAYERS:-in_proj_ba} +export RADIANCE_NVFP4_LMHEAD=${RADIANCE_NVFP4_LMHEAD:-bf16} +export RADIANCE_GDN_PATHS=${RADIANCE_GDN_PATHS:-both} +export RADIANCE_GDN_NANTRACE=${RADIANCE_GDN_NANTRACE:-0} +export RADIANCE_MXFP4_CHECK_MAX_M=${RADIANCE_MXFP4_CHECK_MAX_M:-128} +export RADIANCE_MXFP4_MHIST=${RADIANCE_MXFP4_MHIST:-0} +export VLLM_CACHE_ROOT=/cache/vllm +export TORCHINDUCTOR_CACHE_DIR=/cache/inductor +export TRITON_CACHE_DIR=/cache/triton +export AITER_ROOT_DIR=/cache/aiter +export TRITON_CACHE_AUTOTUNING=1 +# The patched libr4d built by setup-mxfp4.sh, bind-mounted at /r4d. +R4D_SO=${R4D_SO:-/r4d} + +# The norm+quant fusion turns on vLLM's passes (one --compilation-config +# instance; argparse keeps the last, so entries accumulate into one flag). +CC_ITEMS="" +if [ "$RADIANCE_NORMQUANT_FUSION" != "0" ]; then + CC_ITEMS="\"pass_config\":{\"fuse_norm_quant\":true,\"fuse_act_quant\":true}" +fi +EXTRA=${EXTRA:-} +if [ -n "$CC_ITEMS" ]; then + EXTRA="$EXTRA --compilation-config {$CC_ITEMS}" +fi + +# ---------------------------------------------------------------- patch block +# Byte-for-byte the launcher's container-start block. The patches come from +# the /patches mount (the pinned radiance clone). SP must match the image's +# venv layout; the image tag is pinned in the quadlet. +SP=/opt/vllm/lib/python3.12/site-packages +cd /patches +python3 patch_quark_mxfp4.py +python3 patch_nvfp4_mxfp4.py +python3 patch_tp3_pad.py +python3 patch_ar_maxbytes.py +python3 patch_topk_triton_rows.py +python3 patch_dflash_calib.py +python3 patch_dflash_mxfp4_kv.py +python3 patch_rmsquant_fusion.py +python3 patch_verify_head.py +python3 patch_kv_group_size.py +python3 patch_topk_composite.py +python3 patch_gdn_shared_build.py +python3 patch_dflash_selector_topk.py +python3 patch_gdn_merge_inproj.py +python3 patch_dynwidth.py +python3 patch_async_dynwidth.py +python3 patch_step_trace.py +python3 patch_ar_geometry.py +python3 patch_ar_3rank.py +python3 patch_gdn_glue.py +# Non-fatal: fixes content=null on thinking-off requests; not required to serve. +python3 patch_qwen3_thinkoff.py \ + || echo "[radiance] WARNING: thinkoff patch did not apply; thinking-off requests will return empty content" +cp mxfp4-configs/*.json "$SP"/aiter/ops/triton/configs/gemm/ +cp radiance_preamble.py /opt/radiance_preamble.py +cp radiance_nvfp4.py radiance_mxfp4.py radiance_gdn.py radiance_rmsquant.py radiance_drafthead.py \ + radiance_verifyhead.py radiance_gdnmerge.py radiance_aroverlap.py radiance_topk.py \ + radiance_arnq.py radiance_tp3pad.py "$SP"/ +hipcc -O3 -w -std=c++17 -fPIC -shared --offload-arch=gfx1201 $([ "${MXFP4_CUMODE:-0}" = 1 ] && echo -mcumode) \ + $(python3 -m pybind11 --includes) radiance_mxfp4_fp8.hip -o "$SP"/radiance_mxfp4_fp8.so +if [ -n "${R4D_SO:-}" ] && [ -f /r4d/r4d.so ]; then + cp /r4d/r4d.so "$SP"/r4d.so + echo "[radiance] using patched r4d.so from $R4D_SO" +fi +# Leave /patches before exec: a stale radiance_mxfp4_fp8.so there would shadow +# the one just compiled into site-packages (cwd precedes it on sys.path). +cd / +exec /opt/radiance_entrypoint.sh "$SNAP" --served-model-name $SERVED_NAMES \ + --host 0.0.0.0 --port "$PORT" \ + --kv-cache-dtype fp8 --tensor-parallel-size "$TP" \ + --gpu-memory-utilization "$GPU_UTIL" \ + ${KV_MEM:+--kv-cache-memory "$KV_MEM"} \ + --max-model-len "$MAXLEN" --max-num-seqs "$MAXSEQS" --max-num-batched-tokens "$CHUNK" \ + --attention-backend "$ATTN" \ + --speculative-config "$SPEC_CFG" \ + $ASYNC_FLAG $EXTRA \ + --enable-prefix-caching --mamba-cache-mode align --enable-auto-tool-choice \ + --tool-call-parser qwen3_coder --reasoning-parser qwen3 \ + --override-generation-config '{"temperature":0.7,"top_p":0.95,"top_k":20}' \ + --chat-template "$CHAT_TEMPLATE" diff --git a/scripts/download-q8agent.sh b/scripts/.AGENTTRASH/download-q8agent.sh.rocmfpx similarity index 100% rename from scripts/download-q8agent.sh rename to scripts/.AGENTTRASH/download-q8agent.sh.rocmfpx diff --git a/scripts/frank-medium.env b/scripts/.AGENTTRASH/frank-medium.env.rocmfpx similarity index 100% rename from scripts/frank-medium.env rename to scripts/.AGENTTRASH/frank-medium.env.rocmfpx diff --git a/scripts/frank-medium.service b/scripts/.AGENTTRASH/frank-medium.service.rocmfpx similarity index 100% rename from scripts/frank-medium.service rename to scripts/.AGENTTRASH/frank-medium.service.rocmfpx diff --git a/scripts/serve-frank-medium.sh b/scripts/.AGENTTRASH/serve-frank-medium.sh.rocmfpx similarity index 100% rename from scripts/serve-frank-medium.sh rename to scripts/.AGENTTRASH/serve-frank-medium.sh.rocmfpx diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py index af56958..fb8d394 100755 --- a/scripts/deploy-plan.py +++ b/scripts/deploy-plan.py @@ -349,8 +349,16 @@ def services_for_path(path: str, target: str) -> set[str]: "copyparty": {"copyparty"}, "plex": {"plex"}, "tautulli": {"tautulli"}, + "frank-medium": {"frank-medium"}, } _add(services, *quadlet_services.get(parts[1], set())) + if len(parts) >= 2 and parts[1] == "frank-medium": + # The vllm-radiance runtime since 2026-09-15: the quadlet file + # above, plus the entrypoint and healthcheck it mounts straight + # out of the checkout. Every file in the service dir is runtime + # state. + _add(services, "frank-medium") + return services return services if parts[0] == "frankbook-pro": @@ -368,14 +376,7 @@ def services_for_path(path: str, target: str) -> set[str]: if parts[0] == "scripts": name = p.name - if name in {"frank-medium.env", "frank-medium.service", "serve-frank-medium.sh"}: - _add(services, "frank-medium") - elif name == "chat-template.jinja": - # frank-medium reads this template only when its env file points - # at it (the GGUF's embedded template is the default); frank-large - # runs halogen-flash-server, which owns its own chat handling. - _add(services, "frank-medium") - elif name in {"homepage-status.py", "homepage-status.service", "homepage-status.timer"}: + if name in {"homepage-status.py", "homepage-status.service", "homepage-status.timer"}: _add(services, "homepage") # Deployment/test/benchmark scripts and other repo tooling do not # restart a runtime service. diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 89d830a..6ec0ec3 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -991,34 +991,40 @@ wait_for_llm_large_ready() { return 0 } -# deploy_llm_medium — frank-medium replaced frank-small on 2026-09-15 when the -# RTX 3090 left the host for two Radeon AI PRO R9700 cards. The old unit is a -# CUDA-era FreeToken service that can only crash-loop on this hardware, so it -# is retired here (stop, disable, move the installed unit to the trash) before -# its replacement starts. Same failure mode as frank-large's retirement: this -# host's `systemctl disable` has been observed to delete the unit file itself -# (2026-08-26 incident), so both outcomes are tolerated. -retire_root_llm_small() { - local trash="$HOME/.AGENTTRASH/20260915-frank-small-retire/host-units" - run_privileged systemctl stop frank-default-small 2>/dev/null || true - run_privileged systemctl disable frank-default-small 2>/dev/null || true - run_privileged systemctl stop frank-small.service 2>/dev/null || true - run_privileged systemctl disable frank-small.service 2>/dev/null || true - if [[ -e /etc/systemd/system/frank-small.service || -L /etc/systemd/system/frank-small.service ]]; then +# deploy_llm_medium — frank-medium runs as a rootless Quadlet since the +# 2026-09-15 runtime swap to the vllm-radiance MXFP4 stack. It began the day +# as a root llama.cpp unit (the ROCmFPX/Q8_0_ROCMFPX_AGENT deployment from +# the morning GPU swap); that root unit is retired here first — stop, +# disable, move the installed unit into the home trash. Same failure mode as +# frank-large's retirement: this host's `systemctl disable` has been +# observed to delete the unit file itself (2026-08-26 incident), so both +# outcomes are tolerated. +retire_root_llm_medium() { + local trash="$HOME/.AGENTTRASH/20260915-rocmfpx-retire/host-units" + run_privileged systemctl stop frank-medium.service 2>/dev/null || true + run_privileged systemctl disable frank-medium.service 2>/dev/null || true + if [[ -e /etc/systemd/system/frank-medium.service || -L /etc/systemd/system/frank-medium.service ]]; then mkdir -p "$trash" - run_privileged mv -- /etc/systemd/system/frank-small.service \ - "$trash/frank-small.service" - echo "retired root unit frank-small.service -> $trash" + run_privileged mv -- /etc/systemd/system/frank-medium.service \ + "$trash/frank-medium.service" + echo "retired root unit frank-medium.service -> $trash" fi run_privileged systemctl daemon-reload } deploy_llm_medium() { - retire_root_llm_small - relink_unit frank-medium - run_privileged systemctl daemon-reload - run_privileged systemctl enable frank-medium - run_privileged systemctl restart frank-medium + retire_root_llm_medium + # Pull once, then run the localhost/-aliased tag: this host's quadlet + # path cannot resolve registry-prefixed refs from local storage at + # create time (same as frank-large on dankclaw). Pull=never keeps boots + # off the registry. + podman pull docker.io/stilldeadcode/vllm-radiance:0.9.3 >/dev/null 2>&1 || true + podman tag docker.io/stilldeadcode/vllm-radiance:0.9.3 localhost/radiance-vllm:0.9.3 + # The compile cache must exist before the quadlet mounts resolve; the + # one-time setup (setup-mxfp4.sh / PREPARE_ONLY) normally creates it. + mkdir -p /home/dank/.radiance-cache-w4a8-093-gdnm-nqft-fp8s-gnq/{vllm,inductor,triton,aiter} + install_quadlet frank-medium dankserver/frank-medium/quadlet frank-medium + systemctl --user restart frank-medium.service wait_for_health frank-medium http://127.0.0.1:7770/health 1200 } diff --git a/scripts/test-changed.sh b/scripts/test-changed.sh index 2e53f42..e4273a7 100755 --- a/scripts/test-changed.sh +++ b/scripts/test-changed.sh @@ -179,10 +179,13 @@ for path in "${CHANGED[@]}"; do scripts/mount-entropy.sh) add_test tests/mount_entropy.bats ;; - scripts/frank-medium.env|scripts/serve-frank-medium.sh) - add_test tests/llm_units.bats - add_test tests/llm_secrets.bats - add_test tests/scripts_syntax.bats + # frank-medium moved to a rootless quadlet (vllm-radiance MXFP4) on + # 2026-09-15; its unit, entrypoint and healthcheck live in the + # service dir and carry the fleet's GPU and podman posture. + dankserver/frank-medium/*) + add_test tests/frank_medium_quadlet.bats + add_test tests/deployment_scoping.bats + add_test tests/docs_completeness.bats ;; scripts/*.service) add_test tests/llm_units.bats diff --git a/tests/changed_tests.bats b/tests/changed_tests.bats index aac84c2..9828db4 100644 --- a/tests/changed_tests.bats +++ b/tests/changed_tests.bats @@ -21,10 +21,10 @@ load helpers [[ "$output" != *"tests/homepage_pages.bats"* ]] } -@test "changed-test wrapper maps a service script to its focused tests" { - run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths scripts/serve-frank-medium.sh +@test "changed-test wrapper maps the frank-medium quadlet to its focused tests" { + run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths dankserver/frank-medium/quadlet/frank-medium.container [ "$status" -eq 0 ] - [[ "$output" == *"tests/scripts_syntax.bats"* ]] - [[ "$output" == *"tests/llm_units.bats"* ]] - [[ "$output" == *"tests/llm_secrets.bats"* ]] + [[ "$output" == *"tests/frank_medium_quadlet.bats"* ]] + [[ "$output" == *"tests/deployment_scoping.bats"* ]] + [[ "$output" != *"tests/llm_units.bats"* ]] } diff --git a/tests/deployment_scoping.bats b/tests/deployment_scoping.bats index 763093f..fe849b4 100644 --- a/tests/deployment_scoping.bats +++ b/tests/deployment_scoping.bats @@ -28,23 +28,25 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" ! grep -q 'frank_npu\|frank-npu' "$REMOTE_DEPLOY" } -@test "shared chat-template changes map only to the service that reads it" { - # frank-medium reads the tracked template only when its env file points at - # it (the GGUF's embedded template is the default); frank-large runs - # halogen-flash-server, which owns its own chat handling. So a template - # edit is a no-op on Dankclaw and selects frank-medium on Dankserver. +@test "chat-template changes plan no service: the radiance runtime ships its own template" { + # The ROCmFPX llama.cpp deployment optionally read the tracked + # scripts/chat-template.jinja; the vllm-radiance runtime mounts its own + # qwen-fixed-v22.3.jinja out of the pinned clone instead. Nobody reads + # the tracked template any more, so an edit to it deploys nothing. run python3 "$PLANNER" --target dankclaw --paths scripts/chat-template.jinja [ "$status" -eq 0 ] [ -z "$output" ] run python3 "$PLANNER" --target dankserver --paths scripts/chat-template.jinja [ "$status" -eq 0 ] - [ "$output" = "frank-medium" ] + [ -z "$output" ] } -@test "deployment planner maps the dankserver model to frank-medium only" { +@test "deployment planner maps the frank-medium service dir to frank-medium only" { run python3 "$PLANNER" --target dankserver --paths \ - scripts/frank-medium.service + dankserver/frank-medium/quadlet/frank-medium.container \ + dankserver/frank-medium/radiance-entrypoint.sh \ + dankserver/frank-medium/frank-medium-healthcheck.py [ "$status" -eq 0 ] [ "$output" = "frank-medium" ] } diff --git a/tests/frank_medium_quadlet.bats b/tests/frank_medium_quadlet.bats new file mode 100755 index 0000000..d773a27 --- /dev/null +++ b/tests/frank_medium_quadlet.bats @@ -0,0 +1,127 @@ +#!/usr/bin/env bats +# Seam: the frank-medium quadlet on dankserver. +# frank-medium became a rootless Quadlet container on 2026-09-15 when the +# vllm-radiance MXFP4 stack replaced the charlie12345/ROCmFPX llama.cpp +# runtime on the 2x Radeon AI PRO R9700. This seam pins the properties that +# make it work and that no reviewer can see by reading the flags alone: GPU +# device access on the fixed device indices, the upstream launcher's podman +# posture, host networking with the fleet's client port and alias, the +# measured KV pin for this exact hardware signature, boot persistence, and a +# deploy handler that retires the old root unit without deleting anything. + +load helpers + +UNIT="$REPO_ROOT/dankserver/frank-medium/quadlet/frank-medium.container" +ENTRYPOINT="$REPO_ROOT/dankserver/frank-medium/radiance-entrypoint.sh" +HEALTHCHECK="$REPO_ROOT/dankserver/frank-medium/frank-medium-healthcheck.py" +REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" + +@test "frank-medium quadlet exists under dankserver/frank-medium/quadlet/" { + [ -f "$UNIT" ] +} + +@test "frank-medium unit is boot-persistent (WantedBy=default.target)" { + # Generated units cannot be enabled on the podman 5.x v1 quadlet + # generator; [Install] plus linger is the entire boot-persistence story. + grep -q '^WantedBy=default.target$' "$UNIT" +} + +@test "frank-medium unit is self-healing" { + grep -q '^Restart=on-failure$' "$UNIT" + grep -q '^RestartSec=' "$UNIT" +} + +@test "frank-medium image is pinned to an exact version tag" { + grep -Eq '^Image=localhost/radiance-vllm:[0-9]+\.[0-9]+\.[0-9]+$' "$UNIT" + grep -q '^Pull=never$' "$UNIT" +} + +@test "frank-medium can reach the GPU (kfd + DRM render node) on the fixed indices" { + grep -q '^AddDevice=/dev/kfd$' "$UNIT" + grep -q '^AddDevice=/dev/dri$' "$UNIT" + grep -q '^Environment=HIP_VISIBLE_DEVICES=0,1$' "$UNIT" + grep -q '^Environment=ROCR_VISIBLE_DEVICES=0,1$' "$UNIT" +} + +@test "frank-medium runs with the upstream launcher's podman posture" { + # serve-mxfp4.sh runs the image privileged, with host IPC, an unconfined + # seccomp profile and SYS_PTRACE; host networking carries the port. + grep -q -- '--privileged' "$UNIT" + grep -q -- '--ipc=host' "$UNIT" + grep -q -- '--security-opt seccomp=unconfined' "$UNIT" + grep -q -- '--network=host' "$UNIT" +} + +@test "frank-medium keeps the fleet endpoint: host-network port 7770, alias frank-medium" { + # vLLM binds 0.0.0.0:7770 on the host network; no PublishPort exists. + [ "$(grep -c '^PublishPort=' "$UNIT")" -eq 0 ] + grep -q '^Environment=PORT=7770$' "$UNIT" + grep -q '^Environment=SERVED_NAMES=frank-medium$' "$UNIT" +} + +@test "frank-medium pins the measured KV cache for its hardware signature" { + # The upstream repo's shipped kv-profiles.tsv row for + # 2x7551-32624 at maxseqs=8 chunk=8192 maxlen=262144 spec=dflash. + # A pin is only valid at that batch shape; if the shape changes, this + # value must be recalibrated or set to 0 (vLLM profiling). + grep -q '^Environment=KV_MEM=18563072000$' "$UNIT" + grep -q '^Environment=MAXSEQS=8$' "$UNIT" + grep -q '^Environment=CHUNK=8192$' "$UNIT" + grep -q '^Environment=MAXLEN=262144$' "$UNIT" + grep -q '^Environment=SPEC_METHOD=dflash$' "$UNIT" + grep -q '^Environment=SPEC=7$' "$UNIT" +} + +@test "frank-medium mounts the pinned radiance clone and the patched libr4d" { + # The patches are applied at every container start from /patches; the + # image's shipped r4d.so predates the GDN overflow fix and NaNs this + # model, so /r4d (b9e42ab-rx6) must be mounted and copied over it. + grep -q 'Volume=/home/dank/GitPackages/radiance-vllm-mxfp4:/patches:z$' "$UNIT" + grep -q 'Volume=/home/dank/.cache/radiance-libr4d/b9e42ab-rx6:/r4d:z$' "$UNIT" +} + +@test "frank-medium entrypoint is a syntax-clean fork of the upstream launcher" { + [ -f "$ENTRYPOINT" ] + bash -n "$ENTRYPOINT" + # Pinned clone: the entrypoint must name the commit it was forked from so + # a clone update is forced through a re-diff, not drifted into silently. + grep -q '9735329348ae1c9319f01cbe5f7004ec5f4dba63' "$ENTRYPOINT" +} + +@test "frank-medium healthcheck probes the serving endpoint, not the process" { + [ -f "$HEALTHCHECK" ] + python3 -m py_compile "$HEALTHCHECK" + grep -q 'frank-medium-healthcheck.py' "$UNIT" + grep -q '^HealthStartPeriod=' "$UNIT" +} + +@test "frank-medium unit carries no secret material" { + ! grep -Eq '^(Environment|EnvironmentFile)=.*(TOKEN|PASSWORD|SECRET|KEY)' "$UNIT" +} + +@test "the frank-medium checkpoint mount is the designated target" { + grep -q '^Environment=SNAP=/models/Qwen3.8-27B-Uncensored-MXFP4-awq$' "$UNIT" + grep -q '^Volume=/home/dank/models:/models$' "$UNIT" +} + +@test "the deploy handler retires the old root unit without deleting it" { + grep -q 'retire_root_llm_medium' "$REMOTE_DEPLOY" + grep -q 'systemctl stop frank-medium.service' "$REMOTE_DEPLOY" + # Never `rm`: the retired ROCmFPX unit moves into the home trash so the + # llama.cpp era stays recoverable. + grep -q 'mv -- /etc/systemd/system/frank-medium.service' "$REMOTE_DEPLOY" + ! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?/etc/systemd/system/frank-medium' "$REMOTE_DEPLOY" +} + +@test "the deploy handler never tries to enable a generated unit" { + # `systemctl --user enable frank-medium.service` fails on the v1 + # generator ("transient or generated"); the deploy must install and + # restart instead. + ! grep -q 'systemctl --user enable frank-medium.service' "$REMOTE_DEPLOY" + grep -q 'install_quadlet frank-medium dankserver/frank-medium/quadlet frank-medium' "$REMOTE_DEPLOY" + grep -q 'systemctl --user restart frank-medium.service' "$REMOTE_DEPLOY" +} + +@test "the deploy handler re-tags the registry image to the localhost alias" { + grep -q 'podman tag docker.io/stilldeadcode/vllm-radiance:0.9.3 localhost/radiance-vllm:0.9.3' "$REMOTE_DEPLOY" +}