diff --git a/dankserver/frank-medium/proxy/affinity-proxy.py b/dankserver/.AGENTTRASH/20260916-vllm-radiance/affinity-proxy.py similarity index 100% rename from dankserver/frank-medium/proxy/affinity-proxy.py rename to dankserver/.AGENTTRASH/20260916-vllm-radiance/affinity-proxy.py diff --git a/dankserver/frank-medium/quadlet/frank-medium-a.container b/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container similarity index 100% rename from dankserver/frank-medium/quadlet/frank-medium-a.container rename to dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container diff --git a/dankserver/frank-medium/quadlet/frank-medium-b.container b/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-b.container similarity index 100% rename from dankserver/frank-medium/quadlet/frank-medium-b.container rename to dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-b.container diff --git a/dankserver/frank-medium/frank-medium-healthcheck.py b/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-healthcheck.py similarity index 100% rename from dankserver/frank-medium/frank-medium-healthcheck.py rename to dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-healthcheck.py diff --git a/dankserver/frank-medium/quadlet/frank-medium-proxy.container b/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-proxy.container similarity index 100% rename from dankserver/frank-medium/quadlet/frank-medium-proxy.container rename to dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-proxy.container diff --git a/dankserver/frank-medium/quadlet/frank-medium.container b/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium.container similarity index 100% rename from dankserver/frank-medium/quadlet/frank-medium.container rename to dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium.container diff --git a/dankserver/frank-medium/radiance-entrypoint.sh b/dankserver/.AGENTTRASH/20260916-vllm-radiance/radiance-entrypoint.sh similarity index 100% rename from dankserver/frank-medium/radiance-entrypoint.sh rename to dankserver/.AGENTTRASH/20260916-vllm-radiance/radiance-entrypoint.sh diff --git a/dankserver/frank-medium/proxy/routing.py b/dankserver/.AGENTTRASH/20260916-vllm-radiance/routing.py similarity index 100% rename from dankserver/frank-medium/proxy/routing.py rename to dankserver/.AGENTTRASH/20260916-vllm-radiance/routing.py diff --git a/scripts/bench-vllm-corpus.py b/scripts/.AGENTTRASH/bench-vllm-corpus.py similarity index 100% rename from scripts/bench-vllm-corpus.py rename to scripts/.AGENTTRASH/bench-vllm-corpus.py diff --git a/scripts/bench-vllm-serve.sh b/scripts/.AGENTTRASH/bench-vllm-serve.sh similarity index 100% rename from scripts/bench-vllm-serve.sh rename to scripts/.AGENTTRASH/bench-vllm-serve.sh diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py index ecf7119..9d0dd67 100755 --- a/scripts/deploy-plan.py +++ b/scripts/deploy-plan.py @@ -353,7 +353,6 @@ def services_for_path(path: str, target: str) -> set[str]: "copyparty": {"copyparty"}, "plex": {"plex"}, "tautulli": {"tautulli"}, - "frank-medium": {"frank-medium"}, } _add(services, *quadlet_services.get(parts[1], set())) if len(parts) >= 2 and parts[1] == "observability": @@ -367,11 +366,11 @@ def services_for_path(path: str, target: str) -> set[str]: _add(services, "node-exporter") return services if len(parts) >= 2 and parts[1] == "frank-medium": - # The vllm-radiance runtime since 2026-09-15: the quadlet file - # above, plus the entrypoint and healthcheck it mounts straight - # out of the checkout. Every file in the service dir is runtime - # state. - _add(services, "frank-medium") + # frank-medium is a ROOT llama.cpp unit since 2026-09-16: its + # runtime is a native gfx1201 binary, not a container, so there is + # nothing under dankserver/frank-medium/ to plan any more. The + # retired vllm-radiance quadlets live in .AGENTTRASH/, which is + # archive material and plans nothing. return services return services @@ -398,6 +397,12 @@ def services_for_path(path: str, target: str) -> set[str]: # trio that lived here for two days is in scripts/.AGENTTRASH/, # which is archive material and plans nothing. _add(services, "frank-small") + if name in {"frank-medium.env", "frank-medium.service", "serve-frank-medium.sh"}: + # llama.cpp frank-medium (2026-09-16) keeps its unit, env and serve + # files here the same way. The files that BUILD and FETCH its + # runtime and payload are host-side one-offs: they change no + # deployed state, so they select nothing. + _add(services, "frank-medium") # Deployment/test/benchmark scripts and other repo tooling do not # restart a runtime service. return services diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index d545ff0..ca171e5 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1100,62 +1100,72 @@ deploy_llm_small() { wait_for_health frank-small http://127.0.0.1:7771/health 900 } -# deploy_llm_medium — frank-medium runs as a rootless Quadlet since the -# 2026-09-15 runtime swap to the vllm-radiance MXFP4 stack. It began the day -# as a root llama.cpp unit (the ROCmFPX/Q8_0_ROCMFPX_AGENT deployment from -# the morning GPU swap); that root unit is retired here first — stop, -# disable, move the installed unit into the home trash. Same failure mode as -# frank-large's retirement: this host's `systemctl disable` has been -# observed to delete the unit file itself (2026-08-26 incident), so both -# outcomes are tolerated. -retire_root_llm_medium() { - local trash="$HOME/.AGENTTRASH/20260915-rocmfpx-retire/host-units" - run_privileged systemctl stop frank-medium.service 2>/dev/null || true - run_privileged systemctl disable frank-medium.service 2>/dev/null || true - if [[ -e /etc/systemd/system/frank-medium.service || -L /etc/systemd/system/frank-medium.service ]]; then - mkdir -p "$trash" - run_privileged mv -- /etc/systemd/system/frank-medium.service \ - "$trash/frank-medium.service" - echo "retired root unit frank-medium.service -> $trash" - fi - run_privileged systemctl daemon-reload +# deploy_llm_medium — frank-medium is mainline llama.cpp on the two R9700s as a +# ROOT unit since the 2026-09-16 cutover: the runtime is a native gfx1201 HIP +# binary, so there is no image to install and no generated unit to dodge. It +# replaces the vllm-radiance replica pair + routing proxy, which held this +# endpoint for a day and could never use both cards for one model (TP=2 needs +# an RCCL collective, and the second card cannot dispatch one). +# +# Layer split in ONE process is what the successor does instead: layers are +# distributed across the cards and only the activation at the boundary crosses +# PCIe, once per forward pass. That is the "pipeline parallelism" the endpoint +# wanted, with no collective anywhere in it. +LLM_MEDIUM_BIN="${LLM_MEDIUM_BIN:-/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server}" +LLM_MEDIUM_MODEL="${LLM_MEDIUM_MODEL:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}" +LLM_MEDIUM_DRAFT="${LLM_MEDIUM_DRAFT:-/home/dank/models/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q8_0.gguf}" + +# The three vLLM user units are retired HERE, not left to rot: the proxy +# published 7770 on the host network, so the llama-server below cannot bind +# until they are stopped and the port is released. Never `rm` — the unit files +# move into the home trash; the images stay in local storage. +retire_vllm_medium_quadlets() { + local trash="$HOME/.AGENTTRASH/20260916-vllm-radiance-retire" unit + systemctl --user stop frank-medium-a.service frank-medium-b.service \ + frank-medium-proxy.service 2>/dev/null || true + for unit in frank-medium-a frank-medium-b frank-medium-proxy frank-medium; do + if [[ -e "$quadlet_unit_dir/$unit.container" ]]; then + mkdir -p "$trash" + mv -- "$quadlet_unit_dir/$unit.container" "$trash/$unit.container" + echo "retired quadlet $unit.container -> $trash" + fi + done + systemctl --user daemon-reload } deploy_llm_medium() { - retire_root_llm_medium - # Pull once, then run the localhost/-aliased tag: this host's quadlet - # path cannot resolve registry-prefixed refs from local storage at - # create time (same as frank-large on dankclaw). Pull=never keeps boots - # off the registry. - podman pull docker.io/stilldeadcode/vllm-radiance:0.9.3 >/dev/null 2>&1 || true - podman tag docker.io/stilldeadcode/vllm-radiance:0.9.3 localhost/radiance-vllm:0.9.3 - # The compile cache must exist before the quadlet mounts resolve; the - # one-time setup (setup-mxfp4.sh / PREPARE_ONLY) normally creates it. - mkdir -p /home/dank/.radiance-cache-w4a8-093-gdnm-nqft-fp8s-gnq/{vllm,inductor,triton,aiter} - # Three units, not one. TP=2 across both R9700s cannot launch a collective - # on this host — RCCL's kernels need PCIe atomics and the second card's - # chipset path advertises AtomicOpsCap: Routing- — so the endpoint is two - # single-GPU replicas behind a round-robin proxy. frank-medium.container - # stays in the repo as the TP=2 target for a host whose second card sits - # on a CPU root port; it is deliberately NOT installed here. - install_quadlet frank-medium-a dankserver/frank-medium/quadlet frank-medium-a - install_quadlet frank-medium-b dankserver/frank-medium/quadlet frank-medium-b - install_quadlet frank-medium-proxy dankserver/frank-medium/quadlet frank-medium-proxy - # Drop a stale TP=2 unit from an earlier deploy of this service, or the - # generator keeps trying to start a container that can never become - # healthy. Never `rm`: it moves into the home trash. - if [[ -e "$quadlet_unit_dir/frank-medium.container" ]]; then - local trash="$HOME/.AGENTTRASH/20260916-frank-medium-tp2" - mkdir -p "$trash" - systemctl --user stop frank-medium.service 2>/dev/null || true - mv -- "$quadlet_unit_dir/frank-medium.container" "$trash/frank-medium.container" - systemctl --user daemon-reload - echo "retired the TP=2 quadlet -> $trash (see the runbook for why)" + # Refuse rather than restart-loop. A unit whose runtime or payload is + # missing never becomes healthy, and a green deploy would leave the + # endpoint dark with nothing in the deploy log to say why. + local missing=0 + if [[ ! -x "$LLM_MEDIUM_BIN" ]]; then + echo "ERROR: llama-server absent at $LLM_MEDIUM_BIN" >&2 + echo "frank-medium: build it ON the host (gfx1201): bash scripts/build-llama-rdna4.sh" >&2 + missing=1 fi - # The proxy waits on the replicas, but a restart of all three is what a - # deploy does: the replicas reload the model, the proxy reconnects. - systemctl --user restart frank-medium-a.service frank-medium-b.service frank-medium-proxy.service - wait_for_health frank-medium http://127.0.0.1:7770/health 1800 + if [[ ! -f "$LLM_MEDIUM_MODEL" ]]; then + echo "ERROR: target model absent at $LLM_MEDIUM_MODEL" >&2 + echo "frank-medium: fetch it: bash scripts/download-frank-medium-gguf.sh" >&2 + missing=1 + fi + if [[ ! -f "$LLM_MEDIUM_DRAFT" ]]; then + echo "ERROR: DFlash2 drafter absent at $LLM_MEDIUM_DRAFT" >&2 + echo "frank-medium: fetch it: bash scripts/download-frank-medium-gguf.sh" >&2 + missing=1 + fi + [[ "$missing" -eq 0 ]] || return 1 + + retire_vllm_medium_quadlets + # The retired proxy held 7770 on the host network; pasta releases it at + # container teardown and a stopped engine lingers in TIME_WAIT just past + # that. Same gate as frank-large's cutover (2026-09-16 addressed-in-use + # failure). + wait_port_free 7770 || return 1 + relink_unit frank-medium + run_privileged systemctl daemon-reload + run_privileged systemctl enable frank-medium.service + run_privileged systemctl restart frank-medium.service + wait_for_health frank-medium http://127.0.0.1:7770/health 900 } wait_for_health() { diff --git a/scripts/frank-medium.env b/scripts/frank-medium.env new file mode 100644 index 0000000..5d683e9 --- /dev/null +++ b/scripts/frank-medium.env @@ -0,0 +1,146 @@ +# Tunables for frank-medium (ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP +# UD-Q8_K_XL + z-lab/Qwen3.8-27B-DFlash2-GGUF Q8_0 drafter), mainline llama.cpp +# on Dankserver, 2x AMD Radeon AI PRO R9700 (gfx1201). Since 2026-09-16 this +# is the SOLE LLM runtime on the box: the vllm-radiance replica pair and its +# routing proxy are retired. +# +# Hardware: two R9700 (Navi 48, RDNA4, 32,624 MiB each), no XGMI, different +# IOMMU groups -- every cross-card byte goes over PCIe. The Raphael iGPU +# (gfx1036/Radeon 610M) enumerates as HIP device 2 and is hidden below so +# nothing can drift onto shared system RAM. +# +# WHY LAYER SPLIT AND NOT TENSOR (operator call, 2026-09-16): tensor parallel +# needs a cross-card allreduce per layer, and this host cannot dispatch one. +# RCCL's kernels carry a hidden hostcall-buffer parameter, and ROCr refuses it +# on a device without PCIe atomics -- the second card sits behind the B650 +# chipset bridges (AtomicOpsCap: Routing-, read-only), so every collective that +# touches it dies with hipErrorIllegalState. Measured on this box 2026-09-15 +# (llama-bench, pp4096 / tg128, f16 KV, fa on): +# +# single R9700 1160.9 / 19.42 +# layer 2x 1991.2 / 18.76 (859.1 / 17.64 at depth 32768) +# tensor 2x 271.8 / 26.26 (no RCCL: butterfly fallback, -7x prefill) +# +# Layer split distributes layers in order; only the activation at the boundary +# crosses the wire, once per forward pass. That is the "pipeline parallelism" +# this service wanted, and it needs no collective at all. +# +# Runtime: mainline llama.cpp master fb27a525 (2026-09-16), gfx1201, built by +# scripts/build-llama-rdna4.sh against a private ROCm 7.14.0a20260624 toolchain +# that is bundled next to the binary (RPATH $ORIGIN/../lib, no LD_LIBRARY_PATH). +# The ref is pinned by TWO features: DFlash2 (ggml-org/llama.cpp#27342 -- the +# drafter is detected from its GGUF) and qwen35 MTP. Do not float it. +# +# Model: ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF @ UD-Q8_K_XL, +# 29.29 GiB, arch qwen35, Unsloth Dynamic 3.0 (per-tensor types from Unsloth's +# own quant of the same size, on Unsloth's importance matrix). Abliteration is +# orcarouter's single refusal direction projected out of 131 tensors; the vision +# tower is untouched but unused here (text only, see NO_MMPROJ). The file also +# carries the model's MTP head (blk.64.nextn.*), so a file-level switch to +# --spec-type draft-mtp is available without a second copy in VRAM. +# +# Drafter: z-lab/Qwen3.8-27B-DFlash2-GGUF @ Q8_0, 1.92 GiB: 5 layers, hidden +# 5120, block-diffusion proposal with the candidate selector. Decoding stays +# lossless -- the target verifies every drafted token. +# +# Architecture (GGUF arch qwen35): 64 layers as 16 x (3 x (Gated DeltaNet -> +# FFN) + 1 x (Gated Attention -> FFN)) -- 48 linear-attention layers whose +# per-sequence state is CONSTANT-SIZE, 16 full-attention layers at 4 KV heads x +# 256 head_dim. A token therefore costs 2 x 4 x 256 x 2 B = 64 KiB of KV at f16 +# in the attention layers only, plus ~0.16 GiB per sequence of f32 DeltaNet +# state at any depth. +# +# SIZING (per card, 31.86 GiB): weights split ~14.7 GiB; the drafter rides one +# card; KV at CTX_SIZE 131,072 is 8.00 GiB total (~4 GiB per card, since the 16 +# attention layers are split with everything else); the loader's compute buffers +# complete the ledger. Read the loader's own "KV self size" / "compute buffer +# size" lines rather than trusting this arithmetic. +# +# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (HIP reads +# those bare vars; 2026-08-13 incident). llama.cpp takes --host/--port/--temp +# as flags only. + +# llama-server: mainline fb27a525 built for gfx1201 by scripts/build-llama-rdna4.sh. +LLAMA_BIN=/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server + +# Payload, fetched with scripts/download-frank-medium-gguf.sh. +MODEL_DIR=/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF +MODEL_FILE=Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf +DRAFT_MODEL_DIR=/home/dank/models/Qwen3.8-27B-DFlash2-GGUF +DRAFT_MODEL_FILE=Qwen3.8-27B-DFlash2-Q8_0.gguf + +# Fleet identity. 7770 is the fleet's dankserver endpoint; the alias is the id +# at /v1/models and what every client asks for. +ALIAS=frank-medium +BIND_ADDR=0.0.0.0 +LISTEN_PORT=7770 + +# Placement. HIP_VISIBLE_DEVICES hides the iGPU; DEVICES is llama.cpp's own +# list, which is the pair the split runs across. +HIP_VISIBLE_DEVICES=0,1 +DEVICES=ROCm0,ROCm1 +SPLIT_MODE=layer +# Empty = distribute layers by free VRAM (equal here, so 1,1 by hand). +TENSOR_SPLIT=1,1 + +NGL=999 +# Auto-fit has no business resizing a context we set explicitly. +FIT=off +FLASH_ATTN=on + +# One slot at the model's documented-configured window. --parallel divides +# --ctx-size between slots; it does not multiply. The target model ships +# n_ctx_train 262144 -- raising CTX_SIZE is a measurement, not a guess (watch +# the per-card ledger), and this service was deployed at 131072 first. +CTX_SIZE=131072 +PARALLEL=1 + +BATCH_SIZE=2048 +UBATCH_SIZE=512 + +# Threads cover tokenization, sampling and HTTP only: every layer is on a GPU. +NUM_THREADS=8 +THREAD_BATCH=8 + +# f16 K/V: an agentic checkpoint is the wrong place to spend KV precision. +CACHE_TYPE_K=f16 +CACHE_TYPE_V=f16 +CTX_CHECKPOINTS=8 +CACHE_RAM=8192 + +# Sampling defaults for requests that send none -- the values the vLLM +# predecessor served on this endpoint (`--override-generation-config`), so a +# client sees no change across the runtime swap. The model card's own +# recommendation is temp 1.0 / top_p 0.95; the harness sets its own per model in +# models.yml, so this is the floor, not the contract. +TEMPERATURE=0.7 +TOP_P=0.95 +TOP_K=20 +MIN_P=0.0 +PRESENCE_PENALTY=0.0 + +# Reasoning on, projected into reasoning_content (deepseek format) with no +# budget: the model decides how long to think. +REASONING=on +REASONING_FORMAT=deepseek +REASONING_BUDGET=-1 + +# Chat template: empty = the one embedded in the GGUF (Qwen3.8's own). +CHAT_TEMPLATE_FILE= +CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}' + +# Speculative decoding: the DFlash2 block-diffusion drafter. Its GGUF carries +# block_size 8, so at most 7 draft tokens per verification step (llama.cpp +# asserts n_max <= block_size - 1); 7 is also what the drafter was published +# with. The drafter rides ROCm1, the card that ends up holding the target's +# last layers -- the hidden states the proposal consumes. +SPEC_TYPE=draft-dflash +SPEC_DRAFT_N_MAX=7 +SPEC_DRAFT_NGL=999 +SPEC_DRAFT_DEVICE=ROCm1 +SPEC_DRAFT_TYPE_K=f16 +SPEC_DRAFT_TYPE_V=f16 + +# Text only: the server must not autoload a vision tower. Vision stays on +# frank-small (Dankclaw, RTX 3090), which is the fleet's image endpoint. +NO_MMPROJ=1 diff --git a/scripts/frank-medium.service b/scripts/frank-medium.service new file mode 100644 index 0000000..5523942 --- /dev/null +++ b/scripts/frank-medium.service @@ -0,0 +1,20 @@ +[Unit] +Description=llama.cpp Qwen3.8-27B Uncensored UD-Q8_K_XL + DFlash2 drafter (dankserver, frank-medium, 2x Radeon AI PRO R9700) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=dank +Group=dank +# The engine pins staging buffers on the first card; keep the memlock budget +# unbounded so an RLIMIT that is smaller than the split buffers cannot fail the +# load. Parity with the other LLM units on this fleet. +LimitMEMLOCK=infinity +Environment="FRANK_MEDIUM_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-medium.env" +ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-medium.sh +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/serve-frank-medium.sh b/scripts/serve-frank-medium.sh new file mode 100755 index 0000000..3b01d10 --- /dev/null +++ b/scripts/serve-frank-medium.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Serve ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF @ UD-Q8_K_XL as +# frank-medium on Dankserver's two Radeon AI PRO R9700 (gfx1201), with the +# z-lab DFlash2 block-diffusion drafter, using mainline llama.cpp. +# +# Runtime: llama.cpp master fb27a525 (2026-09-16), a gfx1201 HIP build that +# bundles its own ROCm 7.14 runtime (RPATH $ORIGIN/../lib), produced by +# scripts/build-llama-rdna4.sh. Two features pin that ref: DFlash2 +# (ggml-org/llama.cpp#27342) and qwen35 MTP. The charlie12345/ROCmFPX fork +# this deployment replaced has neither (its tip is c49ebdb, 2026-08-22). +# +# Parallelism: --split-mode layer across both cards. Tensor split would need a +# cross-card allreduce and this host cannot dispatch one -- RCCL's kernels +# require a hostcall buffer that ROCr refuses on a card without PCIe atomics, +# and the second R9700 sits behind the chipset (AtomicOpsCap: Routing-). Layer +# split crosses the wire once per forward pass instead of once per layer. +# +# The vLLM replica pair and its routing proxy were retired with this cutover: +# one process, one model, both cards. +# +# Tunables come from frank-medium.env unless FRANK_MEDIUM_ENV_FILE overrides it. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FRANK_MEDIUM_ENV_FILE:-$SCRIPT_DIR/frank-medium.env}" +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +# --- defaults (used when the env file does not set them) ------------------ +: "${LLAMA_BIN:=/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server}" +: "${MODEL_DIR:=/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF}" +: "${MODEL_FILE:=Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}" +: "${DRAFT_MODEL_DIR:=/home/dank/models/Qwen3.8-27B-DFlash2-GGUF}" +: "${DRAFT_MODEL_FILE:=Qwen3.8-27B-DFlash2-Q8_0.gguf}" +: "${ALIAS:=frank-medium}" +: "${BIND_ADDR:=0.0.0.0}" +: "${LISTEN_PORT:=7770}" + +# Devices. HIP_VISIBLE_DEVICES hides the Raphael iGPU (HIP device 2) so nothing +# can drift onto shared system RAM; DEVICES is what llama.cpp splits across. +: "${HIP_VISIBLE_DEVICES:=0,1}" +: "${DEVICES:=ROCm0,ROCm1}" +: "${SPLIT_MODE:=layer}" +: "${TENSOR_SPLIT:=1,1}" +: "${NGL:=999}" +: "${FIT:=off}" +: "${FLASH_ATTN:=on}" + +: "${CTX_SIZE:=131072}" +: "${PARALLEL:=1}" +: "${BATCH_SIZE:=2048}" +: "${UBATCH_SIZE:=512}" +: "${NUM_THREADS:=8}" +: "${THREAD_BATCH:=8}" +: "${CACHE_TYPE_K:=f16}" +: "${CACHE_TYPE_V:=f16}" +: "${CTX_CHECKPOINTS:=8}" +: "${CACHE_RAM:=8192}" + +# Sampling defaults for silent clients: the values the vLLM predecessor served. +: "${TEMPERATURE:=0.7}" +: "${TOP_P:=0.95}" +: "${TOP_K:=20}" +: "${MIN_P:=0.0}" +: "${PRESENCE_PENALTY:=0.0}" + +: "${REASONING:=on}" +: "${REASONING_FORMAT:=deepseek}" +: "${REASONING_BUDGET:=-1}" +: "${CHAT_TEMPLATE_FILE:=}" +: "${CHAT_TEMPLATE_KWARGS:={\"enable_thinking\":true,\"preserve_thinking\":true}}" + +# Speculative decoding: draft-dflash (the DFlash2 drafter is a file, not a +# flag) or draft-mtp (the target GGUF's own MTP head -- no draft file). +: "${SPEC_TYPE:=draft-dflash}" +: "${SPEC_DRAFT_N_MAX:=7}" +: "${SPEC_DRAFT_N_MIN:=0}" +: "${SPEC_DRAFT_P_MIN:=0.0}" +: "${SPEC_DRAFT_P_SPLIT:=0.10}" +: "${SPEC_DRAFT_NGL:=999}" +: "${SPEC_DRAFT_DEVICE:=}" +: "${SPEC_DRAFT_TYPE_K:=f16}" +: "${SPEC_DRAFT_TYPE_V:=f16}" +: "${SPEC_DRAFT_POLL:=1}" +: "${SPEC_DRAFT_POLL_BATCH:=1}" + +NO_MMPROJ="${NO_MMPROJ:-1}" + +export HIP_VISIBLE_DEVICES + +MODEL="$MODEL_DIR/$MODEL_FILE" +DRAFT_MODEL="$DRAFT_MODEL_DIR/$DRAFT_MODEL_FILE" + +if [[ ! -x "$LLAMA_BIN" ]]; then + echo "missing executable runtime: $LLAMA_BIN" >&2 + echo "build it with: bash $SCRIPT_DIR/build-llama-rdna4.sh" >&2 + exit 1 +fi +if [[ ! -r "$MODEL" ]]; then + echo "missing or unreadable target model: $MODEL" >&2 + echo "fetch it with: bash $SCRIPT_DIR/download-frank-medium-gguf.sh" >&2 + exit 1 +fi +if [[ -n "$CHAT_TEMPLATE_FILE" && ! -r "$CHAT_TEMPLATE_FILE" ]]; then + echo "missing or unreadable chat template: $CHAT_TEMPLATE_FILE" >&2 + exit 1 +fi + +# --- speculative decoding ------------------------------------------------- +SPEC_ARGS=() +if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then + SPEC_ARGS=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX") + if [[ "$SPEC_TYPE" == *dflash* ]]; then + if [[ ! -r "$DRAFT_MODEL" ]]; then + echo "missing or unreadable draft model: $DRAFT_MODEL" >&2 + exit 1 + fi + SPEC_ARGS+=( + --spec-draft-model "$DRAFT_MODEL" + --spec-draft-ngl "$SPEC_DRAFT_NGL" + --spec-draft-type-k "$SPEC_DRAFT_TYPE_K" + --spec-draft-type-v "$SPEC_DRAFT_TYPE_V" + ) + fi + if [[ "$SPEC_TYPE" == *dspark* || ( "$SPEC_DRAFT_P_MIN" != "0.0" && "$SPEC_DRAFT_P_MIN" != "0" ) ]]; then + SPEC_ARGS+=(--spec-draft-n-min "$SPEC_DRAFT_N_MIN" --spec-draft-p-min "$SPEC_DRAFT_P_MIN") + fi + if [[ -n "$SPEC_DRAFT_DEVICE" ]]; then + SPEC_ARGS+=(--spec-draft-device "$SPEC_DRAFT_DEVICE") + fi +fi + +# --- template + placement ------------------------------------------------- +TEMPLATE_ARGS=() +if [[ -n "$CHAT_TEMPLATE_FILE" ]]; then + TEMPLATE_ARGS+=(--chat-template-file "$CHAT_TEMPLATE_FILE") +fi +TEMPLATE_ARGS+=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS") + +PLACEMENT_ARGS=(--device "$DEVICES" --split-mode "$SPLIT_MODE" --tensor-split "$TENSOR_SPLIT") + +MMPROJ_ARGS=() +[[ "$NO_MMPROJ" == "1" ]] && MMPROJ_ARGS=(--no-mmproj) + +exec "$LLAMA_BIN" \ + --model "$MODEL" \ + --alias "$ALIAS" \ + --host "$BIND_ADDR" \ + --port "$LISTEN_PORT" \ + --jinja \ + --gpu-layers "$NGL" \ + --fit "$FIT" \ + --parallel "$PARALLEL" \ + --ctx-size "$CTX_SIZE" \ + --batch-size "$BATCH_SIZE" \ + --ubatch-size "$UBATCH_SIZE" \ + --flash-attn "$FLASH_ATTN" \ + --cache-type-k "$CACHE_TYPE_K" \ + --cache-type-v "$CACHE_TYPE_V" \ + --threads "$NUM_THREADS" \ + --threads-batch "$THREAD_BATCH" \ + --ctx-checkpoints "$CTX_CHECKPOINTS" \ + --cache-ram "$CACHE_RAM" \ + --cache-prompt \ + --metrics \ + --temp "$TEMPERATURE" \ + --top-p "$TOP_P" \ + --top-k "$TOP_K" \ + --min-p "$MIN_P" \ + --presence-penalty "$PRESENCE_PENALTY" \ + --reasoning "$REASONING" \ + --reasoning-format "$REASONING_FORMAT" \ + --reasoning-budget "$REASONING_BUDGET" \ + "${TEMPLATE_ARGS[@]}" \ + "${PLACEMENT_ARGS[@]}" \ + "${MMPROJ_ARGS[@]}" \ + "${SPEC_ARGS[@]}" diff --git a/scripts/test-changed.sh b/scripts/test-changed.sh index 4555ca7..042e22b 100755 --- a/scripts/test-changed.sh +++ b/scripts/test-changed.sh @@ -180,13 +180,17 @@ for path in "${CHANGED[@]}"; do scripts/mount-entropy.sh) add_test tests/mount_entropy.bats ;; - # frank-medium moved to a rootless quadlet (vllm-radiance MXFP4) on - # 2026-09-15; its unit, entrypoint and healthcheck live in the - # service dir and carry the fleet's GPU and podman posture. - dankserver/frank-medium/*) - add_test tests/frank_medium_quadlet.bats + # frank-medium is llama.cpp on the two R9700s as a ROOT unit since + # 2026-09-16: a native gfx1201 HIP binary, so (like frank-small) its + # env, unit and serve script all live in scripts/, and this case has to + # precede the generic scripts/*.service fallback below. The retired + # vllm-radiance service dir is under dankserver/.AGENTTRASH/, which is + # archive material and maps to nothing. + scripts/frank-medium.*|scripts/serve-frank-medium.sh) + add_test tests/frank_medium_unit.bats + add_test tests/llm_units.bats add_test tests/deployment_scoping.bats - add_test tests/docs_completeness.bats + add_test tests/scripts_syntax.bats ;; # frank-small is llama.cpp on the 3090 as a ROOT unit: a native CUDA # binary, so no generated quadlet is involved. Its env, unit and serve diff --git a/tests/frank_medium_proxy.bats b/tests/.AGENTTRASH/frank_medium_proxy.bats similarity index 100% rename from tests/frank_medium_proxy.bats rename to tests/.AGENTTRASH/frank_medium_proxy.bats diff --git a/tests/frank_medium_quadlet.bats b/tests/.AGENTTRASH/frank_medium_quadlet.bats similarity index 100% rename from tests/frank_medium_quadlet.bats rename to tests/.AGENTTRASH/frank_medium_quadlet.bats diff --git a/tests/changed_tests.bats b/tests/changed_tests.bats index 9828db4..49c8c08 100644 --- a/tests/changed_tests.bats +++ b/tests/changed_tests.bats @@ -21,10 +21,10 @@ load helpers [[ "$output" != *"tests/homepage_pages.bats"* ]] } -@test "changed-test wrapper maps the frank-medium quadlet to its focused tests" { - run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths dankserver/frank-medium/quadlet/frank-medium.container +@test "changed-test wrapper maps the frank-medium trio to its focused tests" { + run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths scripts/frank-medium.env [ "$status" -eq 0 ] - [[ "$output" == *"tests/frank_medium_quadlet.bats"* ]] + [[ "$output" == *"tests/frank_medium_unit.bats"* ]] [[ "$output" == *"tests/deployment_scoping.bats"* ]] - [[ "$output" != *"tests/llm_units.bats"* ]] + [[ "$output" != *"tests/homepage_pages.bats"* ]] } diff --git a/tests/deployment_scoping.bats b/tests/deployment_scoping.bats index d2cf869..b8fea0f 100644 --- a/tests/deployment_scoping.bats +++ b/tests/deployment_scoping.bats @@ -46,13 +46,20 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" [ -z "$output" ] } -@test "deployment planner maps the frank-medium service dir to frank-medium only" { +@test "deployment planner maps the frank-medium scripts trio to frank-medium only" { run python3 "$PLANNER" --target dankserver --paths \ - dankserver/frank-medium/quadlet/frank-medium.container \ - dankserver/frank-medium/radiance-entrypoint.sh \ - dankserver/frank-medium/frank-medium-healthcheck.py + scripts/frank-medium.env scripts/frank-medium.service scripts/serve-frank-medium.sh [ "$status" -eq 0 ] [ "$output" = "frank-medium" ] + + # The path is not the assertion: with the vllm-radiance service dir + # retired into the trash, a planner rule still naming dankserver/frank-medium + # would plan nothing and look identical to a correct one. Assert the + # invariant (the dir is gone) as well as the mapping above. + [ ! -d "$REPO_ROOT/dankserver/frank-medium" ] + run python3 "$PLANNER" --target dankserver --paths dankserver/frank-medium/quadlet/frank-medium-a.container + [ "$status" -eq 0 ] + [ -z "$output" ] } @test "the small endpoint is Dankclaw's again, and Dankserver's copy stays retired" { diff --git a/tests/frank_medium_unit.bats b/tests/frank_medium_unit.bats new file mode 100755 index 0000000..eceb9c0 --- /dev/null +++ b/tests/frank_medium_unit.bats @@ -0,0 +1,163 @@ +#!/usr/bin/env bats +# Seam: the frank-medium mainline-llama.cpp root unit on dankserver. +# frank-medium became a native llama.cpp service on 2026-09-16, replacing the +# vllm-radiance replica pair + routing proxy that served this endpoint for a +# day. What this seam pins is what the flags alone do not say: boot persistence +# for a NATIVE runtime (no generated quadlet is involved), that BOTH cards work +# on ONE model through a layer split with no collective in the path, that the +# DFlash2 drafter is wired with a depth its block size actually allows, that the +# runtime and payload are pinned rather than floated, and that one slot's KV +# still fits a 32 GiB card beside the weights it holds. + +load helpers + +UNIT="$REPO_ROOT/scripts/frank-medium.service" +SERVE="$REPO_ROOT/scripts/serve-frank-medium.sh" +ENV_FILE="$REPO_ROOT/scripts/frank-medium.env" +BUILD="$REPO_ROOT/scripts/build-llama-rdna4.sh" +FETCH="$REPO_ROOT/scripts/download-frank-medium-gguf.sh" +REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" + +# Measured constants (rocm-smi / HF blob sizes), used by the fit arithmetic. +CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it +MODEL_BYTES=31457990112 # UD-Q8_K_XL target +DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter + +@test "frank-medium's unit, serve script, env and its build/fetch scripts exist" { + [ -f "$UNIT" ] + [ -f "$SERVE" ] + [ -f "$ENV_FILE" ] + [ -f "$BUILD" ] + [ -f "$FETCH" ] +} + +@test "frank-medium is boot-persistent, self-healing and unprivileged" { + grep -q '^WantedBy=multi-user.target$' "$UNIT" + grep -q '^Restart=on-failure$' "$UNIT" + grep -q '^RestartSec=' "$UNIT" + grep -q '^User=dank$' "$UNIT" + grep -q '^LimitMEMLOCK=infinity$' "$UNIT" +} + +@test "frank-medium runs the repo's own serve script with the repo's env" { + grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-medium.sh$' "$UNIT" + grep -q 'FRANK_MEDIUM_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-medium.env' "$UNIT" +} + +@test "both R9700s serve ONE model, layer-split inside one process" { + grep -q '^SPLIT_MODE=layer$' "$ENV_FILE" + grep -q '^DEVICES=ROCm0,ROCm1$' "$ENV_FILE" + grep -q '^TENSOR_SPLIT=1,1$' "$ENV_FILE" + grep -q -- '--split-mode "$SPLIT_MODE"' "$SERVE" + grep -q -- '--device "$DEVICES"' "$SERVE" + grep -q -- '--tensor-split "$TENSOR_SPLIT"' "$SERVE" + # Tensor split is the one shape this host cannot run: it needs a cross-card + # collective, and the second card cannot dispatch one (see the env file). + ! grep -q '^SPLIT_MODE=tensor$' "$ENV_FILE" +} + +@test "the iGPU is hidden so no layer can land on system RAM" { + grep -q '^HIP_VISIBLE_DEVICES=0,1$' "$ENV_FILE" + grep -q '^export HIP_VISIBLE_DEVICES$' "$SERVE" +} + +@test "the DFlash2 drafter is a draft model at a depth its block allows" { + grep -q '^SPEC_TYPE=draft-dflash$' "$ENV_FILE" + grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q8_0\.gguf$' "$ENV_FILE" + grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE" + grep -q -- '--spec-draft-model "$DRAFT_MODEL"' "$SERVE" + # DFlash2 drafts into a block of 8, so at most 7 draft tokens per + # verification step -- llama.cpp asserts n_max <= block_size - 1. + local n_max + n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")" + [ -n "$n_max" ] && [ "$n_max" -ge 1 ] && [ "$n_max" -le 7 ] +} + +@test "the runtime is pinned to a commit and to gfx1201 alone" { + # A floating ref would silently move the service between engine versions, + # and an unpinned arch would build for a sibling GPU that loads and then + # segfaults on these cards. + grep -Eq '^LLAMA_REF="\$\{LLAMA_REF:-[0-9a-f]{40}\}"$' "$BUILD" + grep -Eq '^HIP_ARCH="\$\{HIP_ARCH:-gfx1201\}"$' "$BUILD" + grep -q 'GGML_HIP=ON' "$BUILD" + grep -q 'frank-medium.env' "$SERVE" +} + +@test "the payload is fetched resumably and verified by size" { + grep -q 'UD-Q8_K_XL\.gguf' "$FETCH" + grep -q 'Qwen3\.8-27B-DFlash2-Q8_0\.gguf' "$FETCH" + grep -q 'curl -fL -C -' "$FETCH" + grep -q 'verifying sizes' "$FETCH" +} + +@test "one slot's KV still fits a card beside the weights that card holds" { + local ctx parallel cache_k cache_v + ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" + parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")" + cache_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")" + cache_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")" + [ -n "$ctx" ] && [ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ] + [ "$parallel" -ge 1 ] + [ "$cache_k" = "f16" ] && [ "$cache_v" = "f16" ] + # qwen35 spends KV on its 16 full-attention layers only: 2 (K,V) x 4 heads + # x 256 dim x 2 B = 64 KiB per token at f16. Worst case for the fit is ONE + # card carrying the drafter and every attention layer's KV, with ~2 GiB of + # compute buffers: half the weights, plus that, must stay inside the card. + local per_card + per_card=$(( MODEL_BYTES / 2 + DRAFT_BYTES + ctx * 65536 + 2 * 1024 * 1024 * 1024 )) + [ "$per_card" -le "$CARD_BYTES" ] +} + +@test "frank-medium is text only: the vision tower stays on frank-small" { + grep -q '^NO_MMPROJ=1$' "$ENV_FILE" + grep -q -- '--no-mmproj' "$SERVE" +} + +@test "the fleet endpoint and the predecessor's sampler defaults survive" { + grep -q '^ALIAS=frank-medium$' "$ENV_FILE" + grep -q '^LISTEN_PORT=7770$' "$ENV_FILE" + grep -q -- '--alias "$ALIAS"' "$SERVE" + grep -q -- '--port "$LISTEN_PORT"' "$SERVE" + grep -q -- '--host "$BIND_ADDR"' "$SERVE" + # Prometheus scrapes this endpoint for LLM fleet metrics. + grep -q -- '--metrics' "$SERVE" + # Requests that send no sampler fields get these instead of an error. + grep -q '^TEMPERATURE=0.7$' "$ENV_FILE" + grep -q '^TOP_P=0.95$' "$ENV_FILE" +} + +@test "the deploy handler installs the root unit and refuses a missing payload" { + grep -q 'deploy_llm_medium' "$REMOTE_DEPLOY" + grep -q 'relink_unit frank-medium' "$REMOTE_DEPLOY" + grep -q 'systemctl enable frank-medium.service' "$REMOTE_DEPLOY" + grep -q 'systemctl restart frank-medium.service' "$REMOTE_DEPLOY" + grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY" + grep -q 'build-llama-rdna4.sh' "$REMOTE_DEPLOY" + grep -q 'download-frank-medium-gguf.sh' "$REMOTE_DEPLOY" + grep -q '\[\[ "$missing" -eq 0 \]\] || return 1' "$REMOTE_DEPLOY" +} + +@test "the deploy handler retires the vLLM quadlets and frees the port they held" { + grep -q 'retire_vllm_medium_quadlets' "$REMOTE_DEPLOY" + grep -q 'systemctl --user stop frank-medium-a.service frank-medium-b.service' "$REMOTE_DEPLOY" + grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY" + # No image left to tag: the runtime is a native binary now. + # ...and no image work or quadlet install of the predecessor survives. + ! grep -q 'install_quadlet frank-medium-a' "$REMOTE_DEPLOY" + ! grep -q 'podman tag docker.io/stilldeadcode' "$REMOTE_DEPLOY" +} + +@test "the retired vLLM deployment is gone from the repo, not merely unmapped" { + [ ! -d "$REPO_ROOT/dankserver/frank-medium" ] + [ ! -f "$REPO_ROOT/tests/frank_medium_quadlet.bats" ] + # ...and the archive is present, so the two assertions above cannot pass + # because a path went missing rather than because it stayed missing. + [ -f "$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container" ] + [ -f "$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance/radiance-entrypoint.sh" ] +} + +@test "the serve script refuses to start on an incomplete host" { + grep -q 'frank-medium: missing\|missing executable runtime\|missing or unreadable' "$SERVE" + grep -q 'missing or unreadable target model' "$SERVE" + grep -q 'missing or unreadable draft model' "$SERVE" +}