diff --git a/dankserver/frank-medium/frank-medium-healthcheck.py b/dankserver/frank-medium/frank-medium-healthcheck.py
deleted file mode 100755
index e792150..0000000
--- a/dankserver/frank-medium/frank-medium-healthcheck.py
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env python3
-"""frank-medium container healthcheck: the port's /health must answer 200.
-
-Runs INSIDE the container. The vLLM /health endpoint only answers once the
-engine is up (weights loaded, cudagraphs captured), so a healthy container is
-a serving container, not merely a live process. PORT selects the endpoint so
-the probe follows the unit's Environment=PORT rather than hard-coding the
-fleet port. The HealthCmd in the quadlet splits on spaces into an argv array
-(podman 5.x v1 generator), which is why this lives in a file rather than an
-inline python one-liner.
-"""
-import os
-import sys
-import urllib.request
-
-port = os.environ.get("PORT", "7770")
-try:
- with urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=5) as r:
- sys.exit(0 if r.status == 200 else 1)
-except Exception:
- sys.exit(1)
diff --git a/dankserver/frank-medium/quadlet/frank-medium.container b/dankserver/frank-medium/quadlet/frank-medium.container
deleted file mode 100644
index e72b860..0000000
--- a/dankserver/frank-medium/quadlet/frank-medium.container
+++ /dev/null
@@ -1,186 +0,0 @@
-# Quadlet unit: frank-medium (rootless, boot-persistent).
-#
-# Install path on dankserver: ~/.config/containers/systemd/frank-medium.container
-# Unit name: frank-medium.service (container name: frank-medium)
-#
-# The dankserver LLM endpoint (:7770, alias frank-medium): the vllm-radiance
-# stack (the 0.13.0-e060895 image, PR #51) serving
-# hugypufy/Swift-Qwen3.8-27B-PARO-MXFP6 (ParoQuant MXFP6, W4A8, 25.16 GiB)
-# on the 2x Radeon AI PRO R9700 (gfx1201) as ONE tensor-parallel process
-# (TP=2, R4D one-shot all-reduce) with the z-lab DFlash2 external drafter
-# (SPEC=7) and UNQUANTIZED bf16 KV (KV_DTYPE=auto).
-# The entrypoint is QUANTIZATION-AWARE: it reads the model's weight
-# quantization from config.json and serves the plain FP8 path (no MXFP4
-# patches, --quantization=fp8, ROCM_AITER unified attention), the MXFP4
-# patch-and-serve path, or the ParoQuant MXFP6 stack (PR #51's launch6_p
-# kernel). A model swap is now: change SNAP (+ SPEC_METHOD/SPEC), restart.
-# The MXFP4 checkpoint (just1moremodel/Qwen3.8-27B-Uncensored-MXFP4-awq, the
-# 2026-09-20 to 2026-09-21 serving shape) is the on-disk rollback target.
-#
-# SHAPE. The launcher the upstream repo ships binds the server with
-# --network=host (vLLM takes --host 0.0.0.0 --port 7770 directly), which is
-# what this unit does too — no PublishPort. The fleet contract is unchanged:
-# :7770, served model name frank-medium (SERVED_NAMES), OpenAI /v1.
-#
-# TP=2 IS INSTALLABLE HERE NOW, and it won on measurement. Until 2026-09-20
-# this unit was archived un-installed because the second card sat behind B650
-# chipset bridges advertising `AtomicOpsCap: Routing-`, which stopped RCCL
-# kernel dispatch. The board swap (ProArt B850-Creator WiFi NEO) put both
-# cards on CPU GPP root ports — Gen5 x8 each, every bridge Routing+, both
-# endpoints ReqEn+ — and the collective-free R4D path runs at TP=2 without
-# complaint. The conversation battery (scripts/bench-conversation.py, salted,
-# same 10 prompts, 2026-09-20) measured this shape against the incumbent:
-# decode geomean 95.8 t/s vs 45.7, mean TTFT 0.21 s vs 0.72 s, and 259 t/s
-# aggregate against 4 concurrent streams vs 71.7. Evidence and the llama-side
-# caveats: serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md and
-# Dual-R9700-Cooperation.md.
-#
-# DEVICES. /dev/kfd + /dev/dri through the amdgpu/KFD stack; HIP/ROCR device
-# indices 0,1 (the two R9700s). The 2 GiB Raphael iGPU enumerates as index 2
-# and is excluded both by the index pin and by the upstream MIN_GPU_MIB=8192
-# logic this unit replaces. --privileged --ipc=host --security-opt
-# seccomp=unconfined --cap-add SYS_PTRACE is the upstream launcher's exact
-# podman posture.
-#
-# MOUNTS. /models (rw checkpoints), the HF cache (tokenizer files),
-# /cache (Triton/inductor/aiter/vLLM compile cache — first start compiles for
-# several minutes, later starts reuse), /patches (the pinned radiance clone —
-# its patches are applied at EVERY container start by the entrypoint), /r4d
-# (the patched libr4d build; the image's shipped r4d.so NaNs the
-# gated-delta-net and must never be used), and the entrypoint + healthcheck
-# scripts and the chat template from this checkout (mounted read-only
-# straight out of /home/dank/serverAppConfigs — deploy.sh git-pulls the
-# checkout before dispatching).
-#
-# IMAGE. The unit runs the localhost/-aliased tag: this host's podman quadlet
-# path cannot resolve registry-prefixed refs from local storage at create
-# time (same as frank-large on dankclaw). deploy-remote.sh pulls the
-# docker.io ref and re-tags it before the restart. Pull=never keeps boots off
-# the registry.
-#
-# KEEP IN SYNC: the radiance clone is pinned (e060895...); if it is updated,
-# re-diff dankserver/frank-medium/radiance-entrypoint.sh against the new
-# serve-mxfp4.sh container command and bump the pin deliberately.
-#
-# DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the
-# HIP runtime reads bare TEMP for its JIT cache path (2026-08-13 incident).
-[Unit]
-Description=frank-medium — vllm-radiance ParoQuant MXFP6 Swift Qwen3.8-27B on 2x R9700 (gfx1201)
-After=network-online.target
-
-[Container]
-ContainerName=frank-medium
-Image=localhost/radiance-vllm:0.13.0-e060895
-# vLLM binds 0.0.0.0:7770 on the host network directly; no PublishPort.
-AddDevice=/dev/kfd
-AddDevice=/dev/dri
-GroupAdd=keep-groups
-# The upstream launcher's exact podman posture (rootless; podman-only flags
-# like --replace are not needed under systemd management).
-PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add SYS_PTRACE --network=host
-# The two R9700s. The Raphael iGPU (index 2, 2 GiB) stays out: shared system
-# memory is not a compute target.
-Environment=ROCR_VISIBLE_DEVICES=0,1
-Environment=HIP_VISIBLE_DEVICES=0,1
-# Drafting: the external z-lab DFlash2-FP8 drafter (SPEC_METHOD=dflash). The
-# plain-FP8 checkpoint (rollback) carries its own single-layer MTP head, but
-# DFlash2 beat MTP self-speculation on decode, so the drafter serves either.
-# Chat template: the Qwen-Sharp variant (peculiar-ragdoll v22.5.0) — the terse
-# system prompt (opt out per-request with chat_template_kwargs {"terse":
-# false}), message.reasoning support (vLLM's thinking field), and the
-# tightened tool-call rendering. The predecessor (qwen-sharp-v22.4.0.jinja,
-# mounted beside it) remains as the rollback target: flip CHAT_TEMPLATE.
-Environment=SNAP=/models/Qwen3.8-27B-PARO-MXFP6
-Environment=DRAFTER=/models/Qwen3.8-27B-DFlash2-FP8
-Environment=CHAT_TEMPLATE=/templates/qwen-sharp-v22.5.0.jinja
-# Fleet identity: the id at /v1/models and in every response body. Every
-# client (Hermes, the harness, services.json) points at frank-medium.
-Environment=SERVED_NAMES=frank-medium
-Environment=PORT=7770
-# Fixed 2-card tensor parallel: the tested configuration of the upstream
-# repo; the R4D one-shot all-reduce kernel serves it natively.
-Environment=TP=2
-# Measured production batch shape (upstream README): MAXSEQS 8 peaks
-# aggregate throughput at 573 t/s. MAXLEN is the served window (see KV_DTYPE).
-Environment=MAXSEQS=8
-Environment=CHUNK=8192
-Environment=MAXLEN=262144
-# MAXLEN is the whole pool shared across the MAXSEQS slots: 8 concurrent
-# sessions split 262,144 tokens between them, so a full-window request
-# excludes the others until it finishes (vLLM preempts rather than refuses).
-# The llama deployment it replaced advertised 327,680 across 2 slots.
-#
-# VISION WORKS (verified 2026-09-21: a solid red 48x48 PNG is named "Red").
-# config.json sets language_model_only=true and there is no separate mmproj,
-# but the Qwen3_5ForConditionalGeneration architecture carries the vision tower
-# through the Gemma4 merge path and it is not MXFP4-quantized, so vLLM serves
-# image input on either checkpoint. harness models.yml keeps frank-medium
-# input: [text, image].
-# 0.97, down from 0.98: the startup free-memory check (free >= util x 31.86
-# GiB) sat ~15 MiB inside the threshold on the 2026-09-26 boot (free 31.21
-# GiB vs required 31.22 GiB) — the worker's own HIP context plus the
-# amdgpu per-boot baseline is ~690 MiB at check time and that drifts a few
-# tens of MiB between boots, so 0.98 crash-looped on every restart. 0.97
-# buys ~160 MiB of headroom at the cost of ~5k KV tokens on the
-# auto-sized pool.
-Environment=GPU_UTIL=0.97
-# Speculative decoding: DFlash2 external drafter at depth 7. A/B tested
-# against MTP self-speculation (depth 8) on this FP8 checkpoint: DFlash2
-# wins on decode (100.0 vs 85.9 t/s geomean, 287.1 vs 179.7 t/s burst-4)
-# despite a lower accept length (4.24 vs 5.66) — the small separate drafter
-# steps faster than the target's own MTP layer applied autoregressively.
-Environment=SPEC_METHOD=dflash
-Environment=SPEC=7
-# KV cache: 0 = profile the shape (the documented plain-FP8 path's default;
-# the upstream docker-compose leaves the pin commented out). The measured pin
-# for this shape is 15989735424 (14.89 GiB, upstream, 2x R9700 32GB); after
-# the first healthy start, recalibrate with calibrate-kv.sh to use the extra
-# VRAM on the 34.2 GiB cards.
-Environment=KV_MEM=0
-# KV cache dtype: auto = UNQUANTIZED bf16 KV. The experiment: fp8 KV stacked on
-# MXFP4's already-quantized W4A8 weights compounds attention precision loss, so
-# clean KV should recover output quality. MXFP4's freed weights (19.37 vs 30.9
-# GB) buy the pool back — the full 262,144 native window still fits unquantized
-# (~290k-token f16 pool > 262,144). Flip to fp8 for the measured 943k-token pool.
-Environment=KV_DTYPE=auto
-# Checkpoints (rw: nothing downloads at runtime, but vLLM mmaps the shards;
-# keep parity with the launcher's rw mount).
-Volume=/home/dank/models:/models
-# Tokenizer files ride the HF cache; HF_HUB_OFFLINE=1 is set in the
-# entrypoint.
-Volume=/home/dank/.cache/huggingface:/root/.cache/huggingface
-# Compile cache. Keyed on model + stack version + every graph-changing knob;
-# never share it across configurations (upstream rule).
-Volume=/home/dank/.radiance-cache-mxfp6-013-pq-nqft-fp8s-gnq:/cache
-# The pinned radiance clone: patches applied at every container start.
-Volume=/home/dank/GitPackages/radiance-vllm-mxfp4:/patches:z
-# The patched libr4d (pin b9e42ab + repo extras, key b9e42ab-rx6). The
-# image's shipped r4d.so predates the GDN overflow fix and NaNs this model.
-Volume=/home/dank/.cache/radiance-libr4d/b9e42ab-rx6:/r4d:z
-# Entrypoint + healthcheck from this checkout, mounted read-only.
-Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/radiance-entrypoint.sh:/usr/local/bin/frank-medium-entrypoint.sh:ro
-Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/frank-medium-healthcheck.py:/usr/local/bin/frank-medium-healthcheck.py:ro
-Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/qwen-sharp-v22.4.0.jinja:/templates/qwen-sharp-v22.4.0.jinja:ro
-Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/qwen-sharp-v22.5.0.jinja:/templates/qwen-sharp-v22.5.0.jinja:ro
-# The entrypoint must be the container command; the image's own entrypoint
-# (which the patched script execs at the end) must not run unpatched.
-PodmanArgs=--entrypoint /usr/local/bin/frank-medium-entrypoint.sh
-# 200 only once the engine serves; during first-start compilation the
-# container is legitimately unhealthy. HealthCmd is split on spaces into an
-# argv array by the podman 5.x v1 generator, hence the file-based probe.
-HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py
-HealthInterval=60s
-HealthTimeout=30s
-HealthRetries=3
-HealthStartPeriod=1800s
-Pull=never
-
-[Service]
-# First start compiles Triton/inductor kernels (several minutes) before the
-# engine loads ~9.5 GiB of weights per card.
-TimeoutStartSec=1800
-Restart=on-failure
-RestartSec=10
-
-[Install]
-WantedBy=default.target
diff --git a/dankserver/frank-medium/qwen-sharp-v22.5.0.jinja b/dankserver/frank-medium/qwen-sharp-v22.5.0.jinja
deleted file mode 100644
index 74465e7..0000000
--- a/dankserver/frank-medium/qwen-sharp-v22.5.0.jinja
+++ /dev/null
@@ -1,488 +0,0 @@
-{%- set template_version = "qwen3.8-froggeric-v22.5.0" %}
-{%- set _tool_format = tool_call_format if tool_call_format is defined else 'xml' %}
-{%- set image_count = namespace(value=0) %}
-{%- set video_count = namespace(value=0) %}
-{%- set add_vision_id = add_vision_id if add_vision_id is defined else false %}
-{%- set enable_thinking = enable_thinking if enable_thinking is defined else true %}
-{%- set auto_disable_thinking_with_tools = auto_disable_thinking_with_tools if auto_disable_thinking_with_tools is defined else false %}
-{%- if preserve_reasoning is defined and preserve_reasoning is not none %}
- {%- set _preserve_thinking = preserve_reasoning %}
-{%- elif preserve_thinking is defined and preserve_thinking is not none %}
- {%- set _preserve_thinking = preserve_thinking %}
-{%- else %}
- {%- set _preserve_thinking = true %}
-{%- endif %}
-{%- set max_tool_arg_chars = max_tool_arg_chars if max_tool_arg_chars is defined else 0 %}
-{%- set max_tool_response_chars = max_tool_response_chars if max_tool_response_chars is defined else 0 %}
-{%- set _default_reasoning_effort = 'medium' %}
-{%- set _has_tools = (tools is defined and tools and tools is iterable and tools is not mapping) %}
-{%- set _effort_raw = (reasoning_effort | string | lower) if reasoning_effort is defined and reasoning_effort is not none else _default_reasoning_effort %}
-{%- set _initial_thinking = enable_thinking %}
-{%- if _effort_raw in ('none', 'off') %}
- {%- set _initial_thinking = false %}
- {%- set _initial_effort = 'medium' %}
-{%- elif _effort_raw in ('minimal', 'low') %}
- {%- set _initial_effort = 'low' %}
-{%- elif _effort_raw in ('high', 'xhigh', 'max', 'ultracode', 'extreme') %}
- {%- set _initial_effort = 'xhigh' %}
-{%- else %}
- {%- set _initial_effort = 'medium' %}
-{%- endif %}
-{%- set ns_state = namespace(thinking=_initial_thinking, effort=_initial_effort) %}
-{%- if auto_disable_thinking_with_tools and _has_tools %}
- {%- set ns_state.thinking = false %}
-{%- endif %}
-{%- for msg in messages %}
- {%- if msg.role == 'system' or msg.role == 'developer' or msg.role == 'user' %}
- {%- if msg.content is string %}
- {%- if '<|think_off|>' in msg.content %}
- {%- set ns_state.thinking = false %}
- {%- elif '<|think_on|>' in msg.content %}
- {%- set ns_state.thinking = true %}
- {%- elif '<|think_xhigh|>' in msg.content or '<|think_high|>' in msg.content or '<|think_ultracode|>' in msg.content or '<|think_extreme|>' in msg.content or '<|think_max|>' in msg.content %}
- {%- set ns_state.thinking = true %}
- {%- set ns_state.effort = 'xhigh' %}
- {%- elif '<|think_low|>' in msg.content or '<|think_minimal|>' in msg.content %}
- {%- set ns_state.thinking = true %}
- {%- set ns_state.effort = 'low' %}
- {%- elif '<|think_medium|>' in msg.content %}
- {%- set ns_state.thinking = true %}
- {%- set ns_state.effort = 'medium' %}
- {%- endif %}
- {%- elif msg.content is iterable and msg.content is not mapping %}
- {%- for item in msg.content %}
- {%- if item is string %}
- {%- set _item_text = item %}
- {%- elif item is mapping and 'text' in item and item.text is string %}
- {%- set _item_text = item.text %}
- {%- else %}
- {%- set _item_text = '' %}
- {%- endif %}
- {%- if _item_text %}
- {%- if '<|think_off|>' in _item_text %}
- {%- set ns_state.thinking = false %}
- {%- elif '<|think_on|>' in _item_text %}
- {%- set ns_state.thinking = true %}
- {%- elif '<|think_xhigh|>' in _item_text or '<|think_high|>' in _item_text or '<|think_ultracode|>' in _item_text or '<|think_extreme|>' in _item_text or '<|think_max|>' in _item_text %}
- {%- set ns_state.thinking = true %}
- {%- set ns_state.effort = 'xhigh' %}
- {%- elif '<|think_low|>' in _item_text or '<|think_minimal|>' in _item_text %}
- {%- set ns_state.thinking = true %}
- {%- set ns_state.effort = 'low' %}
- {%- elif '<|think_medium|>' in _item_text %}
- {%- set ns_state.thinking = true %}
- {%- set ns_state.effort = 'medium' %}
- {%- endif %}
- {%- endif %}
- {%- endfor %}
- {%- endif %}
- {%- endif %}
-{%- endfor %}
-{%- set reasoning_instructions = '' %}
-{%- if ns_state.thinking %}
- {%- if ns_state.effort == 'xhigh' %}
- {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}
- {%- elif ns_state.effort == 'low' %}
- {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}
- {%- endif %}
-{%- endif %}
-{%- macro render_content(content, do_vision_count, is_system_content=false) %}
- {%- if content is string %}
- {{- content }}
- {%- elif content is iterable and content is not mapping %}
- {%- for item in content %}
- {%- if item is mapping %}
- {%- if item.type == 'image' or 'image' in item or 'image_url' in item %}
- {%- if is_system_content %}
- {{- raise_exception('System message cannot contain images.') }}
- {%- endif %}
- {%- if do_vision_count %}
- {%- set image_count.value = image_count.value + 1 %}
- {%- endif %}
- {%- if add_vision_id %}
- {{- 'Picture ' ~ image_count.value ~ ': ' }}
- {%- endif %}
- {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
- {%- elif item.type == 'video' or 'video' in item or 'video_url' in item %}
- {%- if is_system_content %}
- {{- raise_exception('System message cannot contain videos.') }}
- {%- endif %}
- {%- if do_vision_count %}
- {%- set video_count.value = video_count.value + 1 %}
- {%- endif %}
- {%- if add_vision_id %}
- {{- 'Video ' ~ video_count.value ~ ': ' }}
- {%- endif %}
- {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
- {%- elif 'text' in item %}
- {{- item.text }}
- {%- else %}
- {{- raise_exception('Unexpected item type in content.') }}
- {%- endif %}
- {%- else %}
- {{- item | string }}
- {%- endif %}
- {%- endfor %}
- {%- elif content is none or content is undefined %}
- {{- '' }}
- {%- else %}
- {{- raise_exception('Unexpected content type.') }}
- {%- endif %}
-{%- endmacro %}
-{%- if not messages %}
- {{- raise_exception('No messages provided.') }}
-{%- endif %}
-{%- set head = namespace(count=0, seen_non_system=false) %}
-{%- for message in messages %}
- {%- set _is_sys = (message.role == 'system' or message.role == 'developer') %}
- {%- if _is_sys and not head.seen_non_system %}
- {%- set head.count = head.count + 1 %}
- {%- else %}
- {%- set head.seen_non_system = true %}
- {%- endif %}
-{%- endfor %}
-{%- set sys_state = namespace(content='') %}
-{%- for message in messages[:head.count] %}
- {%- set _part = render_content(message.content, false, true) | trim %}
- {%- if '<|think_off|>' in _part %}{%- set _part = _part.split('<|think_off|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_on|>' in _part %}{%- set _part = _part.split('<|think_on|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_xhigh|>' in _part %}{%- set _part = _part.split('<|think_xhigh|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_high|>' in _part %}{%- set _part = _part.split('<|think_high|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_ultracode|>' in _part %}{%- set _part = _part.split('<|think_ultracode|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_extreme|>' in _part %}{%- set _part = _part.split('<|think_extreme|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_max|>' in _part %}{%- set _part = _part.split('<|think_max|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_medium|>' in _part %}{%- set _part = _part.split('<|think_medium|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_low|>' in _part %}{%- set _part = _part.split('<|think_low|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_minimal|>' in _part %}{%- set _part = _part.split('<|think_minimal|>') | join('') | trim %}{%- endif %}
- {%- if _part %}
- {%- if sys_state.content %}
- {%- set sys_state.content = sys_state.content ~ '\n\n' ~ _part %}
- {%- else %}
- {%- set sys_state.content = _part %}
- {%- endif %}
- {%- endif %}
-{%- endfor %}
-{%- set _sc = sys_state.content %}
-{%- set _msgs = messages[head.count:] %}
-{%- if ns_state.thinking %}
- {%- set _terse_lead = 'Answer directly, after thinking. Lead with the answer, then only what it needs to be correct and usable.' %}
-{%- else %}
- {%- set _terse_lead = 'Answer directly and concisely. Give the answer with only what it needs to be correct and usable.' %}
-{%- endif %}
-{%- set _terse_core %}
-Never: open with preamble or pleasantries; restate the question; add filler transitions; hedge with niceties; or repeat a point you've already made.
-Always: keep essential steps, caveats, uncertainties, and specifics — never drop correctness or a needed warning for brevity. Keep the final answer lean. Use the least structure that conveys it (plain prose when short; lists or code only when they earn their place). If genuinely uncertain, say so and explain why — never omit uncertainty for the sake of brevity.
-If a user request is genuinely ambiguous, ask a sharp question, don't guess.
-{%- endset %}
-{%- set _terse = _terse_lead ~ '\n' ~ (_terse_core | trim) %}
-{#- `terse` chat-template kwarg: default true, so behaviour is unchanged unless a caller opts out.
- Pass {"terse": false} via chat_template_kwargs to serve the model with only its own system
- prompt -- useful for A/B-ing the terseness effect, or when a downstream prompt conflicts. #}
-{%- set _terse_on = terse if terse is defined else true %}
-{%- if _terse_on %}
- {%- if not _sc %}
- {%- set _sc = _terse | trim %}
- {%- else %}
- {%- set _sc = (_sc | trim) ~ '\n\n' ~ (_terse | trim) %}
- {%- endif %}
-{%- endif %}
-{#- Some runtimes (LM Studio's MLX backend) have no native tool-call parser and prepend their own
- protocol into the system content. Emitting ours as well gives the model two contradictory tool
- protocols and it follows one at random. Detect that marker and stand down; the
- `suppress_tool_instructions` kwarg overrides either way where a runtime can pass template kwargs. #}
-{%- set _runtime_tool_protocol = (_sc is defined and _sc and '[TOOL_REQUEST]' in _sc) %}
-{%- set _suppress_tools = suppress_tool_instructions if suppress_tool_instructions is defined else _runtime_tool_protocol %}
-{%- if _has_tools and not _suppress_tools %}
- {{- '<|im_start|>system\n' }}
- {%- if reasoning_instructions %}
- {{- reasoning_instructions + '\n\n' }}
- {%- endif %}
- {{- '# Tools\n\nYou have access to the following functions:\n\n' }}
- {%- for tool in tools %}
- {{- '\n' }}
- {{- tool | tojson }}
- {%- endfor %}
- {{- '\n' }}
- {%- if _tool_format == 'json' %}
- {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }}
- {%- if ns_state.thinking %}
- {{- '\nBrief explanation of tool call\n\n' }}
- {%- endif %}
- {{- '\n{"name": "example_function_name", "arguments": {"example_parameter_1": "value_1", "example_parameter_2": "This is the value for the second parameter"}}\n\n\n\nReminder:\n' }}
- {%- if ns_state.thinking %}
- {{- '- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n' }}
- {%- endif %}
- {{- '- Function calls MUST follow the specified format: a single JSON object with "name" and "arguments" keys inside XML tags.\n' }}
- {%- if ns_state.thinking %}
- {{- '- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n' }}
- {%- else %}
- {{- '- If you choose to call a tool, you MUST output the block IMMEDIATELY, with NO conversational text before it.\n' }}
- {%- endif %}
- {{- '- The tag MUST be at the very beginning of a new line, with NO spaces or indentation before it.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }}
- {%- else %}
- {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }}
- {%- if ns_state.thinking %}
- {{- '\nBrief explanation of tool call\n\n' }}
- {%- endif %}
- {{- '\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n' }}
- {%- if ns_state.thinking %}
- {{- '- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n' }}
- {%- endif %}
- {{- '- Function calls MUST follow the specified format: an inner block must be nested within XML tags.\n' }}
- {%- if ns_state.thinking %}
- {{- '- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n' }}
- {%- else %}
- {{- '- If you choose to call a tool, you MUST output the block IMMEDIATELY, with NO conversational text before it.\n' }}
- {%- endif %}
- {{- '- The and tags MUST be at the very beginning of a new line, with NO spaces or indentation before them.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }}
- {%- endif %}
- {%- if _sc %}
- {{- '\n\n' + _sc }}
- {%- endif %}
- {{- '<|im_end|>\n' }}
-{%- else %}
- {%- if _sc %}
- {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + _sc + '<|im_end|>\n' }}
- {%- elif reasoning_instructions %}
- {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
- {%- endif %}
-{%- endif %}
-{%- set _last_idx = _msgs | length - 1 %}
-{%- set ns = namespace(multi_step_tool=true, last_query_index=_last_idx) %}
-{%- for message in _msgs[::-1] %}
- {%- set index = (_msgs | length - 1) - loop.index0 %}
- {%- if ns.multi_step_tool and message.role == 'user' %}
- {%- set _rc = render_content(message.content, false) | trim %}
- {%- if not (_rc.startswith('') and _rc.endswith('')) %}
- {%- set ns.multi_step_tool = false %}
- {%- set ns.last_query_index = index %}
- {%- endif %}
- {%- endif %}
-{%- endfor %}
-{%- if ns.multi_step_tool %}
- {%- if _last_idx > 50 %}
- {%- set ns.last_query_index = _last_idx %}
- {%- else %}
- {%- set ns.last_query_index = 0 %}
- {%- endif %}
-{%- endif %}
-{%- set ns2 = namespace(prev_role='', consecutive_failures=0) %}
-{%- for message in _msgs %}
- {%- set is_system = (message.role == "system" or message.role == "developer") %}
- {%- set content = render_content(message.content, true, is_system) | trim %}
- {%- if is_system or message.role == 'user' %}
- {%- if '<|think_off|>' in content %}{%- set content = content.split('<|think_off|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_on|>' in content %}{%- set content = content.split('<|think_on|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_xhigh|>' in content %}{%- set content = content.split('<|think_xhigh|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_high|>' in content %}{%- set content = content.split('<|think_high|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_ultracode|>' in content %}{%- set content = content.split('<|think_ultracode|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_extreme|>' in content %}{%- set content = content.split('<|think_extreme|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_max|>' in content %}{%- set content = content.split('<|think_max|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_medium|>' in content %}{%- set content = content.split('<|think_medium|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_low|>' in content %}{%- set content = content.split('<|think_low|>') | join('') | trim %}{%- endif %}
- {%- if '<|think_minimal|>' in content %}{%- set content = content.split('<|think_minimal|>') | join('') | trim %}{%- endif %}
- {%- endif %}
- {%- if is_system %}
- {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
- {%- elif message.role == 'user' %}
- {%- set ns2.consecutive_failures = 0 %}
- {{- '<|im_start|>user\n' + content + '<|im_end|>\n' }}
- {%- elif message.role == 'assistant' %}
- {%- set reasoning_content = '' %}
- {%- set _explicit_reasoning = '' %}
- {%- if message.reasoning_content is defined and message.reasoning_content is not none %}
- {%- if message.reasoning_content is string %}
- {%- set _explicit_reasoning = message.reasoning_content %}
- {%- else %}
- {%- set _explicit_reasoning = message.reasoning_content | string %}
- {%- endif %}
- {%- elif message.thinking is defined and message.thinking is not none %}
- {%- if message.thinking is string %}
- {%- set _explicit_reasoning = message.thinking %}
- {%- else %}
- {%- set _explicit_reasoning = message.thinking | string %}
- {%- endif %}
- {%- elif message.reasoning is defined and message.reasoning is not none %}
- {%- if message.reasoning is string %}
- {%- set _explicit_reasoning = message.reasoning %}
- {%- else %}
- {%- set _explicit_reasoning = message.reasoning | string %}
- {%- endif %}
- {%- endif %}
- {%- if _explicit_reasoning %}
- {%- set _lead_end = '' %}
- {%- if content.startswith('') and '' in content %}
- {%- set _lead_end = '' %}
- {%- elif content.startswith('') and '' in content %}
- {%- set _lead_end = '' %}
- {%- elif content.startswith('') %}
- {%- set _lead_end = '' %}
- {%- elif content.startswith('') %}
- {%- set _lead_end = '' %}
- {%- endif %}
- {%- if _lead_end %}
- {%- set content = content.split(_lead_end)[-1].lstrip('\n') %}
- {%- endif %}
- {%- set reasoning_content = _explicit_reasoning %}
- {%- else %}
- {%- set _think_end = '' %}
- {%- if content.startswith('') %}
- {%- set _think_end = '' %}
- {%- elif content.startswith('') %}
- {%- set _think_end = '' %}
- {%- elif '\n' in content %}
- {%- set _think_end = '\n' %}
- {%- elif '\n' in content %}
- {%- set _think_end = '\n' %}
- {%- elif '\n' in content %}
- {%- set _think_end = '\n' %}
- {%- elif '\n' in content %}
- {%- set _think_end = '\n' %}
- {%- elif content.startswith('') and '' in content %}
- {%- set _think_end = '' %}
- {%- elif content.startswith('') and '' in content %}
- {%- set _think_end = '' %}
- {%- endif %}
- {%- if _think_end %}
- {%- if 'thinking' in _think_end %}
- {%- set _think_start = '' %}
- {%- else %}
- {%- set _think_start = '' %}
- {%- endif %}
- {%- set reasoning_content = content.split(_think_end)[0].rstrip('\n') %}
- {%- if _think_start in reasoning_content %}
- {%- set reasoning_content = reasoning_content.split(_think_start)[-1].lstrip('\n') %}
- {%- endif %}
- {%- set content = content.split(_think_end)[-1].lstrip('\n') %}
- {%- endif %}
- {%- endif %}
- {%- set reasoning_content = reasoning_content | trim %}
- {%- if (_preserve_thinking or loop.index0 > ns.last_query_index) %}
- {{- '<|im_start|>assistant\n\n' + reasoning_content + '\n\n\n' + content }}
- {%- else %}
- {{- '<|im_start|>assistant\n' + content }}
- {%- endif %}
- {%- if message.tool_calls is defined and message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
- {%- for tool_call in message.tool_calls %}
- {%- if tool_call.function is defined and tool_call.function is not none %}
- {%- set tc = tool_call.function %}
- {%- else %}
- {%- set tc = tool_call %}
- {%- endif %}
- {%- set tc_name = tc.name if (tc.name is defined and tc.name is not none) else '' %}
- {%- if _tool_format == 'json' %}
- {%- if loop.first %}
- {%- if content | trim %}
- {{- '\n\n' }}
- {%- endif %}
- {%- else %}
- {{- '\n' }}
- {%- endif %}
- {%- set _args = '{}' %}
- {%- if tc.arguments is defined and tc.arguments is not none %}
- {%- if tc.arguments is mapping %}
- {%- set _args = tc.arguments | tojson %}
- {%- elif tc.arguments is string %}
- {%- if tc.arguments %}
- {%- set _args = tc.arguments %}
- {%- endif %}
- {%- else %}
- {%- set _args = tc.arguments | tojson %}
- {%- endif %}
- {%- endif %}
- {{- '\n{"name": ' }}{{- tc_name | tojson }}{{- ', "arguments": ' }}{{- _args }}{{- '}\n' }}
- {%- else %}
- {%- if loop.first %}
- {%- if content | trim %}
- {{- '\n\n\n\n' }}
- {%- else %}
- {{- '\n\n' }}
- {%- endif %}
- {%- else %}
- {{- '\n\n\n' }}
- {%- endif %}
- {%- if tc.arguments is defined and tc.arguments is not none %}
- {%- if tc.arguments is mapping %}
- {%- for args_name, args_value in tc.arguments.items() %}
- {{- '\n' }}
- {%- if args_value is string %}
- {%- set _av = args_value %}
- {%- else %}
- {%- set _av = args_value | tojson %}
- {%- endif %}
- {%- if max_tool_arg_chars > 0 and _av | length > max_tool_arg_chars %}
- {{- _av[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_av | length | string) ~ ' chars]' }}
- {%- else %}
- {{- _av }}
- {%- endif %}
- {{- '\n\n' }}
- {%- endfor %}
- {%- else %}
- {%- if tc.arguments is string %}
- {%- set _raw_args = tc.arguments %}
- {%- else %}
- {%- set _raw_args = tc.arguments | tojson %}
- {%- endif %}
- {%- if _raw_args %}
- {%- if max_tool_arg_chars > 0 and _raw_args | length > max_tool_arg_chars %}
- {{- _raw_args[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_raw_args | length | string) ~ ' chars]' }}
- {%- else %}
- {{- _raw_args }}
- {%- endif %}
- {%- endif %}
- {%- endif %}
- {%- endif %}
- {{- '\n' }}
- {%- endif %}
- {%- endfor %}
- {%- endif %}
- {{- '<|im_end|>\n' }}
- {%- elif message.role == 'tool' %}
- {%- set _content_lower = content | lower %}
- {%- set _content_head = _content_lower[:120] %}
- {%- set _is_code_or_grep = ('throw new ' in _content_lower or 'throw error' in _content_lower or 'console.error' in _content_lower or 'logger.error' in _content_lower or 'logging.error' in _content_lower or 'import ' in _content_head or 'def ' in _content_head or 'function ' in _content_head) %}
- {%- set _exit_code_zero = ('exit code: 0' in _content_head or 'process exited with code 0' in _content_head) %}
- {%- set _error_field_ok = ('"error": null' in _content_head or '"error":null' in _content_head or '"error": false' in _content_head or '"error":false' in _content_head or '"error": ""' in _content_head or '"error":""' in _content_head) %}
- {%- set _strong_error = (('"error":' in _content_head and not _error_field_ok) or '"status": "error"' in _content_head or '"status":"error"' in _content_head or 'traceback (most recent call last):' in _content_head or 'command not found' in _content_head or 'invalid syntax' in _content_head or 'fatal:' in _content_head or (('exit code: ' in _content_head or 'process exited with code' in _content_head) and not _exit_code_zero) or _content_head.startswith('exception:') or _content_head.startswith('failed to ')) %}
- {%- set _weak_error = ('error:' in _content_head or 'err!' in _content_head) %}
- {%- set _weak_suppressed = ('$ ' in _content_head or 'took ' in _content_head or content | length >= 600) %}
- {%- if not _is_code_or_grep and (_strong_error or (_weak_error and not _weak_suppressed)) %}
- {%- set ns2.consecutive_failures = ns2.consecutive_failures + 1 %}
- {%- else %}
- {%- set ns2.consecutive_failures = 0 %}
- {%- endif %}
- {%- if ns2.prev_role != 'tool' %}
- {{- '<|im_start|>user' }}
- {%- endif %}
- {%- set _is_json_payload = (_tool_format == 'json' and content | trim | length > 0 and (content | trim)[:1] in ('{', '[')) %}
- {%- if not _is_json_payload and max_tool_response_chars > 0 and content | length > max_tool_response_chars %}
- {%- set content = content[:max_tool_response_chars] + '\n[TRUNCATED - original length ' ~ (content | length | string) ~ ' chars]' %}
- {%- endif %}
- {{- '\n\n' + content }}
- {%- if ns2.consecutive_failures >= 2 %}
- {{- '\n\n⚠️ SYSTEM WARNING: ' ~ ns2.consecutive_failures ~ ' consecutive tool errors detected. Your previous approach is incorrect. You MUST use a fundamentally different approach or corrected arguments.' }}
- {%- elif ns2.consecutive_failures == 1 %}
- {{- '\n\n⚠️ SYSTEM WARNING: The previous tool call returned an error. Diagnose the failure and retry with completely corrected arguments.' }}
- {%- endif %}
- {{- '\n' }}
- {%- if loop.last %}
- {{- '<|im_end|>\n' }}
- {%- else %}
- {%- set _next_role = _msgs[loop.index0 + 1].role %}
- {%- if _next_role != 'tool' %}
- {{- '<|im_end|>\n' }}
- {%- endif %}
- {%- endif %}
- {%- else %}
- {{- '<|im_start|>user\n[' + message.role + ']: ' + content + '<|im_end|>\n' }}
- {%- endif %}
- {%- set ns2.prev_role = message.role %}
-{%- endfor %}
-{%- if add_generation_prompt %}
- {{- '<|im_start|>assistant\n' }}
- {%- if not ns_state.thinking %}
- {{- '\n\n\n\n' }}
- {%- else %}
- {{- '\n' }}
- {%- endif %}
-{%- endif %}
\ No newline at end of file
diff --git a/dankserver/frank-medium/radiance-entrypoint.sh b/dankserver/frank-medium/radiance-entrypoint.sh
deleted file mode 100755
index d0c44c1..0000000
--- a/dankserver/frank-medium/radiance-entrypoint.sh
+++ /dev/null
@@ -1,362 +0,0 @@
-#!/usr/bin/env bash
-# frank-medium container entrypoint: vllm-radiance on 2x Radeon AI PRO R9700
-# (gfx1201). Quantization-aware (config.json quant_method): fp8 -> the plain
-# FP8 path; mxfp4/quark -> the MXFP4 patch-and-serve path; paroquant_mxfp6 ->
-# the ParoQuant W4A8 stack + the MXFP6 kernels (PR #51's launch6_p path).
-#
-# This is a fork of the patch-and-serve block inside radiance-vllm-mxfp4's
-# serve-mxfp4.sh (clone at /home/dank/GitPackages/radiance-vllm-mxfp4, pinned
-# commit 9735329348ae1c9319f01cbe5f7004ec5f4dba63). The published image alone
-# is NOT sufficient for this model: the shipped libr4d predates the
-# gated-delta-net overflow fix and NaNs the output, and the MXFP4 gates need
-# relaxing, so the repo's patches are applied at every container start from
-# the /patches mount. It runs INSIDE the container (executed as the image
-# command by the frank-medium quadlet); the launcher's host-side GPU/TP/KV
-# detection is not needed here because this fleet is fixed 2x gfx1201 and the
-# KV pin is pinned explicitly in the quadlet.
-#
-# KEEP IN SYNC: if the radiance clone is updated, re-diff this file against
-# the new serve-mxfp4.sh container command and against the DRY_RUN output.
-#
-# Env contract: every knob below is ${VAR:-default}; the quadlet sets the
-# fleet-specific ones (SNAP, PORT, SERVED_NAMES, TP, KV_MEM, ...).
-set -euo pipefail
-
-# ---------------------------------------------------------------- serving shape
-: "${SNAP:=/models/Qwen3.8-27B-Uncensored-MXFP4-awq}"
-: "${DRAFTER:=/models/Qwen3.8-27B-DFlash2-FP8}"
-: "${CHAT_TEMPLATE:=/patches/qwen-fixed-v22.3.jinja}"
-: "${PORT:=7770}"
-: "${SERVED_NAMES:=frank-medium}"
-: "${MAXLEN:=262144}"
-: "${MAXSEQS:=8}"
-# KV_MEM=0 is passed THROUGH as --kv-cache-memory 0, which the image's
-# patched KV sizing reads as "profile the shape" (the launcher's rule).
-# Blanking the variable instead omits the flag and makes vLLM size the cache
-# from --gpu-memory-utilization alone: measured on the single-GPU replicas
-# that overshoots and the allocation dies with torch.OutOfMemoryError right
-# after the weights load (2.16 GiB requested, 116 MiB free). Do not "fix"
-# this into an empty expansion.
-: "${KV_MEM:=18563072000}" # shipped kv-profiles.tsv row 2x7551-32624 @ 8/8192/262144/dflash
-: "${KV_DTYPE:=fp8}" # fp8 = quantized KV (2x the tokens); auto = bf16 KV (half the pool)
-: "${SPEC_METHOD:=dflash}"
-if [ "$SPEC_METHOD" = dflash ]; then : "${SPEC:=7}"; else : "${SPEC:=4}"; fi
-: "${MIN_M:=0}" # 0 = the W4A8 kernel serves every M (MIN_M=1 still routes M=1 to aiter)
-: "${R4D_ATTN:=1}"
-: "${ASYNC:=0}"
-: "${GPU_UTIL:=0.98}"
-
-# Quantization-aware: read the model's weight quantization from config.json.
-# fp8 -> the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP
-# self-speculation, ROCM_AITER unified attention with the R4D all-reduce).
-# mxfp4/quark (or unknown) -> the MXFP4 patch-and-serve path (the default).
-QUANT="$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("quantization_config",{}).get("quant_method",""))' "$SNAP/config.json" 2>/dev/null || true)"
-IS_MXFP6=0
-case "$QUANT" in
- fp8) IS_MXFP4=0 ;;
- paroquant_mxfp6) IS_MXFP4=1; IS_MXFP6=1 ;;
- mxfp4|quark|"") IS_MXFP4=1 ;;
- *) IS_MXFP4=1; echo "[radiance] WARNING: quantization '$QUANT'; defaulting to the MXFP4 path" >&2 ;;
-esac
-# MXFP6 (ParoQuant) requires TP>=2 (rad_require_tp2 in the canonical launcher);
-# ${TP:-1} keeps the check set -u safe when the quadlet omits TP.
-if [ "$IS_MXFP6" = "1" ]; then
- [ "${TP:-1}" -ge 2 ] || { echo "frank-medium: paroquant_mxfp6 requires TP>=2 (got ${TP:-1})" >&2; exit 1; }
-fi
-
-if [ "$IS_MXFP4" = "0" ]; then ATTN=ROCM_AITER_UNIFIED_ATTN
-elif [ "$R4D_ATTN" = "1" ]; then ATTN=R4D
-else ATTN=ROCM_AITER_UNIFIED_ATTN; fi
-if [ "$ASYNC" = 1 ]; then ASYNC_FLAG="--async-scheduling"; UNPAD=false; else
- ASYNC_FLAG="--no-async-scheduling"; UNPAD=true; fi
-
-# Drafting config. dflash: the DFlash2-FP8 block-diffusion drafter (its
-# attention backend must support full cuda graphs -> TRITON_ATTN). The
-# disable_padded_drafter_batch unpad is the single-stream lever (~+50%).
-if [ "$SPEC_METHOD" = dflash ]; then
- if [ ! -f "$DRAFTER/config.json" ]; then
- echo "frank-medium: no dflash drafter at $DRAFTER" >&2
- exit 1
- fi
- DRAFT_SAMPLE=${DRAFT_SAMPLE:-greedy}
- SPEC_CFG="{\"method\":\"dflash\",\"model\":\"$DRAFTER\",\"num_speculative_tokens\":$SPEC,\"attention_backend\":\"TRITON_ATTN\",\"disable_padded_drafter_batch\":$UNPAD,\"draft_sample_method\":\"$DRAFT_SAMPLE\"}"
- RADIANCE_DRAFT_RERANK=${RADIANCE_DRAFT_RERANK:-80}
- RADIANCE_VERIFY_HEAD=${RADIANCE_VERIFY_HEAD:-1}
-else
- SPEC_CFG="{\"method\":\"mtp\",\"num_speculative_tokens\":$SPEC,\"attention_backend\":\"$ATTN\",\"disable_padded_drafter_batch\":$UNPAD}"
- RADIANCE_DRAFT_RERANK=${RADIANCE_DRAFT_RERANK:-32}
- RADIANCE_VERIFY_HEAD=${RADIANCE_VERIFY_HEAD:-0}
-fi
-
-# The AR size gate compares raw bf16 bytes: CHUNK x hidden(5120) x 2. Derived
-# so changing CHUNK cannot silently drop prefill back onto RCCL.
-export RADIANCE_AR_MAX_KB=$(( (CHUNK * 5120 * 2) / 1024 + 4096 ))
-
-# ---------------------------------------------------------------- runtime env
-export HF_HUB_OFFLINE=1
-export VLLM_LOGGING_LEVEL=${VLLM_LOGGING_LEVEL:-INFO}
-export VLLM_NO_USAGE_STATS=${VLLM_NO_USAGE_STATS:-1}
-export VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION=1
-export VLLM_ROCM_USE_AITER_MHA=0 VLLM_ROCM_USE_AITER_MLA=0 VLLM_ROCM_USE_AITER_MOE=0
-export VLLM_ROCM_USE_AITER_LINEAR=0 VLLM_ROCM_USE_AITER_FP8BMM=0
-export VLLM_ROCM_USE_AITER_FP4BMM=0 VLLM_ROCM_USE_AITER_RMSNORM=0
-export NCCL_PROTO=Simple
-export RADIANCE_USE_R4D=${RADIANCE_USE_R4D:-1} RADIANCE_USE_R4D_AR=${RADIANCE_USE_R4D_AR:-1}
-export RADIANCE_DFLASH_SELECTOR_TOPK="${RADIANCE_DFLASH_SELECTOR_TOPK:-}"
-export RADIANCE_R4D_REPORT=1
-export RADIANCE_PRESHUFFLE=${RADIANCE_PRESHUFFLE:-1}
-export RADIANCE_MXFP4_SANITIZE="${RADIANCE_MXFP4_SANITIZE:-0}"
-# ---------------------------------------------------------------- mxfp6 profile
-# The ParoQuant W4A8 stack (PR #51, the launch6_p kernel). Mirrors
-# paroquant/run_paroquant.sh's env. Runs BEFORE the MXFP4 base block below so
-# the base block's ${VAR:-default} guards preserve these values where they
-# conflict (GDN_MERGE_INPROJ=0, MXFP4_DECODE_MAX_M=64); the base block fills
-# the rest. Gated on IS_MXFP6 so the mxfp4/fp8 paths are untouched.
-if [ "$IS_MXFP6" = "1" ]; then
- export RADIANCE_PAROQUANT=1
- export RADIANCE_GDN_MERGE_INPROJ=${RADIANCE_GDN_MERGE_INPROJ:-0}
- export RADIANCE_GDN_FUSED_MAX_ITEMS=${RADIANCE_GDN_FUSED_MAX_ITEMS:-32}
- export RADIANCE_USE_R4D_AR_QUANT=${RADIANCE_USE_R4D_AR_QUANT:-1}
- export RADIANCE_FUSE_RMS_QUANT=${RADIANCE_FUSE_RMS_QUANT:-1}
- export RADIANCE_KV_GROUP_OPT=${RADIANCE_KV_GROUP_OPT:-1}
- export RADIANCE_MXFP4_DECODE_MAX_M=${RADIANCE_MXFP4_DECODE_MAX_M:-64}
- export RADIANCE_PQ_CHECKALL=${RADIANCE_PQ_CHECKALL:-}
- export RADIANCE_PQM_CHECKALL=${RADIANCE_PQM_CHECKALL:-}
- export RADIANCE_PQM_CHECK_MAX_M=${RADIANCE_PQM_CHECK_MAX_M:-128}
- export RADIANCE_PQ_CHECK_MAX_M=${RADIANCE_PQ_CHECK_MAX_M:-128}
- export RADIANCE_PQ_DECODE_MAX_M=${RADIANCE_PQ_DECODE_MAX_M:-64}
- export RADIANCE_PQ_WPERM=${RADIANCE_PQ_WPERM:-1}
- export RADIANCE_PQ_DECODE_NT=${RADIANCE_PQ_DECODE_NT:-1}
- export RADIANCE_PQ_ATILED=${RADIANCE_PQ_ATILED:-1}
- export RADIANCE_PQ_AT_LBK=${RADIANCE_PQ_AT_LBK:-128}
- export RADIANCE_PQ_AT_HOIST=${RADIANCE_PQ_AT_HOIST:-1}
- export RADIANCE_PQ_PTOK=${RADIANCE_PQ_PTOK:-1}
- export RADIANCE_PQ_FUSED_TOKQ=${RADIANCE_PQ_FUSED_TOKQ:-1}
- export RADIANCE_PQ_I8=${RADIANCE_PQ_I8:-0}
- export RADIANCE_PQ_PG=${RADIANCE_PQ_PG:-0}
- export RADIANCE_PQ_ZPE=${RADIANCE_PQ_ZPE:-0}
- export RADIANCE_PQ_PG_PRODUCER=${RADIANCE_PQ_PG_PRODUCER:-3}
- export RADIANCE_PQ_ROT_STREAM=${RADIANCE_PQ_ROT_STREAM:-1}
- export RADIANCE_PQ_ROT_STREAM2=${RADIANCE_PQ_ROT_STREAM2:-1}
- export RADIANCE_PQ_ROT_STREAM3=${RADIANCE_PQ_ROT_STREAM3:-0}
- export RADIANCE_PQ_AR_CHECK=${RADIANCE_PQ_AR_CHECK:-0}
- export RADIANCE_PQ_AR_FALLBACK=${RADIANCE_PQ_AR_FALLBACK:-0}
- export RADIANCE_PQ_ROT_V2=${RADIANCE_PQ_ROT_V2:-1}
- export RADIANCE_PQ_HIPCC_FLAGS=${RADIANCE_PQ_HIPCC_FLAGS:-}
- export RADIANCE_PQM_FUSED_TOKQ=${RADIANCE_PQM_FUSED_TOKQ:-1}
- export RADIANCE_PQM_SINGLE_LAUNCH=${RADIANCE_PQM_SINGLE_LAUNCH:-1}
- export RADIANCE_DFLASH_CAPTURE_DIR=${RADIANCE_DFLASH_CAPTURE_DIR:-}
-fi
-if [ "$IS_MXFP4" = "1" ]; then
- export RADIANCE_MXFP4=1 RADIANCE_MXFP4_W4A8=1 RADIANCE_MXFP4_W4A8_MIN_M="$MIN_M"
-else
- export RADIANCE_MXFP4=0 RADIANCE_MXFP4_W4A8=0 RADIANCE_USE_R4D_AR_QUANT=1
-fi
-export RADIANCE_FAST_DRAFT=${RADIANCE_FAST_DRAFT:-1}
-export RADIANCE_DRAFT_TAU=${RADIANCE_DRAFT_TAU:-0.20}
-export RADIANCE_DRAFT_RERANK
-export RADIANCE_VERIFY_HEAD
-export RADIANCE_VERIFY_HEAD_MAX_M=${RADIANCE_VERIFY_HEAD_MAX_M:-32}
-export RADIANCE_MXFP4_DEBUG=${RADIANCE_MXFP4_DEBUG:-0}
-export RADIANCE_MXFP4_PUREQUANT=${RADIANCE_MXFP4_PUREQUANT:-0}
-export RADIANCE_MXFP4_SYNC=${RADIANCE_MXFP4_SYNC:-0}
-export RADIANCE_MXFP4_CLONE=${RADIANCE_MXFP4_CLONE:-0}
-export RADIANCE_MXFP4_CHECKX=${RADIANCE_MXFP4_CHECKX:-0}
-export RADIANCE_MXFP4_PADOUT=${RADIANCE_MXFP4_PADOUT:-0}
-export RADIANCE_MXFP4_TN4_MIN_M=${RADIANCE_MXFP4_TN4_MIN_M:-2048}
-# The decode-kernel band must cover MAXSEQS x (SPEC+1) rows: 64 covers 8x8.
-if [ "${MAXSEQS}" -gt 8 ]; then
- export RADIANCE_MXFP4_DECODE_MAX_M=${RADIANCE_MXFP4_DECODE_MAX_M:-128}
-fi
-export RADIANCE_MXFP4_DECODE_NT=${RADIANCE_MXFP4_DECODE_NT:-1}
-export RADIANCE_MXFP4_A_TILED_MIN_M=${RADIANCE_MXFP4_A_TILED_MIN_M:-513}
-export RADIANCE_MXFP4_WPERM=${RADIANCE_MXFP4_WPERM:-1}
-export RADIANCE_TP_PAD=${RADIANCE_TP_PAD:-0} # TP=2 here; 3 is the only non-zero value upstream supports
-export RADIANCE_TP_PAD_DRAFTER=${RADIANCE_TP_PAD_DRAFTER:-1}
-export RADIANCE_TP_PAD_STRICT=${RADIANCE_TP_PAD_STRICT:-1}
-export RADIANCE_GDN_MERGE_INPROJ=${RADIANCE_GDN_MERGE_INPROJ:-1}
-export RADIANCE_GDN_NORM_QUANT=${RADIANCE_GDN_NORM_QUANT:-1}
-export RADIANCE_GDN_STRIDED_GATES=${RADIANCE_GDN_STRIDED_GATES:-0}
-export RADIANCE_GDN_EMPTY_OUT=${RADIANCE_GDN_EMPTY_OUT:-0}
-export R4D_ATTN_FP8=${R4D_ATTN_FP8:-3}
-export RADIANCE_DFLASH_CALIB="${RADIANCE_DFLASH_CALIB:-}"
-export RADIANCE_DFLASH_CALIB_TOKENS="${RADIANCE_DFLASH_CALIB_TOKENS:-200000}"
-export RADIANCE_MXFP4_SHADOW="${RADIANCE_MXFP4_SHADOW:-}"
-export RADIANCE_MXFP4_KERNEL_N="${RADIANCE_MXFP4_KERNEL_N:-}"
-export RADIANCE_MXFP4_KERNEL_NK="${RADIANCE_MXFP4_KERNEL_NK:-}"
-export RADIANCE_MXFP4_CHECKALL="${RADIANCE_MXFP4_CHECKALL:-}"
-export RADIANCE_MXFP4_DECODE_KS="${RADIANCE_MXFP4_DECODE_KS:-}"
-export RADIANCE_MXFP4_DECODE_BK="${RADIANCE_MXFP4_DECODE_BK:-}"
-export RADIANCE_MXFP4_PERBLOCK_NK="${RADIANCE_MXFP4_PERBLOCK_NK:-}"
-export RADIANCE_MXFP4_REFLINEAR="${RADIANCE_MXFP4_REFLINEAR:-0}"
-export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-}"
-export RADIANCE_MXFP4_EPIFAST="${RADIANCE_MXFP4_EPIFAST:-1}"
-export RADIANCE_AR_OVERLAP=${RADIANCE_AR_OVERLAP:-0}
-export RADIANCE_GDN_FUSED_UPDATE=${RADIANCE_GDN_FUSED_UPDATE:-1}
-export RADIANCE_DYNAMIC_WIDTH=${RADIANCE_DYNAMIC_WIDTH:-1}
-export RADIANCE_DYNW_ALPHA=${RADIANCE_DYNW_ALPHA:-0.35}
-export RADIANCE_DYNW_MARGIN=${RADIANCE_DYNW_MARGIN:-2}
-export RADIANCE_DYNW_MIN=${RADIANCE_DYNW_MIN:-2}
-export RADIANCE_DYNW_MIN_BATCH=${RADIANCE_DYNW_MIN_BATCH:-3}
-export RADIANCE_AR_QNB=${RADIANCE_AR_QNB:-96}
-export RADIANCE_AR_QNT=${RADIANCE_AR_QNT:-1024}
-export MXFP4_CUMODE=${MXFP4_CUMODE:-0}
-export RADIANCE_STEP_TRACE=${RADIANCE_STEP_TRACE:-0}
-export RADIANCE_AR_OVERLAP_MIN_M=${RADIANCE_AR_OVERLAP_MIN_M:-2048}
-export RADIANCE_AR_OVERLAP_SLICES=${RADIANCE_AR_OVERLAP_SLICES:-4}
-export RADIANCE_MXFP4_EPIFAST=${RADIANCE_MXFP4_EPIFAST:-1}
-export RADIANCE_MXFP4_R4D_DECODE_MAX_M=${RADIANCE_MXFP4_R4D_DECODE_MAX_M:-0}
-export RADIANCE_TOPK_TRITON_MIN_ROWS=${RADIANCE_TOPK_TRITON_MIN_ROWS:-1}
-export RADIANCE_SKINNY_GEMM=${RADIANCE_SKINNY_GEMM:-1}
-export RADIANCE_MXFP4_HOIST_QUANT=${RADIANCE_MXFP4_HOIST_QUANT:-1}
-export RADIANCE_MXFP4_TRACED_QUANT=${RADIANCE_MXFP4_TRACED_QUANT:-1}
-export RADIANCE_FP8_STREAM=${RADIANCE_FP8_STREAM:-1}
-export RADIANCE_RMS_QUANT_FUSION=${RADIANCE_RMS_QUANT_FUSION:-1}
-export RADIANCE_NVFP4_MXFP4=${RADIANCE_NVFP4_MXFP4:-0}
-export RADIANCE_NVFP4_EXP=${RADIANCE_NVFP4_EXP:-mse}
-export RADIANCE_NVFP4_FP8_LAYERS=${RADIANCE_NVFP4_FP8_LAYERS:-mxfp4}
-export RADIANCE_NVFP4_BF16_LAYERS=${RADIANCE_NVFP4_BF16_LAYERS:-in_proj_ba}
-export RADIANCE_NVFP4_LMHEAD=${RADIANCE_NVFP4_LMHEAD:-bf16}
-export RADIANCE_GDN_PATHS=${RADIANCE_GDN_PATHS:-both}
-export RADIANCE_GDN_NANTRACE=${RADIANCE_GDN_NANTRACE:-0}
-export RADIANCE_MXFP4_CHECK_MAX_M=${RADIANCE_MXFP4_CHECK_MAX_M:-128}
-export RADIANCE_MXFP4_MHIST=${RADIANCE_MXFP4_MHIST:-0}
-export VLLM_CACHE_ROOT=/cache/vllm
-export TORCHINDUCTOR_CACHE_DIR=/cache/inductor
-export TRITON_CACHE_DIR=/cache/triton
-export AITER_ROOT_DIR=/cache/aiter
-export TRITON_CACHE_AUTOTUNING=1
-# The patched libr4d built by setup-mxfp4.sh, bind-mounted at /r4d.
-R4D_SO=${R4D_SO:-/r4d}
-
-# The norm+quant fusion turns on vLLM's passes (one --compilation-config
-# instance; argparse keeps the last, so entries accumulate into one flag).
-CC_ITEMS=""
-if [ "${RADIANCE_NORMQUANT_FUSION:-1}" != "0" ]; then
- CC_ITEMS="\"pass_config\":{\"fuse_norm_quant\":true,\"fuse_act_quant\":true}"
-fi
-EXTRA=${EXTRA:-}
-if [ -n "$CC_ITEMS" ]; then
- EXTRA="$EXTRA --compilation-config {$CC_ITEMS}"
-fi
-
-if [ "$IS_MXFP4" = "1" ]; then
-# ---------------------------------------------------------------- patch block
-# Byte-for-byte the launcher's container-start block. The patches come from
-# the /patches mount (the pinned radiance clone). SP must match the image's
-# venv layout; the image tag is pinned in the quadlet. The mxfp6 branch
-# mirrors paroquant/run_paroquant.sh (17 patches, 10 modules, + the paroquant
-# kernel module + the sitecustomize registration); the mxfp4 branch keeps the
-# existing 20-patch / 11-module list untouched.
-SP=/opt/vllm/lib/python3.12/site-packages
-cd /patches
-if [ "$IS_MXFP6" = "1" ]; then
- python3 patch_quark_mxfp4.py
- python3 patch_ar_maxbytes.py
- python3 patch_topk_triton_rows.py
- python3 patch_dflash_calib.py
- python3 patch_dflash_mxfp4_kv.py
- python3 patch_rmsquant_fusion.py
- python3 patch_verify_head.py
- python3 patch_kv_group_size.py
- python3 patch_topk_composite.py
- python3 patch_gdn_shared_build.py
- python3 patch_async_dynwidth.py
- python3 patch_step_trace.py
- python3 patch_skinny_gemm.py
- python3 patch_dflash_selector_topk.py
- python3 patch_dynwidth.py
- python3 patch_ar_geometry.py
- python3 patch_gdn_merge_inproj.py
- # Non-fatal: fixes content=null on thinking-off requests; not required to serve.
- python3 patch_qwen3_thinkoff.py \
- || echo "[radiance] WARNING: thinkoff patch did not apply; thinking-off requests will return empty content"
- cp mxfp4-configs/*.json "$SP"/aiter/ops/triton/configs/gemm/
- cp radiance_preamble.py /opt/radiance_preamble.py
- cp radiance_mxfp4.py radiance_gemm.py radiance_gdn.py radiance_gdnmerge.py radiance_rmsquant.py \
- radiance_drafthead.py radiance_verifyhead.py radiance_aroverlap.py radiance_topk.py \
- radiance_arnq.py "$SP"/
- hipcc -O3 -w -std=c++17 -fPIC -shared --offload-arch=gfx1201 $(python3 -m pybind11 --includes) \
- radiance_mxfp4_fp8.hip -o "$SP"/radiance_mxfp4_fp8.so
- # ---- paroquant: build the MXFP6 kernel module and register the quant method ----
- # RADIANCE_PQ_HIPCC_FLAGS: extra compile flags for A/B builds of the kernel
- # module only (e.g. -DPQ_HW_CVT=0). Empty in prod.
- hipcc -O3 -w -std=c++17 -fPIC -shared --offload-arch=gfx1201 $(python3 -m pybind11 --includes) \
- ${RADIANCE_PQ_HIPCC_FLAGS:-} paroquant/radiance_paroquant.hip -o "$SP"/radiance_paroquant_kernel.so
- cp paroquant/radiance_paroquant.py paroquant/radiance_paroquant_mxfp4.py "$SP"/
- cp radiance_dflash_capture.py "$SP"/ 2>/dev/null || true
- # NB: appended to the STDLIB sitecustomize, not site-packages -- Ubuntu ships
- # /usr/lib/python3.12/sitecustomize.py and it shadows any site-packages one, so
- # a file dropped there is silently never imported. Guarded: an unguarded >>
- # appends another copy on every restart of an existing container.
- if ! grep -q radiance_paroquant /usr/lib/python3.12/sitecustomize.py; then
- printf "%s\n" \
- "try:" \
- " import radiance_paroquant # registers the paroquant quantization config" \
- " import radiance_paroquant_mxfp4 # and the MXFP4-weights variant (paroquant_mxfp4)" \
- " import radiance_dflash_capture # drafter training-data capture (inert unless RADIANCE_DFLASH_CAPTURE_DIR)" \
- "except Exception as e:" \
- " import sys" \
- " sys.stderr.write(\"[radiance.paroquant] registration failed: %r\\n\" % (e,))" \
- >> /usr/lib/python3.12/sitecustomize.py
- fi
-else
- python3 patch_quark_mxfp4.py
- python3 patch_nvfp4_mxfp4.py
- python3 patch_tp3_pad.py
- python3 patch_ar_maxbytes.py
- python3 patch_topk_triton_rows.py
- python3 patch_dflash_calib.py
- python3 patch_dflash_mxfp4_kv.py
- python3 patch_rmsquant_fusion.py
- python3 patch_verify_head.py
- python3 patch_kv_group_size.py
- python3 patch_topk_composite.py
- python3 patch_gdn_shared_build.py
- python3 patch_dflash_selector_topk.py
- python3 patch_gdn_merge_inproj.py
- python3 patch_dynwidth.py
- python3 patch_async_dynwidth.py
- python3 patch_step_trace.py
- python3 patch_ar_geometry.py
- python3 patch_ar_3rank.py
- python3 patch_gdn_glue.py
- # Non-fatal: fixes content=null on thinking-off requests; not required to serve.
- python3 patch_qwen3_thinkoff.py \
- || echo "[radiance] WARNING: thinkoff patch did not apply; thinking-off requests will return empty content"
- cp mxfp4-configs/*.json "$SP"/aiter/ops/triton/configs/gemm/
- cp radiance_preamble.py /opt/radiance_preamble.py
- cp radiance_nvfp4.py radiance_mxfp4.py radiance_gdn.py radiance_rmsquant.py radiance_drafthead.py \
- radiance_verifyhead.py radiance_gdnmerge.py radiance_aroverlap.py radiance_topk.py \
- radiance_arnq.py radiance_tp3pad.py "$SP"/
- cp radiance_dflash_capture.py "$SP"/ 2>/dev/null || true
- hipcc -O3 -w -std=c++17 -fPIC -shared --offload-arch=gfx1201 $([ "${MXFP4_CUMODE:-0}" = 1 ] && echo -mcumode) \
- $(python3 -m pybind11 --includes) radiance_mxfp4_fp8.hip -o "$SP"/radiance_mxfp4_fp8.so
-fi
-if [ -n "${R4D_SO:-}" ] && [ -f /r4d/r4d.so ]; then
- cp /r4d/r4d.so "$SP"/r4d.so
- echo "[radiance] using patched r4d.so from $R4D_SO"
-fi
-# Leave /patches before exec: a stale radiance_mxfp4_fp8.so there would shadow
-# the one just compiled into site-packages (cwd precedes it on sys.path).
-cd /
-fi
-# The weight-quantization flag. MXFP4 and paroquant_mxfp6 are auto-detected
-# from the checkpoint's quant_method; FP8 is passed explicitly (the documented
-# plain-FP8 path).
-QUANT_FLAG=""
-if [ "$IS_MXFP4" = "0" ]; then QUANT_FLAG="--quantization fp8"; fi
-exec /opt/radiance_entrypoint.sh "$SNAP" --served-model-name $SERVED_NAMES \
- --host 0.0.0.0 --port "$PORT" \
- --kv-cache-dtype "$KV_DTYPE" --tensor-parallel-size "$TP" \
- $QUANT_FLAG \
- --gpu-memory-utilization "$GPU_UTIL" \
- ${KV_MEM:+--kv-cache-memory "$KV_MEM"} \
- --max-model-len "$MAXLEN" --max-num-seqs "$MAXSEQS" --max-num-batched-tokens "$CHUNK" \
- --attention-backend "$ATTN" \
- --speculative-config "$SPEC_CFG" \
- $ASYNC_FLAG $EXTRA \
- --enable-prefix-caching --mamba-cache-mode align --enable-auto-tool-choice \
- --tool-call-parser qwen3_coder --reasoning-parser qwen3 \
- --override-generation-config '{"temperature":0.7,"top_p":0.95,"top_k":20}' \
- --chat-template "$CHAT_TEMPLATE"
diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py
index 9475e36..4c289f6 100755
--- a/scripts/deploy-plan.py
+++ b/scripts/deploy-plan.py
@@ -63,7 +63,6 @@ SERVICE_ORDER = (
"readarr",
"frank-large",
"frank-small",
- "frank-medium",
"frank-primary",
)
@@ -114,7 +113,6 @@ DANKSERVER_SERVICES = {
"sonarr",
"lidarr",
"readarr",
- "frank-medium",
"frank-primary",
"node-exporter",
}
@@ -373,16 +371,6 @@ def services_for_path(path: str, target: str) -> set[str]:
if name == "node-exporter":
_add(services, "node-exporter")
return services
- if len(parts) >= 2 and parts[1] == "frank-medium":
- # frank-medium is a CONTAINER again since 2026-09-20: the TP=2
- # radiance Quadlet, its entrypoint and its healthcheck all live
- # under this directory, and the container mounts the latter two
- # straight out of this checkout — so any file here changes deployed
- # state. (It was a root llama.cpp unit from 2026-09-16, whose trio
- # lived in scripts/; that shape is archived in
- # scripts/.AGENTTRASH/20260920-llama-layersplit/ and plans nothing.)
- _add(services, "frank-medium")
- return services
if len(parts) >= 2 and parts[1] == "frank-primary":
# frank-primary is the Lucebox DeepSeek V4 Quadlet (2026-09-26
# cutover, replacing frank-medium on :7770): its entrypoint and
@@ -426,6 +414,13 @@ def services_for_path(path: str, target: str) -> set[str]:
# runtime. The download script fetches the payload, which is
# host-side state, so it selects nothing.
_add(services, "frank-large")
+ if name == "qwen-sharp-v22.4.0.jinja":
+ # The Qwen-Sharp chat template frank-large loads via
+ # --chat-template-file. It lived under dankserver/frank-medium/ until
+ # the 2026-09-26 cutover retired that directory; editing it changes
+ # what the live frank-large endpoint renders, so it plans
+ # frank-large.
+ _add(services, "frank-large")
# Deployment/test/benchmark scripts and other repo tooling do not
# restart a runtime service.
return services
@@ -435,7 +430,6 @@ def services_for_path(path: str, target: str) -> set[str]:
return services
return services
-
def changed_paths(repo: str, base: str, head: str) -> list[str]:
result = subprocess.run(
["git", "-C", repo, "diff", "--name-only", "--diff-filter=ACDMRTUXB", base, head],
diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh
index 6e1fa0c..7e309ea 100755
--- a/scripts/deploy-remote.sh
+++ b/scripts/deploy-remote.sh
@@ -95,7 +95,7 @@ service_is_allowed() {
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
- dankserver:frank-medium|dankserver:frank-primary|dankserver:node-exporter|\
+ dankserver:frank-primary|dankserver:node-exporter|\
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
frankbook-pro:readarr)
@@ -1178,159 +1178,6 @@ deploy_llm_small() {
wait_for_health frank-small http://127.0.0.1:7771/health 900
}
-# deploy_llm_medium — frank-medium is the vllm-radiance stack as a
-# rootless Quadlet since the 2026-09-20 cutover: one tensor-parallel (TP=2)
-# engine across both R9700s, R4D one-shot all-reduce, DFlash2-FP8 drafter.
-# The entrypoint is quantization-aware (config.json quant_method): the plain
-# FP8 path, the MXFP4 patch-and-serve path, and the ParoQuant MXFP6 stack
-# (PR #51's launch6_p kernel). It replaced mainline llama.cpp's layer split,
-# which held the endpoint since 2026-09-16; the conversation battery measured
-# radiance at 2.1x decode and a third of the time-to-first-token with 4x the
-# concurrent throughput (scripts/bench-conversation.py, 2026-09-20).
-#
-# WHY THE PAIR IS BACK AND THE TRIO IS NOT: the 2026-09-16 cutover retired this
-# unit because TP=2 could not dispatch a collective on the second card (B650
-# chipset bridges, AtomicOpsCap: Routing-), and the two-replica + routing-proxy
-# trio was the workaround. The board swap cleared the atomics wall, so one
-# TP=2 process is both possible and the fastest shape; the replicas and their
-# affinity proxy stay retired.
-#
-# HOST ASSETS. Nothing here installs the image's payloads: the checkpoints, the
-# pinned radiance patch clone, the patched libr4d build and the Triton compile
-# cache live outside the checkout (see the unit file's mount comments). The
-# gates below refuse a deploy that would start a container guaranteed to die.
-LLM_MEDIUM_IMAGE="${LLM_MEDIUM_IMAGE:-ggz14/vllm-radiance-mxfp4:0.13.0-e060895}"
-LLM_MEDIUM_IMAGE_LOCAL="${LLM_MEDIUM_IMAGE_LOCAL:-localhost/radiance-vllm:0.13.0-e060895}"
-LLM_MEDIUM_SNAP="${LLM_MEDIUM_SNAP:-/home/dank/models/Qwen3.8-27B-PARO-MXFP6}"
-LLM_MEDIUM_DRAFTER="${LLM_MEDIUM_DRAFTER:-/home/dank/models/Qwen3.8-27B-DFlash2-FP8}"
-LLM_MEDIUM_PATCHES="${LLM_MEDIUM_PATCHES:-/home/dank/GitPackages/radiance-vllm-mxfp4}"
-LLM_MEDIUM_R4D="${LLM_MEDIUM_R4D:-/home/dank/.cache/radiance-libr4d/b9e42ab-rx6}"
-LLM_MEDIUM_CACHE="${LLM_MEDIUM_CACHE:-/home/dank/.radiance-cache-mxfp6-013-pq-nqft-fp8s-gnq}"
-
-# The three replica/proxy user units from the 2026-09-16 era are retired HERE,
-# never left to rot: they must not exist alongside the TP=2 unit (the proxy
-# published 7770 on the host network). Never `rm` — unit files move into the
-# home trash; the images stay in local storage. `frank-medium` itself is NOT in
-# the list: it is the unit this handler installs.
-retire_vllm_replica_quadlets() {
- local trash="$HOME/.AGENTTRASH/20260916-vllm-radiance-retire" unit
- systemctl --user stop frank-medium-a.service frank-medium-b.service \
- frank-medium-proxy.service 2>/dev/null || true
- for unit in frank-medium-a frank-medium-b frank-medium-proxy; do
- if [[ -e "$quadlet_unit_dir/$unit.container" ]]; then
- mkdir -p "$trash"
- mv -- "$quadlet_unit_dir/$unit.container" "$trash/$unit.container"
- echo "retired quadlet $unit.container -> $trash"
- fi
- done
- systemctl --user daemon-reload
-}
-
-# The llama.cpp root unit is retired HERE, into the home trash, never `rm`:
-# /etc/systemd/system/frank-medium.service is a relink_unit symlink into this
-# checkout, and the checkout pull that precedes this handler already moved the
-# repo file away, leaving the symlink dangling. Stop it while systemd still has
-# it loaded (an active unit stays loaded without its file), then take both the
-# symlink and its [Install] alias. Nothing can start it at boot afterwards.
-retire_llama_medium_unit() {
- local trash="$HOME/.AGENTTRASH/20260920-llama-layersplit-frank-medium"
- run_privileged systemctl stop frank-medium.service 2>/dev/null || true
- if [[ -L /etc/systemd/system/frank-medium.service ||
- -L /etc/systemd/system/multi-user.target.wants/frank-medium.service ]]; then
- mkdir -p "$trash"
- [[ -L /etc/systemd/system/frank-medium.service ]] &&
- run_privileged mv -- /etc/systemd/system/frank-medium.service \
- "$trash/etc-frank-medium.service.symlink"
- [[ -L /etc/systemd/system/multi-user.target.wants/frank-medium.service ]] &&
- run_privileged mv -- /etc/systemd/system/multi-user.target.wants/frank-medium.service \
- "$trash/etc-frank-medium.wants.symlink"
- echo "retired llama root unit frank-medium.service -> $trash"
- fi
- run_privileged systemctl daemon-reload
-}
-
-deploy_llm_medium() {
- # Refuse rather than restart-loop. A container whose image or payload is
- # missing never becomes healthy, and a green deploy would leave the endpoint
- # dark with nothing in the deploy log to say why.
- local missing=0
- # Model-aware gates: read the serving shape from the quadlet (the source
- # of truth), so a model swap is just "edit the quadlet + deploy". The
- # container /models mount maps to /home/dank/models on the host. The
- # drafter gate applies only to the dflash path; the patched libr4d gate
- # applies to the mxfp4/quark/paroquant_mxfp6 path (the plain FP8 path uses
- # the image's built-in r4d.so and skips the MXFP4 patches).
- local quadlet_file="dankserver/frank-medium/quadlet/frank-medium.container"
- local snap_c snap spec_method quant
- snap_c="$(sed -nE 's/^Environment=SNAP=//p' "$quadlet_file" | head -1)"
- spec_method="$(sed -nE 's/^Environment=SPEC_METHOD=//p' "$quadlet_file" | head -1)"
- snap="/home/dank/models${snap_c#/models}"
- quant="$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("quantization_config",{}).get("quant_method",""))' "$snap/config.json" 2>/dev/null || true)"
- [[ -n "$snap_c" ]] || { echo "ERROR: cannot read SNAP from $quadlet_file" >&2; return 1; }
- if ! podman image exists "$LLM_MEDIUM_IMAGE_LOCAL"; then
- if podman image exists "$LLM_MEDIUM_IMAGE"; then
- podman tag "$LLM_MEDIUM_IMAGE" "$LLM_MEDIUM_IMAGE_LOCAL"
- else
- echo "ERROR: radiance image absent ($LLM_MEDIUM_IMAGE / $LLM_MEDIUM_IMAGE_LOCAL)" >&2
- echo "frank-medium: podman pull $LLM_MEDIUM_IMAGE && podman tag $LLM_MEDIUM_IMAGE $LLM_MEDIUM_IMAGE_LOCAL" >&2
- missing=1
- fi
- fi
- if [[ ! -f "$snap/config.json" ]]; then
- echo "ERROR: model checkpoint absent at $snap" >&2
- missing=1
- fi
- if [[ "$spec_method" == "dflash" && ! -f "$LLM_MEDIUM_DRAFTER/config.json" ]]; then
- echo "ERROR: DFlash2-FP8 drafter absent at $LLM_MEDIUM_DRAFTER" >&2
- missing=1
- fi
- # The patches clone is mounted and its scripts run at EVERY container start
- # on the MXFP4 path; the chat template is the fleet's reasoning-format
- # contract on both paths.
- if [[ ! -f "$LLM_MEDIUM_PATCHES/qwen-fixed-v22.3.jinja" ]]; then
- echo "ERROR: pinned patch clone / chat template absent at $LLM_MEDIUM_PATCHES" >&2
- echo "frank-medium: git clone https://github.com/ggz14/radiance-vllm-mxfp4 $LLM_MEDIUM_PATCHES" >&2
- missing=1
- fi
- # The image's shipped r4d.so predates the gated-delta-net overflow fix and
- # NaNs the MXFP4 model, so the patched build is not optional there. The
- # plain FP8 path uses the image's built-in r4d.so (no MXFP4 patches).
- case "$quant" in
- mxfp4|quark|paroquant_mxfp6)
- if [[ ! -f "$LLM_MEDIUM_R4D/r4d.so" ]]; then
- echo "ERROR: patched libr4d absent at $LLM_MEDIUM_R4D" >&2
- missing=1
- fi
- ;;
- esac
- [[ "$missing" -eq 0 ]] || return 1
- mkdir -p "$LLM_MEDIUM_CACHE"
-
- retire_vllm_replica_quadlets
- retire_llama_medium_unit
- # 7770 is a host-network bind: nothing new can be created until the old
- # holder released it (the same gate as the 2026-09-16 cutover, where the
- # retired proxy held the port and the new engine failed at create). Sweep a
- # stray manual llama-server too — that is how a bench harness leaves the
- # port busy. `pkill -x` matches the process name exactly and can never
- # match the deploying shell; never `pkill -f` here.
- pkill -x llama-server 2>/dev/null || true
- pkill -x VLLM::EngineCor 2>/dev/null || true
- sleep 2
- wait_port_free 7770 || return 1
-
- install_quadlet frank-medium dankserver/frank-medium/quadlet frank-medium
- # No enable: generated units cannot be enabled on this host (v1 quadlet
- # generator); boot persistence is the unit's [Install] WantedBy=default.target.
- systemctl --user restart frank-medium.service
- # Triton/inductor compile on a cold cache runs minutes before the engine
- # loads ~9.5 GiB per card; HealthStartPeriod in the unit is 1800s to match.
- wait_container_healthy frank-medium 1800 || {
- podman logs --tail 40 frank-medium 2>&1 || true
- return 1
- }
-}
-
# deploy_llm_primary — frank-primary is the Lucebox DeepSeek V4 Quadlet
# (the 2026-09-26 cutover that replaces frank-medium on :7770). The model
# is a 377 GiB GGUF and the image is a local build, so the gates are:
@@ -1356,13 +1203,12 @@ deploy_llm_primary() {
fi
[[ "$missing" -eq 0 ]] || return 1
- # Cutover: frank-primary takes :7770 from frank-medium. Stop and mask
- # the radiance unit (masking is mandatory: Restart=on-failure would
- # restart it and re-take the port). Retiring the radiance unit (moving
- # it to the home trash) happens after frank-primary is verified — this
- # handler only releases the port.
- systemctl --user stop frank-medium 2>/dev/null || true
- systemctl --user mask frank-medium
+ # Cutover COMPLETED 2026-09-26: frank-primary took :7770 and the radiance
+ # unit is retired. The host-side unit, its mask and the repo files were all
+ # moved to trash by the parent session, so there is nothing left to stop or
+ # mask here — masking a name that no longer exists would only resurrect it
+ # in the unit search path. The port gate stays, because 7770 is a
+ # host-network bind: nothing new can be created until the holder released it.
# 7770 is a host-network bind: nothing new can be created until the old
# holder released it (the same gate as the 2026-09-16 cutover).
sleep 2
@@ -1566,7 +1412,6 @@ deploy_service() {
marreta) deploy_marreta ;;
frank-large) deploy_llm_large ;;
frank-small) deploy_llm_small ;;
- frank-medium) deploy_llm_medium ;;
frank-primary) deploy_llm_primary ;;
hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
diff --git a/scripts/deploy.sh b/scripts/deploy.sh
index 72daada..efcc350 100755
--- a/scripts/deploy.sh
+++ b/scripts/deploy.sh
@@ -35,7 +35,7 @@ Options:
Examples:
./scripts/deploy.sh dankclaw
./scripts/deploy.sh dankclaw --service searxng
- ./scripts/deploy.sh dankserver --service frank-medium
+ ./scripts/deploy.sh dankserver --service frank-primary
./scripts/deploy.sh dankclaw --dry-run
EOF
}
@@ -146,7 +146,7 @@ valid_service() {
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
dankserver:qbittorrent|dankserver:radarr|dankserver:sonarr|\
dankserver:lidarr|dankserver:readarr|dankserver:node-exporter|\
- dankserver:frank-medium|dankserver:frank-primary|\
+ dankserver:frank-primary|\
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
frankbook-pro:readarr)
diff --git a/scripts/download-frank-medium-gguf.sh b/scripts/download-frank-medium-gguf.sh
deleted file mode 100755
index d7840a4..0000000
--- a/scripts/download-frank-medium-gguf.sh
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/usr/bin/env bash
-# Fetch the frank-medium payload: the target GGUF, the f16 vision projector and
-# the DFlash2 draft model.
-#
-# bash scripts/download-frank-medium-gguf.sh
-# bash scripts/download-frank-medium-gguf.sh --verify-only
-#
-# The target repo (orcarouter) is GATED: anonymous requests get a 401, so a
-# token is REQUIRED for it. The script reads ~/.cache/huggingface/token
-# (mode 600) or the HF_TOKEN environment variable, and passes it to curl
-# out-of-band via a mode-600 -K file (a -H header is world-readable in
-# /proc//cmdline -- the 2026-09-16 HF-token leak). The drafter repo
-# (z-lab) is public; the same token is used for it when present.
-#
-# WHY THESE FILES:
-# * Target: orcarouter/Qwen3.8-27B-Uncensored-GGUF at Q8_0 -- the
-# operator's requested quant (2026-09-19). Arch qwen35, 262144 trained
-# window, and the file carries the model's MTP layer (blk.64.nextn.*), so
-# llama.cpp can also draft from the model itself with no second copy in
-# VRAM.
-# * Draft: z-lab/Qwen3.8-27B-DFlash2-GGUF at Q4_K_M -- the block-diffusion
-# drafter (5 layers, hidden 5120, block_size 8, a 2048-token sliding
-# window). Q4_K_M is 1.06 GiB against Q8_0's 1.92 GiB and the drafter's
-# job is throughput, not output quality (the target verifies every drafted
-# token).
-#
-# The script is resumable (curl -C -) and checks every artifact's exact byte
-# count AND sha256, so a truncated or mis-fetched file fails here instead of
-# load-crashing the server.
-set -euo pipefail
-
-MODEL_REPO="orcarouter/Qwen3.8-27B-Uncensored-GGUF"
-MODEL_DIR="/home/dank/models/Qwen3.8-27B-Uncensored-GGUF"
-MODEL_FILE="Qwen3.8-27B-Uncensored-Q8_0.gguf"
-MODEL_SIZE=29047084416
-MODEL_SHA256=31756fca94beca71ea4b8706d6fdc896dab2a3c6376ab0c1863b98512a24f8d6
-# The f16 vision projector from the same repo; the serve script names it via
-# --mmproj (no autoload in this build), so it must land in MODEL_DIR.
-MM_PROJ_FILE="mmproj-Qwen3.8-27B-Uncensored-f16.gguf"
-MM_PROJ_SIZE=931145984
-MM_PROJ_SHA256=add205b7bfdb3f71f6da36b0a82aa20928dd829a920878c602628cdfbebc5288
-
-DRAFT_REPO="z-lab/Qwen3.8-27B-DFlash2-GGUF"
-DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DFlash2-GGUF"
-DRAFT_FILE="Qwen3.8-27B-DFlash2-Q4_K_M.gguf"
-DRAFT_SIZE=1143006816
-DRAFT_SHA256=1a25c56858e1ebe93f2718ac1d49d1151f9323325c1bbfd6209370f4db131ebd
-
-mode="download"
-[[ "${1:-}" == "--verify-only" ]] && mode="verify"
-
-token="${HF_TOKEN:-}"
-if [[ -z "$token" && -r "$HOME/.cache/huggingface/token" ]]; then
- token="$(cat "$HOME/.cache/huggingface/token")"
-fi
-
-auth_file=""
-cleanup() { [[ -n "$auth_file" ]] && rm -f "$auth_file"; }
-trap cleanup EXIT
-
-dl() { # dl
- mkdir -p "$2"
- echo " fetching $3"
- local args=(-fL -C - -o "$2/$3" --retry 10 --retry-delay 5 --retry-max-time 21600)
- [[ -n "$auth_file" ]] && args+=(-K "$auth_file")
- curl "${args[@]}" "https://huggingface.co/$1/resolve/main/$3"
-}
-
-if [[ "$mode" == "download" ]]; then
- if [[ -z "$token" ]]; then
- echo "ERROR: the target repo is gated and no HF token is available" >&2
- echo " (set HF_TOKEN or write it to ~/.cache/huggingface/token, mode 600)" >&2
- exit 1
- fi
- auth_file="$(mktemp)"
- printf 'header = "Authorization: Bearer %s"\n' "$token" > "$auth_file"
- chmod 600 "$auth_file"
-
- echo "target: $MODEL_REPO (gated)"
- dl "$MODEL_REPO" "$MODEL_DIR" "$MODEL_FILE"
- echo "projector: $MODEL_REPO (gated)"
- dl "$MODEL_REPO" "$MODEL_DIR" "$MM_PROJ_FILE"
- echo "draft: $DRAFT_REPO"
- dl "$DRAFT_REPO" "$DRAFT_DIR" "$DRAFT_FILE"
-fi
-
-echo "verifying sizes and sha256"
-ok=1
-check() { # check
- local act act_sha
- act="$(stat -c%s "$1" 2>/dev/null || echo 0)"
- if [[ "$act" -ne "$2" ]]; then
- echo " BAD-SIZE $1 ($act bytes, want $2)" >&2
- ok=0
- return
- fi
- act_sha="$(sha256sum "$1" | cut -d' ' -f1)"
- if [[ "$act_sha" != "$3" ]]; then
- echo " BAD-SHA $1 ($act_sha)" >&2
- ok=0
- return
- fi
- echo " OK $1 ($act bytes)"
-}
-check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE" "$MODEL_SHA256"
-check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE" "$MM_PROJ_SHA256"
-check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE" "$DRAFT_SHA256"
-[[ "$ok" -eq 1 ]] || exit 1
-echo "all artifacts present, sized and hashed correctly"
diff --git a/scripts/frank-medium-fp8.manifest b/scripts/frank-medium-fp8.manifest
deleted file mode 100644
index d08843d..0000000
--- a/scripts/frank-medium-fp8.manifest
+++ /dev/null
@@ -1,20 +0,0 @@
-# orcarouter/Qwen3.8-27B-Uncensored-FP8 — FP8 E4M3 block-wise (128x128), dynamic act-quant
-# 7 safetensors shards + config/tokenizer/processor. Sizes + LFS sha256 from the
-# HF tree API and X-Linked-ETag (2026-09-20).
-config.json 51350
-generation_config.json 202
-chat_template.jinja 8952
-merges.txt 3353259
-preprocessor_config.json 390
-tokenizer.json 12809320
-tokenizer_config.json 17928
-video_preprocessor_config.json 385
-vocab.json 6722759
-model.safetensors.index.json 156034
-model-00001-of-00007.safetensors 3969317448 141e200c1950c59bc42ae28744f50db8f6ea4c66ec3312b55f1ccad2b60af858
-model-00002-of-00007.safetensors 4939321704 e863e88f53ea5a737581638e41334e9f9871a93053c4251a2cebd4aa6aa1f7c0
-model-00003-of-00007.safetensors 4939863424 7cbf8db425114b9efcfae64364a60e15e223cc5cadb510b7a3a6d3b48c8617cf
-model-00004-of-00007.safetensors 4971328696 37338dc30b490663b641567a57565408fee5f1385e7860dff16dca7be07e1519
-model-00005-of-00007.safetensors 4955596064 fc6fb0dd3675e0cc7e83a454376d12b705a51000f6625cf13f85e384018fc5cb
-model-00006-of-00007.safetensors 4071440632 5fdcbaf80805a750970e076bcd37e6ebe4c3a3d93ce73c767d2854119947952a
-model-00007-of-00007.safetensors 3019999168 1bed352a0625d22dc606de47fd3563f24fd9fa73915c4ddffe7947c4b56c28bf
diff --git a/dankserver/frank-medium/qwen-sharp-v22.4.0.jinja b/scripts/qwen-sharp-v22.4.0.jinja
similarity index 100%
rename from dankserver/frank-medium/qwen-sharp-v22.4.0.jinja
rename to scripts/qwen-sharp-v22.4.0.jinja
diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh
index bfbd20a..0c518fa 100755
--- a/scripts/serve-frank-large.sh
+++ b/scripts/serve-frank-large.sh
@@ -49,7 +49,7 @@ fi
: "${PARALLEL_SLOTS:=1}"
: "${SPEC_DRAFT_N_MAX:=4}"
: "${CACHE_RAM:=8192}"
-TEMPLATE_FILE="${TEMPLATE_FILE:-$SCRIPT_DIR/../dankserver/frank-medium/qwen-sharp-v22.4.0.jinja}"
+TEMPLATE_FILE="${TEMPLATE_FILE:-$SCRIPT_DIR/qwen-sharp-v22.4.0.jinja}"
SERVER_BIN="$RUNTIME_DIR/llama-server"
MODEL_FILE="$MODEL_DIR/Qwen3.8-27B-UD-Q8_K_XL.gguf"
diff --git a/scripts/swap-frank-medium.sh b/scripts/swap-frank-medium.sh
deleted file mode 100755
index 61eb040..0000000
--- a/scripts/swap-frank-medium.sh
+++ /dev/null
@@ -1,124 +0,0 @@
-#!/usr/bin/env bash
-# swap-frank-medium.sh — orchestrate a frank-medium model swap.
-#
-# A swap is four steps, and this script drives all of them:
-# 1. generate a manifest for the new HF repo (hf-manifest.py),
-# 2. download it to the host, resumable + byte/ sha256-verified (hf-fetch.sh),
-# 3. detect the checkpoint's weight quantization and point the quadlet at it
-# (SNAP + the matching SPEC_METHOD/SPEC + KV_MEM),
-# 4. hand off to deploy.sh (pull, restart, health gate).
-#
-# The entrypoint is QUANTIZATION-AWARE: it reads quant_method from the
-# checkpoint's config.json and serves the plain FP8 path (no MXFP4 patches,
-# --quantization=fp8, ROCM_AITER unified attention, MTP self-speculation) or
-# the MXFP4 patch-and-serve path (DFlash2 external drafter). So the ONLY repo
-# change a swap makes is the quadlet's four env vars — this script edits them.
-#
-# Usage:
-# scripts/swap-frank-medium.sh --repo --target \
-# [--generate-manifest] [--manifest ] [--spec N] [--kv MB] \
-# [--apply] [--deploy]
-#
-# --repo the Hugging Face repo to serve
-# --target the host dir the model lands in
-# (/home/dank/models/); the quadlet SNAP is
-# derived as /models/
-# --generate-manifest write scripts/frank-medium-.manifest from the
-# live repo tree (needs the host's HF token)
-# --manifest use an existing manifest instead of generating one
-# --spec N speculative tokens (default 8 for fp8, 7 for mxfp4)
-# --kv MB KV pool pin in MiB (default 0 = auto-size)
-# --apply edit the quadlet (SNAP + SPEC_METHOD + SPEC + KV_MEM)
-# --deploy run deploy.sh after --apply
-#
-# Without --apply the script stops after the download + quantization
-# detection and prints the quadlet lines to set. The download and the
-# quantization detection run on the host (the model dir and the token are
-# there); the quadlet edit runs in the repo.
-
-set -euo pipefail
-
-HOST="dank@192.168.0.235"
-QUADLET="dankserver/frank-medium/quadlet/frank-medium.container"
-MODELS_ROOT="/home/dank/models"
-
-repo="" target="" manifest="" spec="" kv=0 gen=0 apply=0 deploy=0
-
-while [ $# -gt 0 ]; do
- case "$1" in
- --repo) repo="$2"; shift 2 ;;
- --target) target="$2"; shift 2 ;;
- --manifest) manifest="$2"; shift 2 ;;
- --spec) spec="$2"; shift 2 ;;
- --kv) kv="$2"; shift 2 ;;
- --generate-manifest) gen=1; shift ;;
- --apply) apply=1; shift ;;
- --deploy) deploy=1; shift ;;
- *) echo "unknown arg: $1" >&2; exit 2 ;;
- esac
-done
-
-[[ -n "$repo" && -n "$target" ]] || {
- echo "usage: $0 --repo --target [--generate-manifest|--manifest ] [--spec N] [--kv MB] [--apply] [--deploy]" >&2
- exit 2;
-}
-
-name="$(basename "$target")"
-snap_c="/models/$name"
-
-# --- 1. manifest -------------------------------------------------------------
-if [[ "$gen" = 1 ]]; then
- manifest="scripts/frank-medium-${name}.manifest"
- echo "==> Generating manifest -> $manifest"
- scp -q scripts/hf-manifest.py "$HOST:/tmp/hf-manifest.py"
- ssh "$HOST" "python3 /tmp/hf-manifest.py '$repo' /tmp/swap-manifest"
- scp -q "$HOST:/tmp/swap-manifest" "$manifest"
-elif [[ -z "$manifest" ]]; then
- echo "ERROR: pass --generate-manifest or --manifest " >&2
- exit 2
-fi
-[[ -f "$manifest" ]] || { echo "ERROR: manifest not found: $manifest" >&2; exit 2; }
-
-# --- 2. download (resumable, verified) ---------------------------------------
-echo "==> Staging manifest + hf-fetch.sh on $HOST"
-scp -q "$manifest" "$HOST:/tmp/swap-manifest"
-scp -q scripts/hf-fetch.sh "$HOST:/tmp/swap-hf-fetch.sh"
-echo "==> Downloading $repo -> $target"
-ssh "$HOST" "bash /tmp/swap-hf-fetch.sh '$target' /tmp/swap-manifest '$repo'"
-
-# --- 3. detect the quantization on the host ----------------------------------
-quant="$(ssh "$HOST" "python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get(\"quantization_config\",{}).get(\"quant_method\",\"\"))' '$target/config.json' 2>/dev/null || true")"
-echo "==> Detected quantization: ${quant:-}"
-
-case "$quant" in
- fp8) spec_method=dflash; default_spec=7 ;;
- mxfp4|quark|"") spec_method=dflash; default_spec=7 ;;
- *) spec_method=dflash; default_spec=7
- echo "WARNING: unknown quant '$quant'; defaulting to the DFlash2 path" >&2 ;;
-esac
-[[ -n "$spec" ]] || spec="$default_spec"
-
-echo "==> Quadlet env vars for this checkpoint:"
-echo " Environment=SNAP=$snap_c"
-echo " Environment=SPEC_METHOD=$spec_method"
-echo " Environment=SPEC=$spec"
-echo " Environment=KV_MEM=$kv"
-
-# --- 4. apply the quadlet edit (in the repo) ---------------------------------
-if [[ "$apply" = 1 ]]; then
- echo "==> Editing $QUADLET"
- sed -i -E "s|^Environment=SNAP=.*|Environment=SNAP=$snap_c|" "$QUADLET"
- sed -i -E "s|^Environment=SPEC_METHOD=.*|Environment=SPEC_METHOD=$spec_method|" "$QUADLET"
- sed -i -E "s|^Environment=SPEC=.*|Environment=SPEC=$spec|" "$QUADLET"
- sed -i -E "s|^Environment=KV_MEM=.*|Environment=KV_MEM=$kv|" "$QUADLET"
- echo "==> Quadlet updated."
-fi
-
-# --- 5. deploy ----------------------------------------------------------------
-if [[ "$deploy" = 1 ]]; then
- [[ "$apply" = 1 ]] || { echo "ERROR: --deploy requires --apply" >&2; exit 2; }
- echo "==> Deploying (pull, restart, health gate)"
- ./scripts/deploy.sh dankserver --service frank-medium --skip-e2e
- echo "==> Next: run the capability battery"
- echo " python3 scripts/bench-conversation.py http://192.168.0.235:7770 --model frank-medium --salt ${name} --max-tokens 512 --burst 4"
-fi
diff --git a/scripts/test-changed.sh b/scripts/test-changed.sh
index 62b21f9..68752ac 100755
--- a/scripts/test-changed.sh
+++ b/scripts/test-changed.sh
@@ -180,17 +180,30 @@ for path in "${CHANGED[@]}"; do
scripts/mount-entropy.sh)
add_test tests/mount_entropy.bats
;;
- # frank-medium is a rootless quadlet again since the 2026-09-20
- # tensor-parallel cutover: the TP=2 radiance unit, its entrypoint and
- # its healthcheck all live under dankserver/frank-medium/, and the
- # container mounts the latter two straight out of the checkout. The
- # llama.cpp trio that held this endpoint before that date is archive
- # material in scripts/.AGENTTRASH/ and maps to nothing.
- dankserver/frank-medium/*)
- add_test tests/frank_medium_unit.bats
+ # Archive material: a path under any .AGENTTRASH/ directory is
+ # retired-by-policy (deploy-plan.py plans nothing from it), so it
+ # selects no focused test. It still needs its own case: without one
+ # the wrapper refuses retirement commits, whose trash moves show up
+ # as additions, with "no focused-test mapping".
+ .AGENTTRASH/*|*/.AGENTTRASH/*)
+ ;;
+ # frank-primary is the lucebox DS4.1 Quadlet (2026-09-26 cutover on
+ # :7770): its entrypoint and healthcheck live under
+ # dankserver/frank-primary/ and the container mounts them straight out
+ # of the checkout, so any file here changes deployed state. The
+ # radiance quadlet that held this endpoint before that date is archive
+ # material in dankserver/.AGENTTRASH/ and maps to nothing.
+ dankserver/frank-primary/*)
+ add_test tests/frank_primary_unit.bats
add_test tests/deployment_scoping.bats
add_test tests/docs_completeness.bats
;;
+ # frank-large's chat template lives in scripts/ (moved there on
+ # 2026-09-26 when its old home was retired): a template edit changes
+ # what the live endpoint renders, so it selects the large unit seam.
+ scripts/qwen-sharp-v22.4.0.jinja)
+ add_test tests/frank_large_unit.bats
+ ;;
# frank-small is llama.cpp on the 3090 as a ROOT unit: a native CUDA
# binary, so no generated quadlet is involved. Its env, unit and serve
# script all live in scripts/, which is why this case has to precede
diff --git a/shared/grafana/provisioning/dashboards/llm-fleet.json b/shared/grafana/provisioning/dashboards/llm-fleet.json
index ab097c4..d5319e3 100644
--- a/shared/grafana/provisioning/dashboards/llm-fleet.json
+++ b/shared/grafana/provisioning/dashboards/llm-fleet.json
@@ -1,15 +1,14 @@
{
"uid": "llm-fleet",
"title": "LLM fleet \u2014 capacity and reuse",
- "description": "The signals that explain the LLM endpoints under load. The fleet runs two runtimes and the panels read both: vllm:* for frank-medium (Dankserver, vllm-radiance TP=2 across both R9700s since 2026-09-20) and llamacpp:* for frank-small (Dankclaw, RTX 3090) and frank-large (Dankclaw, gfx1151 iGPU \u2014 the CIRU llama.cpp build does export metrics, verified 2026-09-20). The endpoint picker selects between them. See serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md for the numbers these panels are read against, and serverAppConfigs-docs/Infrastructure/Observability.md for the namespace split.",
+ "description": "The signals that explain the LLM endpoints under load. One metric namespace: llamacpp:* for frank-small (Dankclaw, RTX 3090) and frank-large (Dankclaw, gfx1151 iGPU). frank-primary (Dankserver, the Lucebox DS4.1 engine since the 2026-09-26 cutover) exports no metrics and has no panels here. The endpoint picker selects between them. See serverAppConfigs-docs/Infrastructure/Observability.md for the namespace split and the addressability rules.",
"tags": [
"llm",
- "frank-medium",
"capacity"
],
"timezone": "browser",
"schemaVersion": 39,
- "version": 1,
+ "version": 2,
"refresh": "30s",
"time": {
"from": "now-6h",
@@ -26,7 +25,7 @@
"uid": "prometheus"
},
"query": {
- "query": "label_values({__name__=~\"vllm:kv_cache_usage_perc|llamacpp:.*\"}, job)",
+ "query": "label_values({__name__=~\"llamacpp:.*\"}, job)",
"refId": "A"
},
"includeAll": true,
@@ -43,7 +42,7 @@
"id": 1,
"type": "timeseries",
"title": "Prefix cache reuse rate",
- "description": "rate(hits)/rate(queries) in tokens. On the llama.cpp side the prompt counter EXCLUDES cached tokens (its own HELP line), so that side divides by cached + prompt. High means pinned conversations are served from cache: measured 2026-09-20, an identical 13,021-token prompt went 2.30 s cold to 0.68 s warm with 9,888 tokens (76 %) reused. Reuse on frank-medium happens at 1,648-token block granularity (the attention block size is padded to the mamba page size), so shared prefixes shorter than two blocks re-prefill.",
+ "description": "rate(hits)/rate(queries) in tokens. The llama.cpp prompt counter EXCLUDES cached tokens (its own HELP line), so this divides by cached + prompt. High means pinned conversations are served from cache: measured 2026-09-20, an identical 13,021-token prompt went 2.30 s cold to 0.68 s warm with 9,888 tokens (76 %) reused.",
"gridPos": {
"h": 8,
"w": 12,
@@ -71,11 +70,6 @@
"expr": "rate(llamacpp:prompt_tokens_cached_total{job=~\"$job\"}[5m]) / clamp_min(rate(llamacpp:prompt_tokens_cached_total{job=~\"$job\"}[5m]) + rate(llamacpp:prompt_tokens_total{job=~\"$job\"}[5m]), 0.001)",
"legendFormat": "{{job}} reuse (llama.cpp)",
"refId": "A"
- },
- {
- "expr": "job:llm_vllm_prefix_cache_reuse:rate5m{job=~\"$job\"}",
- "legendFormat": "{{job}} reuse (vLLM)",
- "refId": "B"
}
]
},
@@ -83,7 +77,7 @@
"id": 2,
"type": "timeseries",
"title": "Speculative decoding (acceptance)",
- "description": "The one failure no client can see: the drafter silently stops and the endpoint keeps answering at autoregressive speed. llama.cpp: accepted tokens per draft step. vLLM: mean accepted length (1 + accepted/drafts) from the recording rule job:llm_spec_accept_length:rate5m \u2014 measured 3.0-4.3 at boot 2026-09-20 with the DFlash2 drafter.",
+ "description": "The one failure no client can see: the drafter silently stops and the endpoint keeps answering at autoregressive speed. Accepted tokens per draft step, and the mean accepted length (1 + accepted/drafts, the recording rule job:llm_spec_accept_length:rate5m) \u2014 1.0 means the draft round is pure overhead.",
"gridPos": {
"h": 8,
"w": 12,
@@ -137,50 +131,12 @@
"expr": "llamacpp:requests_processing{job=~\"$job\"}",
"legendFormat": "{{job}} processing",
"refId": "B"
- },
- {
- "expr": "vllm:num_requests_waiting{job=~\"$job\"}",
- "legendFormat": "{{job}} waiting (vLLM)",
- "refId": "C"
- },
- {
- "expr": "vllm:num_requests_running{job=~\"$job\"}",
- "legendFormat": "{{job}} running (vLLM)",
- "refId": "D"
}
]
},
{
"id": 4,
"type": "timeseries",
- "title": "KV pool fill (vLLM)",
- "description": "The gauge that was retired 2026-09-16 for want of a llama.cpp equivalent and came back when frank-medium moved to vLLM: vllm:kv_cache_usage_perc (the pool is auto-sized \u2014 943,581 tokens at boot 2026-09-20). Sustained high fill with a collapsing reuse panel is the thrash; preemptions are the visible symptom of it. The llama.cpp side has no fill gauge: its capacity wall is panel 3.",
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 8
- },
- "datasource": {
- "type": "prometheus",
- "uid": "prometheus"
- },
- "targets": [
- {
- "expr": "vllm:kv_cache_usage_perc{job=~\"$job\"}",
- "legendFormat": "{{job}} KV fill %",
- "refId": "A"
- },
- {
- "expr": "rate(vllm:num_preemptions_total{job=~\"$job\"}[5m])",
- "legendFormat": "{{job}} preemptions/s",
- "refId": "B"
- }
- ]
- },
- {
- "id": 5,
- "type": "timeseries",
"title": "Decode and prefill throughput",
"description": "llama.cpp: llamacpp:predicted_tokens_seconds is decode, llamacpp:prompt_tokens_seconds is prompt. vLLM: the token counters, rate'd. The series carry the job label of their scrape, so the endpoint picker selects between the runtimes.",
"gridPos": {
@@ -203,21 +159,11 @@
"expr": "llamacpp:prompt_tokens_seconds{job=~\"$job\"}",
"legendFormat": "{{job}} prefill tok/s (llama.cpp)",
"refId": "B"
- },
- {
- "expr": "rate(vllm:generation_tokens_total{job=~\"$job\"}[5m])",
- "legendFormat": "{{job}} decode tok/s (vLLM)",
- "refId": "C"
- },
- {
- "expr": "rate(vllm:prompt_tokens_total{job=~\"$job\"}[5m])",
- "legendFormat": "{{job}} prefill tok/s (vLLM)",
- "refId": "D"
}
]
},
{
- "id": 6,
+ "id": 5,
"type": "timeseries",
"title": "Decode steps and batch size",
"description": "llama.cpp-only signal \u2014 vLLM does not publish per-step counters. Steps per second and the busy-slot count per step (the effective batch).",
@@ -245,7 +191,7 @@
]
},
{
- "id": 7,
+ "id": 6,
"type": "timeseries",
"title": "Firing fleet alerts",
"description": "The LLM-fleet rules from shared/prometheus/rules/llm-fleet.yml. A firing alert is delivered to Signal: Prometheus -> Alertmanager (dankclaw quadlet) -> the host-side signal-relay user unit -> signal-cli, since 2026-09-17.",
@@ -266,49 +212,6 @@
"refId": "A"
}
]
- },
- {
- "id": 8,
- "type": "timeseries",
- "title": "Request latency p95 (vLLM)",
- "description": "vLLM-only \u2014 the histogram the p95 panel was retired for on 2026-09-16 (llama.cpp exports no latency histogram) is back with the radiance cutover. TTFT is what a fresh prefill costs; ITL is the per-token cadence an agent loop feels; e2e is the whole request.",
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 24
- },
- "datasource": {
- "type": "prometheus",
- "uid": "prometheus"
- },
- "fieldConfig": {
- "defaults": {
- "unit": "s",
- "custom": {
- "fillOpacity": 10,
- "lineWidth": 1
- }
- },
- "overrides": []
- },
- "targets": [
- {
- "expr": "histogram_quantile(0.95, sum by (le, job) (rate(vllm:time_to_first_token_seconds_bucket{job=~\"$job\"}[5m])))",
- "legendFormat": "{{job}} TTFT p95",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.95, sum by (le, job) (rate(vllm:inter_token_latency_seconds_bucket{job=~\"$job\"}[5m])))",
- "legendFormat": "{{job}} ITL p95",
- "refId": "B"
- },
- {
- "expr": "histogram_quantile(0.95, sum by (le, job) (rate(vllm:e2e_request_latency_seconds_bucket{job=~\"$job\"}[5m])))",
- "legendFormat": "{{job}} e2e p95",
- "refId": "C"
- }
- ]
}
]
}
diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml
index a2a828f..94540ed 100644
--- a/shared/prometheus/prometheus.yml
+++ b/shared/prometheus/prometheus.yml
@@ -79,19 +79,11 @@ scrape_configs:
# this container, not the guarded vhosts: these are machine clients and
# Prometheus cannot solve an Anubis challenge.
#
- # Two metric namespaces, because the fleet runs two runtimes:
- #
- # * vllm:* — frank-medium (vllm-radiance TP=2 since 2026-09-20): prompt and
- # generation token counters, num_requests_waiting plus its reason label,
- # num_preemptions_total, the kv_cache_usage_perc fill gauge, the
- # prefix_cache hits/queries pair, the spec_decode drafts/accepted triple,
- # and real latency histograms (time_to_first_token, inter_token,
- # e2e_request_latency).
- # * llamacpp:* — frank-large and frank-small. Both take --metrics and both
- # were verified 2026-09-20 to export requests_deferred (the queue depth,
- # the only form llama.cpp shows the capacity wall in),
- # spec_decode_num_drafts_total, tokens_predicted_total, and the
- # prompt_tokens_cached_total / prompt_tokens_total reuse pair.
+ # One metric namespace: llamacpp:* — frank-large and frank-small, both
+ # llama.cpp with --metrics and both verified 2026-09-20 to export
+ # requests_deferred (the queue depth, the only form llama.cpp shows the
+ # capacity wall in), spec_decode_num_drafts_total, tokens_predicted_total,
+ # and the prompt_tokens_cached_total / prompt_tokens_total reuse pair.
#
# frank-large is scraped over host.containers.internal, not its own LAN address:
# this Prometheus is a bridge-networked container (Network=dankclaw_default),
@@ -102,14 +94,10 @@ scrape_configs:
# "dial tcp 192.168.0.143:7771") even though the endpoint was healthy and
# exporting: the target address was unreachable by construction, not the
# service down. Same host, same rule, so both local endpoints use the gateway.
- # frank-medium stays a LAN address — it is a different machine.
- - job_name: "frank-medium"
- metrics_path: "/metrics"
- static_configs:
- - targets: ["192.168.0.235:7770"]
- labels:
- service: frank-medium
-
+ # (2026-09-26) frank-medium is replaced by frank-primary (lucebox DS4.1),
+ # which exposes no metrics endpoint (/metrics -> 404), so there is no job
+ # for it here. The endpoint's liveness is covered by the homepage probe;
+ # the documented remedy is a blackbox exporter, which is not deployed.
- job_name: "frank-small"
metrics_path: "/metrics"
static_configs:
diff --git a/shared/prometheus/rules/llm-fleet.yml b/shared/prometheus/rules/llm-fleet.yml
index 7d1f62f..54a9932 100644
--- a/shared/prometheus/rules/llm-fleet.yml
+++ b/shared/prometheus/rules/llm-fleet.yml
@@ -1,32 +1,27 @@
# Alerting and recording rules for the LLM fleet.
#
-# Two metric namespaces, because the fleet runs two runtimes:
+# One metric namespace: llamacpp:* — frank-small and frank-large, both
+# llama.cpp with --metrics (the family verified from /metrics on 2026-09-16
+# and 2026-09-20). frank-primary (Dankserver, the Lucebox DS4.1 engine since
+# the 2026-09-26 cutover) answers 404 on /metrics, so nothing here selects
+# it; its liveness is covered by the homepage probe, and the documented
+# remedy is a blackbox exporter (not deployed).
#
-# * llamacpp:* — frank-small (Dankclaw, llama.cpp with --metrics).
-# * vllm:* — frank-medium (Dankserver, vllm-radiance TP=2 since 2026-09-20).
+# Every expression is keyed to a series that EXISTS. Nothing matches on
+# absence: when an endpoint's runtime leaves the fleet, its selectors are
+# deleted rather than left vacuous (the vllm: group went with frank-medium's
+# radiance runtime on 2026-09-26).
#
-# Every expression is keyed to a series that EXISTS, verified against
-# 192.168.0.235:7770/metrics and 192.168.0.143:7771/metrics on 2026-09-20.
-# Nothing here matches on absence: when frank-medium moved back to vLLM the
-# llamacpp: selectors narrowed to frank-small rather than going vacuous, and
-# the vllm: group below was added beside them, not in place of them.
+# What llamacpp:* can and cannot see:
#
-# What each namespace can and cannot see:
-#
-# * KV pressure. vLLM exports a fill gauge (vllm:kv_cache_usage_perc) and a
-# preemption counter; llama.cpp exports neither, so its capacity wall is
+# * KV pressure: llama.cpp exports no fill gauge, so the capacity wall is
# observed as queueing (llamacpp:requests_deferred).
-# * Prefix reuse. Both export a hits/queries pair counted in tokens, but the
-# units are NOT interchangeable: llama.cpp's prompt_tokens_total EXCLUDES
-# cached tokens (its own HELP line says so), so that side must divide by
-# cached + prompt to get a fraction, while vLLM divides hits by queries.
-# The two recording rules below keep that straight, and the collapse alert
-# watches both sides with one threshold.
-# * Speculative decoding. Both export drafts and accepted tokens, so
-# LlmSpecDecodeOff covers both runtimes.
-# * Latency histograms exist only on vLLM (e2e, TTFT, inter-token). No
-# p95 alert is defined for the llama.cpp side because there is nothing to
-# compute one from.
+# * Prefix reuse: counted in tokens, and prompt_tokens_total EXCLUDES
+# cached tokens (its own HELP line says so), so a reuse fraction divides
+# cached by cached + prompt.
+# * Speculative decoding: drafts and accepted tokens are both counters,
+# so LlmSpecDecodeOff watches the pair.
+# * No latency histograms exist, so there is no p95 alert.
#
# IMPORTANT: delivery. Since 2026-09-17 a firing alert reaches Frank on
# Signal: Prometheus pushes to Alertmanager (dankclaw/observability/quadlet/
@@ -37,32 +32,24 @@ groups:
- name: llm-fleet
interval: 30s
rules:
- # Recording rules so the dashboard and the collapse alert below share one
- # expression. Both namespaces export hits/queries pairs in TOKENS; the
- # rate of the quotient is the reuse rate. Kept as two records because the
- # series names differ and a vacuous joint query is worse than two honest
- # ones.
+ # Recording rules so the dashboard and the collapse alert below share
+ # one expression. llamacpp's prompt counter EXCLUDES cached tokens, so
+ # the reuse fraction is cached / (cached + prompt).
- record: job:llm_prefix_cache_reuse:rate5m
expr: |
rate(llamacpp:prompt_tokens_cached_total[5m])
/ clamp_min(rate(llamacpp:prompt_tokens_cached_total[5m])
+ rate(llamacpp:prompt_tokens_total[5m]), 0.001)
- - record: job:llm_vllm_prefix_cache_reuse:rate5m
- expr: |
- rate(vllm:prefix_cache_hits_total[5m])
- / clamp_min(rate(vllm:prefix_cache_queries_total[5m]), 0.001)
-
# Mean accepted draft length (1.0 = the drafter proposes and nothing is
- # taken, so the draft round is pure overhead). Measured 3.0-4.3 on
- # frank-medium at boot with the DFlash2 drafter, 2026-09-20.
+ # taken, so the draft round is pure overhead).
- record: job:llm_spec_accept_length:rate5m
expr: |
- 1 + rate(vllm:spec_decode_num_accepted_tokens_total[5m])
- / clamp_min(rate(vllm:spec_decode_num_drafts_total[5m]), 0.001)
+ 1 + rate(llamacpp:spec_decode_num_accepted_tokens_total[5m])
+ / clamp_min(rate(llamacpp:spec_decode_num_drafts_total[5m]), 0.001)
- # The capacity wall on the llama.cpp side, in the only form it exports:
- # work waiting for a slot. Past here the next large request queues.
+ # The capacity wall, in the only form llama.cpp exports: work waiting
+ # for a slot. Past here the next large request queues.
- alert: LlmRequestsQueueing
expr: llamacpp:requests_deferred > 0
for: 10m
@@ -75,51 +62,15 @@ groups:
fleet is genuinely busy or one endpoint is carrying it alone. With
--parallel 1 a second concurrent request always shows up here.
- # The capacity wall on the vLLM side, in its own two terms: requests the
- # scheduler has not admitted, and requests it admitted and then PREEMPTED
- # (a preemption throws away the KV and re-prefills, so its rate is the
- # cost of running the pool full). frank-medium auto-sizes its pool to
- # 943,581 tokens = 3.59 concurrent full 262,144-token conversations, so
- # saturation is four such conversations, not two.
- - alert: LlmRequestsWaiting
- expr: vllm:num_requests_waiting > 0
- for: 10m
- labels:
- severity: warning
- annotations:
- summary: "{{ $labels.job }}: vLLM has not admitted work for 10 minutes"
- description: >-
- The scheduler is holding requests back. Check the reason label on
- vllm:num_requests_waiting_by_reason (capacity = no KV room, defer =
- waiting for a better batch shape) and the preemption counter
- vllm:num_preemptions_total.
-
- - alert: LlmKvCacheSaturated
- expr: vllm:kv_cache_usage_perc > 0.95
- for: 10m
- labels:
- severity: warning
- annotations:
- summary: "{{ $labels.job }}: KV pool above 95 % for 10 minutes"
- description: >-
- The pool is full, which is where preemption and re-prefill start:
- decode slows before anything errors. The pool is auto-sized by the
- quadlet (no KV_CACHE_MEMORY_MB pin); the boot ledger prints the
- token count — `podman logs frank-medium | grep "GPU KV cache size"`
- — and it was 943,581 tokens on 2026-09-20.
-
# The thrash signature measured on 2026-09-16: requests keep arriving but
# almost none are served from cache, which is what several concurrent
- # 50-80k-token conversations do to this fleet. One threshold, two
- # namespaces, and a traffic floor per side so an idle endpoint cannot
- # fire it on a 0/0.
+ # 50-80k-token conversations do to this fleet. One threshold with a
+ # traffic floor so an idle endpoint cannot fire it on a 0/0.
- alert: LlmPrefixCacheCollapse
expr: >-
- (job:llm_prefix_cache_reuse:rate5m < 0.20
+ job:llm_prefix_cache_reuse:rate5m < 0.20
and rate(llamacpp:prompt_tokens_total[5m])
- + rate(llamacpp:prompt_tokens_cached_total[5m]) > 0.05)
- or (job:llm_vllm_prefix_cache_reuse:rate5m < 0.20
- and rate(vllm:prefix_cache_queries_total[5m]) > 1)
+ + rate(llamacpp:prompt_tokens_cached_total[5m]) > 0.05
for: 15m
labels:
severity: info
@@ -132,14 +83,11 @@ groups:
# The speculative path is the one failure no client can see: the server
# answers normally, just at half the speed, when the drafter is gone.
- # Drafts and accepts are counters in both runtimes, so a rate that stays
- # at zero while the endpoint is serving tokens means nothing is drafted.
+ # Drafts and accepts are counters, so a rate that stays at zero while
+ # the endpoint is serving tokens means nothing is drafted.
- alert: LlmSpecDecodeOff
- expr: >-
- (rate(llamacpp:spec_decode_num_drafts_total[5m]) == 0
- and rate(llamacpp:tokens_predicted_total[5m]) > 0.1)
- or (rate(vllm:spec_decode_num_drafts_total[5m]) == 0
- and rate(vllm:generation_tokens_total[5m]) > 0.1)
+ expr: rate(llamacpp:spec_decode_num_drafts_total[5m]) == 0
+ and rate(llamacpp:tokens_predicted_total[5m]) > 0.1
for: 15m
labels:
severity: warning
@@ -147,11 +95,9 @@ groups:
summary: "{{ $labels.job }}: serving without speculative decoding"
description: >-
Tokens are being produced but no draft has been proposed for 15
- minutes. frank-small (llama.cpp) should log "adding speculative
- implementation" at load; frank-medium (vllm-radiance) should be
- counting vllm:spec_decode_num_drafts_total and reporting 3+ mean
- accept length (job:llm_spec_accept_length:rate5m). Either way the
- endpoint is decoding autoregressively.
+ minutes. The llama.cpp load log should carry an "adding
+ speculative implementation" line; the endpoint is decoding
+ autoregressively.
# The endpoint, not a replica: one process serves each port now, so a
# restart of the unit is a scrape gap, not a failover.
@@ -163,10 +109,9 @@ groups:
annotations:
summary: "{{ $labels.job }}: scrape target down for 2 minutes"
description: >-
- Prometheus cannot reach the LLM endpoint. frank-medium is a
- rootless quadlet: `systemctl --user is-active frank-medium` on
- Dankserver, then `podman ps -a --filter name=frank-medium`.
- frank-small is a root unit: `systemctl is-active frank-small` on
- Dankclaw. All three endpoints are scraped: frank-large's CIRU
- llama.cpp build does export the llamacpp:* family (verified
- 2026-09-20), so this alert covers the whole fleet.
+ Prometheus cannot reach the LLM endpoint. frank-small and
+ frank-large are root units on Dankclaw: `systemctl is-active
+ frank-small` / `frank-large`. Both scraped endpoints export the
+ llamacpp:* family. frank-primary (Dankserver) is not scraped —
+ its runtime exports no metrics; check the homepage probe or
+ `systemctl --user is-active frank-primary` on Dankserver.
diff --git a/tests/changed_tests.bats b/tests/changed_tests.bats
index 19c730c..e3a2291 100644
--- a/tests/changed_tests.bats
+++ b/tests/changed_tests.bats
@@ -21,18 +21,22 @@ load helpers
[[ "$output" != *"tests/homepage_pages.bats"* ]]
}
-@test "changed-test wrapper maps the frank-medium dir to its focused tests" {
+@test "changed-test wrapper maps the frank-primary dir to its focused tests" {
run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths \
- dankserver/frank-medium/quadlet/frank-medium.container
+ dankserver/frank-primary/quadlet/frank-primary.container
[ "$status" -eq 0 ]
- [[ "$output" == *"tests/frank_medium_unit.bats"* ]]
+ [[ "$output" == *"tests/frank_primary_unit.bats"* ]]
[[ "$output" == *"tests/deployment_scoping.bats"* ]]
[[ "$output" == *"tests/docs_completeness.bats"* ]]
[[ "$output" != *"tests/homepage_pages.bats"* ]]
- # The llama trio's old path must no longer select the container seam: the
- # mapping was removed with the runtime, and a stale case here would keep
- # firing tests for an archived service.
- run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths scripts/frank-medium.env
+ # The retired frank-medium tree must no longer select any LLM seam: the
+ # mappings were removed with the service, and a stale case would keep
+ # firing tests for an archived runtime. Both archive shapes — the radiance
+ # Quadlet era and the llama.cpp layersplit trio before it — map to nothing.
+ run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths \
+ dankserver/.AGENTTRASH/20260926-frank-medium-radiance/frank-medium.container \
+ scripts/.AGENTTRASH/20260920-llama-layersplit/serve-frank-medium.sh.llamacpp-layersplit
[ "$status" -eq 0 ]
[[ "$output" != *"tests/frank_medium_unit.bats"* ]]
+ [[ "$output" != *"tests/frank_primary_unit.bats"* ]]
}
diff --git a/tests/deployment_flow.bats b/tests/deployment_flow.bats
index c20c58a..5582197 100644
--- a/tests/deployment_flow.bats
+++ b/tests/deployment_flow.bats
@@ -67,9 +67,10 @@ load helpers
! grep -q 'no proxy was stopped or replaced' "$REPO_ROOT/scripts/deploy-remote.sh"
}
-@test "remote runner has separate handlers for both inference services" {
+@test "remote runner has a handler per inference service" {
grep -q 'deploy_llm_large' "$REPO_ROOT/scripts/deploy-remote.sh"
- grep -q 'deploy_llm_medium' "$REPO_ROOT/scripts/deploy-remote.sh"
+ grep -q 'deploy_llm_primary' "$REPO_ROOT/scripts/deploy-remote.sh"
+ grep -q 'deploy_llm_small' "$REPO_ROOT/scripts/deploy-remote.sh"
}
@test "inference restarts are absent from the top-level dispatcher" {
diff --git a/tests/deployment_scoping.bats b/tests/deployment_scoping.bats
index 6f950ce..9458ac3 100644
--- a/tests/deployment_scoping.bats
+++ b/tests/deployment_scoping.bats
@@ -46,25 +46,19 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
[ -z "$output" ]
}
-@test "deployment planner maps the frank-medium container dir to frank-medium only" {
- run python3 "$PLANNER" --target dankserver --paths \
- dankserver/frank-medium/quadlet/frank-medium.container \
- dankserver/frank-medium/radiance-entrypoint.sh \
- dankserver/frank-medium/frank-medium-healthcheck.py
- [ "$status" -eq 0 ]
- [ "$output" = "frank-medium" ]
-
- # The path is not the assertion: both retired shapes are still in the tree
- # as archive, and a rule matching either would restart a service nobody
- # runs. Assert the negatives explicitly, plus the llama trio's old paths,
- # whose mapping was removed with the runtime.
- run python3 "$PLANNER" --target dankserver --paths \
- dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container
- [ "$status" -eq 0 ]
- [ -z "$output" ]
+@test "the retired frank-medium tree plans nothing, from any archive" {
+ # frank-medium left the live tree on 2026-09-26 — first the radiance
+ # Quadlet (and its vLLM replica archive before it), then the llama.cpp
+ # layersplit trio and its fetch tooling. Every shape is archive material
+ # now, and a planner rule matching any of them would restart a service
+ # nobody maintains.
run python3 "$PLANNER" --target dankserver --paths \
+ dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container \
+ dankserver/.AGENTTRASH/20260926-frank-medium-radiance/frank-medium.container \
+ dankserver/.AGENTTRASH/20260926-frank-medium-radiance/radiance-entrypoint.sh \
scripts/.AGENTTRASH/20260920-llama-layersplit/frank-medium.env.llamacpp-layersplit \
- scripts/frank-medium.env
+ scripts/.AGENTTRASH/20260926-frank-medium-radiance/download-frank-medium-gguf.sh \
+ scripts/.AGENTTRASH/20260926-frank-medium-radiance/swap-frank-medium.sh
[ "$status" -eq 0 ]
[ -z "$output" ]
}
@@ -202,7 +196,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
# The runner may restart inference only from the service-dispatch path. It
# must not contain a host-wide restart loop.
grep -q 'deploy_llm_large' "$REMOTE_DEPLOY"
- grep -q 'deploy_llm_medium' "$REMOTE_DEPLOY"
+ grep -q 'deploy_llm_primary' "$REMOTE_DEPLOY"
grep -q 'deploy_service' "$REMOTE_DEPLOY"
! grep -qE 'for .*frank-(small|large|npu)-?' "$REMOTE_DEPLOY"
}
diff --git a/tests/frank_medium_unit.bats b/tests/frank_medium_unit.bats
deleted file mode 100755
index f290440..0000000
--- a/tests/frank_medium_unit.bats
+++ /dev/null
@@ -1,219 +0,0 @@
-#!/usr/bin/env bats
-# Seam: the frank-medium vllm-radiance container on dankserver.
-# frank-medium became a CONTAINER again on 2026-09-20, when the board swap put
-# both R9700s on CPU root ports and a measured three-arm comparison retired the
-# mainline-llama.cpp layer split (scripts/.AGENTTRASH/20260920-llama-layersplit/).
-# What this seam pins is what the flags alone do not say: boot persistence for a
-# GENERATED unit (this host's v1 quadlet generator cannot enable units, so the
-# [Install] section is the only thing that survives a reboot), that BOTH cards
-# work on ONE model through tensor parallelism — the thing the pre-2026-09-20
-# hardware could not do — the KV pin's coupling to the batch shape it was
-# calibrated at, and that the pool still holds one full-window request.
-
-load helpers
-
-UNIT="$REPO_ROOT/dankserver/frank-medium/quadlet/frank-medium.container"
-ENTRY="$REPO_ROOT/dankserver/frank-medium/radiance-entrypoint.sh"
-HEALTH="$REPO_ROOT/dankserver/frank-medium/frank-medium-healthcheck.py"
-REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
-LLAMA_TRASH="$REPO_ROOT/scripts/.AGENTTRASH/20260920-llama-layersplit"
-REPLICA_TRASH="$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance"
-
-# Measured constants, used by the fit arithmetic.
-CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it (32624 MiB)
-KV_BYTES_PER_TOKEN=70656 # --kv-cache-memory 18563072000 => "262,720 tokens"
-env_value() {
- sed -nE "s/^Environment=$1=(-?[0-9]+)$/\1/p" "$UNIT" | head -n1
-}
-# GPU_UTIL is a fraction, not a count; the integer reader would drop it.
-env_ratio() {
- sed -nE "s/^Environment=$1=([0-9.]+)$/\1/p" "$UNIT" | head -n1
-}
-# SPEC_METHOD is a string (dflash|mtp), not a count; the integer reader drops it.
-env_str() {
- sed -nE "s/^Environment=$1=(.+)$/\1/p" "$UNIT" | head -n1
-}
-
-@test "frank-medium's unit, entrypoint and healthcheck exist" {
- [ -f "$UNIT" ]
- [ -x "$ENTRY" ]
- [ -f "$HEALTH" ]
-}
-
-@test "frank-medium is boot-persistent and self-healing" {
- # Generated units cannot be enabled on this host, so WantedBy in the file is
- # the whole boot story (AGENTS.md: Boot Persistence & Self-Healing).
- grep -q '^WantedBy=default.target$' "$UNIT"
- grep -q '^Restart=on-failure$' "$UNIT"
- grep -q '^RestartSec=' "$UNIT"
- # First start compiles Triton/inductor kernels before the engine loads ~9.5
- # GiB per card; a shorter timeout kills a healthy boot.
- grep -q '^TimeoutStartSec=1800$' "$UNIT"
- grep -q '^HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py$' "$UNIT"
- grep -q '^HealthStartPeriod=1800s$' "$UNIT"
- # Boots must not depend on the registry.
- grep -q '^Pull=never$' "$UNIT"
-}
-
-@test "both R9700s serve ONE model as one tensor-parallel process" {
- grep -q '^Environment=TP=2$' "$UNIT"
- grep -q '^Environment=ROCR_VISIBLE_DEVICES=0,1$' "$UNIT"
- grep -q '^Environment=HIP_VISIBLE_DEVICES=0,1$' "$UNIT"
- grep -q -- '--tensor-parallel-size "$TP"' "$ENTRY"
- # The iGPU (index 2, 2 GiB) must stay out of the visible set: a third rank
- # would take layers onto system RAM.
- ! grep -q '^Environment=.*VISIBLE_DEVICES=0,1,2' "$UNIT"
- # TP needs the all-reduce to actually run. R4D is the collective-free path
- # this host measured; the entrypoint must still select it, and the RCCL
- # fallback must not be re-disabled by the pre-board-swap workarounds.
- grep -q 'RADIANCE_USE_R4D_AR' "$ENTRY"
- grep -q -- '--attention-backend "$ATTN"' "$ENTRY"
- ! grep -q 'NCCL_P2P_DISABLE' "$UNIT"
- ! grep -q 'RCCL_MSCCL_ENABLE' "$UNIT"
-}
-
-@test "HIP never reads one of these tunables as its own setting" {
- # HIP reads bare TEMP, HOST and GPU_MAX_HEAP_SIZE (2026-08-13 incident:
- # TEMP=1.0 sent the JIT cache to path "1.0" and segfaulted every load).
- # Serve scripts export env files with `set -a`; a container inherits the
- # same hazard from Environment=.
- ! grep -qE '^Environment=(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$UNIT"
-}
-
-@test "the speculative drafter matches the serving shape" {
- local spec_method spec maxseqs chunk
- spec_method="$(env_str SPEC_METHOD)"
- spec="$(env_value SPEC)"
- maxseqs="$(env_value MAXSEQS)"
- chunk="$(env_value CHUNK)"
- [ -n "$spec_method" ] && [ -n "$spec" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ]
- # The chunk is the AR size gate's input (CHUNK x hidden x 2 bytes); a chunk
- # larger than this row's silently changes which all-reduce path runs.
- [ "$chunk" -le 8192 ]
- case "$spec_method" in
- dflash)
- # DFlash2 drafts at most 7 tokens per step; deeper is a different
- # drafter's contract and costs acceptance, not just VRAM.
- [ "$spec" -eq 7 ]
- # The MXFP4 / ParoQuant-MXFP6 decode kernel band covers MAXSEQS x
- # (SPEC+1) rows, and the shipped default covers 64 (the mxfp6 profile
- # block exports the same RADIANCE_MXFP4_DECODE_MAX_M=64). Exceeding it
- # without the entrypoint's >8 bump is a launch failure at the worst
- # moment (first concurrent request).
- [ $(( maxseqs * (spec + 1) )) -le 64 ]
- grep -q 'RADIANCE_MXFP4_DECODE_MAX_M' "$ENTRY"
- ;;
- mtp)
- # MTP self-speculation at depth 8 (the documented plain-FP8 path; the
- # model's own single-layer MTP head, applied autoregressively).
- [ "$spec" -eq 8 ]
- ;;
- *)
- false "unknown SPEC_METHOD: $spec_method"
- ;;
- esac
-}
-
-@test "the KV pool holds a full window and fits beside the weights" {
- local kv_mem maxseqs chunk maxlen spec kv_dtype
- kv_mem="$(env_value KV_MEM)"
- maxseqs="$(env_value MAXSEQS)"
- chunk="$(env_value CHUNK)"
- maxlen="$(env_value MAXLEN)"
- spec="$(env_value SPEC)"
- kv_dtype="$(env_str KV_DTYPE)"
- [ -n "$kv_mem" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ] && [ -n "$maxlen" ]
- # KV_MEM=0 = profile the shape (the documented plain-FP8 path's default;
- # the upstream docker-compose leaves the pin commented out). The pool is
- # auto-sized to fit, so there is no pin to verify — only that the shape
- # is set and the model's weights fit beside an auto-sized pool.
- if [ "$kv_mem" -eq 0 ]; then
- # MXFP4 + unquantized (KV_DTYPE=auto -> bf16) KV: MXFP4's freed weights
- # (19.37 vs 30.9 GB) buy the pool back, so the full 262,144 native window
- # still fits clean (~290k f16 pool); vLLM boots or the pool is too small.
- [ "$maxseqs" -eq 8 ] && [ "$chunk" -eq 8192 ] && [ "$maxlen" -eq 262144 ]
- [ "$kv_dtype" = "auto" ]
- return 0
- fi
- # Pinned pool: the pin must hold at least ONE full-window request or the
- # endpoint refuses its own --max-model-len, and it must fit on one card
- # beside the weights.
- grep -q 'kv-profiles.tsv' "$UNIT"
- [ $(( kv_mem / KV_BYTES_PER_TOKEN )) -ge "$maxlen" ]
- # Weights per card: MXFP4 ~9.5 GiB (19 GB / 2), FP8 ~15.5 GiB (30.9 GB / 2).
- local weights util util_pct floor
- case "$(env_str SPEC_METHOD)" in
- dflash) weights=10200547328 ;;
- *) weights=16646128640 ;;
- esac
- util="$(env_ratio GPU_UTIL)"
- [ -n "$util" ]
- util_pct="${util#0.}"
- floor=$(( 1024 * 1024 * 1024 )) # 1 GiB per card
- [ $(( weights + kv_mem / 2 )) -le $(( CARD_BYTES * util_pct / 100 - floor )) ]
-}
-
-@test "the fleet endpoint contract survives the runtime swap" {
- grep -q '^Environment=PORT=7770$' "$UNIT"
- grep -q '^Environment=SERVED_NAMES=frank-medium$' "$UNIT"
- grep -q -- '--port "$PORT"' "$ENTRY"
- grep -q -- '--served-model-name $SERVED_NAMES' "$ENTRY"
- # Host network: vLLM binds the port itself, so no PublishPort (and no
- # pasta bind, which is what made the retired proxy's port replacement
- # fail at container create).
- grep -q -- '--network=host' "$UNIT"
- ! grep -q '^PublishPort=' "$UNIT"
- # Tools and reasoning are part of the harness contract; both parsers must
- # be selected, and thinking must land in its own field.
- grep -q -- '--enable-auto-tool-choice' "$ENTRY"
- grep -q -- '--tool-call-parser qwen3_coder' "$ENTRY"
- grep -q -- '--reasoning-parser qwen3' "$ENTRY"
- grep -q -- '--enable-prefix-caching' "$ENTRY"
- # Requests that send no sampler fields get the predecessor's defaults.
- grep -q '"temperature":0.7' "$ENTRY"
- grep -q '"top_p":0.95' "$ENTRY"
-}
-
-@test "the healthcheck proves the engine serves, not merely that a process lives" {
- grep -q '"/health"' "$HEALTH" || grep -q '/health' "$HEALTH"
- grep -q 'status == 200' "$HEALTH"
- # PORT-aware, and it must not hard-code a replica port.
- grep -q 'os.environ.get("PORT"' "$HEALTH"
- ! grep -qE '7771|7772' "$HEALTH"
-}
-
-@test "the deploy handler gates on host assets and installs the quadlet" {
- grep -q 'deploy_llm_medium' "$REMOTE_DEPLOY"
- grep -q 'install_quadlet frank-medium dankserver/frank-medium/quadlet frank-medium' "$REMOTE_DEPLOY"
- grep -q 'wait_container_healthy frank-medium' "$REMOTE_DEPLOY"
- grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
- grep -q '\[\[ "$missing" -eq 0 \]\] || return 1' "$REMOTE_DEPLOY"
- # The image is only pulled when absent, and re-tagged to the name the unit
- # actually runs (the quadlet path cannot resolve registry refs at create).
- grep -q 'podman image exists "$LLM_MEDIUM_IMAGE_LOCAL"' "$REMOTE_DEPLOY"
- grep -q 'podman tag "$LLM_MEDIUM_IMAGE" "$LLM_MEDIUM_IMAGE_LOCAL"' "$REMOTE_DEPLOY"
- # The patched libr4d is not optional (the image's own build NaNs this model).
- grep -q 'patched libr4d absent' "$REMOTE_DEPLOY"
-}
-
-@test "a generated unit is never asked to enable, and the llama unit is gone" {
- # podman 5.x v1 generator: generated units cannot be enabled; asking makes
- # the deploy fail on a unit that is already boot-persistent via [Install].
- ! grep -q 'systemctl --user enable frank-medium' "$REMOTE_DEPLOY"
- # ...and the root llama unit must be retired, not relinked: relinking would
- # resurrect a service whose repo file no longer exists.
- ! grep -q 'relink_unit frank-medium' "$REMOTE_DEPLOY"
- grep -q 'retire_llama_medium_unit' "$REMOTE_DEPLOY"
-}
-
-@test "the llama layer split and the replica trio are archived, not deleted" {
- [ -f "$LLAMA_TRASH/frank-medium.service.llamacpp-layersplit" ]
- [ -f "$LLAMA_TRASH/frank-medium.env.llamacpp-layersplit" ]
- [ -f "$LLAMA_TRASH/serve-frank-medium.sh.llamacpp-layersplit" ]
- [ ! -f "$REPO_ROOT/scripts/frank-medium.env" ]
- [ -f "$REPLICA_TRASH/frank-medium-a.container" ]
- [ -f "$REPLICA_TRASH/frank-medium-proxy.container" ]
- # The replicas stay retired: the handler stops them but never installs them.
- grep -q 'systemctl --user stop frank-medium-a.service frank-medium-b.service' "$REMOTE_DEPLOY"
- ! grep -q 'install_quadlet frank-medium-a' "$REMOTE_DEPLOY"
-}
diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats
new file mode 100755
index 0000000..3746526
--- /dev/null
+++ b/tests/frank_primary_unit.bats
@@ -0,0 +1,94 @@
+#!/usr/bin/env bats
+# Seam: frank-primary — the Dankserver LLM endpoint on :7770.
+#
+# This file asserts the LUCEBOX shape (the 2026-09-26 cutover that replaced
+# frank-medium): a rootless Quadlet running the lucebox DS4.1 engine on the
+# two R9700s, with the model mmapped out of /home/dank/models and the
+# entrypoint/healthcheck mounted straight out of this checkout.
+
+load helpers
+
+UNIT="$REPO_ROOT/dankserver/frank-primary/quadlet/frank-primary.container"
+EP="$REPO_ROOT/dankserver/frank-primary/frank-primary-entrypoint.sh"
+HC="$REPO_ROOT/dankserver/frank-primary/frank-primary-healthcheck.py"
+
+@test "the frank-primary quadlet file and its mounted scripts exist" {
+ [ -f "$UNIT" ]
+ [ -x "$EP" ]
+ [ -f "$HC" ]
+ # The healthcheck is python; a syntax error would leave the container
+ # permanently unhealthy with the failure hidden inside HealthCmd.
+ python3 -m py_compile "$HC"
+}
+
+@test "frank-primary is boot-persistent and self-healing" {
+ # Rootless Podman does not auto-start bare containers and restart:
+ # unless-stopped does not survive shutdown's explicit stop. The unit must
+ # carry both the boot symlink and a restart policy (2026-08-26 incident:
+ # a compose-only Caddy never came back after a reboot).
+ grep -q '^WantedBy=default.target' "$UNIT"
+ grep -qE '^Restart=(always|on-failure)' "$UNIT"
+}
+
+@test "frank-primary binds the host network and needs no published port" {
+ # luce_server binds 0.0.0.0:7770 directly, so the container takes the
+ # host netns. A PublishPort would be dead config.
+ grep -q -- '--network=host' "$UNIT"
+ ! grep -q '^PublishPort=' "$UNIT"
+}
+
+@test "frank-primary reaches both R9700s and hides the Raphael iGPU" {
+ # /dev/kfd + /dev/dri via AddDevice; HIP_VISIBLE_DEVICES=0,1 keeps the
+ # host's Raphael iGPU (index 2) out of the enumeration so the engine's
+ # device picks cannot land on it.
+ grep -q '^AddDevice=/dev/kfd' "$UNIT"
+ grep -q '^AddDevice=/dev/dri' "$UNIT"
+ grep -q '^Environment=HIP_VISIBLE_DEVICES=0,1$' "$UNIT"
+}
+
+@test "frank-primary never names a bare HIP-reserved env var" {
+ # The HIP runtime reads bare TEMP for its JIT cache path (the 2026-08-13
+ # incident: TEMP=1.0 sent the JIT cache to path "1.0" and segfaulted every
+ # load). HOST and GPU_MAX_HEAP_SIZE are the same class. The unit's
+ # Environment= lines must not define them; the entrypoint exports
+ # GPU_MAX_HW_QUEUES (explicitly not GPU_MAX_HEAP_SIZE).
+ ! grep -qE '^Environment=(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$UNIT"
+ ! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$EP"
+}
+
+@test "the unit mounts the entrypoint and healthcheck from the checkout" {
+ # deploy.sh git-pulls this checkout before dispatching, so a Volume
+ # reference is how an edit here reaches the container without an image
+ # rebuild. Absolute host paths, read-only.
+ grep -q 'Volume=/home/dank/serverAppConfigs/dankserver/frank-primary/frank-primary-entrypoint.sh:/usr/local/bin/frank-primary-entrypoint.sh:ro' "$UNIT"
+ grep -q 'Volume=/home/dank/serverAppConfigs/dankserver/frank-primary/frank-primary-healthcheck.py:/usr/local/bin/frank-primary-healthcheck.py:ro' "$UNIT"
+}
+
+@test "first-start patience is configured on both sides of the health check" {
+ # Cold start page-ins the dense weights and builds the graphs; the
+ # container must not be killed as unhealthy while that happens, and the
+ # deployer must wait the same order of magnitude.
+ grep -q '^HealthStartPeriod=1800s' "$UNIT"
+ grep -q '^TimeoutStartSec=1800$' "$UNIT"
+ grep -q 'wait_container_healthy frank-primary 1800' \
+ "$REPO_ROOT/scripts/deploy-remote.sh"
+}
+
+@test "the deployer refuses to start without image and complete payload" {
+ # A 377 GiB partial download must never be served: the handler gates on
+ # the image being present and the model file existing at its exact
+ # declared byte count (stat -c %s, never wc -c — wc read the whole stream
+ # for minutes on 2026-09-26).
+ grep -q 'deploy_llm_primary() {' "$REPO_ROOT/scripts/deploy-remote.sh"
+ grep -q 'podman image exists localhost/lucebox-rocm:gfx1201' \
+ "$REPO_ROOT/scripts/deploy-remote.sh"
+ grep -q 'want_size=405349859328' "$REPO_ROOT/scripts/deploy-remote.sh"
+ grep -q 'stat -c %s' "$REPO_ROOT/scripts/deploy-remote.sh"
+}
+
+@test "the deployer gates on the port actually being free" {
+ # 7770 is a host-network bind: the previous holder must release it before
+ # the new container can be created, or pasta's bind fails at create time
+ # and the failure hides behind Restart=.
+ grep -q 'wait_port_free 7770' "$REPO_ROOT/scripts/deploy-remote.sh"
+}
diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats
index 829d03d..ea8d27a 100755
--- a/tests/observability_quadlet.bats
+++ b/tests/observability_quadlet.bats
@@ -315,19 +315,24 @@ PAIRS
@test "prometheus scrapes every LLM endpoint that exports metrics" {
# The LLM endpoints are the services whose behaviour this fleet argues about
- # most (KV pressure, prefix-cache reuse, queue depth). Since frank-medium
- # moved to radiance/vLLM on 2026-09-20 there are two metric namespaces, and
- # the three addresses differ by HOW each endpoint is reachable from a
- # container: netavark DNATs *published* container ports only, so
- # frank-medium — a rootless Quadlet on another host that publishes 7770 — is
- # its LAN address, while the two Dankclaw endpoints are HOST processes and
- # answer only as host.containers.internal. Addressing those two by LAN IP
- # left both targets red by construction until the 2026-09-20 /targets audit.
+ # most (KV pressure, prefix-cache reuse, queue depth). Only the two Dankclaw
+ # llama.cpp endpoints export metrics today: frank-primary (Dankserver, the
+ # Lucebox DS4.1 engine since the 2026-09-26 cutover) answers 404 on
+ # /metrics, so a job for it would be a permanently-down target — no job,
+ # and its LAN address must not sneak back in with one.
+ # The two scraped endpoints are HOST processes reached through
+ # host.containers.internal: this Prometheus is a bridge-networked
+ # container, and on this netavark host a container can only reach the
+ # host's LAN IP at a PUBLISHED port — a host-process bind like
+ # llama-server's 0.0.0.0:7770 is refused at 192.168.0.143:7770 while it
+ # answers at 169.254.1.2:7770 (why frank-small stayed red by construction
+ # for days until the 2026-09-20 /targets audit).
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
- grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-medium not scraped"; return 1; }
+ ! grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-primary is scraped but exports no metrics"; return 1; }
+
grep -q '"host.containers.internal:7771"' "$cfg" || { echo "frank-small not scraped"; return 1; }
grep -q '"host.containers.internal:7770"' "$cfg" || { echo "frank-large not scraped"; return 1; }
- for job in frank-medium frank-small frank-large; do
+ for job in frank-small frank-large; do
grep -q "job_name: \"$job\"" "$cfg" || { echo "no scrape job for $job"; return 1; }
done
# None of them is reached through a guarded vhost: Prometheus cannot solve
@@ -369,45 +374,30 @@ import re, yaml
d = yaml.safe_load(open('$rules'))
names = [r.get('alert') or r.get('record') for g in d['groups'] for r in g['rules']]
for want in ('job:llm_prefix_cache_reuse:rate5m',
- 'job:llm_vllm_prefix_cache_reuse:rate5m',
'job:llm_spec_accept_length:rate5m',
- 'LlmRequestsQueueing', 'LlmRequestsWaiting',
- 'LlmKvCacheSaturated', 'LlmPrefixCacheCollapse',
+ 'LlmRequestsQueueing', 'LlmPrefixCacheCollapse',
'LlmSpecDecodeOff', 'LlmEndpointDown'):
assert want in names, want
# Every rule must be keyed to a series that actually exists. A rule naming a
-# series nobody exports evaluates nothing forever and errors never — the failure
-# this seam exists for. Since 2026-09-20 two engines are scraped, so both
-# namespaces are legal, but each name must be on the measured list.
+# series nobody exports evaluates nothing forever and errors never — the
+# failure this seam exists for. Since the 2026-09-26 cutover the only scraped
+# LLM runtime is llama.cpp (frank-small, frank-large); the vllm: family left
+# with frank-medium's radiance runtime and must not survive in any expr.
exprs = [r.get('expr', '') for g in d['groups'] for r in g['rules']]
-# llama.cpp families measured from /metrics on 2026-09-16 (frank-small,
-# frank-large), vLLM families measured from /metrics on 2026-09-20
-# (frank-medium, radiance/vLLM). Anything else is a guess.
+# llama.cpp families measured from /metrics on 2026-09-16 and 2026-09-20
+# (frank-small, frank-large). Anything else is a guess.
live = {'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total',
'llamacpp:requests_deferred', 'llamacpp:requests_processing',
'llamacpp:spec_decode_num_drafts_total',
'llamacpp:spec_decode_num_accepted_tokens_total',
+ 'llamacpp:spec_decode_num_draft_tokens_total',
'llamacpp:tokens_predicted_total',
- 'vllm:kv_cache_usage_perc', 'vllm:num_requests_running',
- 'vllm:num_requests_waiting', 'vllm:num_preemptions_total',
- 'vllm:generation_tokens_total', 'vllm:prompt_tokens_total',
- 'vllm:spec_decode_num_accepted_tokens_total',
- 'vllm:spec_decode_num_draft_tokens_total',
- 'vllm:spec_decode_num_drafts_total',
- 'vllm:prefix_cache_hits_total', 'vllm:prefix_cache_queries_total',
- 'vllm:request_success_total'}
+ 'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds',
+ 'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode'}
used = {m for e in exprs
for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', e)}
assert used <= live, used - live
-# Engine-independent rules must cover BOTH families, or the cutover silently
-# blinds one endpoint: the two rules that describe an engine-agnostic failure
-# name a series from each namespace.
-joined = '\n'.join(exprs)
-for pair in (('LlmSpecDecodeOff', 'llamacpp:spec_decode_num_drafts_total',
- 'vllm:spec_decode_num_drafts_total'),
- ('LlmPrefixCacheCollapse', 'llamacpp:prompt_tokens_total',
- 'vllm:prefix_cache_queries_total')):
- assert any(pair[1] in e and pair[2] in e for e in exprs), pair[0]
+assert not any(':vllm' in e for e in exprs), 'vllm: series in rules but no vllm endpoint is scraped'
print('rules ok:', len(names))
"
# A rule file that nothing loads evaluates nothing, and Prometheus reports