diff --git a/scripts/build-frank-primary-rad.sh b/scripts/build-frank-primary-rad.sh new file mode 100755 index 0000000..895b32c --- /dev/null +++ b/scripts/build-frank-primary-rad.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# build-frank-primary-rad.sh — build frank-primary's .rad container from the +# OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint, at NATIVE precision. +# +# bash scripts/build-frank-primary-rad.sh # convert +# bash scripts/build-frank-primary-rad.sh --plan-only # declare, no I/O +# bash scripts/build-frank-primary-rad.sh --restage # redo staging +# bash scripts/build-frank-primary-rad.sh --force # rebuild existing +# +# WHY A STAGING DIR (the one non-obvious step): rad-convert selects its +# architecture plugin by the checkpoint's quantisation DESCRIPTOR — +# derived from config.json's quantization_config (core/format/checkpoint.cpp: +# "fp8" here). No shipped plugin claims (qwen4exp, "fp8"): the shipped +# qwen4exp plugin claims (qwen4exp, ""), the descriptor recipe-built +# containers carry (they are built from the bf16 upstream, which has no +# quantization_config at all). The match is exact with no fallback, so an +# unmodified convert refuses before declare. The descriptor is only a +# selection key; per-weight encodings are data — the checkpoint reader +# derives them from the tensors themselves (X.weight_scale_inv is the +# scale plane; checkpoint.h). Staging drops the quantization_config key +# (everything else is a relative symlink into the pristine download) so +# the convert selects the shipped plugin. EVERY weight is then copied through +# in its own encoding — block-FP8 codes + bf16 scale planes stay block-FP8 — +# except TWO value-preserving reshapes that stack-qwen38-official-fp8.py +# performs on the staging dir first, and ONE recipe rule that finishes the +# PLE table: +# * experts: the plugin maps the STACKED layout (mlp.experts.gate_up_proj +# [E,2F,H] with the _scale_inv plane at the same index) while the official +# repo ships per-expert unfused tensors — staging byte-stacks them +# (gate rows first, no requantisation). +# * PLE: the plugin's fp8 branch wants the table codes beside a ONE-ELEMENT +# bf16 scale plane; the official repo ships 128 SCALELESS fp8 shards plus +# one bf16 scalar. Staging decodes to bf16 with the checkpoint's own +# scale, and the convert's rtn rule re-encodes at that same scale — +# every original code is restored exactly and the scale plane lands in +# the form the plugin declares. +# +# WHY A CONTAINER AT ALL: a checkpoint dir has no mapped/file tiers, and a +# weight only a host kernel reads (the n-gram / PLE table, 51B params) +# cannot be served from one (GUIDE §3.4). rad-convert also bakes the +# tokenizer and the checkpoint's chat template; the runtime replaces the +# template with froggeric's (--override-chat-template). +# +# The convert runs rad-convert inside the digest-pinned radiance image +# (the host has no ROCm and needs none): /home/dank/models is mounted at +# /models so the staging dir's RELATIVE symlinks resolve inside the +# container. Output lands beside the raw download, owned by dank +# (rootless podman maps container-root to the invoking user). +set -euo pipefail + +SRC="/home/dank/models/Qwen/Qwen3.8-Flash-Next-FP8" +STAGE="/home/dank/models/stage-qwen4exp-fp8" +OUT_DIR="/home/dank/models/rad" +OUT="$OUT_DIR/qwen4exp-fp8-native.rad" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/frank-primary.env" +IMAGE="$FRANK_PRIMARY_IMAGE" + +plan_only=0 restage=0 force=0 +for a in "$@"; do + case "$a" in + --plan-only) plan_only=1 ;; + --restage) restage=1 ;; + --force) force=1 ;; + *) echo "unknown flag: $a" >&2; exit 2 ;; + esac +done + +if [ ! -f "$SRC/config.json" ]; then + echo "ERROR: raw checkpoint missing: $SRC (run scripts/download-frank-primary.sh --fetch)" >&2 + exit 1 +fi +echo "verifying raw checkpoint first..." +bash "$SCRIPT_DIR/download-frank-primary.sh" + +if [ -f "$OUT" ] && [ "$plan_only" -eq 0 ] && [ "$force" -eq 0 ]; then + echo "container already exists: $OUT" + echo "rebuild it with --force (the old file moves to ~/.AGENTTRASH)" + exit 0 +fi + +# ---- staging ---------------------------------------------------------------- +if [ "$restage" -eq 1 ] && [ -e "$STAGE" ]; then + TRASH="$HOME/.AGENTTRASH/$(date +%Y%m%d)-frank-primary-stage" + mkdir -p "$TRASH" + mv "$STAGE" "$TRASH/stage-$(date +%H%M%S)" + echo "old staging dir moved to $TRASH" +fi +if [ ! -f "$STAGE/config.json" ]; then + mkdir -p "$STAGE" + python3 - "$SRC/config.json" "$STAGE/config.json" <<'PY' +import json, sys +src, dst = sys.argv[1], sys.argv[2] +cfg = json.load(open(src)) +dropped = cfg.pop("quantization_config", None) +json.dump(cfg, open(dst, "w"), indent=2) +print(f"staging config.json written (quantization_config dropped: {dropped is not None})") +PY + # Everything else is a RELATIVE symlink — relative so the targets resolve + # inside the convert container (the stage's parent is mounted at /models). + for f in "$SRC"/*; do + name="$(basename "$f")" + [ "$name" = "config.json" ] && continue + ln -sfn "../Qwen/Qwen3.8-Flash-Next-FP8/$name" "$STAGE/$name" + done + echo "staging dir ready: $STAGE ($(ls "$STAGE" | wc -l) entries)" +fi + +# ---- checkpoint reshape (stacked experts + bf16 PLE table) ------------------- +# stack-qwen38-official-fp8.py reshapes the staged checkpoint in place +# (idempotent) and writes the PLE scale to $STAGE/ple_scale.txt — see the +# script's docstring for why each transform is value-preserving. +echo "stacking experts + merging the PLE table (value-preserving)..." +python3 "$SCRIPT_DIR/stack-qwen38-official-fp8.py" --stage "$STAGE" +PLE_SCALE="$(tr -d '\n' < "$STAGE/ple_scale.txt")" +echo "PLE fixed scale: $PLE_SCALE" + +# ---- convert ---------------------------------------------------------------- +# rad-convert needs no GPU: declare resolves kernels and asks each about the +# encoding it will read — CPU-side layout questions only. (If a plugin load +# ever demands a card, add --device /dev/kfd --device /dev/dri to this run.) +# --plan-only runs the same declare phase and exits before a byte is written. +echo "rad-convert: $SRC (staged) -> $OUT" +mkdir -p "$OUT_DIR" +CONVERT_ARGS=(/models/stage-qwen4exp-fp8 -o /models/rad/qwen4exp-fp8-native.rad + --home /opt/radiance/share/radiance) +[ "$plan_only" -eq 1 ] && CONVERT_ARGS+=(--plan-only -v) +# THE ONE RECIPE RULE (see header): the PLE table arrives from staging as a +# bf16 merge of the checkpoint's own fp8 codes decoded with the checkpoint's +# own scale; rtn re-encodes it at that same scale — the codes come back +# exactly and the plugin's one-element scale plane is produced. +CONVERT_ARGS+=(--quant "blk.*.ple_ngram.weight=rtn:codes=fp8_e4m3,block=*x*,scale=bf16,rule=fixed,scale_value=$PLE_SCALE") +[ "$force" -eq 1 ] && [ -f "$OUT" ] && { + TRASH="$HOME/.AGENTTRASH/$(date +%Y%m%d)-frank-primary-rad" + mkdir -p "$TRASH" + mv "$OUT" "$TRASH/qwen4exp-fp8-native.rad.$(date +%H%M%S)" + echo "previous container moved to $TRASH" +} +podman run --rm \ + --entrypoint rad-convert \ + -v /home/dank/models:/models \ + "$IMAGE" "${CONVERT_ARGS[@]}" + +if [ "$plan_only" -eq 1 ]; then + echo "plan-only run complete (no container written)" + exit 0 +fi + +# ---- record ----------------------------------------------------------------- +echo "rad-info summary:" +podman run --rm --entrypoint rad-info -v /home/dank/models:/models:ro \ + "$IMAGE" "/models/rad/qwen4exp-fp8-native.rad" | head -20 +MIB=$(( $(stat -c %s "$OUT") / 1048576 )) +echo "container: $OUT (${MIB} MiB; env floor $FRANK_PRIMARY_RAD_MIN_MIB MiB)" +if [ "$MIB" -lt "$FRANK_PRIMARY_RAD_MIN_MIB" ]; then + echo "ERROR: container below the env floor — the convert was truncated" >&2 + exit 1 +fi +echo "build complete" diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 662fa18..0c26a41 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1207,12 +1207,14 @@ deploy_llm_large() { # # It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is # not shared: frank-large's engine runs on the gfx1151 iGPU. -LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/Ornith-Q8/Ornith-1.5-9B-Q8_0.gguf}" +LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/Swift15-IQ3S/Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}" LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" -# The projector IS the vision capability here: the Ornith GGUF ships no -# `v.*` tensors, so a deploy that starts without this file silently ships a +# The projector IS the vision capability here: no Swift repo ships one (the +# GGUF carries no vision tensors), so the file is unsloth's mmproj built +# from the shared Qwen3.8-27B base — proven byte-identical to Swift's own +# vision stack. A deploy that starts without this file silently ships a # text-only endpoint. Checked unconditionally, like frank-large's. -LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/Ornith-Q8/mmproj-Ornith-1.5-9B-BF16.gguf}" +LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/Swift15-IQ3S/mmproj-Qwen3.8-27B-F16.gguf}" deploy_llm_small() { # Refuse rather than restart-loop: a unit whose runtime or payload is @@ -1250,23 +1252,24 @@ deploy_llm_small() { echo "frank-small: health answered by unit MainPID $main_pid" } -# deploy_llm_primary — frank-primary is the FreeToken engine (PR #587 ROCm -# build, gfx1201) serving the orcarouter Qwen3.8-Flash-Next-Uncensored-FP8 -# payload (131 safetensors shards) on ONE R9700 as a ROOT unit whose serve -# script drives `podman run` directly (the image carries `ft serve`). -# 2026-10-05 cutover from the tcclaviger vLLM fork (pre-cutover tag -# frank-primary-vllm-shapeG-20261003); before that, 2026-09-29 cutover from -# the R9V radiance fork, whose launcher/shim/setup-state machinery is -# retired. Same port and alias as the endpoints it replaced; the harness -# models.yml id is unchanged. +# deploy_llm_primary — frank-primary is the radiance engine (StillDeadcode, +# digest-pinned stilldeadcode/radiance image) serving the OFFICIAL +# Qwen/Qwen3.8-Flash-Next-FP8 at native block-FP8 precision from the +# rad-convert container qwen4exp-fp8-native.rad, on BOTH R9700s (TP=2) as a +# ROOT unit whose serve script drives `podman run` directly (the engine is +# the image entrypoint; the image lives in dank's rootless podman storage). +# 2026-10-05 cutover from FreeToken (pre-cutover tag +# frank-primary-freetoken-pr587-20261005); before that, the tcclaviger vLLM +# fork (tag frank-primary-vllm-shapeG-20261003). Same port and alias; the +# harness models.yml id is unchanged. # -# Gates: the runtime image must exist in dank's rootless storage -# (FRANK_PRIMARY_IMAGE — a LOCAL tag: the image is built on the host from -# scripts/docker/frank-primary-freetoken.Dockerfile, which pins the -# FreeToken PR-head SHA and the base image), the payload dir must carry -# config.json and the expected shard count (FRANK_PRIMARY_MODEL_SHARDS), -# and the unit env must be complete. Per-file sha256 verification was the -# fetch script's job at download time, never re-hashed here. +# Gates: the image must exist under its digest ref in dank's rootless +# storage (FRANK_PRIMARY_IMAGE), the .rad container file must exist at the +# env's size floor (FRANK_PRIMARY_MODEL_FILE >= FRANK_PRIMARY_RAD_MIN_MIB — +# a truncated convert is a guaranteed long-timeout boot), and the unit env +# must be complete. Per-file sha256 verification is the fetch script's job +# at download time and rad-convert/rad-info's job at build time, never +# re-hashed here. LLM_PRIMARY_CONTAINER=frank-primary LLM_PRIMARY_LEGACY_CONTAINERS="frank-primary-qwen frank-primary-r9v qwen-fp8 glm-probe" @@ -1275,44 +1278,36 @@ deploy_llm_primary() { # missing never becomes healthy, and a green deploy would leave the # endpoint dark with nothing in the log to say why. local env_file="/home/dank/serverAppConfigs/scripts/frank-primary.env" - local image model_dir shards count + local image model_dir model_file rad_min mib if [[ ! -f "$env_file" ]]; then echo "ERROR: frank-primary env file absent: $env_file" >&2 return 1 fi image="$(sed -n 's/^FRANK_PRIMARY_IMAGE=//p' "$env_file" | tail -1)" model_dir="$(sed -n 's/^FRANK_PRIMARY_MODEL_DIR=//p' "$env_file" | tail -1)" - shards="$(sed -n 's/^FRANK_PRIMARY_MODEL_SHARDS=//p' "$env_file" | tail -1)" - if [[ -z "$image" || -z "$model_dir" || -z "$shards" ]]; then - echo "ERROR: frank-primary.env missing FRANK_PRIMARY_IMAGE/MODEL_DIR/MODEL_SHARDS" >&2 + model_file="$(sed -n 's/^FRANK_PRIMARY_MODEL_FILE=//p' "$env_file" | tail -1)" + rad_min="$(sed -n 's/^FRANK_PRIMARY_RAD_MIN_MIB=//p' "$env_file" | tail -1)" + if [[ -z "$image" || -z "$model_dir" || -z "$model_file" || -z "$rad_min" ]]; then + echo "ERROR: frank-primary.env missing FRANK_PRIMARY_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2 return 1 fi - # The image gate is EXISTENCE IN STORAGE, not a registry digest: the - # FreeToken image is built ON THE HOST (a local tag carries NO - # RepoDigests entry — the old grep refused exactly this shape), and - # pinning lives in the Dockerfile (pinned PR-head SHA + digest-pinned - # base image). `podman image exists` accepts a local tag and a - # name@digest ref alike, in one command — no pipefail/grep race (the - # 2026-09-30 lesson) for either shape. + # The image gate is the digest ref itself: a name@sha256 ref exists in + # storage only if THAT manifest was pulled. One command — no + # pipefail/grep race (the 2026-09-30 lesson). if ! podman image exists "$image"; then - echo "ERROR: runtime image absent from rootless storage: $image" >&2 - echo "frank-primary: build it (scripts/docker/frank-primary-freetoken.Dockerfile) or pull it" >&2 + echo "ERROR: runtime image absent from storage: $image" >&2 + echo "frank-primary: podman pull docker://stilldeadcode/radiance:latest" >&2 return 1 fi - if [[ ! -f "$model_dir/config.json" ]]; then - echo "ERROR: payload config missing: $model_dir/config.json" >&2 - echo "frank-primary: re-run scripts/download-frank-primary.sh" >&2 + if [[ ! -f "$model_dir/$model_file" ]]; then + echo "ERROR: .rad container missing: $model_dir/$model_file" >&2 + echo "frank-primary: run scripts/build-frank-primary-rad.sh" >&2 return 1 fi - if [[ ! -f "$model_dir/model.safetensors.index.json" ]]; then - echo "ERROR: weight-map index missing: $model_dir/model.safetensors.index.json" >&2 - echo "frank-primary: re-run scripts/download-frank-primary.sh" >&2 - return 1 - fi - count="$(find "$model_dir" -maxdepth 1 -name 'model-*-of-*.safetensors' 2>/dev/null | wc -l)" - if [[ "$count" != "$shards" ]]; then - echo "ERROR: payload shard count wrong: $model_dir (found $count, want $shards)" >&2 - echo "frank-primary: re-run scripts/download-frank-primary.sh" >&2 + mib=$(( $(stat -c %s "$model_dir/$model_file") / 1048576 )) + if [[ "$mib" -lt "$rad_min" ]]; then + echo "ERROR: .rad container too small: $model_dir/$model_file is ${mib} MiB, want >= $rad_min" >&2 + echo "frank-primary: rebuild it (scripts/build-frank-primary-rad.sh --force)" >&2 return 1 fi @@ -1337,18 +1332,16 @@ deploy_llm_primary() { run_privileged systemctl daemon-reload run_privileged systemctl enable frank-primary.service run_privileged systemctl restart frank-primary.service - # Cold start: FP8 page-in plus torch compile/graph capture (~15 min cold, - # measured 2026-09-29). Parity with every predecessor on this port. - # FreeToken's /health returns 200 with {"status":"loading"} while the - # engine loads (the FastAPI front end binds before the scheduler is - # up), so a bare 200 false-greens — measured 2026-10-06: "healthy after - # 6 s" while 131 shards were still paging in. vLLM 503'd during load, - # which made the bare curl honest there; FreeToken needs the BODY read. - # Only status:"ok" counts. (The MainPID check below still guards the - # stray-port case.) + # Cold start: .rad load + PLE pin + expert host-pool upload (the + # publisher's compose healthcheck allows a 15-minute start_period). + # Parity with every predecessor on this port. The HTTP listener binds + # AFTER the full load (main.cpp: engine.start() then engine.serve()), + # so during load the port REFUSES connections — a bare curl is honest + # here, unlike FreeToken's 200-while-loading body. (The MainPID check + # below still guards the stray-port case.) local i ok=0 for ((i=1; i<=900; i++)); do - if curl -fsS http://127.0.0.1:7770/health 2>/dev/null | grep -q '"status":"ok"'; then + if curl -fsS http://127.0.0.1:7770/health 2>/dev/null >/dev/null; then echo "frank-primary engine healthy after approximately $((i * 2))s" ok=1 break @@ -1356,7 +1349,7 @@ deploy_llm_primary() { sleep 2 done if [[ "$ok" != 1 ]]; then - echo "FAIL: frank-primary never reached /health status ok in 1800s" >&2 + echo "FAIL: frank-primary /health never answered 200 in 1800s" >&2 return 1 fi # False-green hardening (the 2026-09-26 lesson): a bare curl against the diff --git a/scripts/docker/frank-primary-freetoken.Dockerfile b/scripts/docker/frank-primary-freetoken.Dockerfile deleted file mode 100644 index 2d9e4b1..0000000 --- a/scripts/docker/frank-primary-freetoken.Dockerfile +++ /dev/null @@ -1,63 +0,0 @@ -# frank-primary-freetoken.Dockerfile — FreeToken (PR #587 ROCm) serving image -# for the frank-primary endpoint (dankserver :7770, 1x R9700 gfx1201). -# -# Built on dankserver with rootless podman (dank user), same storage the -# service's `podman run` reads: -# podman build -t frank-primary-freetoken:pr587-77a7201 \ -# -f scripts/docker/frank-primary-freetoken.Dockerfile scripts/docker/ -# -# Base: the PR's own docs/install_amd.md names this exact image as the -# known-good ROCm 7.14 stack for gfx1201 (torch 2.11.0+rocm7.14, triton-rocm -# 3.7 — the original pin; FreeToken main+PR587 accepts torch>=2.11,<2.16 and -# triton>=3.6,<3.9). There is no rocm7.14+torch2.14 image on Docker Hub (the -# 2.14 images are rocm10.1.0 — an untested pair for this PR), so the PR doc's -# verbatim recommendation is the deployed stack. -# -# FreeToken source: FlashML-org/FreeToken PR #587 at pinned head -# 77a7201f98f00d93f618c6cf8b81ffe8360e0899 (branch rocm714_torch214, base -# main 581fdca3). The PR is ROCm-gated end to end; the NVIDIA/CUDA path is -# byte-identical to main. -# -# Build law from docs/install_amd.md: -# - `pip install --no-build-isolation --no-deps -e .` — the image's ROCm -# torch is preserved and used to compile the C++/HIP extensions. -# - NEVER a plain `pip install -e .` on ROCm: the dependency set is -# authored for NVIDIA (flashlib pulls numba + nvidia-cutlass-dsl; torch -# metadata pulls cuda-bindings) and the resolve can REPLACE the ROCm -# torch with a CUDA build. -# - Runtime deps installed explicitly, curated by the same doc. flashinfer -# / sgl_kernel / nvidia-cutlass-dsl have no ROCm builds and are -# intentionally absent — FreeToken gates them off and falls back to its -# Triton kernels on ROCm. -# -# Base is digest-pinned (2026-10-05): sha256:a223aee1… is the manifest this -# build was qualified against; a re-pushed tag cannot change the stack. The -# readable tag rides along for humans (podman verifies the digest matches). -FROM rocm/pytorch:rocm7.14_ubuntu24.04_py3.12_pytorch_release_2.11.0@sha256:a223aee17aef5d21c3b9f63436dd19d27d1c665ec8b2f40011c9546cabae2a80 - -# Pin the PR head, not the branch: a future push to rocm714_torch214 would -# otherwise silently change the deployed engine. -# The pip-ROCm torch in this image ships libhiprtc as the VERSIONED -# libhiprtc.so.7 only; the PR's `_cpu_moe` extension links `-lhiprtc`, and -# ld looks for the unversioned dev symlink in the ALREADY-LINKED -# _rocm_sdk_core/lib dir. Create it there (first build failure 2026-10-05: -# `ld: cannot find -lhiprtc`). -RUN ln -s libhiprtc.so.7 /opt/venv/lib/python3.12/site-packages/_rocm_sdk_core/lib/libhiprtc.so -RUN git clone https://github.com/FlashML-org/FreeToken.git /opt/FreeToken \ - && cd /opt/FreeToken \ - && git fetch origin pull/587/head \ - && git checkout 77a7201f98f00d93f618c6cf8b81ffe8360e0899 - -WORKDIR /opt/FreeToken -RUN python -m pip install --no-build-isolation --no-deps -e . - -# Curated runtime dependencies (docs/install_amd.md). transformers is pinned -# to the pyproject window (>=5.16,<5.17); the rest are the doc's portable set. -# torch / torchvision / triton-rocm come from the base image, 1:1 aligned. -RUN python -m pip install \ - "transformers>=5.16,<5.17" tokenizers safetensors huggingface_hub \ - fastapi uvicorn pydantic openai httpx anyio \ - einops gguf msgpack pyzmq typer rich apache-tvm-ffi \ - partial-json-parser \ - && python -m pip install --no-deps flashlib==0.3.0 \ - && python -m pip install ninja diff --git a/scripts/download-frank-primary.sh b/scripts/download-frank-primary.sh index 8347940..083b918 100755 --- a/scripts/download-frank-primary.sh +++ b/scripts/download-frank-primary.sh @@ -1,88 +1,111 @@ #!/usr/bin/env bash -# Payload inventory/verification for frank-primary: -# orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8 (compressed-tensors FP8, 131 -# shards + weight-map index, ~172.8 GiB) — the tcclaviger-stack payload that -# became frank-primary on 2026-09-29. +# download-frank-primary.sh — payload fetch/verify for frank-primary: +# the OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint (131 safetensors +# shards + weight-map index + tokenizer + vision preprocessing configs, +# ~172.8 GB). 2026-10-05: replaces the orcarouter uncensored payload of the +# FreeToken era (that script lived here; its repo was gated and needed +# HF_TOKEN — this one is UNGATED and needs no token at all). # -# bash scripts/download-frank-primary.sh --verify-only (the ONLY mode) -# bash scripts/download-frank-primary.sh --fetch +# bash scripts/download-frank-primary.sh --fetch resume/complete +# bash scripts/download-frank-primary.sh verify only # -# THIS REPO IS GATED (401 anonymous — measured 2026-09-29): anonymous fetch -# is impossible and the tree API masks LFS hashes, so a byte-exact upstream -# manifest cannot be built without the authed endpoint. The payload on -# /home/dank/models was fetched with the operator's HF token (out of band: -# HF_TOKEN in the environment, never in argv) at bring-up time. -# -# Modes: -# --fetch resume any missing/truncated shard via `hf download` with -# HF_TOKEN from the environment (NEVER passed on the command -# line — a -H auth header is world-readable in -# /proc//cmdline, the 2026-09-16 leak). -# --verify-only default: local integrity check — shard count vs the -# weight-map index, every file nonzero and its size equal to -# the index's declared total, plus sha256 of every shard -# (slow, ~10 min for 173 GiB). +# Fetch uses the vendored scripts/hfget.sh (StillDeadcode's Apache-2.0 +# listing/resume downloader, curl+jq only): per-file size checks against +# the Hub listing, `-C -` resume, pickle-format duplicates skipped. +# Verify (default) is the AGENTS.md law in three parts: every listed file +# present with the listed size, the shard count matches the weight-map +# index, and every shard's sha256 equals the Hub's LFS oid. Verify needs +# only the public API. set -euo pipefail -REPO="orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8" -DEST="/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8" +REPO="Qwen/Qwen3.8-Flash-Next-FP8" +DEST="/home/dank/models/Qwen/Qwen3.8-Flash-Next-FP8" EXPECTED_SHARDS=131 mode="verify" [[ "${1:-}" == "--fetch" ]] && mode="fetch" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [[ "$mode" == "fetch" ]]; then - : "${HF_TOKEN:?--fetch needs HF_TOKEN in the environment (never argv)}" - export HF_TOKEN - mkdir -p "$DEST" - hf download "$REPO" --local-dir "$DEST" + # HFGET_JOBS=4 is hfget's default (a few streams saturate the link). + bash "$SCRIPT_DIR/hfget.sh" "$REPO" "$DEST" main fi echo "verifying local payload: $DEST" fail=0 -# The weight-map index is the source of truth for the shard set. INDEX="$DEST/model.safetensors.index.json" if [[ ! -f "$INDEX" ]]; then echo " BAD weight-map index missing: $INDEX" >&2 exit 1 fi -# Every index entry must exist, nonzero, and sized equal to the map's total. -python3 - "$INDEX" "$DEST" <<'PY' || fail=1 -import json, os, sys -index, dest = sys.argv[1], sys.argv[2] -wm = json.load(open(index))["weight_map"] -files = sorted(set(wm.values())) -print(f" index maps {len(wm)} tensors across {len(files)} files") +python3 - "$REPO" "$DEST" <<'PY' || fail=1 +import hashlib, json, os, sys, urllib.request + +repo, dest = sys.argv[1], sys.argv[2] + +def listing(): + """The tree API, following the Link cursor exactly like hfget.sh.""" + out, api, pages = [], ( + f"https://huggingface.co/api/models/{repo}/tree/main" + "?recursive=true&expand=true"), 0 + while api: + with urllib.request.urlopen(api) as r: + out += json.load(r) + api = None + link = r.headers.get("Link", "") + for part in link.split(","): + if 'rel="next"' in part: + api = part[part.find("<") + 1:part.find(">")] + pages += 1 + if pages > 200: + raise SystemExit("listing did not terminate") + return out + +files = [e for e in listing() if e.get("type") == "file"] +sizes = {e["path"]: e.get("size", 0) for e in files} +lfs = {e["path"]: e["lfs"]["oid"] for e in files if e.get("lfs")} + bad = 0 -total = 0 -for f in files: - p = os.path.join(dest, f) - if not os.path.isfile(p) or os.path.getsize(p) == 0: - print(f" BAD {f}: missing or zero bytes" ) +for path, want in sorted(sizes.items()): + p = os.path.join(dest, path) + if not os.path.isfile(p): + print(f" BAD {path}: missing") bad += 1 - else: - total += os.path.getsize(p) -print(f" {'OK' if bad == 0 else 'FAIL'}: {len(files) - bad}/{len(files)} files present, {total/2**30:.1f} GiB total") -raise SystemExit(1 if bad else 0) + elif os.path.getsize(p) != want: + print(f" BAD {path}: {os.path.getsize(p)} bytes, listing says {want}") + bad += 1 +print(f" {'OK' if bad == 0 else 'FAIL'}: {len(sizes) - bad}/{len(sizes)} files present at listed size") +if bad: + raise SystemExit(1) + +# Shard set against the weight-map index. +wm = json.load(open(os.path.join(dest, "model.safetensors.index.json")))["weight_map"] +shards = sorted(set(wm.values())) +print(f" index maps {len(wm)} tensors across {len(shards)} shards") +if len(shards) != 131: + print(f" BAD shard count {len(shards)}, want 131") + raise SystemExit(1) + +# Content: every LFS-tracked file hashed against the Hub oid. The safetensors +# shards dominate the bytes; tokenizer/config files ride the same check when +# the Hub tracks them in LFS (they are plain blobs here, already size-checked). +todo = [p for p in shards if p in lfs] +for i, path in enumerate(todo): + h = hashlib.sha256() + with open(os.path.join(dest, path), "rb") as f: + for chunk in iter(lambda: f.read(1 << 24), b""): + h.update(chunk) + if h.hexdigest() != lfs[path]: + print(f" BAD {path}: sha256 mismatch (want {lfs[path]})") + bad += 1 + if (i + 1) % 16 == 0: + print(f" ...hashed {i + 1}/{len(todo)}") +print(f" {'OK' if bad == 0 else 'FAIL'}: {len(todo) - bad}/{len(todo)} shards sha256-match the Hub") +if bad: + raise SystemExit(1) PY -# Shard count gate (what the serve script and deployer check). -n="$(find "$DEST" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)" -if [[ "$n" != "$EXPECTED_SHARDS" ]]; then - echo " BAD shard count: found $n, want $EXPECTED_SHARDS" >&2 - fail=1 -else - echo " OK shard count $n" -fi - -if [[ "${2:-}" == "--hash" || "${VERIFY_HASH:-0}" == "1" ]]; then - echo "hashing every shard (slow)..." - cd "$DEST" - sha256sum -c --quiet model.safetensors.index.json 2>/dev/null || \ - sha256sum model-*-of-00131.safetensors > /dev/null # presence check; content hash list printed on demand -fi - [[ "$fail" -eq 0 ]] || exit 1 -echo "payload inventory OK: $EXPECTED_SHARDS shards + index present" +echo "payload verified: 131 shards + index + tokenizer + configs, sizes and sha256 checked" diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index 266c51d..cae68a8 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -1,82 +1,135 @@ -# frank-primary.env — FreeToken (PR #587 ROCm) serving the orcarouter -# Qwen3.8-Flash-Next-Uncensored-FP8 payload on dankserver :7770. +# frank-primary.env — radiance (StillDeadcode, Apache-2.0) serving the +# OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint at NATIVE precision +# (modelopt block-FP8 throughout, no requantisation) on BOTH R9700s (TP=2), +# dankserver :7770. # -# 2026-10-05 cutover from the tcclaviger RDNA4 vLLM fork (git tag -# frank-primary-vllm-shapeG-20261003 is the pre-cutover state; the 362-line -# vLLM-era env is in that tag). The engine is FreeToken at PR head -# 77a7201f98f00d93f618c6cf8b81ffe8360e0899 (branch rocm714_torch214), built -# on the host into a local image by -# scripts/docker/frank-primary-freetoken.Dockerfile — the PR's own -# docs/install_amd.md stack (ROCm 7.14 base image, torch 2.11.0+rocm7.14, -# triton 3.7) for gfx1201. +# 2026-10-05 cutover from FreeToken (PR #587 ROCm, one card, experts in +# pinned host RAM; git tag frank-primary-freetoken-pr587-20261005 is the +# pre-cutover state — its unit/env/serve trio and Dockerfile are all in the +# tag and rebuildable). Before that: tcclaviger vLLM fork (tag +# frank-primary-vllm-shapeG-20261003). # -# WHY FREETOKEN: the vLLM fork's expert offload planned VRAM/host tiers per -# boot from an estimator (util 0.982 law, arena 17.77 GiB / 7,762 slots, -# ~40k-token GDN state faults) and 1.0 was unbootable. FreeToken keeps ALL -# expert weights in pinned HOST RAM (~120 GiB) with an LRU GPU cache, sizes -# KV from an explicit token pool, and serves on ONE R9700 (32 GB) with the -# second card idle. RAM fit: 246 GiB total, service stopped frees the vLLM -# container's ~170 GiB; weights ~120 GiB + PLE 47.7 GiB + overhead ≈ 190 GiB. +# WHY RADIANCE: the engine is C++/HIP with libr4d kernels built for +# gfx1201, drives `podman run` directly from the digest-pinned +# stilldeadcode/radiance image (287 MiB; host needs only the amdgpu driver), +# and its qwen4exp plugin reads the modelopt FP8 encoding NATIVELY +# (`X.weight_scale_inv` is the scale plane — checkpoint.h). The PLE n-gram +# table (51B params, 47.7 GiB) is served from pinned host RAM via io_uring +# reads — Frank's call: "Native FP8. Experts from ram. Pin the PLE to ram." +# The 4-bit-codebook publisher shape (qwen3.8-next-flash-fp8-iq4r-moe.rad, +# --host-pool-mib 12288) remains the documented fallback if the native +# container proves unbootable. # -# PAYLOAD: unchanged — /home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8, -# 131 safetensors shards. Quant is standard modelopt block-FP8: 75,264 -# weight_scale_inv tensors (block 128x128), zero input_scale (FreeToken #462 -# covers exactly this export shape), one weight_scale on the PLE ngram -# embedding. FreeToken loads HF safetensors directly; no FTW conversion. +# PAYLOAD: /home/dank/models/rad/qwen4exp-fp8-native.rad — a rad-convert +# NO-RECIPE pure copy of the official 131-shard checkpoint (every weight +# keeps its block-FP8 encoding; ~161 GiB). The convert runs against a +# staging dir whose config.json drops `quantization_config`: the descriptor +# string is only the plugin SELECTION key (spec §2.4 — exact match, no +# fallback), and no shipped plugin claims (qwen4exp, "fp8"); the shipped +# qwen4exp plugin claims (qwen4exp, "") which is what recipe-built +# containers carry. Per-weight encodings are data, not the descriptor — +# removing the key changes nothing about what the weights hold. Built by +# scripts/build-frank-primary-rad.sh (staging + convert + rad-info). # -# SERVING FACTS (docs/cli.md, verified against main+PR587): -# - `ft serve` loads the checkpoint directly; `--served-model-name` keeps the -# harness id; reasoning/tool parsers are AUTO (reasoning lands in -# reasoning_content — models.yml flips reasoningContentField to match). -# - KV: --page-size must stay 1 for GDN/SWA models (default); --cache-type -# radix auto-picks the GDN-aware variant. -# - MoE: --moe-strategy auto -> offload (fp8 experts CANNOT use -# --moe-cpu-layers; only bf16/nvfp4/mxfp4 have a CPU executor). -# --moe-cache-rate holds the GPU LRU fraction of all experts. -# - `ft ctl cache --kv --moe` grows the pools LIVE on a running server -# (shrinking needs a restart); `ft ctl stats` reports pool occupancy. -# - KV MATH: 12 full-attention layers x 2 KV heads x 256 head_dim x 2 (K+V) -# x 2 B = 24 KiB/token (f16 KV; the 36 GDN layers use fixed per-seq state, -# not per-token KV). See FRANK_PRIMARY_FT_NUM_TOKENS below for the pool -# size and its measured page-size law. -FRANK_PRIMARY_IMAGE=localhost/frank-primary-freetoken:pr587-77a7201 -FRANK_PRIMARY_MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8 -FRANK_PRIMARY_MODEL_SHARDS=131 +# SERVING FACTS (docs/GUIDE.md, docs/DOCKER.md, compose/flashnext.yaml): +# - The HTTP listener binds AFTER the full load (main.cpp: engine.start() +# then engine.serve()) — during load the port refuses connections, so a +# bare curl /health gate is honest. /health == /ping: 200 JSON while +# serving, 503 only while stopping (core/server/admin.cpp:385). +# - Reasoning: --reasoning-format defaults to auto, emitting thinking in +# `reasoning_content` (config.cpp). Tool calls: the qwen3.8 dialect, from +# the chat format. Both need no parser flags. +# - Sampling: the OpenAI endpoint accepts temperature, top_p, top_k, min_p, +# typical_p, presence_penalty, frequency_penalty, repetition_penalty, +# repeat_last_n, chat_template_kwargs, reasoning_effort, +# enable_thinking, preserve_thinking (core/server/oai.cpp:194-205) — +# models.yml's extraBody set is fully inside it. Unknown params are +# refused (400), which is why nothing unverified rides the wire. +# - MTP: the official checkpoint carries the mtp.* head (3,101 tensors, +# mtp_num_hidden_layers=1) and the plugin probes draft depth 3 — the +# publisher's throughput peak for this model. Stated explicitly with +# --num-speculative-tokens 3. +# - The fused TP all-reduce is declared over max-num-seqs x +# (1 + --num-speculative-tokens) rows; 2 x 4 = 8 rows fits either wire. +FRANK_PRIMARY_IMAGE=stilldeadcode/radiance@sha256:4c3f484564bb2872784e46b0a59cbd71becc9c1e746e301a3596ffae253c322e +# The .rad container FILE (mounted under /models) and the state dir for the +# prefix-cache disk tier (mounted rw at /data). The model dir holds ONLY +# .rad files; the raw HF download and its staging dir live beside it and +# are never mounted. +FRANK_PRIMARY_MODEL_DIR=/home/dank/models/rad +FRANK_PRIMARY_MODEL_FILE=qwen4exp-fp8-native.rad +FRANK_PRIMARY_STATE_DIR=/home/dank/radiance-state # Container identity is the SERVICE name (Frank's call, 2026-10-05): one -# name across systemctl and podman ps. The vLLM-era name frank-primary-qwen -# moved into the legacy list so every deploy cleans it up. +# name across systemctl and podman ps. The runtime changes; the name does +# not. The legacy list retires every predecessor identity that could hold +# the GPUs or the port. FRANK_PRIMARY_CONTAINER=frank-primary FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-qwen frank-primary-r9v qwen-fp8 glm-probe" FRANK_PRIMARY_SERVED_NAME=frank-primary -FRANK_PRIMARY_PUBLISH=7770:7770 +FRANK_PRIMARY_PORT=7770 +# Frank's chat-template call: froggeric's fixed Qwen3.8 template (the +# repo's root chat_template.jinja, vendored here; the checkpoint's own +# template is baked into the .rad by rad-convert and this overrides it). +# Radiance derives the reasoning/tool-call reply format from it. +FRANK_PRIMARY_TEMPLATE=/home/dank/serverAppConfigs/scripts/qwen38-froggeric-v22.5.jinja -# --- FreeToken engine knobs (first-boot conservative, resize live) ----- -# Single GPU: the R9700 at index 0. HIP_VISIBLE_DEVICES=0 hides card 1 so -# the in-container index 0 IS the right card; --gpu 0 selects it. -FRANK_PRIMARY_FT_GPU=0 -# KV pool in TOKENS (not bytes — vLLM-era FRANK_PRIMARY_KV_BYTES is gone). -# 299,968 tokens x 24 KiB = 7.0 GiB; clears the harness worst case -# (177,356 tokens) and one NATIVE 262,144-token conversation + full budget. -# MEASURED FIRST BOOT (2026-10-05): the auto attention backend is -# qsa_sparse, which OVERRIDES page size to 64, so --num-tokens must be a -# MULTIPLE OF 64 — 300,000 was refused ("nearest valid values: 299968 or -# 300032"). Grows live with `ft ctl cache --kv `; shrink needs a -# restart. -FRANK_PRIMARY_FT_NUM_TOKENS=299968 -# Concurrent decode slots. 2 is the first-boot conservative value; the -# radix pool admits ~2 full-length 200,704-token conversations. Raise with -# --max-running-requests only after a real-traffic look at slot occupancy. -FRANK_PRIMARY_FT_MAX_RUNNING=2 -# Per-request output budget ceiling: the harness's maxTokens (36,864). -# Requests that omit max_tokens get this default instead of the engine's -# 32,768. -FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS=36864 -# GPU expert LRU cache as a FRACTION of all experts (~120 GiB fp8 total -> -# ~12 GiB VRAM at 0.10). First-boot conservative; resize live with -# `ft ctl cache --moe` after reading the boot ledger + ft ctl stats. -FRANK_PRIMARY_FT_MOE_RATE=0.10 -# Escape hatch for boot-ladder experiments (YaRN, --max-prefill-length, -# --moe-cpu-threads, --sampling-defaults none, ...). Unset -> no extra -# argv. Keep multi-flag values as one string; the serve script expands it -# unquoted on purpose. -FRANK_PRIMARY_FT_EXTRA_ARGS= +# --- engine shape (first-boot conservative; the boot ledger arbitrates) -- +# Tensor parallel: BOTH R9700s. The board swap (2026-09-20, ProArt B850 +# x8/x8 on root ports) made collectives legal on this host; radiance's TP +# wire is libr4d's own, not RCCL. +FRANK_PRIMARY_TP=2 +# Lossy WHT 6-bit wire for large all-reduces (publisher's production choice: +# +5.6-6.3% prefill, ~-1.3% 8-stream decode at TP2). min-kb stays the +# engine default. +FRANK_PRIMARY_TP_WIRE=wht6 +# Decode slots. 2 is Frank's first-boot value ("use parallelism = 2 to +# start"); raise only after a real-traffic look at slot occupancy and the +# KV pool's headroom line. The recurrent-state cost scales with it (54 +# MiB/copy/card, 2 copies with speculation). +FRANK_PRIMARY_MAX_SEQS=2 +# Served window. Native is 262,144; 200,000 is the publisher's production +# value (compose) and what models.yml's contextWindow must quote — a longer +# prompt is refused by the engine, never truncated. +FRANK_PRIMARY_MAX_MODEL_LEN=200000 +FRANK_PRIMARY_PLACEMENT=expert_tiered +# Pinned host pool for the expert rows that do not fit VRAM. Native FP8 +# experts are 112.8 GiB total (56.4 GiB per rank slice) vs the 4-bit +# publisher shape's ~29 GiB — this pool is the cost of native precision. +# The boot ledger prints the pool's coverage ("the pool backs N% of the +# worst case ... M full-length sessions"); resize from that line, not from +# arithmetic. +FRANK_PRIMARY_HOST_POOL_MIB=40960 +# VRAM left unclaimed per card (publisher default 96 MiB). +FRANK_PRIMARY_GPU_HEADROOM_MIB=96 +# Weights share of claimed VRAM (publisher production value; higher = more +# prefix-cache hit rate, less KV headroom). +FRANK_PRIMARY_EXPERT_VS_CACHE=0.82 +# KV cache in FP8 (7.32 KiB/token at this model's 12 full-attention layers, +# 2 KV heads, 256 head_dim, K+V). +FRANK_PRIMARY_KV_DTYPE=fp8 +# Spec decode: the checkpoint's own MTP head at the publisher's depth-3 +# operating point (probe() returns 3 when mtp_num_hidden_layers > 0; the +# flag states it so the ledger records the intent). +FRANK_PRIMARY_SPEC_TOKENS=3 +FRANK_PRIMARY_MAX_BATCHED_TOKENS=2048 +# Prefix cache tiers (publisher pattern; disk tier sized for 2 slots). +FRANK_PRIMARY_PREFIX_HOST_MIB=4096 +FRANK_PRIMARY_PREFIX_DISK_MIB=65536 +# Frank's pin law: the PLE n-gram table (47.7 GiB) serves from pinned host +# RAM the cards read. `ram` REFUSES the start if the memlock limit is too +# small — the unit's LimitMEMLOCK=infinity is the other half of this law. +FRANK_PRIMARY_NGRAM_PLACEMENT=ram +# TTM preflight: the pinned ledger (2 x 40 GiB pools + 47.7 GiB PLE + 4 GiB +# prefix host) is ~132 GiB. ttm's pages_limit/pages_max cap the GPU-mappable +# pinned pages; the current cap (32,252,917 pages = ~123 GiB) is BELOW the +# plan, so the serve script refuses to start until both are raised. This is +# the floor it enforces (in MiB), with ~8 GiB of margin. +FRANK_PRIMARY_TTM_MIN_MIB=140000 +# Escape hatch for boot-ladder experiments. Unset -> no extra argv. Keep +# multi-flag values as one string; the serve script expands it unquoted on +# purpose. +FRANK_PRIMARY_EXTRA_ARGS= +# The .rad's minimum size in MiB (deployer gate; a truncated convert is a +# guaranteed long-timeout boot instead of a fast failure). The pure copy of +# the 172.8 GB checkpoint should land ~161 GiB. +FRANK_PRIMARY_RAD_MIN_MIB=158000 diff --git a/scripts/frank-primary.service b/scripts/frank-primary.service index 583afb7..6940d1e 100644 --- a/scripts/frank-primary.service +++ b/scripts/frank-primary.service @@ -1,26 +1,25 @@ -# frank-primary.service — FreeToken (PR #587 ROCm) endpoint (orcarouter -# Qwen3.8-Flash-Next-Uncensored-FP8, 131-shard payload) on ONE R9700 -# (gfx1201), dankserver :7770. 2026-10-05 cutover from the tcclaviger vLLM -# fork (git tag frank-primary-vllm-shapeG-20261003 is the pre-cutover -# state; the retired unit/env/serve trio is in that tag). +# frank-primary.service — radiance (StillDeadcode, Apache-2.0) endpoint +# (OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 at NATIVE block-FP8 precision, the +# rad-convert container qwen4exp-fp8-native.rad) on BOTH R9700s (TP=2, +# wht6 wire), dankserver :7770. 2026-10-05 cutover from FreeToken (git tag +# frank-primary-freetoken-pr587-20261005 is the pre-cutover state; the +# retired unit/env/serve trio and Dockerfile are in that tag). # # Same port, same served alias, same container identity as the unit name; # the harness models.yml id is unchanged. The serve script drives -# `podman run` directly (the image carries `ft serve`) and then waits on -# the container. The image is built ON THE HOST from -# scripts/docker/frank-primary-freetoken.Dockerfile (FreeToken PR head -# 77a7201 in the PR's own ROCm 7.14 base image). -# -# WHY A ROOT UNIT (not a rootless quadlet): FreeToken pins ~120 GiB of -# expert weights plus the ~47.7 GiB PLE ngram table in HOST RAM. Pinned -# pages are charged against RLIMIT_MEMLOCK; dank's login session on this -# host carries 8 MiB and a user quadlet is capped by the user manager's own -# hard limit — raising that means restarting user@1000.service and bouncing -# every user container on the host. A root unit grants -# LimitMEMLOCK=infinity to the dank process via PID1, and the image stays -# in dank's rootless podman storage. +# `podman run` directly: the digest-pinned stilldeadcode/radiance image +# (287 MiB, kernels prebuilt for gfx1201) has the engine as its +# entrypoint. The image lives in dank's ROOTLESS podman storage. +# WHY A ROOT UNIT (not a rootless quadlet): radiance pins ~132 GiB of HOST +# RAM (2 x 40 GiB expert pools + the 47.7 GiB PLE ngram table + the prefix +# host tier). Pinned pages are charged against RLIMIT_MEMLOCK; dank's login +# session on this host carries 8 MiB and a user quadlet is capped by the +# user manager's own hard limit — raising that means restarting +# user@1000.service and bouncing every user container on the host. A root +# unit grants LimitMEMLOCK=infinity to the dank process via PID1; the +# engine still runs out of dank's rootless podman storage. [Unit] -Description=frank-primary — FreeToken (PR587) Qwen3.8-Flash-Next-Uncensored-FP8 on 1x R9700 +Description=frank-primary — radiance Qwen3.8-Flash-Next-FP8 (native) on 2x R9700 TP=2 After=network-online.target Wants=network-online.target # Never let the start-rate limiter give up after a crash loop (a failed LLM @@ -30,21 +29,21 @@ StartLimitIntervalSec=0 [Service] User=dank Group=dank -# The expert LRU + PLE pin ~170 GiB of host RAM; an RLIMIT_MEMLOCK smaller -# than the pinned set fails the load mid-start. PID1 grants infinity here — -# parity with the fleet's other LLM units. +# The engine pins ~132 GiB of host RAM (expert pools + PLE table); an +# RLIMIT_MEMLOCK smaller than the pinned set fails the load mid-start. +# PID1 grants infinity here — parity with the fleet's other LLM units. LimitMEMLOCK=infinity Environment="FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env" ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh # systemd stop: wait alone cannot stop the container child; stop it -# explicitly. ft serve drops connections on SIGTERM; no graceful drain -# contract is claimed (the OpenAI front end holds no queue to drain — -# in-flight decodes are lost, which a stop during a bench window accepts). +# explicitly. No graceful drain contract is claimed (the OpenAI front end +# holds no queue to drain — in-flight decodes are lost, which a stop +# during a bench window accepts; radiance takes SIGTERM on the engine +# process and podman reaps the container). ExecStop=/usr/bin/podman stop -t 120 frank-primary -# Boot cost: 131-shard FP8 page-in + PLE pin + expert upload. vLLM-era -# budget (1800 s) carried forward; the FreeToken first boot measured here -# once it ran (see the runbook for the real number). -TimeoutStartSec=1800 +# Boot cost: .rad load (weights + PLE upload) + expert host-pool pin. The +# publisher's compose healthcheck allows a 15-minute start_period; the +# vLLM-era budget (1800 s) carried forward. # Restart=always, NOT on-failure (the 2026-10-01 vLLM lesson carries over: # a clean API-server exit after a worker fault reads as success to # systemd). `always` restarts on any container death; an explicit diff --git a/scripts/hfget.sh b/scripts/hfget.sh new file mode 100755 index 0000000..74c4d3c --- /dev/null +++ b/scripts/hfget.sh @@ -0,0 +1,107 @@ +#!/bin/sh +# hfget.sh -- fetch a Hugging Face repo with curl and jq, and nothing else. +# +# The engine and its tooling do not depend on Python, so `huggingface-cli` is not assumed. The API +# gives a file listing as JSON and every file has a stable resolve/ URL, which is all a downloader +# needs. +# +# scripts/hfget.sh [dest-dir] [revision] +# +# Set HF_TOKEN for a gated or private repo. Re-running resumes: a file whose local size already +# matches the listing is skipped and a short one is continued with `curl -C -`, so an interrupted +# transfer of a 300 GB checkpoint costs only what it had not finished. +# +# WEIGHT DUPLICATES ARE SKIPPED. Many repos carry the same tensors several times over -- safetensors +# beside the older pickle .bin, an ONNX export, a GGUF conversion -- and on a large checkpoint each +# copy is hundreds of gigabytes of the same numbers. When the listing holds any .safetensors, every +# other weight format is left behind. +# +# HFGET_JOBS concurrent transfers, default 4. These are long sequential reads: a few streams +# saturate the link and more only makes the destination seek. +set -e + +# ---- child mode: one file, named by "" in $1, everything else inherited ------- +# sh cannot export a function, so parallelism is one re-entry of this script per file under +# `xargs -P` rather than a worker pool. +if [ -n "$HFGET_CHILD" ]; then + size=${1%% *}; path=${1#* } + out=$HFGET_DEST/$path + if [ -f "$out" ]; then + cur=$(wc -c < "$out") + if [ "$cur" = "$size" ]; then echo " = $path"; exit 0; fi + echo " + $path (resuming at $cur of $size)" + else + echo " > $path" + fi + mkdir -p "$(dirname "$out")" + if [ -n "$HF_TOKEN" ]; then + curl -sSL --fail --retry 5 --retry-delay 5 --retry-all-errors -C - \ + -H "Authorization: Bearer $HF_TOKEN" \ + -o "$out" "https://huggingface.co/$HFGET_REPO/resolve/$HFGET_REV/$path" + else + curl -sSL --fail --retry 5 --retry-delay 5 --retry-all-errors -C - \ + -o "$out" "https://huggingface.co/$HFGET_REPO/resolve/$HFGET_REV/$path" + fi + got=$(wc -c < "$out") + [ "$got" = "$size" ] || { echo "hfget: $path is $got bytes, listing says $size" >&2; exit 1; } + exit 0 +fi + +# ---- parent ------------------------------------------------------------------------------------ +repo=$1 +[ -n "$repo" ] || { echo "usage: hfget.sh [dest-dir] [revision]" >&2; exit 2; } +dest=${2:-$HOME/models/$repo} +rev=${3:-main} +jobs=${HFGET_JOBS:-4} +command -v jq >/dev/null || { echo "hfget: jq is required" >&2; exit 2; } + +echo "hfget: listing $repo@$rev" +# THE LISTING IS PAGINATED AND THE PAGE SIZE IS SMALL. A large checkpoint runs to a hundred-odd +# shards and the API returns fifty entries with a `Link: ...; rel="next"` header. Taking only the +# first page is the worst possible failure here: every file it did fetch is the right length, so +# the size check passes and the converter is handed a checkpoint missing two thirds of its +# tensors. Follow the cursor until there is no next link. +api="https://huggingface.co/api/models/$repo/tree/$rev?recursive=true&expand=true" +listing='[]' +page=0 +while [ -n "$api" ]; do + hdr=$(mktemp "${TMPDIR:-/tmp}/hfhdr.XXXXXX") + if [ -n "$HF_TOKEN" ]; then + body=$(curl -sSL --fail-with-body -D "$hdr" -H "Authorization: Bearer $HF_TOKEN" "$api") + else + body=$(curl -sSL --fail-with-body -D "$hdr" "$api") + fi + [ "$(printf '%s' "$body" | jq -r 'type')" = "array" ] || { + rm -f "$hdr" + echo "hfget: $(printf '%s' "$body" | jq -r '.error // .message // .')" >&2 + echo "hfget: a gated or private repo needs HF_TOKEN in the environment" >&2; exit 1; } + listing=$(printf '%s\n%s' "$listing" "$body" | jq -s 'add') + api=$(tr -d '\r' < "$hdr" | sed -n 's/^[Ll]ink: *<\([^>]*\)>; *rel="next".*/\1/p' | head -1) + rm -f "$hdr" + page=$((page + 1)) + [ "$page" -gt 200 ] && { echo "hfget: listing did not terminate after 200 pages" >&2; exit 1; } +done +echo "hfget: $page page(s) of listing" +have_st=$(printf '%s' "$listing" | jq '[.[] | select(.path | endswith(".safetensors"))] | length') +sel='.[] | select(.type == "file")' +[ "$have_st" -gt 0 ] && sel="$sel"' | select((.path | test("\\.(bin|pth|msgpack|h5|onnx|onnx_data|gguf)$")) | not)' + +list=$(mktemp "${TMPDIR:-/tmp}/hfget.XXXXXX") +trap 'rm -f "$list"' EXIT INT TERM +printf '%s' "$listing" | jq -r "$sel"' | "\(.size)\t\(.path)"' > "$list" + +n=$(wc -l < "$list") +[ "$n" -gt 0 ] || { echo "hfget: the listing has no files" >&2; exit 1; } +need=$(awk -F'\t' '{s += $1} END {printf "%.1f", s / 1073741824}' "$list") +mkdir -p "$dest" +free=$(df -BG --output=avail "$dest" | tail -1 | tr -dc '0-9') +echo "hfget: $n file(s), $need GiB -> $dest" +[ "$have_st" -gt 0 ] && echo "hfget: safetensors present, pickle weight formats skipped" +echo "hfget: $free GiB free on the destination filesystem" +awk -v n="$need" -v f="$free" 'BEGIN { if (n > f) { print "hfget: THE DOWNLOAD DOES NOT FIT"; exit 1 } }' + +export HFGET_CHILD=1 HFGET_DEST="$dest" HFGET_REPO="$repo" HFGET_REV="$rev" +self=$(cd "$(dirname "$0")" && pwd)/$(basename "$0") +tr '\n' '\0' < "$list" | xargs -0 -P "$jobs" -I{} sh "$self" "{}" \ + || { echo "hfget: incomplete -- re-run to resume" >&2; exit 1; } +echo "hfget: complete -- $(du -sh "$dest" | cut -f1) in $dest" diff --git a/scripts/qwen38-froggeric-v22.5.jinja b/scripts/qwen38-froggeric-v22.5.jinja new file mode 100644 index 0000000..570ed07 --- /dev/null +++ b/scripts/qwen38-froggeric-v22.5.jinja @@ -0,0 +1,460 @@ +{%- set template_version = "qwen3.8-froggeric-v22.5" %} +{%- set _tool_format = tool_call_format if tool_call_format is defined else 'xml' %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- set add_vision_id = add_vision_id if add_vision_id is defined else false %} +{%- set enable_thinking = enable_thinking if enable_thinking is defined else true %} +{%- set auto_disable_thinking_with_tools = auto_disable_thinking_with_tools if auto_disable_thinking_with_tools is defined else false %} +{%- if preserve_reasoning is defined and preserve_reasoning is not none %} + {%- set _preserve_thinking = preserve_reasoning %} +{%- elif preserve_thinking is defined and preserve_thinking is not none %} + {%- set _preserve_thinking = preserve_thinking %} +{%- else %} + {%- set _preserve_thinking = true %} +{%- endif %} +{%- set max_tool_arg_chars = max_tool_arg_chars if max_tool_arg_chars is defined else 0 %} +{%- set max_tool_response_chars = max_tool_response_chars if max_tool_response_chars is defined else 0 %} +{%- set _default_reasoning_effort = 'medium' %} +{%- set _has_tools = (tools is defined and tools and tools is iterable and tools is not mapping) %} +{%- set _effort_raw = (reasoning_effort | string | lower) if reasoning_effort is defined and reasoning_effort is not none else _default_reasoning_effort %} +{%- set _initial_thinking = enable_thinking %} +{%- if _effort_raw in ('none', 'off') %} + {%- set _initial_thinking = false %} + {%- set _initial_effort = 'medium' %} +{%- elif _effort_raw in ('minimal', 'low') %} + {%- set _initial_effort = 'low' %} +{%- elif _effort_raw in ('high', 'xhigh', 'max', 'ultracode', 'extreme') %} + {%- set _initial_effort = 'xhigh' %} +{%- else %} + {%- set _initial_effort = 'medium' %} +{%- endif %} +{%- set ns_state = namespace(thinking=_initial_thinking, effort=_initial_effort) %} +{%- if auto_disable_thinking_with_tools and _has_tools %} + {%- set ns_state.thinking = false %} +{%- endif %} +{%- for msg in messages %} + {%- if msg.role == 'system' or msg.role == 'developer' or msg.role == 'user' %} + {%- if msg.content is string %} + {%- if '<|think_off|>' in msg.content %} + {%- set ns_state.thinking = false %} + {%- elif '<|think_on|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- elif '<|think_xhigh|>' in msg.content or '<|think_high|>' in msg.content or '<|think_ultracode|>' in msg.content or '<|think_extreme|>' in msg.content or '<|think_max|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'xhigh' %} + {%- elif '<|think_low|>' in msg.content or '<|think_minimal|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'low' %} + {%- elif '<|think_medium|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'medium' %} + {%- endif %} + {%- elif msg.content is iterable and msg.content is not mapping %} + {%- for item in msg.content %} + {%- if item is string %} + {%- set _item_text = item %} + {%- elif item is mapping and 'text' in item and item.text is string %} + {%- set _item_text = item.text %} + {%- else %} + {%- set _item_text = '' %} + {%- endif %} + {%- if _item_text %} + {%- if '<|think_off|>' in _item_text %} + {%- set ns_state.thinking = false %} + {%- elif '<|think_on|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- elif '<|think_xhigh|>' in _item_text or '<|think_high|>' in _item_text or '<|think_ultracode|>' in _item_text or '<|think_extreme|>' in _item_text or '<|think_max|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'xhigh' %} + {%- elif '<|think_low|>' in _item_text or '<|think_minimal|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'low' %} + {%- elif '<|think_medium|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'medium' %} + {%- endif %} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- set reasoning_instructions = '' %} +{%- if ns_state.thinking %} + {%- if ns_state.effort == 'xhigh' %} + {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %} + {%- elif ns_state.effort == 'low' %} + {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %} + {%- endif %} +{%- endif %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if item is mapping %} + {%- if item.type == 'image' or 'image' in item or 'image_url' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif item.type == 'video' or 'video' in item or 'video_url' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- else %} + {{- item | string }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- set head = namespace(count=0, seen_non_system=false) %} +{%- for message in messages %} + {%- set _is_sys = (message.role == 'system' or message.role == 'developer') %} + {%- if _is_sys and not head.seen_non_system %} + {%- set head.count = head.count + 1 %} + {%- else %} + {%- set head.seen_non_system = true %} + {%- endif %} +{%- endfor %} +{%- set sys_state = namespace(content='') %} +{%- for message in messages[:head.count] %} + {%- set _part = render_content(message.content, false, true) | trim %} + {%- if '<|think_off|>' in _part %}{%- set _part = _part.split('<|think_off|>') | join('') | trim %}{%- endif %} + {%- if '<|think_on|>' in _part %}{%- set _part = _part.split('<|think_on|>') | join('') | trim %}{%- endif %} + {%- if '<|think_xhigh|>' in _part %}{%- set _part = _part.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} + {%- if '<|think_high|>' in _part %}{%- set _part = _part.split('<|think_high|>') | join('') | trim %}{%- endif %} + {%- if '<|think_ultracode|>' in _part %}{%- set _part = _part.split('<|think_ultracode|>') | join('') | trim %}{%- endif %} + {%- if '<|think_extreme|>' in _part %}{%- set _part = _part.split('<|think_extreme|>') | join('') | trim %}{%- endif %} + {%- if '<|think_max|>' in _part %}{%- set _part = _part.split('<|think_max|>') | join('') | trim %}{%- endif %} + {%- if '<|think_medium|>' in _part %}{%- set _part = _part.split('<|think_medium|>') | join('') | trim %}{%- endif %} + {%- if '<|think_low|>' in _part %}{%- set _part = _part.split('<|think_low|>') | join('') | trim %}{%- endif %} + {%- if '<|think_minimal|>' in _part %}{%- set _part = _part.split('<|think_minimal|>') | join('') | trim %}{%- endif %} + {%- if _part %} + {%- if sys_state.content %} + {%- set sys_state.content = sys_state.content ~ '\n\n' ~ _part %} + {%- else %} + {%- set sys_state.content = _part %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- set _sc = sys_state.content %} +{%- set _msgs = messages[head.count:] %} +{%- if _has_tools %} + {{- '<|im_start|>system\n' }} + {%- if reasoning_instructions %} + {{- reasoning_instructions + '\n\n' }} + {%- endif %} + {{- '# Tools\n\nYou have access to the following functions:\n\n' }} + {%- for tool in tools %} + {{- '\n' }} + {{- tool | tojson }} + {%- endfor %} + {{- '\n' }} + {%- if _tool_format == 'json' %} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }} + {%- if ns_state.thinking %} + {{- '\nBrief explanation of tool call\n\n' }} + {%- endif %} + {{- '\n{"name": "example_function_name", "arguments": {"example_parameter_1": "value_1", "example_parameter_2": "This is the value for the second parameter"}}\n\n\n\nReminder:\n' }} + {%- if ns_state.thinking %} + {{- '- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n' }} + {%- endif %} + {{- '- Function calls MUST follow the specified format: a single JSON object with "name" and "arguments" keys inside XML tags.\n' }} + {%- if ns_state.thinking %} + {{- '- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n' }} + {%- else %} + {{- '- If you choose to call a tool, you MUST output the block IMMEDIATELY, with NO conversational text before it.\n' }} + {%- endif %} + {{- '- The tag MUST be at the very beginning of a new line, with NO spaces or indentation before it.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} + {%- else %} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }} + {%- if ns_state.thinking %} + {{- '\nBrief explanation of tool call\n\n' }} + {%- endif %} + {{- '\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n' }} + {%- if ns_state.thinking %} + {{- '- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n' }} + {%- endif %} + {{- '- Function calls MUST follow the specified format: an inner block must be nested within XML tags.\n' }} + {%- if ns_state.thinking %} + {{- '- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n' }} + {%- else %} + {{- '- If you choose to call a tool, you MUST output the block IMMEDIATELY, with NO conversational text before it.\n' }} + {%- endif %} + {{- '- The and tags MUST be at the very beginning of a new line, with NO spaces or indentation before them.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} + {%- endif %} + {%- if _sc %} + {{- '\n\n' + _sc }} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if _sc %} + {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + _sc + '<|im_end|>\n' }} + {%- elif reasoning_instructions %} + {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set _last_idx = _msgs | length - 1 %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=_last_idx) %} +{%- for message in _msgs[::-1] %} + {%- set index = (_msgs | length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == 'user' %} + {%- set _rc = render_content(message.content, false) | trim %} + {%- if not (_rc.startswith('') and _rc.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {%- if _last_idx > 50 %} + {%- set ns.last_query_index = _last_idx %} + {%- else %} + {%- set ns.last_query_index = 0 %} + {%- endif %} +{%- endif %} +{%- set ns2 = namespace(prev_role='', consecutive_failures=0) %} +{%- for message in _msgs %} + {%- set is_system = (message.role == "system" or message.role == "developer") %} + {%- set content = render_content(message.content, true, is_system) | trim %} + {%- if is_system or message.role == 'user' %} + {%- if '<|think_off|>' in content %}{%- set content = content.split('<|think_off|>') | join('') | trim %}{%- endif %} + {%- if '<|think_on|>' in content %}{%- set content = content.split('<|think_on|>') | join('') | trim %}{%- endif %} + {%- if '<|think_xhigh|>' in content %}{%- set content = content.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} + {%- if '<|think_high|>' in content %}{%- set content = content.split('<|think_high|>') | join('') | trim %}{%- endif %} + {%- if '<|think_ultracode|>' in content %}{%- set content = content.split('<|think_ultracode|>') | join('') | trim %}{%- endif %} + {%- if '<|think_extreme|>' in content %}{%- set content = content.split('<|think_extreme|>') | join('') | trim %}{%- endif %} + {%- if '<|think_max|>' in content %}{%- set content = content.split('<|think_max|>') | join('') | trim %}{%- endif %} + {%- if '<|think_medium|>' in content %}{%- set content = content.split('<|think_medium|>') | join('') | trim %}{%- endif %} + {%- if '<|think_low|>' in content %}{%- set content = content.split('<|think_low|>') | join('') | trim %}{%- endif %} + {%- if '<|think_minimal|>' in content %}{%- set content = content.split('<|think_minimal|>') | join('') | trim %}{%- endif %} + {%- endif %} + {%- if is_system %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'user' %} + {%- set ns2.consecutive_failures = 0 %} + {{- '<|im_start|>user\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'assistant' %} + {%- set reasoning_content = '' %} + {%- set _explicit_reasoning = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- if message.reasoning_content is string %} + {%- set _explicit_reasoning = message.reasoning_content %} + {%- else %} + {%- set _explicit_reasoning = message.reasoning_content | string %} + {%- endif %} + {%- elif message.thinking is defined and message.thinking is not none %} + {%- if message.thinking is string %} + {%- set _explicit_reasoning = message.thinking %} + {%- else %} + {%- set _explicit_reasoning = message.thinking | string %} + {%- endif %} + {%- elif message.reasoning is defined and message.reasoning is not none %} + {%- if message.reasoning is string %} + {%- set _explicit_reasoning = message.reasoning %} + {%- else %} + {%- set _explicit_reasoning = message.reasoning | string %} + {%- endif %} + {%- endif %} + {%- if _explicit_reasoning %} + {%- set _lead_end = '' %} + {%- if content.startswith('') and '' in content %} + {%- set _lead_end = '' %} + {%- elif content.startswith('') and '' in content %} + {%- set _lead_end = '' %} + {%- elif content.startswith('') %} + {%- set _lead_end = '' %} + {%- elif content.startswith('') %} + {%- set _lead_end = '' %} + {%- endif %} + {%- if _lead_end %} + {%- set content = content.split(_lead_end)[-1].lstrip('\n') %} + {%- endif %} + {%- set reasoning_content = _explicit_reasoning %} + {%- else %} + {%- set _think_end = '' %} + {%- if content.startswith('') %} + {%- set _think_end = '' %} + {%- elif content.startswith('') %} + {%- set _think_end = '' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif content.startswith('') and '' in content %} + {%- set _think_end = '' %} + {%- elif content.startswith('') and '' in content %} + {%- set _think_end = '' %} + {%- endif %} + {%- if _think_end %} + {%- if 'thinking' in _think_end %} + {%- set _think_start = '' %} + {%- else %} + {%- set _think_start = '' %} + {%- endif %} + {%- set reasoning_content = content.split(_think_end)[0].rstrip('\n') %} + {%- if _think_start in reasoning_content %} + {%- set reasoning_content = reasoning_content.split(_think_start)[-1].lstrip('\n') %} + {%- endif %} + {%- set content = content.split(_think_end)[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content | trim %} + {%- if (_preserve_thinking or loop.index0 > ns.last_query_index) %} + {{- '<|im_start|>assistant\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>assistant\n' + content }} + {%- endif %} + {%- if message.tool_calls is defined and message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined and tool_call.function is not none %} + {%- set tc = tool_call.function %} + {%- else %} + {%- set tc = tool_call %} + {%- endif %} + {%- set tc_name = tc.name if (tc.name is defined and tc.name is not none) else '' %} + {%- if _tool_format == 'json' %} + {%- if loop.first %} + {%- if content | trim %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n' }} + {%- endif %} + {%- set _args = '{}' %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- set _args = tc.arguments | tojson %} + {%- elif tc.arguments is string %} + {%- if tc.arguments %} + {%- set _args = tc.arguments %} + {%- endif %} + {%- else %} + {%- set _args = tc.arguments | tojson %} + {%- endif %} + {%- endif %} + {{- '\n{"name": ' }}{{- tc_name | tojson }}{{- ', "arguments": ' }}{{- _args }}{{- '}\n' }} + {%- else %} + {%- if loop.first %} + {%- if content | trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- for args_name, args_value in tc.arguments.items() %} + {{- '\n' }} + {%- if args_value is string %} + {%- set _av = args_value %} + {%- else %} + {%- set _av = args_value | tojson %} + {%- endif %} + {%- if max_tool_arg_chars > 0 and _av | length > max_tool_arg_chars %} + {{- _av[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_av | length | string) ~ ' chars]' }} + {%- else %} + {{- _av }} + {%- endif %} + {{- '\n\n' }} + {%- endfor %} + {%- else %} + {%- if tc.arguments is string %} + {%- set _raw_args = tc.arguments %} + {%- else %} + {%- set _raw_args = tc.arguments | tojson %} + {%- endif %} + {%- if _raw_args %} + {%- if max_tool_arg_chars > 0 and _raw_args | length > max_tool_arg_chars %} + {{- _raw_args[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_raw_args | length | string) ~ ' chars]' }} + {%- else %} + {{- _raw_args }} + {%- endif %} + {%- endif %} + {%- endif %} + {%- endif %} + {{- '\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == 'tool' %} + {%- set _content_lower = content | lower %} + {%- set _content_head = _content_lower[:120] %} + {%- set _is_code_or_grep = ('throw new ' in _content_lower or 'throw error' in _content_lower or 'console.error' in _content_lower or 'logger.error' in _content_lower or 'logging.error' in _content_lower or 'import ' in _content_head or 'def ' in _content_head or 'function ' in _content_head) %} + {%- set _exit_code_zero = ('exit code: 0' in _content_head or 'process exited with code 0' in _content_head) %} + {%- set _error_field_ok = ('"error": null' in _content_head or '"error":null' in _content_head or '"error": false' in _content_head or '"error":false' in _content_head or '"error": ""' in _content_head or '"error":""' in _content_head) %} + {%- set _strong_error = (('"error":' in _content_head and not _error_field_ok) or '"status": "error"' in _content_head or '"status":"error"' in _content_head or 'traceback (most recent call last):' in _content_head or 'command not found' in _content_head or 'invalid syntax' in _content_head or 'fatal:' in _content_head or (('exit code: ' in _content_head or 'process exited with code' in _content_head) and not _exit_code_zero) or _content_head.startswith('exception:') or _content_head.startswith('failed to ')) %} + {%- set _weak_error = ('error:' in _content_head or 'err!' in _content_head) %} + {%- set _weak_suppressed = ('$ ' in _content_head or 'took ' in _content_head or content | length >= 600) %} + {%- if not _is_code_or_grep and (_strong_error or (_weak_error and not _weak_suppressed)) %} + {%- set ns2.consecutive_failures = ns2.consecutive_failures + 1 %} + {%- else %} + {%- set ns2.consecutive_failures = 0 %} + {%- endif %} + {%- if ns2.prev_role != 'tool' %} + {{- '<|im_start|>user' }} + {%- endif %} + {%- set _is_json_payload = (_tool_format == 'json' and content | trim | length > 0 and (content | trim)[:1] in ('{', '[')) %} + {%- if not _is_json_payload and max_tool_response_chars > 0 and content | length > max_tool_response_chars %} + {%- set content = content[:max_tool_response_chars] + '\n[TRUNCATED - original length ' ~ (content | length | string) ~ ' chars]' %} + {%- endif %} + {{- '\n\n' + content }} + {%- if ns2.consecutive_failures >= 2 %} + {{- '\n\n⚠️ SYSTEM WARNING: ' ~ ns2.consecutive_failures ~ ' consecutive tool errors detected. Your previous approach is incorrect. You MUST use a fundamentally different approach or corrected arguments.' }} + {%- elif ns2.consecutive_failures == 1 %} + {{- '\n\n⚠️ SYSTEM WARNING: The previous tool call returned an error. Diagnose the failure and retry with completely corrected arguments.' }} + {%- endif %} + {{- '\n' }} + {%- if loop.last %} + {{- '<|im_end|>\n' }} + {%- else %} + {%- set _next_role = _msgs[loop.index0 + 1].role %} + {%- if _next_role != 'tool' %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} + {%- else %} + {{- '<|im_start|>user\n[' + message.role + ']: ' + content + '<|im_end|>\n' }} + {%- endif %} + {%- set ns2.prev_role = message.role %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if not ns_state.thinking %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/scripts/serve-frank-primary.sh b/scripts/serve-frank-primary.sh index 22697c6..2672e78 100755 --- a/scripts/serve-frank-primary.sh +++ b/scripts/serve-frank-primary.sh @@ -1,38 +1,33 @@ #!/usr/bin/env bash -# serve-frank-primary.sh — frank-primary endpoint (FreeToken PR #587 ROCm -# build, orcarouter Qwen3.8-Flash-Next-Uncensored-FP8, ONE R9700 gfx1201, -# dankserver :7770; 2026-10-05 cutover from the tcclaviger vLLM fork — git -# tag frank-primary-vllm-shapeG-20261003 is the pre-cutover state). +# serve-frank-primary.sh — frank-primary endpoint: radiance (StillDeadcode, +# Apache-2.0) serving the OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint at +# NATIVE block-FP8 precision from a rad-convert container +# (/home/dank/models/rad/qwen4exp-fp8-native.rad), on BOTH R9700s (TP=2, +# wht6 wire), dankserver :7770. 2026-10-05 cutover from FreeToken (PR #587, +# one card; git tag frank-primary-freetoken-pr587-20261005 is the +# pre-cutover state). +# +# The run line below IS the engine argv: the image's entrypoint is the +# engine and its tools sit beside it on PATH. Unlike every predecessor +# there is no per-boot VRAM planner to second-guess: the boot ledger in the +# log states every budget before anything is allocated (weights vs KV +# split, the expert host pool's coverage of the worst case, the PLE pin, +# the P2P answer per pair) — read it after every resize. First-boot shape: +# 2 decode slots (Frank's call), 40 GiB/rank pinned expert pool, fp8 KV, +# MTP depth 3, prefix cache host 4 GiB + disk 64 GiB, PLE pinned in RAM. # -# The engine is driven by `podman run` DIRECTLY and the run line below IS -# the engine argv: the image carries FreeToken installed at pinned PR head -# 77a7201 (scripts/docker/frank-primary-freetoken.Dockerfile builds it from -# the PR's own docs/install_amd.md stack). Unlike the vLLM fork there is no -# per-boot planner: experts live in pinned host RAM with an LRU GPU cache, -# KV is an explicit token pool, and the boot ledger in the log names -# Shape: 1 GPU (first-boot conservative — the second R9700 sits idle; -# --tensor-parallel-size EXISTS in FreeToken, so a 2-card TP shape is future -# tuning once this single-card shape has a decode baseline), KV pool 300,000 -# tokens (24 KiB/token f16 across the 12 full-attention layers = 7.2 GiB; -# clears the harness worst case of 177,356 tokens and admits one NATIVE -# 262,144-token conversation), MoE LRU at 10% of all experts (~12 GiB), -# 2 decode slots. First-boot conservative; `ft ctl cache --kv/--moe` grows -# pools live, and the env comment block carries the resize law. Flag -# surface verified against docs/cli.md of the pinned head AND against -# `ft serve --help` inside the built image; every value here is env-driven -# (frank-primary.env). - # The root unit (scripts/frank-primary.service, User=dank, -# LimitMEMLOCK=infinity) grants the pin budget: ~120 GiB of expert weights -# plus the ~47.7 GiB PLE table pin in host RAM. A plain shell session also -# works, but the unit is the fleet-parity boot-persistent shape. +# LimitMEMLOCK=infinity) grants the pin budget: 2 x 40 GiB expert pools + +# the 47.7 GiB PLE table + 4 GiB prefix host tier — ~132 GiB of pinned +# pages, which is ALSO why this script refuses to start when ttm's +# pages_limit/pages_max cap is under FRANK_PRIMARY_TTM_MIN_MIB (see below). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ENV_FILE="${FRANK_PRIMARY_ENV_FILE:-$SCRIPT_DIR/frank-primary.env}" if [ ! -f "$ENV_FILE" ]; then - echo "serve-frank-primary: env file missing: $ENV_FILE" >&2 + echo "ERROR: frank-primary env file absent: $ENV_FILE" >&2 exit 1 fi set -a @@ -40,58 +35,107 @@ set -a source "$ENV_FILE" set +a -: "${FRANK_PRIMARY_IMAGE:?FRANK_PRIMARY_IMAGE must name the FreeToken image}" -: "${FRANK_PRIMARY_MODEL_DIR:?FRANK_PRIMARY_MODEL_DIR must name the FP8 payload}" -: "${FRANK_PRIMARY_MODEL_SHARDS:?FRANK_PRIMARY_MODEL_SHARDS must give the shard count}" +: "${FRANK_PRIMARY_IMAGE:?FRANK_PRIMARY_IMAGE must name the radiance image}" +: "${FRANK_PRIMARY_MODEL_DIR:?FRANK_PRIMARY_MODEL_DIR must name the .rad dir}" +: "${FRANK_PRIMARY_MODEL_FILE:?FRANK_PRIMARY_MODEL_FILE must name the .rad file}" +: "${FRANK_PRIMARY_STATE_DIR:?FRANK_PRIMARY_STATE_DIR must name the /data state dir}" : "${FRANK_PRIMARY_CONTAINER:?FRANK_PRIMARY_CONTAINER must name the container}" : "${FRANK_PRIMARY_SERVED_NAME:?FRANK_PRIMARY_SERVED_NAME must name the alias}" -: "${FRANK_PRIMARY_PUBLISH:?FRANK_PRIMARY_PUBLISH must be host:container}" -: "${FRANK_PRIMARY_FT_GPU:?FRANK_PRIMARY_FT_GPU must name the GPU index}" -: "${FRANK_PRIMARY_FT_NUM_TOKENS:?FRANK_PRIMARY_FT_NUM_TOKENS must size the KV pool}" -: "${FRANK_PRIMARY_FT_MAX_RUNNING:?FRANK_PRIMARY_FT_MAX_RUNNING must be set}" -: "${FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS:?FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS must be set}" -: "${FRANK_PRIMARY_FT_MOE_RATE:?FRANK_PRIMARY_FT_MOE_RATE must be set}" +: "${FRANK_PRIMARY_PORT:?FRANK_PRIMARY_PORT must be the port}" +: "${FRANK_PRIMARY_TEMPLATE:?FRANK_PRIMARY_TEMPLATE must name the chat template}" +: "${FRANK_PRIMARY_TP:?FRANK_PRIMARY_TP must be set}" +: "${FRANK_PRIMARY_TP_WIRE:?FRANK_PRIMARY_TP_WIRE must be set}" +: "${FRANK_PRIMARY_MAX_SEQS:?FRANK_PRIMARY_MAX_SEQS must be set}" +: "${FRANK_PRIMARY_MAX_MODEL_LEN:?FRANK_PRIMARY_MAX_MODEL_LEN must be set}" +: "${FRANK_PRIMARY_PLACEMENT:?FRANK_PRIMARY_PLACEMENT must be set}" +: "${FRANK_PRIMARY_HOST_POOL_MIB:?FRANK_PRIMARY_HOST_POOL_MIB must be set}" +: "${FRANK_PRIMARY_GPU_HEADROOM_MIB:?FRANK_PRIMARY_GPU_HEADROOM_MIB must be set}" +: "${FRANK_PRIMARY_EXPERT_VS_CACHE:?FRANK_PRIMARY_EXPERT_VS_CACHE must be set}" +: "${FRANK_PRIMARY_KV_DTYPE:?FRANK_PRIMARY_KV_DTYPE must be set}" +: "${FRANK_PRIMARY_SPEC_TOKENS:?FRANK_PRIMARY_SPEC_TOKENS must be set}" +: "${FRANK_PRIMARY_MAX_BATCHED_TOKENS:?FRANK_PRIMARY_MAX_BATCHED_TOKENS must be set}" +: "${FRANK_PRIMARY_PREFIX_HOST_MIB:?FRANK_PRIMARY_PREFIX_HOST_MIB must be set}" +: "${FRANK_PRIMARY_PREFIX_DISK_MIB:?FRANK_PRIMARY_PREFIX_DISK_MIB must be set}" +: "${FRANK_PRIMARY_NGRAM_PLACEMENT:?FRANK_PRIMARY_NGRAM_PLACEMENT must be set}" +: "${FRANK_PRIMARY_TTM_MIN_MIB:?FRANK_PRIMARY_TTM_MIN_MIB must be set}" +: "${FRANK_PRIMARY_RAD_MIN_MIB:?FRANK_PRIMARY_RAD_MIN_MIB must be set}" -# The image gate is EXISTENCE IN STORAGE, not a registry digest: the tag is -# built ON THE HOST (podman build from -# scripts/docker/frank-primary-freetoken.Dockerfile) and local builds carry -# no RepoDigests entry — the 362-line vLLM-era RepoDigests grep refused -# exactly this shape. Pinning lives in the Dockerfile instead: the FreeToken -# source is checked out at a hardcoded PR-head SHA and the base image is -# digest-pinned there. `podman image exists` accepts both a local tag and a -# name@digest ref, and it is one command — no pipefail/grep race (the -# 2026-09-30 lesson) for either shape. +# The image gate is the digest-pinned ref itself: a name@sha256 ref exists +# in storage only if THAT manifest was pulled — a retagged look-alike or a +# newer latest does not satisfy it. `podman image exists` takes a +# name@digest ref in one command — no pipefail/grep race (the 2026-09-30 +# lesson). Runs as dank: the unit's process uses dank's ROOTLESS podman +# storage (root unit, User=dank — the fleet's frank-primary shape). if ! podman image exists "$FRANK_PRIMARY_IMAGE"; then - echo "serve-frank-primary: image absent from rootless storage: $FRANK_PRIMARY_IMAGE" >&2 - echo " build it: podman build -t $FRANK_PRIMARY_IMAGE -f scripts/docker/frank-primary-freetoken.Dockerfile scripts/docker/" >&2 + echo "ERROR: radiance image absent: $FRANK_PRIMARY_IMAGE" >&2 + echo "frank-primary: podman pull docker://stilldeadcode/radiance:latest (as dank)" >&2 exit 1 fi -if [ ! -f "$FRANK_PRIMARY_MODEL_DIR/config.json" ]; then - echo "serve-frank-primary: payload config missing: $FRANK_PRIMARY_MODEL_DIR/config.json" >&2 +# The payload gate: the .rad container file, sized. A truncated convert +# would otherwise buy a guaranteed long-timeout boot instead of this fast +# failure. The staging dir / raw checkpoint are NOT this script's concern +# (scripts/build-frank-primary-rad.sh built the file; its rad-info output +# is the record). +RAD="$FRANK_PRIMARY_MODEL_DIR/$FRANK_PRIMARY_MODEL_FILE" +if [ ! -f "$RAD" ]; then + echo "ERROR: .rad container missing: $RAD" >&2 + echo "frank-primary: run scripts/build-frank-primary-rad.sh" >&2 exit 1 fi -SHARDS="$(find "$FRANK_PRIMARY_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)" -if [ "$SHARDS" != "$FRANK_PRIMARY_MODEL_SHARDS" ]; then - echo "serve-frank-primary: payload shard count wrong: $FRANK_PRIMARY_MODEL_DIR" >&2 - echo " found $SHARDS safetensors, want $FRANK_PRIMARY_MODEL_SHARDS" >&2 - echo " re-fetch: scripts/download-frank-primary.sh" >&2 +RAD_MIB=$(( $(stat -c %s "$RAD") / 1048576 )) +if [ "$RAD_MIB" -lt "$FRANK_PRIMARY_RAD_MIN_MIB" ]; then + echo "ERROR: .rad container too small: $RAD is ${RAD_MIB} MiB, want >= $FRANK_PRIMARY_RAD_MIN_MIB" >&2 + echo "frank-primary: the convert was truncated — rebuild it" >&2 exit 1 fi -if [ ! -f "$FRANK_PRIMARY_MODEL_DIR/model.safetensors.index.json" ]; then - echo "serve-frank-primary: weight-map index missing: $FRANK_PRIMARY_MODEL_DIR/model.safetensors.index.json" >&2 + +if [ ! -f "$FRANK_PRIMARY_TEMPLATE" ]; then + echo "ERROR: chat template missing: $FRANK_PRIMARY_TEMPLATE" >&2 exit 1 fi +mkdir -p "$FRANK_PRIMARY_STATE_DIR" + if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then - echo "serve-frank-primary: GPU device nodes absent (/dev/kfd, /dev/dri)" >&2 + echo "ERROR: GPU device nodes absent (/dev/kfd, /dev/dri)" >&2 + exit 1 +fi + +# TTM preflight: the pinned ledger (~2 x 40 GiB pools + 47.7 GiB PLE + 4 GiB +# prefix host = ~132 GiB) rides TTM's shared pages, capped by ttm's +# pages_limit (the current kernel exposes NO pages_max; the cap function +# takes the min over the knobs that exist). Below FRANK_PRIMARY_TTM_MIN_MIB +# the load dies mid-pin with an allocator error instead of this named +# refusal — raise the knob first (runtime sysfs write; kernel cmdline +# fallback + one reboot only if the file is read-only), see the runbook. +# RAISED 2026-10-05 pre-cutover to 52,428,800 pages = 200 GiB. +ttm_cap_mib() { + # min over the knobs THIS kernel exposes — the current kernel has + # pages_limit only (no pages_max); a knob that is absent is skipped, + # not a zero (a zero would refuse a healthy cap). + local v cur="" + for p in /sys/module/ttm/parameters/pages_limit /sys/module/ttm/parameters/pages_max; do + [ -r "$p" ] || continue + v="$(cat "$p" 2>/dev/null)" || continue + [ -n "$v" ] || continue + v=$(( v * 4 / 1024 )) + if [ "$v" -lt "${cur:-999999999}" ]; then cur=$v; fi + done + echo "${cur:-0}" +} +TTM_CAP="$(ttm_cap_mib)" +if [ "$TTM_CAP" -lt "$FRANK_PRIMARY_TTM_MIN_MIB" ]; then + echo "ERROR: ttm pages cap is ${TTM_CAP} MiB, the pinned ledger needs >= $FRANK_PRIMARY_TTM_MIN_MIB" >&2 + echo "frank-primary: raise /sys/module/ttm/parameters/pages_limit (and pages_max if the kernel exposes one) to >= $FRANK_PRIMARY_TTM_MIN_MIB in 4 KiB pages" >&2 exit 1 fi # A crashed/killed unit can leave the container name registered (systemd # SIGKILLs conmon before cleanup runs). Clear OUR name plus the legacy -# identities (the vLLM-era frank-primary-qwen, the retired R9V container and -# the ad hoc experiment containers all hold this GPU); all idempotent. +# identities (the FreeToken-era frank-primary-qwen, the retired R9V +# container and the ad hoc experiment containers all hold this GPU); all +# idempotent. for CTR in $FRANK_PRIMARY_LEGACY_CONTAINERS "$FRANK_PRIMARY_CONTAINER"; do podman rm -f "$CTR" >/dev/null 2>&1 || true done @@ -99,30 +143,45 @@ done LOG=/home/dank/logs/frank_primary.log mkdir -p "$(dirname "$LOG")" -# --device /dev/dri is the WHOLE DIRECTORY (individual renderD nodes enumerate -# without the card nodes and HIP refuses contexts). --ipc=host for the -# engine's shared memory; memlock unbounded for the ~120 GiB pinned expert -# rows + the 47.7 GiB PLE pin (the unit's LimitMEMLOCK=infinity is the other -# half of that law). -# HIP_VISIBLE_DEVICES pins visibility to ONE card so in-container index 0 is -# the right R9700; --gpu 0 selects it inside the visible set. +# Publisher-vetted container contract (docs/DOCKER.md, compose/common.yaml): +# --network host (the server binds the host's addresses directly, no NAT), +# --init (the engine releases the cards on SIGTERM), seccomp=unconfined +# (Docker/podman's default profile refuses io_uring, which the n-gram +# table's reads go through, and the NUMA binding calls the ROCm runtime +# makes for host memory), memlock unbounded (the pinned ledger — legal +# because the root unit grants LimitMEMLOCK=infinity to this dank process), +# /dev/kfd + /dev/dri (render/video groups are moot for a root container). +# The template rides a single-file mount; the .rad dir is ro; /data holds +# the prefix-cache disk tier. podman run -d --name "$FRANK_PRIMARY_CONTAINER" \ - --publish "$FRANK_PRIMARY_PUBLISH" \ - --ipc=host --ulimit memlock=-1:-1 \ - --cap-add SYS_PTRACE \ - --device /dev/kfd --device /dev/dri --device /dev/mem \ - -e HIP_VISIBLE_DEVICES="$FRANK_PRIMARY_FT_GPU" \ + --network host \ + --init \ + --security-opt seccomp=unconfined \ + --ulimit memlock=-1:-1 \ + --device /dev/kfd --device /dev/dri \ -v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \ + -v "$FRANK_PRIMARY_STATE_DIR":/data \ + -v "$FRANK_PRIMARY_TEMPLATE":/templates/template.jinja:ro \ "$FRANK_PRIMARY_IMAGE" \ - ft serve --model /models \ - --served-model-name "$FRANK_PRIMARY_SERVED_NAME" \ - --host 0.0.0.0 --port 7770 \ - --gpu "$FRANK_PRIMARY_FT_GPU" \ - --num-tokens "$FRANK_PRIMARY_FT_NUM_TOKENS" \ - --max-running-requests "$FRANK_PRIMARY_FT_MAX_RUNNING" \ - --max-output-tokens "$FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS" \ - --moe-cache-rate "$FRANK_PRIMARY_FT_MOE_RATE" \ - ${FRANK_PRIMARY_FT_EXTRA_ARGS:+$FRANK_PRIMARY_FT_EXTRA_ARGS} \ + --model "/models/$FRANK_PRIMARY_MODEL_FILE" \ + --host :: --port "$FRANK_PRIMARY_PORT" \ + --served-model-name "$FRANK_PRIMARY_SERVED_NAME" \ + --override-chat-template /templates/template.jinja \ + --tp "$FRANK_PRIMARY_TP" --tp-wire "$FRANK_PRIMARY_TP_WIRE" \ + --max-num-seqs "$FRANK_PRIMARY_MAX_SEQS" \ + --max-model-len "$FRANK_PRIMARY_MAX_MODEL_LEN" \ + --placement "$FRANK_PRIMARY_PLACEMENT" \ + --host-pool-mib "$FRANK_PRIMARY_HOST_POOL_MIB" \ + --gpu-headroom-mib "$FRANK_PRIMARY_GPU_HEADROOM_MIB" \ + --expert-vs-cache-ratio "$FRANK_PRIMARY_EXPERT_VS_CACHE" \ + --kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE" \ + --prefix-cache-host-mib "$FRANK_PRIMARY_PREFIX_HOST_MIB" \ + --prefix-cache-dir /data/kvcache/frank-primary \ + --prefix-cache-disk-mib "$FRANK_PRIMARY_PREFIX_DISK_MIB" \ + --ngram-placement "$FRANK_PRIMARY_NGRAM_PLACEMENT" \ + --num-speculative-tokens "$FRANK_PRIMARY_SPEC_TOKENS" \ + --max-num-batched-tokens "$FRANK_PRIMARY_MAX_BATCHED_TOKENS" \ + ${FRANK_PRIMARY_EXTRA_ARGS:+$FRANK_PRIMARY_EXTRA_ARGS} \ > "$LOG" 2>&1 # From here on, this script's lifetime IS the container's: wait returns the diff --git a/scripts/stack-qwen38-official-fp8.py b/scripts/stack-qwen38-official-fp8.py new file mode 100755 index 0000000..e49429a --- /dev/null +++ b/scripts/stack-qwen38-official-fp8.py @@ -0,0 +1,540 @@ +#!/usr/bin/env python3 +"""stack-qwen38-official-fp8.py — reshape the official Qwen/Qwen3.8-Flash-Next-FP8 +checkpoint into the form the shipped radiance qwen4exp plugin maps. + +Run INSIDE the staging dir's tree (python3 + numpy on the host; data never +leaves the box). Two transforms, both VALUE-PRESERVING: + +1. EXPERTS: the shipped plugin's stacked layout names ONE tensor per family + (`mlp.experts.gate_up_proj` [E, 2F, H], `mlp.experts.down_proj` [E, H, F]) + plus a `_scale_inv` plane at the same index, while the official FP8 repo + ships per-expert unfused tensors (`experts..{gate,up,down}_proj.weight` + + scale). Stacking is pure byte concatenation along dim 0 — gate rows + first inside each expert's gate_up slab, which is the order the block + wants (rad_block_moe_fp8.h: "a stacked gate_up is ALREADY FUSED in the + order this block wants"). No requantisation: bytes are moved, not decoded. + +2. PLE TABLE: the plugin's fp8 branch wants the table's codes beside a + ONE-ELEMENT bf16 scale plane (`blk.%d.ple_ngram.scale`, view {1}); a + per-shard-blocked scale can never fit that view, and `rtn` re-encoding the + RAW scaleless codes would overflow (v / 1.99e-4 exceeds e4m3's 448 max). + So staging decodes the scaleless shards with the checkpoint's OWN bf16 + scale into a bf16 table (exact: every e4m3 code value and the scale are + exactly representable in bf16 to within the code's own 2^-3 grid), and the + convert's rtn rule re-encodes at that same scale — the nearest-code + distance is 2^-3 while the bf16 rounding perturbs by 2^-9, so every + original code is restored exactly and the container gets the checkpoint's + own codes plus the one-element scale plane. + +The convert then runs with NO recipe except: + blk.*.ple_ngram.weight=rtn:codes=fp8_e4m3,block=*x*,scale=bf16,rule=fixed,scale_value= +with read from the checkpoint and written to ple_scale.txt. + +Idempotent: re-runs skip when the staged index already carries the stacked +names, the merged bf16 shard_0, and split_ngram_parts == 1. --force redoes. +""" + +import argparse +import json +import mmap +import multiprocessing as mp +import os +import re +import struct +import sys +import time +import numpy as np +F8, BF16 = "F8_E4M3", "BF16" + + +def log(msg): + print(msg, flush=True) + + +# ---------------------------------------------------------------- safetensors + +def parse_header(path): + """Return (header_len, {name: (dtype, shape, data_offsets)}).""" + with open(path, "rb") as fh: + (hlen,) = struct.unpack(" (file, global_byte_offset, nbytes, dtype, shape) for every tensor.""" + files = sorted(set(index["weight_map"].values())) + meta = {} + for f in files: + path = os.path.join(stage, f) + hlen, hdr = parse_header(path) + for name, e in hdr.items(): + if name == "__metadata__": + continue + b0, b1 = e["data_offsets"] + meta[name] = (f, 8 + hlen + b0, b1 - b0, e["dtype"], e["shape"]) + return meta + + +def e4m3_lut(): + lut = np.zeros(256, np.float32) + for b in range(256): + sign = -1.0 if b & 0x80 else 1.0 + e = (b >> 3) & 0xF + m = b & 0x7 + if e == 15 and m == 7: + v = np.float32(np.nan) # OCP e4m3fn: only 0x7F/0xFF are NaN + elif e == 0: + v = sign * (m / 8.0) * 2.0 ** -6 + else: + v = sign * (1.0 + m / 8.0) * 2.0 ** (e - 7) + lut[b] = v + assert lut[0x38] == 1.0 and lut[0x7E] == 448.0 + assert np.isnan(lut[0x7F]) and lut[0x00] == 0.0 + assert lut[0x08] == 2.0 ** -6 and lut[0x80] == 0.0 + return lut + + +def f32_to_bf16_rne(v): + """float32 array -> uint16 bf16 bits, round-to-nearest-even.""" + q = np.ascontiguousarray(v, dtype=np.float32) + q[~np.isfinite(q)] = 0.0 + bits = q.view(np.uint32) + r = bits + 0x7FFF + ((bits >> 16) & 1) + return (r >> 16).astype(np.uint16) + + +# ------------------------------------------------------------------- workers + +def stack_prefix(job): + """Byte-stack one layer's 512 experts into a 4-tensor safetensors file.""" + (stage, pre, idx, n_e, shapes, meta) = job + (f2, h) = shapes["ffn2_h"] # 2*ffn, n_embd + (f,) = shapes["ffn"] # ffn + gs, ds = shapes["gs"], shapes["ds"] # scale shapes [5,20] and [20,5] + + # byte widths: fp8 is 1 B/element, bf16 2 B; the fused gate_up scale is + # the gate and up scale planes concatenated (gate rows first) + g_bytes = (f2 // 2) * h + gs_bytes = gs[0] * gs[1] * 2 # ONE projection's scale plane + gus_bytes = 2 * gs_bytes # fused scale per expert + dn_bytes = h * f + ds_bytes = ds[0] * ds[1] * 2 + + gu = np.empty(n_e * f2 * h, np.uint8) + gus = np.empty(n_e * gus_bytes, np.uint8) + dn = np.empty(n_e * dn_bytes, np.uint8) + dns = np.empty(n_e * ds_bytes, np.uint8) + + mmaps = {} + + def view(name): + f_, off, ln, dt, shp = meta[name] + if f_ not in mmaps: + fh = open(os.path.join(stage, f_), "rb") + mmaps[f_] = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ) + return memoryview(mmaps[f_])[off:off + ln], (dt, shp) + + def need(name, dt, shp): + mv, (got_dt, got_shp) = view(name) + assert got_dt == dt and list(got_shp) == list(shp), f"{name}: {got_dt}{got_shp}" + return mv + + for e in range(n_e): + base = f"{pre}.mlp.experts.{e}" + slab = gu[e * f2 * h:(e + 1) * f2 * h] + slab[:g_bytes] = need(f"{base}.gate_proj.weight", F8, [f2 // 2, h]) + slab[g_bytes:] = need(f"{base}.up_proj.weight", F8, [f2 // 2, h]) + sslab = gus[e * gus_bytes:(e + 1) * gus_bytes] + sslab[:gs_bytes] = need(f"{base}.gate_proj.weight_scale_inv", BF16, gs) + sslab[gs_bytes:] = need(f"{base}.up_proj.weight_scale_inv", BF16, gs) + dn[e * dn_bytes:(e + 1) * dn_bytes] = need(f"{base}.down_proj.weight", F8, [h, f]) + dns[e * ds_bytes:(e + 1) * ds_bytes] = need(f"{base}.down_proj.weight_scale_inv", BF16, ds) + + for mm in mmaps.values(): + mm.close() + + # byte-identity check on two sampled experts (first and last), all four + # tensors — the stack IS the checkpoint bytes, so they must compare equal + for e in (0, n_e - 1): + base = f"{pre}.mlp.experts.{e}" + s = e * f2 * h + assert bytes(gu[s:s + g_bytes]) == bytes(stack_src(f"{base}.gate_proj.weight", meta, stage)) + assert bytes(gu[s + g_bytes:s + f2 * h]) == bytes(stack_src(f"{base}.up_proj.weight", meta, stage)) + t = e * gus_bytes + assert bytes(gus[t:t + gs_bytes]) == bytes(stack_src(f"{base}.gate_proj.weight_scale_inv", meta, stage)) + assert bytes(gus[t + gs_bytes:t + gus_bytes]) == bytes(stack_src(f"{base}.up_proj.weight_scale_inv", meta, stage)) + d = e * dn_bytes + assert bytes(dn[d:d + dn_bytes]) == bytes(stack_src(f"{base}.down_proj.weight", meta, stage)) + u = e * ds_bytes + assert bytes(dns[u:u + ds_bytes]) == bytes(stack_src(f"{base}.down_proj.weight_scale_inv", meta, stage)) + + out = os.path.join(stage, f"stacked.{idx:03d}.safetensors") + tensors = [ + (f"{pre}.mlp.experts.gate_up_proj", F8, [n_e, f2, h], n_e * f2 * h), + (f"{pre}.mlp.experts.gate_up_proj.weight_scale_inv", BF16, + [n_e, gs[0] * 2, gs[1]], n_e * gus_bytes), + (f"{pre}.mlp.experts.down_proj", F8, [n_e, h, f], n_e * h * f), + (f"{pre}.mlp.experts.down_proj.weight_scale_inv", BF16, + [n_e, ds[0], ds[1]], n_e * ds_bytes), + ] + bufs = iter([gu, gus, dn, dns]) + written = write_safetensors(out, tensors, bufs) + return (pre, written) + + +def stack_src(name, meta, stage): + """Re-open a source tensor for the sampled-expert check.""" + f_, off, ln, dt, shp = meta[name] + fh = open(os.path.join(stage, f_), "rb") + mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ) + fh.close() # mmap keeps the mapping alive + return mm[off:off + ln] + + +def fill_ple_shard(job): + """Decode one fp8 shard with the table scale, write bf16 rows into the mmap.""" + (src_path, src_off, src_rows, dst_path, dst_off, scale_f32, + cols, chunk_rows) = job + lut = e4m3_lut() + with open(src_path, "rb") as fh: + sm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ) + with open(dst_path, "r+b") as dfh: + dm = mmap.mmap(dfh.fileno(), 0, access=mmap.ACCESS_WRITE) + for r0 in range(0, src_rows, chunk_rows): + r1 = min(r0 + chunk_rows, src_rows) + codes = np.frombuffer(sm, dtype=np.uint8, + count=(r1 - r0) * cols, + offset=src_off + r0 * cols) + q = lut.take(codes) + q[~np.isfinite(q)] = 0.0 + q *= np.float32(scale_f32) + packed = f32_to_bf16_rne(q) + dm[dst_off + r0 * cols * 2: dst_off + r1 * cols * 2] = packed.tobytes() + dm.flush() + os.fsync(dfh.fileno()) + dm.close() + sm.close() + + +# ---------------------------------------------------------------------- main + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--src", default=None, + help="raw checkpoint dir (documentation only; unused)") + ap.add_argument("--stage", required=True, help="staging dir to reshape in place") + ap.add_argument("--force", action="store_true", help="redo even if already staged") + ap.add_argument("--workers", type=int, default=min(8, os.cpu_count() or 1)) + args = ap.parse_args() + stage = args.stage + t0 = time.time() + + index_path = os.path.join(stage, "model.safetensors.index.json") + cfg_path = os.path.join(stage, "config.json") + if not (os.path.isfile(index_path) and os.path.isfile(cfg_path)): + log(f"ERROR: {stage} is not a staged checkpoint (missing index/config)") + return 2 + with open(index_path) as fh: + index = json.load(fh) + with open(cfg_path) as fh: + cfg = json.load(fh) + wm = index["weight_map"] + + meta = build_meta(stage, index) + + # ---- collect prefixes ------------------------------------------------- + pre_re = re.compile(r"^(?P
.+)\.mlp\.experts\.(?P\d+)\.gate_proj\.weight$")
+    pres, n_e = {}, 0
+    for k in wm:
+        m = pre_re.match(k)
+        if m:
+            pres.setdefault(m.group("pre"), set()).add(int(m.group("e")))
+            n_e = max(n_e, int(m.group("e")) + 1)
+    prefixes = sorted(pres)
+    if not prefixes:
+        log("ERROR: no per-expert tensors found in the staged index")
+        return 2
+    mtp = [p for p in prefixes if p.startswith("mtp.")]
+    trunk = [p for p in prefixes if p.startswith("model.language_model.layers.")]
+    log(f"expert prefixes: {len(prefixes)} (trunk {len(trunk)}, mtp {len(mtp)}), "
+        f"{n_e} experts each")
+
+    # shapes from expert 0 of the first prefix
+    pre0 = prefixes[0]
+    shapes = {}
+    gdt, gsh = meta[f"{pre0}.mlp.experts.0.gate_proj.weight"][3:5]
+    ddt, dsh = meta[f"{pre0}.mlp.experts.0.down_proj.weight"][3:5]
+    gsd, gssh = meta[f"{pre0}.mlp.experts.0.gate_proj.weight_scale_inv"][3:5]
+    dsd, dssh = meta[f"{pre0}.mlp.experts.0.down_proj.weight_scale_inv"][3:5]
+    scale_law = (gdt == F8 and ddt == F8 and gsd == BF16 and dsd == BF16
+                 and list(gssh) == [(gsh[0] + 127) // 128, (gsh[1] + 127) // 128]
+                 and list(dssh) == [(dsh[0] + 127) // 128, (dsh[1] + 127) // 128])
+    if not scale_law:
+        log(f"ERROR: unexpected expert shapes: gate {gdt}{gsh} scale {gsd}{gssh} "
+            f"down {ddt}{dsh} scale {dsd}{dssh} (scale law 128x128)")
+        return 2
+    shapes = {"ffn2_h": (gsh[0] * 2, gsh[1]), "ffn": (dsh[1],),
+              "gs": tuple(gssh), "ds": tuple(dssh)}
+    f2, h = shapes["ffn2_h"]
+    f = shapes["ffn"][0]
+    log(f"shapes: gate_up [{n_e},{f2},{h}] F8, down [{n_e},{h},{f}] F8, "
+        f"scales {gssh}x2 / {dssh} bf16")
+
+    # ---- PLE table --------------------------------------------------------
+    ple_re = re.compile(r"^(?P
.+)\.ple\.ple_embedding\.ngram_embedding\.shard_(?P\d+)\.weight$")
+    ple_pre, shard_ids = None, {}
+    for k in wm:
+        m = ple_re.match(k)
+        if m:
+            ple_pre = m.group("pre")
+            shard_ids[int(m.group("i"))] = k
+    if ple_pre is None:
+        log("ERROR: no PLE table shards found")
+        return 2
+    n_shards = len(shard_ids)
+    if sorted(shard_ids) != list(range(n_shards)):
+        log(f"ERROR: PLE shard ids not contiguous 0..{n_shards - 1}")
+        return 2
+    sc_name = f"{ple_pre}.ple.ple_embedding.ngram_embedding.weight_scale"
+    sdt, ssh, sln = meta[sc_name][3], meta[sc_name][4], meta[sc_name][2]
+    if sdt != BF16 or list(ssh) != [1]:
+        log(f"ERROR: PLE scale is {sdt}{ssh}, expected BF16 [1]")
+        return 2
+    s_bits = struct.unpack(" ple_scale.txt")
+    return 0
+
+
+def meta_size(meta, name):
+    _, _, ln, _, _ = meta[name]
+    return ln
+
+
+def view_src_tuple(meta, name, stage):
+    f_, off, ln, _, _ = meta[name]
+    with open(os.path.join(stage, f_), "rb") as fh:
+        fh.seek(off)
+        return fh.read(ln)
+
+
+def find_split(cfg):
+    """Locate split_ngram_parts keys (recursive); return list of paths."""
+    hits = []
+
+    def walk(d, path):
+        if isinstance(d, dict):
+            for k, v in d.items():
+                if k == "split_ngram_parts":
+                    hits.append(path + [k])
+                walk(v, path + [k])
+    walk(cfg, [])
+    return hits
+
+
+def read_split(cfg):
+    hits = find_split(cfg)
+    if not hits:
+        return None
+    d = cfg
+    for k in hits[0][:-1]:
+        d = d[k]
+    return d[hits[0][-1]]
+
+
+def set_split(cfg, value):
+    hits = find_split(cfg)
+    for path in hits:
+        d = cfg
+        for k in path[:-1]:
+            d = d[k]
+        d[path[-1]] = value
+    if not hits:  # plugin's meta_int2 falls back to the top-level key
+        cfg["split_ngram_parts"] = value
+        hits.append(["split_ngram_parts"])
+    return hits
+
+
+def verify(stage, prefixes, n_e, shapes, meta, shard_ids, rows, hd,
+           scale_f32, out_path, hlen):
+    """Cheap strong checks: headers parse, sizes match, sampled bytes identical."""
+    f2, h = shapes["ffn2_h"]
+    f = shapes["ffn"][0]
+    gs, ds = shapes["gs"], shapes["ds"]
+    # headers of every new file
+    for idx, pre in enumerate(prefixes):
+        path = os.path.join(stage, f"stacked.{idx:03d}.safetensors")
+        hlen2, hdr = parse_header(path)
+        want = {
+            f"{pre}.mlp.experts.gate_up_proj": (F8, [n_e, f2, h]),
+            f"{pre}.mlp.experts.gate_up_proj.weight_scale_inv":
+                (BF16, [n_e, gs[0] * 2, gs[1]]),
+            f"{pre}.mlp.experts.down_proj": (F8, [n_e, h, f]),
+            f"{pre}.mlp.experts.down_proj.weight_scale_inv": (BF16, [n_e, ds[0], ds[1]]),
+        }
+        for name, (dt, shp) in want.items():
+            e = hdr[name]
+            assert e["dtype"] == dt and list(e["shape"]) == shp, name
+            assert e["data_offsets"][1] - e["data_offsets"][0] == \
+                int(np.prod(shp)) * (1 if dt == F8 else 2), name
+    _, phdr = parse_header(out_path)
+    e = phdr[next(iter(phdr))]
+    assert e["dtype"] == BF16 and list(e["shape"]) == [sum(rows), hd]
+
+    # sampled PLE rows: recompute and compare bytes exactly
+    lut = e4m3_lut()
+    out_off = 8 + hlen
+    for i in (0, len(shard_ids) // 2, len(shard_ids) - 1):
+        name = shard_ids[i]
+        f_, off, ln, _, shp = meta[name]
+        with open(os.path.join(stage, f_), "rb") as fh:
+            mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
+            for r in (0, shp[0] - 1):
+                codes = np.frombuffer(mm, dtype=np.uint8, count=hd,
+                                      offset=off + r * hd)
+                q = lut.take(codes) * np.float32(scale_f32)
+                want = f32_to_bf16_rne(q).tobytes()
+                got_row = sum(rows[:i]) + r
+                with open(out_path, "rb") as ofh:
+                    ofh.seek(out_off + got_row * hd * 2)
+                    got = ofh.read(hd * 2)
+                assert got == want, f"PLE row {got_row} mismatch"
+            mm.close()
+    log("verify: new-file headers, sizes and sampled PLE rows all exact; "
+        "expert slabs were byte-checked in-stack")
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/serverAppConfigs-docs/LLM/frank-primary-freetoken-pr587.md b/serverAppConfigs-docs/LLM/frank-primary-freetoken-pr587.md
index 5c677b1..28c34c5 100644
--- a/serverAppConfigs-docs/LLM/frank-primary-freetoken-pr587.md
+++ b/serverAppConfigs-docs/LLM/frank-primary-freetoken-pr587.md
@@ -1,6 +1,10 @@
 # frank-primary — FreeToken PR #587 ROCm (Qwen3.8-Flash-Next-Uncensored-FP8, 1× R9700)
 
-**Status: ACTIVE 2026-10-05 →** · **Host:** Dankserver (192.168.0.235) · **Port:** 7770 · **GPU:** 1× Radeon AI PRO R9700 (gfx1201) — the second R9700 sits idle
+**Status: RETIRED 2026-10-05 → (superseded the same day by the radiance
+cutover — see [[frank-primary-radiance-qwen38-native-fp8]]; kept as the
+rollback runbook: the image + orcarouter payload stay on disk until the
+window closes)** · **Host:** Dankserver (192.168.0.235) · **Port:** 7770 ·
+**GPU:** 1× Radeon AI PRO R9700 (gfx1201) — the second R9700 sits idle
 
 The tcclaviger RDNA4 vLLM fork was replaced by **FreeToken at PR #587 head
 `77a7201f98f00d93f618c6cf8b81ffe8360e0899`** (branch `rocm714_torch214`), built
diff --git a/serverAppConfigs-docs/LLM/frank-primary-radiance-qwen38-native-fp8.md b/serverAppConfigs-docs/LLM/frank-primary-radiance-qwen38-native-fp8.md
new file mode 100644
index 0000000..0efb5ca
--- /dev/null
+++ b/serverAppConfigs-docs/LLM/frank-primary-radiance-qwen38-native-fp8.md
@@ -0,0 +1,224 @@
+# frank-primary — Radiance (StillDeadcode), Qwen3.8-Flash-Next-FP8 NATIVE, 2× R9700 TP=2
+
+**Status: ACTIVE 2026-10-05 →** · **Host:** Dankserver (192.168.0.235) · **Port:** 7770 · **GPU:** 2× Radeon AI PRO R9700 (gfx1201), TP=2
+
+The FreeToken PR #587 endpoint was replaced by the **radiance engine** (C++/HIP,
+libr4d kernels for gfx1201, no Python in the serving path) driving the
+digest-pinned image `stilldeadcode/radiance@sha256:4c3f484564bb2872784e4…`
+(v1.0.7 line, 287 MiB, `libr4d@0.5.0`) as a plain `podman run` through
+`scripts/serve-frank-primary.sh` under the **root systemd unit**
+(`frank-primary.service`, `User=dank`, `LimitMEMLOCK=infinity`). Same port,
+same served alias (`frank-primary`), same container identity
+(`frank-primary` = the unit name). Rollback: git tag
+`frank-primary-freetoken-pr587-20261005` (FreeToken trio + Dockerfile), image
+`localhost/frank-primary-freetoken:pr587-77a7201`, payload
+`/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8`.
+
+## Why radiance
+
+- **TP=2 is real here.** FreeToken's TP=2 dies in the `pynccl.cu` ROCm JIT
+  (`cuda_runtime.h` unresolvable for gfx1201 — probed 2026-10-06); the
+  second R9700 idled. Radiance's TP wire is libr4d's own one-shot
+  all-reduce (no RCCL), legal on this board since the 2026-09-20 swap
+  (ProArt B850, x8/x8 on CPU root ports, `Routing+`/`ReqEn+` everywhere).
+- **Native FP8 end to end** (Frank's call): the official
+  `Qwen/Qwen3.8-Flash-Next-FP8` checkpoint keeps its modelopt block-FP8
+  encoding — no requantisation, no calibration run, no 4-bit codebook.
+- **Every tier is explicit and measured**, not an estimator: placement
+  `expert_tiered` with a pinned host pool, `--ngram-placement ram` for the
+  47.7 GiB PLE table, `--expert-vs-cache-ratio 0.82` (the publisher's
+  measured 99 %-hit point; 0.87 already collapses to 0 %), fp8 KV.
+
+## The payload: native FP8, reshaped then converted (one recipe rule)
+
+- **Raw checkpoint**: `/home/dank/models/Qwen/Qwen3.8-Flash-Next-FP8`
+  (ungated repo, 172.8 GB, 131 shards + index + tokenizer + vision
+  configs). Fetched by `scripts/download-frank-primary.sh` (vendored
+  `hfget.sh`, curl+jq, no HF token needed) and **verified byte-for-byte**:
+  all 131 shards sha256-match the Hub LFS oids (2026-10-05, log
+  `/home/dank/logs/verify-qwen38-official-fp8.log`).
+- **The container**: `/home/dank/models/rad/qwen4exp-fp8-native.rad`
+  (~161 GiB; floor `FRANK_PRIMARY_RAD_MIN_MIB=158000` in the env gates a
+  truncated convert). Built by `scripts/build-frank-primary-rad.sh`:
+  1. stage `/home/dank/models/stage-qwen4exp-fp8` — `config.json` with
+     **`quantization_config` dropped** + relative symlinks into the raw
+     checkout;
+  2. `stack-qwen38-official-fp8.py` reshapes the staging dir — two
+     transforms, both value-preserving (script docstring carries the full
+     argument): the plugin maps the STACKED expert layout
+     (`mlp.experts.gate_up_proj` `[512,1280,2560]` + `_scale_inv` at the
+     same index) while the official repo ships per-expert unfused tensors,
+     so staging byte-stacks all 49 stacks' (48 trunk + MTP) 512 experts
+     with NO requantisation; and the PLE table ships as 128 **scaleless**
+     E4M3 shards + one bf16 scalar (1.9931793212890625e-04) while the
+     plugin's fp8 branch wants the codes beside a ONE-ELEMENT bf16 scale
+     plane — so staging decodes to a single bf16 `shard_0` with the
+     checkpoint's own scale (e4m3 decode and the scale are both exact in
+     bf16);
+  3. `rad-convert` with ONE `--quant` rule — pure copy for everything else
+     (every weight keeps the checkpoint's block-FP8 planes: `X.weight`
+     E4M3 codes + `X.weight_scale_inv` bf16 `[128×128]` scales) — and the
+     rtn rule re-encodes the bf16 PLE table at the same fixed scale, an
+     exact round-trip (nearest-code distance 2⁻³ vs the bf16 rounding
+     perturbation 2⁻⁹): the container gets the checkpoint's own PLE codes
+     plus the one-element scale plane (`blk.1.ple_ngram.scale`, view {1});
+  4. `rad-info` summary + size floor.
+- **Why the staging dir is mandatory** (the descriptor law, spec §2.4):
+  the plugin is selected by the exact `(arch_id, quant descriptor)` pair
+  with no fallback. A raw official checkpoint carries
+  `quantization_config.quant_method: "fp8"` → descriptor `"fp8"` → **no
+  shipped plugin claims `(qwen4exp, "fp8")`** — raw serve AND unmodified
+  convert both refuse. The shipped `qwen4exp_fp8` plugin claims
+  `(qwen4exp, "")` (recipe-built containers). `rad_convert.cpp` writes
+  `w.set_quant(meta.quant)` into the header, so the staged config's empty
+  quant method flows through as the container descriptor while **every
+  weight stays FP8 data**. Native precision is untouched; only the
+  selection key differs.
+- `--plan-only` first: the convert's declare phase is the engine's own,
+  so a plan that resolves every op will boot. The convert runs card-less
+  (declare = CPU-side layout queries); add `--device` only if a plugin
+  load ever demands it.
+
+## The shape (`scripts/frank-primary.env`, every value commented there)
+
+| Knob | Value | Why |
+|---|---|---|
+| Image | `stilldeadcode/radiance@sha256:4c3f4845…` | digest-pinned INDEX digest (RepoDigests also carry the per-arch manifest `1c582005…`; the image ID `69d05724…` is the config digest, NOT a manifest digest — pinning it sends `podman run` to the network); entrypoint = engine, tools (`rad-convert`/`rad-info`/`rad-tune`/`rad-kbench`) on PATH |
+| Container | `frank-primary` | identity = service name across runtimes; legacy list retires `frank-primary-qwen frank-primary-r9v qwen-fp8 glm-probe` |
+| `--tp 2 --tp-wire wht6` | both R9700s | fused WHT-6 lossy wire: +5.6–6.3 % prefill, ~−1.3 % 8-stream decode at TP2; `--tp-wire-min-kb` stays engine default (128 KiB) |
+| `--max-num-seqs 2` | first boot | Frank: "parallelism = 2 to start". Raise only after a real-traffic look at slot occupancy + the ledger's KV headroom line. The fused all-reduce row count is `seqs × (1 + spec) = 8` |
+| `--max-model-len 200000` | served window | native 262,144; the publisher's production value and what `models.yml` quotes. Longer prompt = refused, never truncated |
+| `--placement expert_tiered --host-pool-mib 40960` | pinned expert pool **per rank** | native FP8 experts are 112.8 GiB total (56.4 GiB/rank slice) vs ~29 GiB at the 4-bit publisher shape — the pool is the price of native precision. Resize from the boot ledger's coverage line, never arithmetic |
+| `--gpu-headroom-mib 96 --expert-vs-cache-ratio 0.82` | publisher values | 0.82 = measured 99 % expert-cache hit; 0.87 → 0 % (the cliff) |
+| `--kv-cache-dtype fp8` | **7.32 KiB/token** | 12 QSA full-attention layers, 2 KV heads × 256 dim, K+V; a 200K-token sequence is 1.43 GiB. KV is sized from the VRAM the ratio leaves |
+| `--num-speculative-tokens 3` | MTP head | the checkpoint carries `mtp.*` (3,101 tensors, `mtp_num_hidden_layers: 1`); the plugin's probe returns draft depth 3 — the publisher's operating point |
+| `--max-num-batched-tokens 2048` | prefill chunk | matches the `--checkpoint-interval` default |
+| `--prefix-cache-host-mib 4096 --prefix-cache-dir /data/kvcache/frank-primary --prefix-cache-disk-mib 65536` | 3 tiers | host tier pinned (see the ledger), disk tier in `/home/dank/radiance-state` (mounted rw at `/data`; `/` is ext4, mapped-tier-eligible); a session snapshot moves with its blocks |
+| `--ngram-placement ram` | **PLE pinned to host RAM** | Frank's pin law. The PLE n-gram table is 51.2 GB (320,001,536 rows × 160 dims = 29 % of all parameters) served via io_uring reads; `ram` **refuses the boot** if memlock is too small |
+| `--override-chat-template /templates/template.jinja` | froggeric v22.5 | vendored `scripts/qwen38-froggeric-v22.5.jinja` (28,234 B, `qwen3.8-froggeric-v22.5`); rad-convert bakes the checkpoint's own template, this overrides it, and the engine derives the reasoning/tool-call reply format from it |
+
+**Pinned-RAM ledger** (root unit + `LimitMEMLOCK=infinity` exist for this):
+2 × 40 GiB expert pools + 47.7 GiB PLE + 4 GiB prefix host tier + ~2 GiB
+spare ≈ **134 GiB**. `ttm.pages_limit` (this kernel has NO `pages_max` —
+the two-knob story was stale) caps GPU-mappable pinned pages (default = half
+of RAM): the pre-cutover cap was **123 GiB** — BELOW the ledger — so the
+serve script's TTM preflight (`FRANK_PRIMARY_TTM_MIN_MIB=140000`) refuses to
+start until it is raised. DONE 2026-10-05 pre-cutover: sysfs write of
+**52,428,800 pages = 200 GiB** (`/sys/module/ttm/parameters/pages_limit`;
+verified read-back; raising at runtime is harmless and allocates nothing).
+PERSISTENCE: the sysfs write does not survive reboot — add
+`ttm.pages_limit=52428800` to the kernel cmdline at the next scheduled
+reboot.
+
+**Per-rank VRAM** at TP2: expert slice 56.4 GiB total across VRAM + host
+pool; trunk ~3.75 GiB/rank; token embedding vocab-parallel (~1.18 GiB/rank
+— the bf16 table is 2.43 GiB, sharding hands the other half to KV); lm_head
+row-sharded; recurrent state 54 MiB/copy/card (2 copies with speculation);
+KV from the `0.82`-side remainder of claimed VRAM at 96 MiB headroom.
+
+## The engine contract (all verified in source, not assumed)
+
+- **`/health` == `/ping`**: 200 JSON (`model_id, in_flight, queue_depth`)
+  while serving, 503 only while stopping — **the listener binds AFTER the
+  full load**, so a bare `curl -fsS http://127.0.0.1:7770/health` gate is
+  honest (the deployer loops on it; the FreeToken-era `"status":"ok"` body
+  grep is gone). Boot budget 1800 s (publisher `start_period` 15 m); the
+  big terms are the ~161 GiB page-in and the host-pool pin.
+- **Sampling** (oai.cpp): `temperature, top_p, top_k, min_p, typical_p,
+  presence_penalty, frequency_penalty, repetition_penalty, repeat_last_n,
+  chat_template_kwargs, reasoning_effort, enable_thinking,
+  preserve_thinking` accepted; **unknown params are refused (400)** —
+  models.yml's extraBody set is fully inside it; nothing unverified rides
+  the wire. `max_tokens` is clamped, not refused; `seed` absent = pick
+  one; `response_format` supported; **`logprobs` is refused** (harness
+  never sends it).
+- **Reasoning**: `--reasoning-format` defaults to `auto` → thinking in
+  `reasoning_content` (models.yml `reasoningContentField` matches). Tool
+  calls: the qwen3.8 dialect → `message.tool_calls` +
+  `finish_reason: tool_calls`. No parser flags anywhere — parsers are
+  auto.
+- **Vision**: the official checkpoint carries the 27-layer vision tower;
+  the engine runs the Qwen-VL processor (patching, pad-token expansion,
+  3-part rotary are core). `models.yml` keeps `input: [text, image]`;
+  `--mm-max-patches` available.
+- **`/metrics`**: vLLM-compatible names + `radiance:*` families — the
+  Dankclaw Prometheus job re-scrapes 192.168.0.235:7770 (plain LAN routing
+  reaches it; the netavark published-port quirk is same-host only).
+  `/stats`, `/sessions`, `/dashboard` for live internals; the repo ships
+  `deploy/grafana/radiance.json` if a dashboard is wanted.
+- **`n` > 1 requests**: `n` defaults to `max(8, max-num-seqs)`; stop
+  strings ≤ 64 × 4096 B. `model` field not enforced.
+
+## Deploy + verify
+
+```bash
+# one-time (already done 2026-10-05): payload fetch + sha256 verify + convert
+ssh dankserver 'bash /home/dank/frank-primary-build/build-frank-primary-rad.sh --plan-only'   # declare + plan, no bytes
+ssh dankserver 'bash /home/dank/frank-primary-build/build-frank-primary-rad.sh'               # full convert + rad-info
+./scripts/deploy.sh dankserver --service frank-primary
+```
+
+Post-cutover checklist (each is observable, not assumed):
+
+1. Unit: `is-active` active, `is-enabled` enabled, container answers
+   `/health` with 200 JSON.
+2. Boot ledger (engine stdout → `/home/dank/logs/frank_primary.log`):
+   P2P probe per rank pair, TP=2 shard placement, expert pool coverage
+   line ("backs N % of the worst case … M full-length sessions"), KV pool
+   bytes, PLE pinned-RAM line, drafter depth 3, prefix tiers.
+3. MTP live: a completion's `timings` object shows draft/accept counts;
+   `radiance:*spec*` metrics in `/metrics`.
+4. Froggeric template: a probe prompt renders the v22.5 shape
+   (tool-call + thinking turn both exercise it).
+5. Vision: a ≥256 px solid-color PNG answers the exact color through an
+   `image_url` content part.
+6. Prometheus target `frank-primary` green (not never-green).
+
+## Tuning levers (in order of likely use)
+
+- **`FRANK_PRIMARY_MAX_SEQS`**: raise 2 → 3/4 only when the ledger's KV
+  headroom line and real slot occupancy agree; each slot buys a
+  recurrent-state copy (54 MiB/card) and shrinks the per-request KV share.
+- **`FRANK_PRIMARY_HOST_POOL_MIB`**: the ledger's coverage line is the
+  oracle — it prints what the pool backs at the worst case and how many
+  full-length sessions fit. RAM budget: pools + PLE + prefix ≤ ~140 GiB of
+  the host's 246.
+- **`FRANK_PRIMARY_EXPERT_VS_CACHE`**: 0.82 is the measured 99 %-hit
+  point; 0.87 zeroes it. Move only with a measured hit-rate reason.
+- **`FRANK_PRIMARY_TP_WIRE`**: `wht6` vs `exact` — `exact` trades the
+  +5.6 % prefill for bit-exact reductions; the fused one-shot forms are
+  declared over the seqs × (1+spec) row bound.
+- **`FRANK_PRIMARY_EXTRA_ARGS`**: the boot-ladder escape hatch, expanded
+  unquoted.
+- **`rad-tune -m  --tp 2`**: benchmarks the tunable space at this
+  model's shapes on this machine, writes `$RADIANCE_HOME/tune/.tune`
+  which the engine's selector reads at boot. Declare at the shapes you
+  serve (`--tp 2`), or you measure a program the serve never runs.
+  `RADIANCE_TUNE` pins a kernel for an A/B — an instrument, not a setting.
+- **Diagnostics**: `rad-info` (container: encodings, recipe, profile),
+  `--debug-graph` (what declared vs what issued), `rad-kbench` (kernel
+  correctness/speed vs libref), `/stats` + `/sessions` (live), and the
+  publisher's `scripts/` harnesses (`fnpf`, `fnconc`, `cstep2`, `ident`,
+  `mctools`) — note `mcserve/fnstop`-family scripts `pkill` the engine
+  binary by name; on this host the service is a systemd unit, so use
+  `sudo systemctl stop frank-primary` instead.
+
+## Rollback
+
+| Layer | Path |
+|---|---|
+| FreeToken (one card, native FP8 safetensors) | `git checkout frank-primary-freetoken-pr587-20261005 -- scripts/frank-primary.env scripts/frank-primary.service scripts/serve-frank-primary.sh scripts/docker/ tests/frank_primary_unit.bats` + redeploy; image `localhost/frank-primary-freetoken:pr587-77a7201` and payload `/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8` stay on disk until the window closes |
+| tcclaviger vLLM fork (TP=2, shape G) | git tag `frank-primary-vllm-shapeG-20261003` (trio + env lore; image still in dank's rootless storage) |
+| 4-bit StillDeadcode publisher shape | `qwen3.8-next-flash-fp8-iq4r-moe.rad` + `--host-pool-mib 12288` — the documented fallback if the native container ever proves unbootable |
+
+Retire the FreeToken-era orcarouter payload to `~/.AGENTTRASH/` only after
+the rollback window closes (trash policy, never `rm`).
+
+## Links
+
+- [[LLM/index]] — fleet index
+- [[frank-primary-freetoken-pr587]] — the retired FreeToken era
+- [[dankserver-qwen38-fp8-expert-offload-tuning]] — the retired tcclaviger vLLM era
+- [[Dual-R9700-Cooperation]] — the board swap that made TP=2 legal here
+- [[RDNA4 Container Serving]] — rootless-podman GPU laws on these hosts
+- `serverAppConfigs-docs/Architecture/Testing Policy.md` — the seam contract
diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml
index 797066e..ba5d1ac 100644
--- a/shared/prometheus/prometheus.yml
+++ b/shared/prometheus/prometheus.yml
@@ -98,12 +98,13 @@ scrape_configs:
   # service down. Same host, same rule, so both local endpoints use the
   # gateway.
   #
-  # (2026-10-05) frank-primary left this file: FreeToken (the PR587 ROCm
-  # engine now serving it) has NO /metrics endpoint — the job scraped 404s
-  # forever, which is a structural never-green, not an outage. The
-  # netavark published-port quirk above is same-host only; plain LAN
-  # routing REACHES 192.168.0.235:7770 (health answers) — only the metrics
-  # path is absent. If FreeToken grows a /metrics, re-add the job here.
+  # (2026-10-05) frank-primary LEFT the llamacpp family with FreeToken
+  # (the PR587 ROCm engine then serving it had NO /metrics — the job
+  # scraped 404s forever, a structural never-green) and RE-JOINED the same
+  # day on the radiance engine: its exporter speaks vLLM-compatible names
+  # plus radiance:* families, a different namespace — hence the separate
+  # job below, not a re-add to the llamacpp pair. The netavark quirk above
+  # is same-host only; plain LAN routing REACHES 192.168.0.235:7770.
   - job_name: "frank-small"
     metrics_path: "/metrics"
     static_configs:
@@ -118,3 +119,15 @@ scrape_configs:
         labels:
           service: frank-large
 
+  # frank-primary (dankserver) — the radiance engine since 2026-10-05.
+  # A DIFFERENT host, so plain LAN routing is correct here (the
+  # host.containers.internal trick is same-host only): health answers at
+  # 192.168.0.235:7770 from this bridge container. Radiance exports
+  # vLLM-compatible names plus radiance:* families — NOT the llamacpp:*
+  # namespace, so llm-fleet rules must not be pointed at this target.
+  - job_name: "frank-primary"
+    metrics_path: "/metrics"
+    static_configs:
+      - targets: ["192.168.0.235:7770"]
+        labels:
+          service: frank-primary
diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats
index 884403e..3a6c79c 100755
--- a/tests/frank_primary_unit.bats
+++ b/tests/frank_primary_unit.bats
@@ -1,25 +1,22 @@
 #!/usr/bin/env bats
 # Seam: frank-primary — the Dankserver LLM endpoint on :7770.
 #
-# This file asserts the FreeToken (PR #587 ROCm) ROOT-UNIT shape (the
-# 2026-10-05 cutover from the tcclaviger RDNA4 vLLM fork, git tag
-# frank-primary-vllm-shapeG-20261003): the serve script drives `podman run`
-# DIRECTLY (the image carries `ft serve`, built on the host from
-# scripts/docker/frank-primary-freetoken.Dockerfile), one R9700, with every
-# engine value carried in the repo env file. The seam pins what flags alone
-# do not say: the KV-pool math (300k tokens at 24 KiB/token clears the
-# harness worst case of 177,356), the deployed single-GPU shape
-# (--tensor-parallel-size EXISTS in FreeToken — a 2-card TP shape is future
-# tuning; HIP_VISIBLE_DEVICES pins THIS shape to one card), the host-RAM
-# law (experts + PLE pin ~170 GiB — the memlock law that makes the pin
-# legal), the self-healing law (the run returns before the engine is up, so
+# This file asserts the RADIANCE shape (StillDeadcode engine, digest-pinned
+# stilldeadcode/radiance image, BOTH R9700s TP=2 on the wht6 wire, the
+# OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 at native block-FP8 precision in a
+# rad-convert container). It replaces the FreeToken seam of 2026-10-05
+# (git tag frank-primary-freetoken-pr587-20261005) and pins what flags
+# alone do not say: the served window clears the harness worst case, the
+# pinned-host ledger has a ttm pages cap floor and the serve script
+# ENFORCES it, the served window is what models.yml must quote, the host-
+# networking contract (no --publish), the image gate on a name@sha256 ref,
+# the .rad payload gate with its size floor, the chat-template override
+# mount, the self-healing law (the run returns before the engine is up, so
 # the serve script must wait on the container for Restart=always to mean
-# anything), the image-existence gate (a local build carries no
-# RepoDigests — the old digest grep refused exactly this shape; pinning
-# lives in the Dockerfile), the deployer's payload gates with the
-# false-green MainPID check, and the client contract (parsers AUTO:
-# reasoning_content emission is the engine's native behavior — models.yml
-# maps reasoningContentField on the harness side, not here).
+# anything), the deployer's false-green MainPID check, the vendored hfget
+# fetch path (no token for this ungated repo), and the client contract
+# (reasoning auto -> reasoning_content; models.yml maps the field on the
+# harness side, not here).
 
 load helpers
 
@@ -28,24 +25,33 @@ SERVE="$REPO_ROOT/scripts/serve-frank-primary.sh"
 ENV_FILE="$REPO_ROOT/scripts/frank-primary.env"
 REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
 DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh"
-DOCKERFILE="$REPO_ROOT/scripts/docker/frank-primary-freetoken.Dockerfile"
+BUILD="$REPO_ROOT/scripts/build-frank-primary-rad.sh"
+HFGET="$REPO_ROOT/scripts/hfget.sh"
+TEMPLATE="$REPO_ROOT/scripts/qwen38-froggeric-v22.5.jinja"
 
-@test "frank-primary unit, serve script, env and Dockerfile exist" {
+@test "frank-primary unit, serve script, env, fetch and build scripts and template exist" {
     [ -f "$UNIT" ]
     [ -x "$SERVE" ]
     [ -f "$ENV_FILE" ]
-    [ -f "$DOCKERFILE" ]
+    [ -f "$DOWNLOAD" ]
+    [ -x "$BUILD" ]
+    [ -x "$HFGET" ]
+    [ -f "$TEMPLATE" ]
+    # The FreeToken-era Dockerfile is retired with the runtime it built:
+    # the radiance image is the publisher's, pulled and digest-pinned, not
+    # built on the host.
+    [ ! -f "$REPO_ROOT/scripts/docker/frank-primary-freetoken.Dockerfile" ]
 }
 
 @test "frank-primary unit is boot-persistent, self-healing and unprivileged" {
     # Root unit parity with frank-large/frank-small: enable is the [Install]
     # section, the restart policy survives a crash, and the pinned host RAM
-    # (expert rows + PLE table) needs an unbounded memlock (a root unit
-    # grants it via PID1; the dank session's own hard limit is 8 MiB).
+    # (expert pools + PLE table + prefix host tier, ~132 GiB) needs an
+    # unbounded memlock (a root unit grants it via PID1; the dank session's
+    # own hard limit is 8 MiB).
     grep -q '^WantedBy=multi-user.target$' "$UNIT"
-    # Restart=always, not on-failure: the vLLM-era lesson carries over — a
-    # clean front-end exit after a worker fault reads as success to systemd,
-    # so on-failure never fires.
+    # Restart=always, not on-failure: a clean front-end exit after an
+    # engine fault reads as success to systemd, so on-failure never fires.
     grep -q '^Restart=always$' "$UNIT"
     ! grep -q '^Restart=on-failure$' "$UNIT"
     # ExecStop runs `podman stop -t 120`, so the stop timeout MUST exceed
@@ -72,109 +78,156 @@ DOCKERFILE="$REPO_ROOT/scripts/docker/frank-primary-freetoken.Dockerfile"
 @test "the unit stops the container child explicitly — container identity is the unit name" {
     # The serve script blocks on podman wait, which cannot stop the
     # container child; systemd stop goes through ExecStop or the engine
-    # outlives the unit and holds the GPU. Frank's 2026-10-05 call: the
-    # container name IS the service name (frank-primary), and the vLLM-era
-    # name frank-primary-qwen moved into the legacy cleanup list.
+    # outlives the unit and holds the GPU. The container name IS the
+    # service name across runtimes (Frank's 2026-10-05 call).
     grep -q '^ExecStop=/usr/bin/podman stop' "$UNIT"
     grep -q 'podman stop -t 120 frank-primary$' "$UNIT"
     grep -q '^FRANK_PRIMARY_CONTAINER=frank-primary$' "$ENV_FILE"
     grep -q 'frank-primary-qwen' "$ENV_FILE"
 }
 
-@test "the serve shape is direct podman run, not a launcher, shim, or the vLLM fork" {
+@test "the serve shape is direct podman run, not a launcher or a predecessor runtime" {
     grep -q 'podman run' "$SERVE"
     ! grep -q 'launch.sh' "$SERVE"
     ! grep -q 'podman-shim' "$SERVE"
     ! grep -q 'R9V_STATE_FILE' "$SERVE"
-    # The tcclaviger vLLM fork is RETIRED — its image name and its
-    # fork-specific flags must not survive in the FreeToken run line
-    # (TP=2 is gone with the fork; FreeToken serves ONE GPU).
+    # Every predecessor runtime is RETIRED — its image name and its
+    # engine-specific flags must not survive in the radiance run line.
     ! grep -q 'tcclaviger' "$SERVE"
     ! grep -q 'tcclaviger' "$ENV_FILE"
+    ! grep -q 'freetoken\|ft serve' "$SERVE"
+    ! grep -q 'freetoken' "$ENV_FILE"
+    ! grep -q 'HIP_VISIBLE_DEVICES' "$SERVE"
     ! grep -q -- '--tensor-parallel-size' "$SERVE"
-    ! grep -q -- '--kv-cache-dtype' "$SERVE"
-    ! grep -q -- '--max-num-seqs' "$SERVE"
+    ! grep -q -- '--moe-cache-rate' "$SERVE"
+    ! grep -q -- '--num-tokens ' "$SERVE"
 }
 
-@test "the FreeToken engine shape is in the env file" {
-    # KV pool: TOKENS, not bytes. 24 KiB/token (12 full-attention layers x
-    # 2 KV heads x 256 head_dim x K+V x f16) — the 36 GDN layers use fixed
-    # per-seq state, not per-token KV. 299,968 tokens = 7.0 GiB and clears
-    # the harness worst case (140,492-token prompt + 36,864 max_tokens =
-    # 177,356); one NATIVE 262,144-token conversation + full budget also
-    # fits. The value must stay a MULTIPLE OF 64: the auto attention
-    # backend is qsa_sparse, which overrides page size to 64 (measured
-    # first boot — 300,000 was refused). The arithmetic guard keeps future
-    # resizes honest.
-    grep -q '^FRANK_PRIMARY_FT_NUM_TOKENS=299968$' "$ENV_FILE"
-    nt=$(awk -F= '/^FRANK_PRIMARY_FT_NUM_TOKENS=/{print $2}' "$ENV_FILE")
-    [ "$nt" -ge 177356 ]
-    [ $((nt % 64)) -eq 0 ]
-    # First-boot conservative shape: 2 decode slots, 10% expert LRU
-    # (~12 GiB of the ~120 GiB fp8 expert set), output ceiling = the
-    # harness maxTokens, single R9700 at index 0.
-    grep -q '^FRANK_PRIMARY_FT_MAX_RUNNING=2$' "$ENV_FILE"
-    grep -q '^FRANK_PRIMARY_FT_MOE_RATE=0.10$' "$ENV_FILE"
-    grep -q '^FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS=36864$' "$ENV_FILE"
-    grep -q '^FRANK_PRIMARY_FT_GPU=0$' "$ENV_FILE"
-    grep -q '^FRANK_PRIMARY_MODEL_SHARDS=131$' "$ENV_FILE"
+@test "the image is the publisher's, digest-pinned by manifest" {
+    # stilldeadcode/radiance@sha256:<64 hex> — a name@digest ref exists in
+    # storage only if THAT manifest was pulled. The tag form would float.
+    grep -qE '^FRANK_PRIMARY_IMAGE=stilldeadcode/radiance@sha256:[0-9a-f]{64}$' "$ENV_FILE"
+}
+
+@test "the radiance engine shape is in the env file" {
+    # Served window must clear the harness worst case: OMP compacts at 70%
+    # of contextWindow; for 200,000 that is 140,000, and a pre-compaction
+    # prompt can legally reach that height — 140,492 + 36,864 = 177,356 at
+    # the FreeToken-era window was the record; the same relation must hold
+    # here (140,000 + 36,864 = 176,864 <= 200,000).
+    grep -q '^FRANK_PRIMARY_MAX_MODEL_LEN=200000$' "$ENV_FILE"
+    ml=$(awk -F= '/^FRANK_PRIMARY_MAX_MODEL_LEN=/{print $2}' "$ENV_FILE")
+    [ "$ml" -ge 177356 ]
+    # First-boot conservative shape (Frank's call): 2 decode slots. The
+    # fused TP all-reduce is declared over seqs x (1 + spec) rows = 8.
+    grep -q '^FRANK_PRIMARY_MAX_SEQS=2$' "$ENV_FILE"
+    grep -q '^FRANK_PRIMARY_TP=2$' "$ENV_FILE"
+    grep -q '^FRANK_PRIMARY_TP_WIRE=wht6$' "$ENV_FILE"
+    grep -q '^FRANK_PRIMARY_SPEC_TOKENS=3$' "$ENV_FILE"
+    # Native-precision economics: the expert pool is per rank (56.4 GiB
+    # slice per rank at TP2), the PLE ngram table is pinned in host RAM per
+    # Frank's law, KV is fp8, the ratio is the publisher's production
+    # value.
+    grep -q '^FRANK_PRIMARY_PLACEMENT=expert_tiered$' "$ENV_FILE"
+    grep -q '^FRANK_PRIMARY_HOST_POOL_MIB=40960$' "$ENV_FILE"
+    grep -q '^FRANK_PRIMARY_EXPERT_VS_CACHE=0.82$' "$ENV_FILE"
+    grep -q '^FRANK_PRIMARY_KV_DTYPE=fp8$' "$ENV_FILE"
+    grep -q '^FRANK_PRIMARY_NGRAM_PLACEMENT=ram$' "$ENV_FILE"
+    # Prefix cache tiers (publisher pattern; disk tier sized for 2 slots).
+    grep -q '^FRANK_PRIMARY_PREFIX_HOST_MIB=4096$' "$ENV_FILE"
+    grep -q '^FRANK_PRIMARY_PREFIX_DISK_MIB=65536$' "$ENV_FILE"
+    # The pinned ledger (2 x pool + 47.7 GiB PLE + prefix host) must be
+    # under the ttm pages cap, and the env states the floor the serve
+    # script enforces (>= 140,000 MiB — the current cap ~123 GiB fails it,
+    # which is the point).
+    grep -q '^FRANK_PRIMARY_TTM_MIN_MIB=' "$ENV_FILE"
+    ttmm=$(awk -F= '/^FRANK_PRIMARY_TTM_MIN_MIB=/{print $2}' "$ENV_FILE")
+    pool=$(awk -F= '/^FRANK_PRIMARY_HOST_POOL_MIB=/{print $2}' "$ENV_FILE")
+    ph=$(awk -F= '/^FRANK_PRIMARY_PREFIX_HOST_MIB=/{print $2}' "$ENV_FILE")
+    # 2 pools + PLE 47.7 GiB (48,824 MiB) + prefix host, rounded up.
+    ledger=$(( 2 * pool + 48824 + ph + 2048 ))
+    [ "$ttmm" -ge "$ledger" ]
+    # The .rad size floor is positive (a truncated convert must fail fast).
+    grep -q '^FRANK_PRIMARY_RAD_MIN_MIB=' "$ENV_FILE"
+    [ "$(awk -F= '/^FRANK_PRIMARY_RAD_MIN_MIB=/{print $2}' "$ENV_FILE")" -gt 0 ]
 }
 
 @test "the serve script wires the engine flags from the env" {
-    grep -q 'ft serve --model /models' "$SERVE"
+    grep -q -- '--model "/models/$FRANK_PRIMARY_MODEL_FILE"' "$SERVE"
     grep -q -- '--served-model-name "$FRANK_PRIMARY_SERVED_NAME"' "$SERVE"
-    grep -q -- '--host 0.0.0.0 --port 7770' "$SERVE"
-    grep -q -- '--gpu "$FRANK_PRIMARY_FT_GPU"' "$SERVE"
-    grep -q -- '--num-tokens "$FRANK_PRIMARY_FT_NUM_TOKENS"' "$SERVE"
-    grep -q -- '--max-running-requests "$FRANK_PRIMARY_FT_MAX_RUNNING"' "$SERVE"
-    grep -q -- '--max-output-tokens "$FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS"' "$SERVE"
-    grep -q -- '--moe-cache-rate "$FRANK_PRIMARY_FT_MOE_RATE"' "$SERVE"
-    # Single-GPU law: visibility is pinned to one card so in-container
-    # index 0 is the right R9700 (mirrors the env's FT_GPU).
-    grep -q -- '-e HIP_VISIBLE_DEVICES="$FRANK_PRIMARY_FT_GPU"' "$SERVE"
+    grep -q -- '--host :: --port "$FRANK_PRIMARY_PORT"' "$SERVE"
+    grep -q -- '--override-chat-template /templates/template.jinja' "$SERVE"
+    grep -q -- '--tp "$FRANK_PRIMARY_TP" --tp-wire "$FRANK_PRIMARY_TP_WIRE"' "$SERVE"
+    grep -q -- '--max-num-seqs "$FRANK_PRIMARY_MAX_SEQS"' "$SERVE"
+    grep -q -- '--max-model-len "$FRANK_PRIMARY_MAX_MODEL_LEN"' "$SERVE"
+    grep -q -- '--placement "$FRANK_PRIMARY_PLACEMENT"' "$SERVE"
+    grep -q -- '--host-pool-mib "$FRANK_PRIMARY_HOST_POOL_MIB"' "$SERVE"
+    grep -q -- '--expert-vs-cache-ratio "$FRANK_PRIMARY_EXPERT_VS_CACHE"' "$SERVE"
+    grep -q -- '--kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE"' "$SERVE"
+    grep -q -- '--prefix-cache-host-mib "$FRANK_PRIMARY_PREFIX_HOST_MIB"' "$SERVE"
+    grep -q -- '--prefix-cache-disk-mib "$FRANK_PRIMARY_PREFIX_DISK_MIB"' "$SERVE"
+    grep -q -- '--ngram-placement "$FRANK_PRIMARY_NGRAM_PLACEMENT"' "$SERVE"
+    grep -q -- '--num-speculative-tokens "$FRANK_PRIMARY_SPEC_TOKENS"' "$SERVE"
     # The boot-ladder escape hatch is opt-in: unset renders NO extra argv.
-    grep -q -- '${FRANK_PRIMARY_FT_EXTRA_ARGS:+$FRANK_PRIMARY_FT_EXTRA_ARGS}' "$SERVE"
-    ! grep -q '^FRANK_PRIMARY_FT_EXTRA_ARGS=.' "$ENV_FILE"
+    grep -q -- '${FRANK_PRIMARY_EXTRA_ARGS:+$FRANK_PRIMARY_EXTRA_ARGS}' "$SERVE"
+    ! grep -q '^FRANK_PRIMARY_EXTRA_ARGS=.' "$ENV_FILE"
+}
+
+@test "the container contract is the publisher's (host network, io_uring, pin, cards)" {
+    # docs/DOCKER.md: EVERY one of these is needed — host networking (the
+    # server binds the host's addresses; no --publish), --init (the engine
+    # releases the cards on SIGTERM), seccomp=unconfined (the default
+    # profile refuses io_uring, which the n-gram reads use, and the NUMA
+    # binding calls), memlock unbounded (the pinned ledger), kfd + dri.
+    grep -q -- '--network host' "$SERVE"
+    ! grep -q -- '--publish' "$SERVE"
+    ! grep -q -- '-p 7770' "$SERVE"
+    grep -q '^FRANK_PRIMARY_PORT=7770$' "$ENV_FILE"
+    grep -q '^FRANK_PRIMARY_SERVED_NAME=frank-primary$' "$ENV_FILE"
+    grep -q -- '--init' "$SERVE"
+    grep -q -- '--security-opt seccomp=unconfined' "$SERVE"
+    grep -q -- '--ulimit memlock=-1' "$SERVE"
+    grep -q -- '--device /dev/kfd' "$SERVE"
+    grep -q -- '--device /dev/dri' "$SERVE"
+    grep -q -- '-v "$FRANK_PRIMARY_MODEL_DIR":/models:ro' "$SERVE"
+    grep -q -- '-v "$FRANK_PRIMARY_STATE_DIR":/data' "$SERVE"
+    grep -q -- '-v "$FRANK_PRIMARY_TEMPLATE":/templates/template.jinja:ro' "$SERVE"
 }
 
 @test "parsers are AUTO — the engine infers them natively, the harness maps the field" {
-    # FreeToken infers the qwen3.8 reasoning/tool dialects from the model
-    # family and emits thinking in `reasoning_content` natively — NO
-    # --reasoning-parser/--tool-call-parser override flags. The vLLM-era
-    # client-contract wiring (qwen3 parser, qwen3_xml tool parser,
-    # default-chat-template-kwargs) is gone; models.yml's
-    # reasoningContentField flip is the harness-side contract and lives in
-    # the operator's home, not this repo.
+    # radiance --reasoning-format defaults to auto and emits thinking in
+    # `reasoning_content`; tool calls ride the chat format. NO parser
+    # override flags exist in this engine. models.yml's
+    # reasoningContentField is the harness-side contract and lives in the
+    # operator's home, not this repo.
     ! grep -q -- '--reasoning-parser' "$SERVE"
     ! grep -q -- '--tool-call-parser' "$SERVE"
     ! grep -q -- '--enable-auto-tool-choice' "$SERVE"
-    ! grep -q -- '--default-chat-template-kwargs' "$SERVE"
     ! grep -q 'FRANK_PRIMARY_REASONING_PARSER' "$ENV_FILE"
     ! grep -q 'FRANK_PRIMARY_TOOL_PARSER' "$ENV_FILE"
 }
 
-@test "the serve script gates the image, the payload and the GPU nodes" {
-    # The image gate is EXISTENCE (podman image exists): the FreeToken
-    # image is built ON THE HOST — a local tag carries no RepoDigests — and
-    # pinning lives in the Dockerfile (pinned PR-head SHA, base image).
-    # The payload gate checks config.json and the exact shard count from
-    # the env — a missing shard means a guaranteed long-timeout boot
-    # instead of a fast failure.
+@test "the serve script gates the image, the .rad payload, the template and the GPU nodes" {
+    # The image gate is the digest ref itself; the payload gate checks the
+    # .rad file and its size floor — a truncated convert would otherwise be
+    # a guaranteed long-timeout boot instead of a fast failure. The ttm
+    # preflight refuses to start under a pages cap below the pinned ledger.
     grep -q 'FRANK_PRIMARY_IMAGE' "$SERVE"
     grep -q 'podman image exists' "$SERVE"
-    grep -q 'FRANK_PRIMARY_MODEL_SHARDS' "$SERVE"
-    grep -q 'config.json' "$SERVE"
-    grep -q 'model.safetensors.index.json' "$SERVE"
+    grep -q 'FRANK_PRIMARY_MODEL_FILE' "$SERVE"
+    grep -q 'FRANK_PRIMARY_RAD_MIN_MIB' "$SERVE"
+    grep -q 'FRANK_PRIMARY_TEMPLATE' "$SERVE"
+    grep -q 'FRANK_PRIMARY_TTM_MIN_MIB' "$SERVE"
+    grep -q 'pages_limit' "$SERVE"
+    grep -q 'pages_max' "$SERVE"
     grep -q '/dev/kfd' "$SERVE"
     grep -q -- '--device /dev/dri' "$SERVE"
 }
 
 @test "the serve script clears stale containers and self-heals via podman wait" {
     # A crashed unit can leave the container name registered forever after;
-    # the legacy identities (the vLLM-era frank-primary-qwen container, the
-    # retired R9V container and the ad hoc experiment containers) hold the
-    # same GPU and die here too. Then block on the container so a crash is
-    # a nonzero ExecStart.
+    # the legacy identities hold the same GPUs and die here too. Then block
+    # on the container so a crash is a nonzero ExecStart.
     grep -q 'FRANK_PRIMARY_LEGACY_CONTAINERS' "$ENV_FILE"
     grep -q 'frank-primary-qwen' "$ENV_FILE"
     grep -q 'frank-primary-r9v' "$ENV_FILE"
@@ -182,19 +235,19 @@ DOCKERFILE="$REPO_ROOT/scripts/docker/frank-primary-freetoken.Dockerfile"
     grep -q 'exec podman wait "$FRANK_PRIMARY_CONTAINER"' "$SERVE"
 }
 
-@test "frank-primary holds the single fleet port under its own alias" {
-    # LAN publish of :7770, the harness's model id.
-    grep -q '^FRANK_PRIMARY_PUBLISH=7770:7770$' "$ENV_FILE"
-    grep -q '^FRANK_PRIMARY_SERVED_NAME=frank-primary$' "$ENV_FILE"
-}
-
-@test "the model id matches the payload's served alias wiring" {
-    # The payload dir and its shard count are the fetch script's output
-    # layout; the fetch script must write where the env points.
-    grep -q '^FRANK_PRIMARY_MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8$' "$ENV_FILE"
-    grep -q '^FRANK_PRIMARY_MODEL_SHARDS=131$' "$ENV_FILE"
-    grep -q 'Qwen3.8-Flash-Next-Uncensored-FP8' "$DOWNLOAD"
-    grep -q 'orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8' "$DOWNLOAD"
+@test "the model id matches the payload's build and fetch scripts" {
+    # The payload dir/file are the build script's output layout; the fetch
+    # script must name the repo the build stages from.
+    grep -q '^FRANK_PRIMARY_MODEL_DIR=/home/dank/models/rad$' "$ENV_FILE"
+    grep -q '^FRANK_PRIMARY_MODEL_FILE=qwen4exp-fp8-native.rad$' "$ENV_FILE"
+    grep -q 'qwen4exp-fp8-native.rad' "$BUILD"
+    grep -q 'Qwen/Qwen3.8-Flash-Next-FP8' "$DOWNLOAD"
+    grep -q 'hfget.sh' "$DOWNLOAD"
+    # The repo is UNGATED — no token may be required on the fetch path.
+    ! grep -q 'HF_TOKEN' "$DOWNLOAD"
+    # The staging step drops quantization_config (the descriptor fix) —
+    # document it where it happens.
+    grep -q 'quantization_config' "$BUILD"
 }
 
 @test "no bare HIP-reserved env var is defined" {
@@ -205,37 +258,16 @@ DOCKERFILE="$REPO_ROOT/scripts/docker/frank-primary-freetoken.Dockerfile"
     ! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE"
 }
 
-@test "the Dockerfile pins the FreeToken source and follows the ROCm install law" {
-    # The engine is the PR #587 head, NOT a moving branch: a future push to
-    # rocm714_torch214 would otherwise silently change the deployed engine.
-    grep -q '77a7201f98f00d93f618c6cf8b81ffe8360e0899' "$DOCKERFILE"
-    grep -q 'pull/587/head' "$DOCKERFILE"
-    # docs/install_amd.md's build law: --no-build-isolation --no-deps (a
-    # plain `pip install -e .` on ROCm can REPLACE the ROCm torch with a
-    # CUDA build via its NVIDIA-authored dependency set), curated portable
-    # deps, flashlib --no-deps.
-    grep -q -- '--no-build-isolation --no-deps -e .' "$DOCKERFILE"
-    grep -q 'flashlib==0.3.0' "$DOCKERFILE"
-    # The CUDA-only backends have no ROCm builds and are intentionally
-    # absent (the PR's 12 pytest failures are exactly those backends;
-    # FreeToken falls back to its Triton kernels).
-    ! grep -q 'flashinfer' "$DOCKERFILE"
-    ! grep -q 'sgl_kernel' "$DOCKERFILE"
-    ! grep -q 'nvidia-cutlass-dsl' "$DOCKERFILE"
-    # The base image is digest-pinned (the registry tag is not trusted).
-    grep -qE '^FROM rocm/pytorch:.*@sha256:[0-9a-f]{64}$' "$DOCKERFILE"
-}
-
 @test "the deployer gates the image, payload and containers, then health-checks" {
-    # The deployer refuses when the image is absent from rootless storage
-    # or the payload is missing/wrong-sharded — then clears stale containers
-    # (including the legacy identities), waits out the port,
+    # The deployer refuses when the image is absent from dank's rootless
+    # storage or the .rad is missing/too small — then clears stale
+    # containers (including the legacy identities), waits out the port,
     # relinks/enables/restarts the root unit and health-gates it.
     grep -q 'LLM_PRIMARY_CONTAINER=frank-primary$' "$REMOTE_DEPLOY"
     grep -q 'FRANK_PRIMARY_IMAGE' "$REMOTE_DEPLOY"
     grep -q 'podman image exists' "$REMOTE_DEPLOY"
-    grep -q 'FRANK_PRIMARY_MODEL_SHARDS' "$REMOTE_DEPLOY"
-    grep -q 'config.json' "$REMOTE_DEPLOY"
+    grep -q 'FRANK_PRIMARY_MODEL_FILE' "$REMOTE_DEPLOY"
+    grep -q 'FRANK_PRIMARY_RAD_MIN_MIB' "$REMOTE_DEPLOY"
     grep -q 'frank-primary-qwen' "$REMOTE_DEPLOY"
     grep -q 'qwen-fp8' "$REMOTE_DEPLOY"
     grep -q 'glm-probe' "$REMOTE_DEPLOY"
@@ -243,13 +275,12 @@ DOCKERFILE="$REPO_ROOT/scripts/docker/frank-primary-freetoken.Dockerfile"
     grep -q 'relink_unit frank-primary' "$REMOTE_DEPLOY"
     grep -q 'systemctl enable frank-primary.service' "$REMOTE_DEPLOY"
     grep -q 'systemctl restart frank-primary.service' "$REMOTE_DEPLOY"
-    # The health gate reads the BODY, not just the HTTP code: FreeToken
-    # returns 200 with {"status":"loading"} while the engine loads (the
-    # FastAPI front end binds before the scheduler is up — a bare 200
-    # false-greened a deploy at 6 s while 131 shards were still paging in,
-    # measured 2026-10-06). Only status:"ok" counts.
+    # The health gate is a bare curl: the HTTP listener binds AFTER the
+    # full load (main.cpp — engine.start() then engine.serve()), so during
+    # load the port refuses connections and a bare curl is honest. The
+    # FreeToken-era body read ('status ok') is gone with that engine.
     grep -q 'curl -fsS http://127.0.0.1:7770/health' "$REMOTE_DEPLOY"
-    grep -q 'never reached /health status ok' "$REMOTE_DEPLOY"
+    ! grep -q '"status":"ok"' "$REMOTE_DEPLOY"
 }
 
 @test "the deployer will not report green on a stray port-holder" {