From 095128b669a1908e8581b46b6af698b3c27f1837 Mon Sep 17 00:00:00 2001 From: Frank Date: Sun, 27 Sep 2026 04:05:46 -0700 Subject: [PATCH] frank-primary: R9V radiance cutover wiring (launcher-driven serve + state gates) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the vLLM 0.30 UVA-offload wiring (never deployed; llama.cpp b11030 endpoint replaced by Frank's order, downtime window open) with the R9V radiance-vLLM shape on the same root unit: - serve-frank-primary.sh: layer the on-host qualified setup state (104 tuned R9V_* keys, /home/dank/r9v-state/uncensored/setup.json) under repo production overrides, then exec the fork's own launcher through the docker->podman shim and block on docker wait so a crashed engine is a nonzero ExecStart (launch.sh --detach alone would fake green). Gates: pinned image by OCI manifest digest in RepoDigests, PLE at pinned byte count (stat -c %s), all GGUF shards + mmproj. - frank-primary.env: thin production overrides only (LAN bind 0.0.0.0, :7770, alias frank-primary, container frank-primary-r9v, GPU BDFs); tuned engine shape stays in the setup state by design. - frank-primary.service: same root-unit shape (User=dank, LimitMEMLOCK=infinity for the tiered-expert pin), plus ExecStop podman stop — docker wait cannot stop the container child. - deploy-remote.sh deploy_llm_primary: state-driven gates (state ready, image digest present in rootless storage, PLE bytes, shard files) replace the fp030 shard-index gate; false-green MainPID check kept. - tests/frank_primary_unit.bats: seam rewritten for the R9V shape. --- scripts/deploy-remote.sh | 81 ++++++++----- scripts/frank-primary.env | 114 +++--------------- scripts/frank-primary.service | 44 ++++--- scripts/serve-frank-primary.sh | 203 +++++++++++++++++---------------- tests/frank_primary_unit.bats | 189 +++++++++++++----------------- 5 files changed, 282 insertions(+), 349 deletions(-) diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index a0afbb2..404b46e 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1183,42 +1183,70 @@ deploy_llm_small() { wait_for_health frank-small http://127.0.0.1:7771/health 900 } -# deploy_llm_primary — frank-primary is vLLM 0.30 (locally built ROCm image, -# gfx1201) serving Qwen3.8-Flash-Next-FP8 on the two R9700s as a ROOT unit -# whose serve script execs a rootless-podman container as dank. Same port and -# alias as the llama.cpp endpoint it replaces (2026-09-27 cutover); the -# harness models.yml and homepage cards are unchanged. +# deploy_llm_primary — frank-primary is the R9V radiance-vLLM fork (docker- +# pinned prebuilt, gfx1201, driven through the docker->podman shim) serving +# Qwen3.8-Flash-Next-Uncensored IQ4_XS with the MTP4 drafter on the two +# R9700s as a ROOT unit whose serve script execs the fork's launcher through +# rootless podman as dank. Same port and alias as the llama.cpp endpoint it +# replaced (2026-09-27 cutover); the harness models.yml id is unchanged. # -# Gates: the image must exist in dank's rootless storage, and the payload -# must be complete — every shard named by the model's own -# model.safetensors.index.json present on disk (131 shards, 172.78 GiB; -# per-shard sha256 + byte-count verification is download-frank-primary-fp8.sh's -# job at fetch time, never re-hashed here). -LLM_PRIMARY_IMAGE=localhost/frank-primary:vllm030 -LLM_PRIMARY_ROOT=/home/dank/models/Qwen3.8-Flash-Next-FP8 -LLM_PRIMARY_CONTAINER=frank-primary-vllm +# Gates: the on-host R9V setup state must exist and be ready (it carries the +# pinned image digest and every payload path), the pinned runtime image must +# be loadable from dank's rootless storage by its OCI manifest digest, the +# PLE payload must match its pinned byte count, and every GGUF shard + the +# mmproj must exist (per-file sha256 verification was setup's job at fetch +# time, never re-hashed here). +LLM_PRIMARY_STATE=/home/dank/r9v-state/uncensored/setup.json +LLM_PRIMARY_CONTAINER=frank-primary-r9v deploy_llm_primary() { # Refuse rather than restart-loop: a unit whose image or payload is # missing never becomes healthy, and a green deploy would leave the # endpoint dark with nothing in the log to say why. - if ! podman image inspect "$LLM_PRIMARY_IMAGE" >/dev/null 2>&1; then - echo "ERROR: image absent: $LLM_PRIMARY_IMAGE" >&2 - echo "frank-primary: build the fp030 image first (see the runbook)" >&2 + if [[ ! -f "$LLM_PRIMARY_STATE" ]]; then + echo "ERROR: R9V setup state absent: $LLM_PRIMARY_STATE" >&2 + echo "frank-primary: run the profile setup on this host first" >&2 return 1 fi - local want_shards have_shards - want_shards="$(python3 - "$LLM_PRIMARY_ROOT/model.safetensors.index.json" <<'PY' -import json, sys -print(len(set(json.load(open(sys.argv[1]))["weight_map"].values()))) + local state_json image_digest model_dir ple_path ple_bytes + state_exports="$(python3 - "$LLM_PRIMARY_STATE" <<'PY' +import json, shlex, sys +state = json.load(open(sys.argv[1])) +config = state.get("config") or {} +if state.get("ready") is not True: + sys.exit("R9V setup state is not ready") +for key in ("R9V_IMAGE", "R9V_MODEL_DIR", "R9V_PLE_PATH", "R9V_PLE_EXPECTED_BYTES", + "R9V_TARGET_REL", "R9V_TARGET_SHARD2_REL", "R9V_TARGET_SHARD3_REL", "R9V_MMPROJ_REL"): + print(f"{key}={shlex.quote(config.get(key, ''))}") PY )" || return 1 - have_shards="$(find "$LLM_PRIMARY_ROOT" -maxdepth 1 -name '*.safetensors' | wc -l)" - if [[ -z "$want_shards" || "$have_shards" != "$want_shards" ]]; then - echo "ERROR: payload incomplete at $LLM_PRIMARY_ROOT (have $have_shards shards, index names $want_shards)" >&2 - echo "frank-primary: run scripts/download-frank-primary-fp8.sh on this host" >&2 + eval "$state_exports" + if [[ -z "$R9V_IMAGE" || -z "$R9V_MODEL_DIR" || -z "$R9V_PLE_PATH" ]]; then + echo "ERROR: R9V setup state is missing image/model/PLE paths" >&2 return 1 fi + if ! podman images --format '{{.RepoDigests}}' | grep -qF "${R9V_IMAGE#sha256:}"; then + echo "ERROR: pinned R9V runtime image absent from rootless storage: $R9V_IMAGE" >&2 + echo "frank-primary: load the image bundle (./r9v setup ...)" >&2 + return 1 + fi + if [[ -n "$R9V_PLE_EXPECTED_BYTES" ]]; then + ple_bytes="$(stat -c %s "$R9V_PLE_PATH" 2>/dev/null || echo 0)" + if [[ "$ple_bytes" != "$R9V_PLE_EXPECTED_BYTES" ]]; then + echo "ERROR: PLE payload wrong size: $R9V_PLE_PATH (expected $R9V_PLE_EXPECTED_BYTES, found $ple_bytes)" >&2 + echo "frank-primary: re-run the PLE derivation (./r9v setup ...)" >&2 + return 1 + fi + fi + local rel + for rel in "$R9V_TARGET_REL" "$R9V_TARGET_SHARD2_REL" "$R9V_TARGET_SHARD3_REL" "$R9V_MMPROJ_REL"; do + [[ -z "$rel" ]] && continue + if [[ ! -f "$R9V_MODEL_DIR/$rel" ]]; then + echo "ERROR: payload file missing: $R9V_MODEL_DIR/$rel" >&2 + echo "frank-primary: re-run the profile setup to fetch/verify the payload" >&2 + return 1 + fi + done # A stale container with the same name makes `podman run --name` fail at # start; remove any leftover from a failed previous attempt. @@ -1236,8 +1264,9 @@ PY run_privileged systemctl daemon-reload run_privileged systemctl enable frank-primary.service run_privileged systemctl restart frank-primary.service - # Cold start: triton/inductor compile on first boot plus reading+pinning - # ~173 GiB of weights. Parity with every predecessor on this port. + # Cold start: the launcher's preflight doctor plus payload page-in + # (~93 GiB cold; expert cache fills after /health). Parity with every + # predecessor on this port. wait_for_health frank-primary http://127.0.0.1:7770/health 1800 || return 1 # False-green hardening (the 2026-09-26 lesson): a bare curl against the # port cannot tell WHO answers — a stray probe holding :7770 satisfies diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index 0bb9d23..63a1040 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -1,96 +1,20 @@ -# frank-primary.env — tunables for the vLLM 0.30 frank-primary endpoint -# (Qwen3.8-Flash-Next-FP8, 2x R9700 gfx1201, dankserver :7770). +# frank-primary.env — R9V radiance-vLLM production overrides (dankserver :7770). # -# Paths and numbers only — NEVER credentials. Do not name a tunable TEMP, HOST -# or GPU_MAX_HEAP_SIZE: the HIP runtime reads those bare names (2026-08-13). -# -# Container paths: the model tree is mounted at /models (host -# /home/dank/models); the compile caches live under /cache (host -# /home/dank/.cache/frank-primary-vllm, keyed on model + stack + graph knobs — -# never share it across configurations). - -# --- identity --------------------------------------------------------------- -SERVED_NAME=frank-primary -BIND_HOST=0.0.0.0 -PORT=7770 - -# --- model ------------------------------------------------------------------ -# Host-side dir: /home/dank/models/Qwen3.8-Flash-Next-FP8 (131 safetensors -# shards, 172.78 GiB, verified against model.safetensors.index.json). -MODEL_DIR=/models/Qwen3.8-Flash-Next-FP8 - -# --- placement -------------------------------------------------------------- -# Fixed 2-card tensor parallel (the R9700 pair; the Raphael iGPU is index 2 -# and excluded by the visibility pins). -ROCR_VISIBLE_DEVICES=0,1 -HIP_VISIBLE_DEVICES=0,1 -TP=2 -# 0.97 of 31.86 GiB/card (34,208,743,424 B) — leaves ~1 GiB for the ROCm -# baseline. Raise only with a measured reason. -GPU_UTIL=0.97 -# The model's native window. -CTX_SIZE=262144 -# Sequences the scheduler may hold. 8 is the proven batch shape on this host -# (radiance era); revisit with a bench after bring-up. -MAX_SEQS=8 - -# --- weight offloading (the UVA shape) --------------------------------------- -# --offload-backend uva: matched weights live in PINNED host RAM and are read -# in place over PCIe by the kernels (zero-copy), instead of being resident -# on the cards. is_uva_available() = is_pin_memory_available() on ROCm; the -# pin needs the unit's LimitMEMLOCK=infinity (a root unit grants it via -# PID1 — the dankserver user session defaults to 8 MiB and cannot raise it). -# If pinning is unavailable the offloader falls back to a copy path — the -# load log MUST be checked for the UVA path, not just /health. -# --cpu-offload-gb is PER GPU (GiB). 2x R9700 total pinned budget ≈ 160 GiB of -# the 246 GiB host RAM; the rest covers the OS, llama.cpp-era page cache -# drain and the pinned staging buffers. Do not raise without checking -# MemAvailable at load time. -# --cpu-offload-params: name SEGMENTS (dot-delimited components) of tensors -# to offload. Measured from the payload headers (2026-09-26): `ple` matches -# 138 tensors = 131 n-gram table shards + 7 per-layer conv/proj weights, -# 47.75 GiB; `experts` matches 150,528 tensors, 114.86 GiB. Combined -# 162.60 GiB — with TP=2 each rank needs ~81.3 GiB of pinned host RAM, -# hence OFFLOAD_GB=82 per GPU (164 GiB total; 80/card left 2.6 GiB of -# experts resident, inflating the on-card weight share). Resident after -# the full offload: 10.16 GiB of dense/attention/MTP/ViT/lm_head weights. -# Matching is greedy-up-to-budget in traversal order; unmatched params are -# never offloaded. -OFFLOAD_BACKEND=uva -OFFLOAD_GB=82 -OFFLOAD_PARAMS=ple,experts - -# --- engine ----------------------------------------------------------------- -# No aiter wheel in this image (gfx9-only upstream); the AMD qwen4_exp path -# runs its in-tree triton kernels. Deterministic rather than relying on the -# availability probe. -VLLM_ROCM_USE_AITER=0 -# Compile/inductor/triton caches (container /cache, TRITON_CACHE_DIR etc.) -CACHE_ROOT=/cache - -# --- parsers / template (the harness wire contract) --------------------------- -# vLLM 0.30 emits the thinking channel as `reasoning` (the deprecated -# `reasoning_content` is renamed server-side, chat_completion/protocol.py:554). -# The qwen3 reasoning parser extracts the template's think-block tags into -# that channel; the qwen3_xml tool parser structures tool-call XML output into -# tool_calls (auto-tool-choice enables the structured path for API clients). -# The family precedent is the host's own Ornith-era serve scripts -# (~/vllm/failedAttempts/OrnithFP8.sh); the FP8 tokenizer's template -# consumes enable_thinking + preserve_thinking (verified in tokenizer_config). -TOOL_PARSER=qwen3_xml -REASONING_PARSER=qwen3 -CHAT_TEMPLATE_KWARGS='{"enable_thinking": true, "preserve_thinking": true}' - -# --- speculative decode ------------------------------------------------------ -# The hybrid stack (GatedDeltaNet + QSA short-conv) uses the v1 engine's -# mamba-capable attention backends, negotiated by the model, not by env. -# MTP drafting comes from the model's own mtp.* weights via -# --speculative-config, only when SPEC_CONFIG is non-empty. Bring-up runs -# with it empty on purpose: measure the base shape clean, then A/B; never -# change two knobs at once. -SPEC_CONFIG= - - -# --- container --------------------------------------------------------------- -IMAGE=localhost/frank-primary:vllm030 -CONTAINER_NAME=frank-primary-vllm +# The full tuned engine shape (104 R9V_* keys: tiered-expert placement, +# CPU offload budgets 146.25/78.75 GiB per rank, CED on, MTP4, TP=2, +# qwen3_coder tool parser, 196608 model len) lives on-host in the R9V setup +# state — /home/dank/r9v-state/uncensored/setup.json — written by the +# profile's setup run (PASS=28/FAIL=0, all 20 artifacts sha256-verified). +# serve-frank-primary.sh layers that state first and THIS FILE second, so the +# keys below win. Never copy tuned values from the state into this file — +# the state is the single source of truth for engine shape. +R9V_HOME=/home/dank/GitPackages/radiance-vllm-qwen4exp +R9V_STATE_FILE=/home/dank/r9v-state/uncensored/setup.json +# Fleet identity: LAN bind, the single frank-primary port, the harness's +# model id, our own container name (the launcher default r9v-qwen38-flash-next +# is the scratch identity). +R9V_HOST_BIND=0.0.0.0 +R9V_HOST_PORT=7770 +R9V_SERVED_MODEL_NAME=frank-primary +R9V_CONTAINER_NAME=frank-primary-r9v +R9V_EXPECTED_GPU_BDFS=0000:03:00.0,0000:07:00.0 diff --git a/scripts/frank-primary.service b/scripts/frank-primary.service index 85d2e2a..cd2b0f6 100644 --- a/scripts/frank-primary.service +++ b/scripts/frank-primary.service @@ -1,34 +1,40 @@ -# frank-primary.service — vLLM 0.30 endpoint (Qwen3.8-Flash-Next-FP8, -# UVA offload shape) on the 2x R9700 pair, dankserver :7770. +# frank-primary.service — R9V radiance-vLLM endpoint (Qwen3.8-Flash-Next- +# Uncensored IQ4_XS, MTP4, TP=2, tiered experts) on the 2x R9700 pair, +# dankserver :7770. # -# Replaces the llama.cpp frank-primary unit of 2026-09-26 (same port, same -# served alias; harness models.yml unchanged). The serve script execs a -# podman container (image localhost/frank-primary:vllm030) as dank. +# Replaces the llama.cpp frank-primary unit (same port, same served alias; +# harness models.yml unchanged in id, retuned for the 196608 window). The +# serve script drives the R9V fork's own launcher through the docker->podman +# shim and then waits on the container, so a crashed engine is a nonzero +# ExecStart and Restart=on-failure is real. # -# WHY A ROOT UNIT (not a rootless quadlet): the UVA offload pins ~160 GiB of -# host RAM (PLE n-gram table + routed experts). Pinned pages are charged -# against RLIMIT_MEMLOCK; dank's login session on this host carries 8 MiB and -# a user quadlet is capped by the user manager's own hard limit — raising -# that means restarting user@1000.service and bouncing every user container -# on the host. A root unit grants LimitMEMLOCK=infinity to the dank process -# via PID1, no user-manager restart, and the image stays in dank's rootless -# podman storage (podman run as dank reads ~/.local/share/containers). +# WHY A ROOT UNIT (not a rootless quadlet): the tiered-expert shape pins tens +# of GiB of host RAM (the UVA expert host copy plus the 16 GiB PLE pinned +# reserve). Pinned pages are charged against RLIMIT_MEMLOCK; dank's login +# session on this host carries 8 MiB and a user quadlet is capped by the user +# manager's own hard limit — raising that means restarting user@1000.service +# and bouncing every user container on the host. A root unit grants +# LimitMEMLOCK=infinity to the dank process via PID1, and the image stays in +# dank's rootless podman storage (the shim drives podman as dank). [Unit] -Description=frank-primary — vLLM 0.30 Qwen3.8-Flash-Next-FP8 (UVA offload) on 2x R9700 +Description=frank-primary — R9V radiance-vLLM Qwen3.8-Flash-Next-Uncensored IQ4_XS on 2x R9700 After=network-online.target Wants=network-online.target [Service] User=dank Group=dank -# The UVA offloader pins the offloaded weights in host RAM; an RLIMIT_MEMLOCK -# smaller than the pinned set fails the load mid-start. PID1 grants infinity -# here — parity with the fleet's other LLM units. +# The tiered-expert offloader pins host RAM; an RLIMIT_MEMLOCK smaller than +# the pinned set fails the load mid-start. PID1 grants infinity here — +# parity with the fleet's other LLM units. LimitMEMLOCK=infinity Environment="FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env" ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh -# Cold start: Triton/inductor compile (several minutes on first boot) plus -# reading+pinning ~173 GiB of weights. Parity with the radiance-era timeout. +# systemd stop: wait alone cannot stop the container child; stop it +# explicitly. vLLM 0.26 drains gracefully on SIGTERM. +ExecStop=/usr/bin/podman stop -t 120 frank-primary-r9v +# Boot cost: the launcher's preflight doctor (~1 min) after a warm page +# cache; cold (post-reboot) payload page-in of ~93 GiB dominates. TimeoutStartSec=1800 Restart=on-failure RestartSec=10 diff --git a/scripts/serve-frank-primary.sh b/scripts/serve-frank-primary.sh index 20e8720..2427fe6 100755 --- a/scripts/serve-frank-primary.sh +++ b/scripts/serve-frank-primary.sh @@ -1,21 +1,29 @@ #!/usr/bin/env bash -# serve-frank-primary.sh — frank-primary endpoint (vLLM 0.30, -# Qwen3.8-Flash-Next-FP8 on the 2x R9700 pair, dankserver :7770). +# serve-frank-primary.sh — frank-primary endpoint (R9V radiance-vLLM fork, +# Qwen3.8-Flash-Next-Uncensored IQ4_XS + MTP4 drafter, 2x R9700, dankserver +# :7770). # -# Validates the image, the model payload and the env tunables, then execs the -# container. The unit (scripts/frank-primary.service, root systemd, User=dank, -# LimitMEMLOCK=infinity) is what makes the UVA pin budget legal — a plain -# shell here carries the 8 MiB session limit and the pinned offload fails. +# The engine is the upstream R9V launcher (scripts/launch.sh in the fork +# checkout), NOT a hand-frozen podman argv: launch.sh owns the resolved +# docker/podman args (binds, overlays, device passthrough, publish lines) +# and the fork's qualification was measured through it. This script's job: # -# OFFLOAD SHAPE (the only legal fit for a ~125B-param FP8 model on 2x 32 GiB -# cards): --offload-backend uva pins the PLE n-gram table (51.2B params) and -# the routed experts in host RAM and reads them in place over PCIe. The -# first start MUST be watched for the UVA engagement log (not just /health): -# a silent fallback to the copy path turns the endpoint into a slideshow. +# 1. layer the on-host R9V setup state (the 104-key tuned shape the +# profile setup qualified) under this repo's frank-primary.env +# overrides (bind/port/alias/container); +# 2. gate on the pinned runtime image (by its OCI manifest digest, the +# podman RepoDigest), the exact-size PLE payload, and the GGUF shards; +# 3. exec the launcher, then wait on the container: launch.sh --detach +# exits as soon as the container is created, so without the wait the +# unit would be "green" while the engine crashes — the wait is what +# keeps systemd's Restart=on-failure self-healing real. # -# NO --model-draft, no draft GGUF: vLLM's MTP drafting comes from the model's -# own mtp.* weights via --speculative-config, and only when SPEC_CONFIG is -# non-empty. Bring-up runs with it empty on purpose. +# The root unit (scripts/frank-primary.service, User=dank, +# LimitMEMLOCK=infinity) grants the pin budget for the tiered-expert UVA +# host copy (~43 GiB pinned) and the 16 GiB PLE pinned reserve. A plain +# shell session also works (proven 2026-09-27: podman rootless inherits +# usable limits here), but the unit is the fleet-parity boot-persistent +# shape. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -30,100 +38,101 @@ set -a source "$ENV_FILE" set +a -: "${SERVED_NAME:?SERVED_NAME must be set}" -: "${BIND_HOST:?BIND_HOST must be set}" -: "${PORT:?PORT must be set}" -: "${MODEL_DIR:?MODEL_DIR must be set}" -: "${TP:?TP must be set}" -: "${GPU_UTIL:?GPU_UTIL must be set}" -: "${CTX_SIZE:?CTX_SIZE must be set}" -: "${MAX_SEQS:?MAX_SEQS must be set}" -: "${OFFLOAD_BACKEND:?OFFLOAD_BACKEND must be set}" -: "${OFFLOAD_GB:?OFFLOAD_GB must be set}" -: "${OFFLOAD_PARAMS:?OFFLOAD_PARAMS must be set}" -: "${VLLM_ROCM_USE_AITER:?VLLM_ROCM_USE_AITER must be set}" -: "${IMAGE:?IMAGE must be set}" -: "${CONTAINER_NAME:?CONTAINER_NAME must be set}" -: "${ROCR_VISIBLE_DEVICES:?ROCR_VISIBLE_DEVICES must be set}" -: "${TOOL_PARSER:?TOOL_PARSER must be set}" -: "${REASONING_PARSER:?REASONING_PARSER must be set}" -: "${CHAT_TEMPLATE_KWARGS:?CHAT_TEMPLATE_KWARGS must be set}" -: "${CACHE_ROOT:?CACHE_ROOT must be set}" +: "${R9V_HOME:?R9V_HOME must name the fork checkout}" +: "${R9V_STATE_FILE:?R9V_STATE_FILE must name the setup state}" +: "${R9V_HOST_BIND:?R9V_HOST_BIND must be set}" +: "${R9V_HOST_PORT:?R9V_HOST_PORT must be set}" +: "${R9V_SERVED_MODEL_NAME:?R9V_SERVED_MODEL_NAME must be set}" +: "${R9V_CONTAINER_NAME:?R9V_CONTAINER_NAME must be set}" -if [ "$OFFLOAD_BACKEND" != "uva" ]; then - echo "serve-frank-primary: OFFLOAD_BACKEND=$OFFLOAD_BACKEND — this deployment is only qualified for uva" >&2 +LAUNCHER="$R9V_HOME/scripts/launch.sh" +if [ ! -x "$LAUNCHER" ]; then + echo "serve-frank-primary: R9V launcher missing: $LAUNCHER" >&2 exit 1 fi -if ! podman image inspect "$IMAGE" > /dev/null 2>&1; then - echo "serve-frank-primary: image missing: $IMAGE — build it before serving" >&2 +# The docker->podman shim is REQUIRED: the launcher and its doctor speak +# docker (OCI-manifest-digest image IDs, LogConfig fields) that podman's +# native CLI answers differently. The shim carries those laws. +SHIM_DIR="$R9V_HOME/.podman-shim" +if [ ! -x "$SHIM_DIR/docker" ]; then + echo "serve-frank-primary: docker->podman shim missing: $SHIM_DIR/docker" >&2 + exit 1 +fi +export PATH="$SHIM_DIR:$PATH" + +# --- layer 1: the qualified setup state (production overrides never come +# from here; keys already defined in the env file are skipped) ------------- +STATE_FILE="$R9V_STATE_FILE" +if [ ! -f "$STATE_FILE" ]; then + echo "serve-frank-primary: R9V setup state missing: $STATE_FILE" >&2 + echo " run the profile setup first (r9v setup qwen38-mtp4-uncensored ...)" >&2 + exit 1 +fi +STATE_EXPORTS="$(python3 - "$STATE_FILE" <<'PY' +import json, shlex, sys, os +state = json.load(open(sys.argv[1])) +config = state.get("config") or {} +if state.get("ready") is not True: + sys.exit("R9V setup state is not ready; rerun setup before serving") +env_keys = {k for k in os.environ if k.startswith("R9V_")} +for key in sorted(config): + if key.startswith("R9V_") and key not in env_keys: + print(f"export {key}={shlex.quote(config[key])}") +PY +)" +eval "$STATE_EXPORTS" + +: "${R9V_IMAGE:?state must carry R9V_IMAGE}" +: "${R9V_MODEL_DIR:?state must carry R9V_MODEL_DIR}" +: "${R9V_PLE_PATH:?state must carry R9V_PLE_PATH}" + +# --- gates: never serve a half-finished payload or a missing image -------- +# Runtime image by pinned OCI manifest digest (podman keys images by config +# digest but preserves the manifest digest in RepoDigests — the shim's ID law). +if ! podman images --format '{{.RepoDigests}}' | grep -qF "${R9V_IMAGE#sha256:}"; then + echo "serve-frank-primary: pinned R9V image absent: $R9V_IMAGE" >&2 + echo " load the bundle: ./r9v setup qwen38-mtp4-uncensored --state-dir ..." >&2 exit 1 fi -# The model payload is verified by the fetch/verify step (131 shards, index -# cross-checked). The serve-time check is the load-bearing minimum: the -# wrapper config and the shard index must exist, or the container start is a -# guaranteed 20-minute timeout instead of a fast failure. -HOST_MODEL_DIR="/home/dank/models/${MODEL_DIR#/models/}" -if [ ! -f "$HOST_MODEL_DIR/config.json" ] || [ ! -f "$HOST_MODEL_DIR/model.safetensors.index.json" ]; then - echo "serve-frank-primary: model payload incomplete at $HOST_MODEL_DIR (config.json or index missing)" >&2 - echo "serve-frank-primary: run scripts/download-frank-primary-fp8.sh on this host" >&2 - exit 1 +# PLE payload at its pinned byte count (stat -c %s, never wc -c), plus the +# three target GGUF shards and the mmproj the launcher mounts. +PLE_BYTES="${R9V_PLE_EXPECTED_BYTES:-}" +if [ -n "$PLE_BYTES" ]; then + ACTUAL="$(stat -c %s "$R9V_PLE_PATH" 2>/dev/null || echo 0)" + if [ "$ACTUAL" != "$PLE_BYTES" ]; then + echo "serve-frank-primary: PLE payload wrong size: $R9V_PLE_PATH" >&2 + echo " expected $PLE_BYTES, found $ACTUAL" >&2 + exit 1 + fi +else + if [ ! -f "$R9V_PLE_PATH" ]; then + echo "serve-frank-primary: PLE payload missing: $R9V_PLE_PATH" >&2 + exit 1 + fi fi +for rel in "$R9V_TARGET_REL" "$R9V_TARGET_SHARD2_REL" "$R9V_TARGET_SHARD3_REL" "$R9V_MMPROJ_REL"; do + if [ -z "$rel" ]; then continue; fi + if [ ! -f "$R9V_MODEL_DIR/$rel" ]; then + echo "serve-frank-primary: payload file missing: $R9V_MODEL_DIR/$rel" >&2 + exit 1 + fi +done if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then - echo "serve-frank-primary: /dev/kfd or /dev/dri missing — GPU stack not up" >&2 + echo "serve-frank-primary: GPU device nodes absent (/dev/kfd, /dev/dri)" >&2 exit 1 fi -mkdir -p /home/dank/.cache/frank-primary-vllm - -SERVE_ARGS=("$MODEL_DIR" - --served-model-name "$SERVED_NAME" - --host "$BIND_HOST" - --port "$PORT" - --tensor-parallel-size "$TP" - --gpu-memory-utilization "$GPU_UTIL" - --max-model-len "$CTX_SIZE" - --max-num-seqs "$MAX_SEQS" - --offload-backend "$OFFLOAD_BACKEND" - --cpu-offload-gb "$OFFLOAD_GB" - --cpu-offload-params "$OFFLOAD_PARAMS" - --enable-auto-tool-choice --tool-call-parser "$TOOL_PARSER" - --reasoning-parser "$REASONING_PARSER" - --default-chat-template-kwargs "$CHAT_TEMPLATE_KWARGS" -) - -# SPEC_CONFIG (optional, JSON) is how MTP drafting is enabled after the -# bring-up measurement: --speculative-config '{"method":"mtp",...}'. -if [ -n "${SPEC_CONFIG:-}" ]; then - SERVE_ARGS+=(--speculative-config "$SPEC_CONFIG") -fi -# A crashed/killed unit leaves the container name registered (systemd can -# SIGKILL conmon before the --rm cleanup runs); podman run would then die -# on name-in-use forever. Clear any orphan of OUR name first — idempotent. -podman rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true - -exec podman run --rm \ - --name "$CONTAINER_NAME" \ - --userns=keep-id \ - --network=host \ - --ipc=host \ - --security-opt seccomp=unconfined \ - --cap-add SYS_PTRACE \ - --device /dev/kfd \ - --device /dev/dri \ - --group-add keep-groups \ - -e ROCR_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" \ - -e HIP_VISIBLE_DEVICES="$HIP_VISIBLE_DEVICES" \ - -e VLLM_ROCM_USE_AITER="$VLLM_ROCM_USE_AITER" \ - -e NCCL_P2P_DISABLE="$NCCL_P2P_DISABLE" \ - -e TRITON_CACHE_DIR="$CACHE_ROOT/triton" \ - -e TORCHINDUCTOR_CACHE_DIR="$CACHE_ROOT/inductor" \ - -e HOME="$CACHE_ROOT" \ - -e TRITON_ALLOW_NON_CONST_GLOBALS=1 \ - --volume /home/dank/models:/models \ - --volume /home/dank/.cache/frank-primary-vllm:"$CACHE_ROOT" \ - "$IMAGE" \ - "${SERVE_ARGS[@]}" +# A crashed/killed unit can leave the container name registered (systemd +# SIGKILLs conmon before cleanup runs); the launcher refuses an existing +# name forever after. Clear any orphan of OUR name first — idempotent. +docker rm -f "$R9V_CONTAINER_NAME" >/dev/null 2>&1 || true +# launch.sh detaches after preflight; from here on, this script's lifetime +# IS the container's: wait returns the container's exit code, so a crash is +# a nonzero ExecStart and Restart=on-failure fires. (systemd stop goes +# through ExecStop in the unit — docker wait alone cannot stop the child.) +"$LAUNCHER" +exec docker wait "$R9V_CONTAINER_NAME" diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats index 2f5536e..5635580 100755 --- a/tests/frank_primary_unit.bats +++ b/tests/frank_primary_unit.bats @@ -1,17 +1,17 @@ #!/usr/bin/env bats # Seam: frank-primary — the Dankserver LLM endpoint on :7770. # -# This file asserts the vLLM 0.30 ROOT-UNIT shape (the 2026-09-27 swap that -# replaced the llama.cpp b11030 endpoint): a locally built ROCm gfx1201 -# image (localhost/frank-primary:vllm030) serving Qwen3.8-Flash-Next-FP8 -# (Qwen4ExpForConditionalGeneration, 131 shards, 172.78 GiB) on the two -# R9700s with the UVA offload shape. The seam pins what the flags alone do -# not say: the offload budget arithmetic (the bandwidth-huge tensors — the -# 47.75 GiB PLE family and the 114.86 GiB routed experts — are pinned in -# host RAM and read in place, never streamed), the memlock law that makes -# the pin legal, the parser contract that keeps the harness wire formats -# (tool_calls + the reasoning channel) working, HIP env hygiene, and the -# deployer's image/payload gates with the false-green MainPID check. +# This file asserts the R9V radiance-vLLM ROOT-UNIT shape (the 2026-09-27 +# swap that replaced the llama.cpp b11030 endpoint): the R9V fork's own +# launcher (scripts/launch.sh in the pinned fork checkout) driven through a +# docker->podman shim, with the qualified on-host setup state layered under +# the repo's production overrides. The seam pins what the flags alone do not +# say: the state-layering contract (104 tuned R9V_* keys live on-host; the +# repo env carries only production overrides), the self-healing law (the +# launcher detaches, so the serve script must wait on the container for +# Restart=on-failure to mean anything), the memlock law that makes the +# tiered-expert pin legal, and the deployer's state/payload/image gates with +# the false-green MainPID check. load helpers @@ -28,9 +28,9 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" @test "frank-primary is boot-persistent, self-healing and unprivileged" { # Root unit parity with frank-large/frank-small: enable is the [Install] - # section, the restart policy survives a crash, and the ~164 GiB UVA pin - # needs an unbounded memlock (a root unit grants it via PID1; the dank - # session's own hard limit is 8 MiB and cannot be raised without + # section, the restart policy survives a crash, and the tiered-expert + # pin needs an unbounded memlock (a root unit grants it via PID1; the + # dank session's own hard limit is 8 MiB and cannot be raised without # restarting user@1000.service). grep -q '^WantedBy=multi-user.target$' "$UNIT" grep -q '^Restart=on-failure$' "$UNIT" @@ -44,126 +44,91 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" grep -q 'FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env' "$UNIT" } -@test "the serve shape is vLLM: no llama.cpp-ism survives the cutover" { - # Clean cutover: the GGUF era's flags would silently no-op or crash a vLLM - # server, so their absence is part of the contract. +@test "the unit stops the container child explicitly" { + # The serve script blocks on docker wait, which cannot stop the + # container child; systemd stop goes through ExecStop or the engine + # outlives the unit and holds both GPUs. + grep -q '^ExecStop=/usr/bin/podman stop' "$UNIT" + grep -q 'frank-primary-r9v' "$UNIT" +} + +@test "the serve shape is the R9V launcher, not a frozen podman run" { + # The launcher owns the resolved container args (binds, overlays, device + # passthrough); a frozen copy of its argv would drift on every fork + # update. The llama.cpp era's flags would also silently no-op here. + grep -q 'launch.sh' "$SERVE" ! grep -q -- '--model-draft' "$SERVE" ! grep -q -- '--spec-type' "$SERVE" ! grep -q -- '--mmproj' "$SERVE" ! grep -q -- '--override-tensor' "$SERVE" - ! grep -q '\.gguf' "$SERVE" - ! grep -q '\.gguf' "$ENV_FILE" + ! grep -q 'podman run' "$SERVE" } -@test "the bandwidth-huge tensors are pinned in host RAM, not streamed" { - # The UVA shape (the x4-pair law: streaming experts over PCIe loses, - # computing them in host RAM does not care): --offload-backend uva pins - # the matched weights with zero-copy reads. Measured from the payload - # headers (2026-09-26): `ple` matches 138 tensors (131 n-gram table - # shards + 7 per-layer conv/proj weights) = 47.75 GiB; `experts` matches - # 150,528 tensors = 114.86 GiB; combined 162.60 GiB. cpu-offload-gb is - # PER GPU: under TP=2 each rank needs ~81.3 GiB, hence 82 — 80/card - # would strand 2.6 GiB of experts on the cards. - grep -q '^OFFLOAD_BACKEND=uva$' "$ENV_FILE" - grep -q '^OFFLOAD_GB=82$' "$ENV_FILE" - grep -q '^OFFLOAD_PARAMS=ple,experts$' "$ENV_FILE" - grep -q -- '--offload-backend "$OFFLOAD_BACKEND"' "$SERVE" - grep -q -- '--cpu-offload-gb "$OFFLOAD_GB"' "$SERVE" - grep -q -- '--cpu-offload-params "$OFFLOAD_PARAMS"' "$SERVE" - # auto/prefetch are the copy paths: this deployment is only qualified - # for uva, so the serve script refuses anything else. - grep -q 'only qualified for uva' "$SERVE" +@test "the tuned engine shape comes from the on-host setup state, not the repo" { + # The profile setup qualified a 104-key shape (tiered-expert placement, + # per-rank offload budgets, CED, MTP4, parser wiring). Copying it into + # the repo would drift; the serve script layers it and skips keys the + # env file defines, so production overrides always win. + grep -q 'R9V_STATE_FILE=/home/dank/r9v-state/uncensored/setup.json' "$ENV_FILE" + grep -q 'state must carry R9V_MODEL_DIR' "$SERVE" + grep -q 'is not ready' "$SERVE" + grep -q 'key not in env_keys' "$SERVE" } -@test "the engine shape: TP=2 over both R9700s, full native window, batch 8" { - grep -q '^TP=2$' "$ENV_FILE" - grep -q '^GPU_UTIL=0.97$' "$ENV_FILE" - grep -q '^CTX_SIZE=262144$' "$ENV_FILE" - grep -q '^MAX_SEQS=8$' "$ENV_FILE" - grep -q -- '--tensor-parallel-size "$TP"' "$SERVE" - grep -q -- '--gpu-memory-utilization "$GPU_UTIL"' "$SERVE" - grep -q -- '--max-model-len "$CTX_SIZE"' "$SERVE" - grep -q -- '--max-num-seqs "$MAX_SEQS"' "$SERVE" +@test "the serve script gates the pinned image, PLE bytes and GGUF shards" { + # The image is answered by its OCI manifest digest (podman keeps the + # manifest digest in RepoDigests, not as the image ID — the shim's ID + # law). The PLE payload is checked at its pinned byte count + # (stat -c %s, never wc -c). A missing shard means a guaranteed + # long-timeout boot instead of a fast failure. + grep -q 'R9V_PLE_EXPECTED_BYTES' "$SERVE" + grep -q 'stat -c %s' "$SERVE" + grep -q 'R9V_TARGET_SHARD2_REL' "$SERVE" + grep -q 'R9V_MMPROJ_REL' "$SERVE" + grep -q 'RepoDigests' "$SERVE" + grep -q '/dev/kfd' "$SERVE" } -@test "the payload is the FP8 tree, vision served from the model's own config" { - # Qwen4ExpForConditionalGeneration: vLLM auto-loads the ViT and the - # mtp.* weights from the payload config — no explicit projector flag - # exists in this runtime (the llama.cpp --mmproj trap does not apply). - grep -q '^MODEL_DIR=/models/Qwen3.8-Flash-Next-FP8$' "$ENV_FILE" - grep -q '^IMAGE=localhost/frank-primary:vllm030$' "$ENV_FILE" +@test "the serve script clears the stale container and self-heals via docker wait" { + # launch.sh detaches and refuses an existing container name; a crashed + # unit can leave the name registered forever after. Clear our own name + # first (idempotent), then block on the container so a crash is a + # nonzero ExecStart. + grep -q 'docker rm -f "$R9V_CONTAINER_NAME"' "$SERVE" + grep -q 'exec docker wait "$R9V_CONTAINER_NAME"' "$SERVE" } @test "frank-primary holds the single fleet port under its own alias" { - grep -q '^PORT=7770$' "$ENV_FILE" - grep -q '^BIND_HOST=0.0.0.0$' "$ENV_FILE" - grep -q '^SERVED_NAME=frank-primary$' "$ENV_FILE" - grep -q -- '--served-model-name "$SERVED_NAME"' "$SERVE" - grep -q -- '--host "$BIND_HOST"' "$SERVE" - grep -q -- '--port "$PORT"' "$SERVE" -} - -@test "tool_calls and the reasoning channel are wired for the harness" { - # The wire contract: vLLM 0.30 emits the thinking channel as `reasoning` - # (the deprecated `reasoning_content` is renamed server-side), the qwen3 - # parser extracts the template's think block into it, and qwen3_xml + - # auto-tool-choice structure tool calls. Precedent: the host's own - # Ornith-era scripts (~/vllm/failedAttempts/OrnithFP8.sh); the FP8 - # template consumes enable_thinking/preserve_thinking (verified in - # tokenizer_config.json). - grep -q '^TOOL_PARSER=qwen3_xml$' "$ENV_FILE" - grep -q '^REASONING_PARSER=qwen3$' "$ENV_FILE" - grep -q -- '--enable-auto-tool-choice --tool-call-parser "$TOOL_PARSER"' "$SERVE" - grep -q -- '--reasoning-parser "$REASONING_PARSER"' "$SERVE" - # The template kwargs are JSON: the env value must be single-quoted or - # bash splits it and CHAT_TEMPLATE_KWARGS arrives empty (the -ot lesson). - grep -q "^CHAT_TEMPLATE_KWARGS='{\"enable_thinking\": true, \"preserve_thinking\": true}'$" "$ENV_FILE" - grep -q -- '--default-chat-template-kwargs "$CHAT_TEMPLATE_KWARGS"' "$SERVE" -} - -@test "speculative decode is env-wired and OFF at bring-up" { - # MTP drafting comes from the model's own mtp.* weights via - # --speculative-config; bring-up measures the base shape clean, so - # SPEC_CONFIG is empty and the flag is only added when non-empty. - grep -q '^SPEC_CONFIG=$' "$ENV_FILE" - grep -q -- '--speculative-config "$SPEC_CONFIG"' "$SERVE" -} - -@test "both R9700s are visible and the Raphael iGPU is not" { - grep -q '^ROCR_VISIBLE_DEVICES=0,1$' "$ENV_FILE" - grep -q '^HIP_VISIBLE_DEVICES=0,1$' "$ENV_FILE" - grep -q -- '-e ROCR_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"' "$SERVE" - grep -q -- '-e HIP_VISIBLE_DEVICES="$HIP_VISIBLE_DEVICES"' "$SERVE" + # The scratch identity (r9v-qwen38-flash-next on 127.0.0.1:7772) must + # never leak into production: LAN bind, :7770, the harness's model id. + grep -q '^R9V_HOST_BIND=0.0.0.0$' "$ENV_FILE" + grep -q '^R9V_HOST_PORT=7770$' "$ENV_FILE" + grep -q '^R9V_SERVED_MODEL_NAME=frank-primary$' "$ENV_FILE" + grep -q '^R9V_CONTAINER_NAME=frank-primary-r9v$' "$ENV_FILE" + grep -q '^R9V_EXPECTED_GPU_BDFS=0000:03:00.0,0000:07:00.0$' "$ENV_FILE" } @test "no bare HIP-reserved env var is defined" { # HIP reads bare TEMP for its JIT cache path (2026-08-13: TEMP=1.0 # segfaulted every load), and HOST/GPU_MAX_HEAP_SIZE are the same - # class. The bind address is BIND_HOST, not HOST. + # class. The bind address is R9V_HOST_BIND, not HOST. ! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE" - grep -q '^BIND_HOST=' "$ENV_FILE" ! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE" } -@test "the serve script refuses to start on a half-finished payload or missing image" { - # A payload missing its wrapper config or shard index turns the - # container start into a guaranteed 20-minute timeout; the checks fail - # fast with a pointer at the fetch script instead. - grep -q 'config.json or index missing' "$SERVE" - grep -q 'download-frank-primary-fp8.sh' "$SERVE" - grep -q 'podman image inspect' "$SERVE" -} - -@test "the deployer gates the image and the full shard set before starting" { - # The deployer refuses when the image or any index-named shard is - # absent (never serve a partial download), clears any stale container - # that would break `podman run --name`, and waits out the port before - # the new engine binds it. - grep -q 'LLM_PRIMARY_IMAGE=localhost/frank-primary:vllm030' "$REMOTE_DEPLOY" - grep -q 'LLM_PRIMARY_ROOT=/home/dank/models/Qwen3.8-Flash-Next-FP8' "$REMOTE_DEPLOY" - grep -q 'weight_map' "$REMOTE_DEPLOY" - grep -q 'download-frank-primary-fp8.sh' "$REMOTE_DEPLOY" - grep -q "podman rm -f \"\$LLM_PRIMARY_CONTAINER\"" "$REMOTE_DEPLOY" +@test "the deployer gates the setup state, image digest, PLE bytes and shards" { + # The deployer refuses when the on-host setup state is missing or not + # ready, when the pinned image is absent from rootless storage, when + # the PLE payload is off its pinned byte count, or when any GGUF shard + # or the mmproj is missing — then clears stale containers, waits out + # the port, relinks/enables/restarts the root unit and health-gates it. + grep -q 'LLM_PRIMARY_STATE=/home/dank/r9v-state/uncensored/setup.json' "$REMOTE_DEPLOY" + grep -q 'LLM_PRIMARY_CONTAINER=frank-primary-r9v' "$REMOTE_DEPLOY" + grep -q 'is not ready' "$REMOTE_DEPLOY" + grep -q "podman images --format '{{.RepoDigests}}'" "$REMOTE_DEPLOY" + grep -q 'stat -c %s' "$REMOTE_DEPLOY" + grep -q 'R9V_TARGET_SHARD2_REL' "$REMOTE_DEPLOY" + grep -q 'podman rm -f "$LLM_PRIMARY_CONTAINER"' "$REMOTE_DEPLOY" grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY" grep -q 'relink_unit frank-primary' "$REMOTE_DEPLOY" grep -q 'systemctl enable frank-primary.service' "$REMOTE_DEPLOY"