diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index fa934c7..d0cd3fa 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1201,85 +1201,76 @@ deploy_llm_small() { wait_for_health frank-small http://127.0.0.1:7771/health 900 } -# deploy_llm_primary — frank-primary is the R9V radiance-vLLM fork (docker- -# pinned prebuilt, gfx1201, driven through the docker->podman shim) serving -# the unsloth Qwen3.8-Flash-Next Q8_0 payload (6 GGUF shards) with the MTP4 -# drafter on the two R9700s as a ROOT unit whose serve script execs the -# fork's launcher through rootless podman as dank. Same port and alias as -# the endpoints it replaced (the 2026-09-27 cutover from uncensored -# IQ4_XS); the harness models.yml id is unchanged. The uncensored state dir -# (/home/dank/r9v-state/uncensored) stays on disk as the rollback target. +# deploy_llm_primary — frank-primary is the tcclaviger RDNA4 vLLM fork +# (digest-pinned image, gfx1201) serving the orcarouter +# Qwen3.8-Flash-Next-Uncensored-FP8 payload (131 safetensors shards) with +# MTP4 spec decode and q8/q8 KV on the two R9700s as a ROOT unit whose serve +# script drives `podman run` directly (the image entrypoint execs +# `vllm serve`). 2026-09-29 cutover from the R9V radiance fork, whose +# launcher/shim/setup-state machinery is retired (state dirs stay on disk as +# rollback; pre-cutover tag frank-primary-r9v-q8-20260929). Same port and +# alias as the endpoints it replaced; the harness models.yml id is unchanged. # -# Gates: the on-host R9V setup state must exist and be ready (it carries the -# pinned image digest and every payload path), the pinned runtime image must -# be loadable from dank's rootless storage by its OCI manifest digest, the -# PLE payload must match its pinned byte count, and every GGUF shard + the -# mmproj must exist (per-file sha256 verification was setup's job at fetch -# time, never re-hashed here). -LLM_PRIMARY_STATE=/home/dank/r9v-state/q8xl/setup.json -LLM_PRIMARY_CONTAINER=frank-primary-r9v +# Gates: the pinned runtime image must be loadable from dank's rootless +# storage by its OCI manifest digest (the env file's FRANK_PRIMARY_IMAGE), +# the payload dir must carry config.json and the expected shard count +# (FRANK_PRIMARY_MODEL_SHARDS), and the unit env must be complete. Per-file +# sha256 verification was the fetch script's job at download time, never +# re-hashed here. +LLM_PRIMARY_CONTAINER=frank-primary-qwen +LLM_PRIMARY_LEGACY_CONTAINERS="frank-primary-r9v qwen-fp8" deploy_llm_primary() { # Refuse rather than restart-loop: a unit whose image or payload is # missing never becomes healthy, and a green deploy would leave the # endpoint dark with nothing in the log to say why. - if [[ ! -f "$LLM_PRIMARY_STATE" ]]; then - echo "ERROR: R9V setup state absent: $LLM_PRIMARY_STATE" >&2 - echo "frank-primary: run the profile setup on this host first" >&2 + local env_file="/home/dank/serverAppConfigs/scripts/frank-primary.env" + local image model_dir shards count + if [[ ! -f "$env_file" ]]; then + echo "ERROR: frank-primary env file absent: $env_file" >&2 return 1 fi - local state_json image_digest model_dir ple_path ple_bytes - state_exports="$(python3 - "$LLM_PRIMARY_STATE" <<'PY' -import json, shlex, sys -state = json.load(open(sys.argv[1])) -config = state.get("config") or {} -if state.get("ready") is not True: - sys.exit("R9V setup state is not ready") -for key in ("R9V_IMAGE", "R9V_MODEL_DIR", "R9V_PLE_PATH", "R9V_PLE_EXPECTED_BYTES", - "R9V_TARGET_REL", "R9V_TARGET_SHARD2_REL", "R9V_TARGET_SHARD3_REL", - "R9V_TARGET_SHARD4_REL", "R9V_TARGET_SHARD5_REL", "R9V_TARGET_SHARD6_REL", - "R9V_MMPROJ_REL"): - print(f"{key}={shlex.quote(config.get(key, ''))}") -PY -)" || return 1 - eval "$state_exports" - if [[ -z "$R9V_IMAGE" || -z "$R9V_MODEL_DIR" || -z "$R9V_PLE_PATH" ]]; then - echo "ERROR: R9V setup state is missing image/model/PLE paths" >&2 + image="$(sed -n 's/^FRANK_PRIMARY_IMAGE=//p' "$env_file" | tail -1)" + model_dir="$(sed -n 's/^FRANK_PRIMARY_MODEL_DIR=//p' "$env_file" | tail -1)" + shards="$(sed -n 's/^FRANK_PRIMARY_MODEL_SHARDS=//p' "$env_file" | tail -1)" + if [[ -z "$image" || -z "$model_dir" || -z "$shards" ]]; then + echo "ERROR: frank-primary.env missing FRANK_PRIMARY_IMAGE/MODEL_DIR/MODEL_SHARDS" >&2 return 1 fi - if ! podman images --format '{{.RepoDigests}}' | grep -qF "${R9V_IMAGE#sha256:}"; then - echo "ERROR: pinned R9V runtime image absent from rootless storage: $R9V_IMAGE" >&2 - echo "frank-primary: load the image bundle (./r9v setup ...)" >&2 + if ! podman images --format '{{.RepoDigests}}' | grep -qF "${image#*@}"; then + echo "ERROR: pinned runtime image absent from rootless storage: $image" >&2 + echo "frank-primary: fetch it (podman pull $image)" >&2 return 1 fi - if [[ -n "$R9V_PLE_EXPECTED_BYTES" ]]; then - ple_bytes="$(stat -c %s "$R9V_PLE_PATH" 2>/dev/null || echo 0)" - if [[ "$ple_bytes" != "$R9V_PLE_EXPECTED_BYTES" ]]; then - echo "ERROR: PLE payload wrong size: $R9V_PLE_PATH (expected $R9V_PLE_EXPECTED_BYTES, found $ple_bytes)" >&2 - echo "frank-primary: re-run the PLE derivation (./r9v setup ...)" >&2 - return 1 - fi + if [[ ! -f "$model_dir/config.json" ]]; then + echo "ERROR: payload config missing: $model_dir/config.json" >&2 + echo "frank-primary: re-run scripts/download-frank-primary.sh" >&2 + return 1 + fi + if [[ ! -f "$model_dir/model.safetensors.index.json" ]]; then + echo "ERROR: weight-map index missing: $model_dir/model.safetensors.index.json" >&2 + echo "frank-primary: re-run scripts/download-frank-primary.sh" >&2 + return 1 + fi + count="$(find "$model_dir" -maxdepth 1 -name 'model-*-of-*.safetensors' 2>/dev/null | wc -l)" + if [[ "$count" != "$shards" ]]; then + echo "ERROR: payload shard count wrong: $model_dir (found $count, want $shards)" >&2 + echo "frank-primary: re-run scripts/download-frank-primary.sh" >&2 + return 1 fi - local rel - for rel in "$R9V_TARGET_REL" "${R9V_TARGET_SHARD2_REL:-}" "${R9V_TARGET_SHARD3_REL:-}" \ - "${R9V_TARGET_SHARD4_REL:-}" "${R9V_TARGET_SHARD5_REL:-}" "${R9V_TARGET_SHARD6_REL:-}" \ - "$R9V_MMPROJ_REL"; do - [[ -z "$rel" ]] && continue - if [[ ! -f "$R9V_MODEL_DIR/$rel" ]]; then - echo "ERROR: payload file missing: $R9V_MODEL_DIR/$rel" >&2 - echo "frank-primary: re-run the profile setup to fetch/verify the payload" >&2 - return 1 - fi - done # A stale container with the same name makes `podman run --name` fail at # start; remove any leftover from a failed previous attempt. - podman rm -f "$LLM_PRIMARY_CONTAINER" >/dev/null 2>&1 || true # A stopped unit can leave its container running: systemd's cgroup kill # races conmon's --rm cleanup, and the orphan holds the port and GPUs. - # Stop the unit, then remove the container BEFORE waiting on the port. + # Stop the unit, then remove the containers BEFORE waiting on the port. + # The legacy identities (the retired R9V container and the ad hoc + # experiment container qwen-fp8) hold the same GPUs and die here too. run_privileged systemctl stop frank-primary.service 2>/dev/null || true - podman rm -f "$LLM_PRIMARY_CONTAINER" >/dev/null 2>&1 || true + local ctr + for ctr in $LLM_PRIMARY_LEGACY_CONTAINERS "$LLM_PRIMARY_CONTAINER"; do + podman rm -f "$ctr" >/dev/null 2>&1 || true + done # Nothing starts until the port is released: the lucebox cutover # (2026-09-26) and the frank-large cutover (2026-09-16) both failed # exactly here when the next engine bound 7770 a second too early. @@ -1288,9 +1279,8 @@ PY run_privileged systemctl daemon-reload run_privileged systemctl enable frank-primary.service run_privileged systemctl restart frank-primary.service - # Cold start: the launcher's preflight doctor plus payload page-in - # (~93 GiB cold; expert cache fills after /health). Parity with every - # predecessor on this port. + # Cold start: FP8 page-in plus torch compile/graph capture (~15 min cold, + # measured 2026-09-29). Parity with every predecessor on this port. wait_for_health frank-primary http://127.0.0.1:7770/health 1800 || return 1 # False-green hardening (the 2026-09-26 lesson): a bare curl against the # port cannot tell WHO answers — a stray probe holding :7770 satisfies diff --git a/scripts/download-frank-primary.sh b/scripts/download-frank-primary.sh index f212aa0..8347940 100755 --- a/scripts/download-frank-primary.sh +++ b/scripts/download-frank-primary.sh @@ -1,92 +1,88 @@ #!/usr/bin/env bash -# Fetch the frank-primary payload: the Qwen3.8-Flash-Next Q8_0 target (six -# split parts), the shared Q8_0 MTP drafter head and the BF16 vision -# projector. +# Payload inventory/verification for frank-primary: +# orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8 (compressed-tensors FP8, 131 +# shards + weight-map index, ~172.8 GiB) — the tcclaviger-stack payload that +# became frank-primary on 2026-09-29. # -# bash scripts/download-frank-primary.sh -# bash scripts/download-frank-primary.sh --verify-only +# bash scripts/download-frank-primary.sh --verify-only (the ONLY mode) +# bash scripts/download-frank-primary.sh --fetch # -# The repo (unsloth/Qwen3.8-Flash-Next-GGUF) is PUBLIC and anonymous: no HF -# token is needed, and none is accepted on the command line (a -H auth -# header is world-readable in /proc//cmdline -- the 2026-09-16 leak). +# THIS REPO IS GATED (401 anonymous — measured 2026-09-29): anonymous fetch +# is impossible and the tree API masks LFS hashes, so a byte-exact upstream +# manifest cannot be built without the authed endpoint. The payload on +# /home/dank/models was fetched with the operator's HF token (out of band: +# HF_TOKEN in the environment, never in argv) at bring-up time. # -# WHY THESE FILES: -# * Target: Q8_0/ -- unsloth's Q8_0 of Qwen3.8-Flash-Next (125B total / -# 6B active MoE + the 51B n-gram embedding table). Q_8 is the -# operator's requested quant (2026-09-26 swap off the lucebox DS4.1 -# engine). The six parts ARE the payload: part 1's header advertises -# file_count=6, so they are never concatenated (concatenation trips -# "invalid split file name"). -# * Drafter: MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf -- the shared -# MTP head (1 layer). "shared" means it borrows the target's embedding -# tensors instead of carrying its own copy: 2.6 GiB instead of 3.9, -# and unsloth's own MTP guide names it as the recommended head. It is -# loaded via -md (this build never autoloads drafters or projectors). -# * Vision: mmproj-BF16.gguf -- the BF16 projector, RDNA4-native per the -# frank-small precedent. Loaded via --mmproj explicitly. -# -# The script is resumable (curl -C -) and checks every artifact's exact byte -# count AND sha256 against the Hub's declared values, so a truncated or -# mis-fetched file fails here instead of load-crashing the server. Sizes and -# hashes were captured from the resolve endpoint's x-linked-etag/x-linked-size -# on 2026-09-26. -# -# Total: 191,919,144,448 bytes (178.74 GiB), of which 175.30 GiB is the -# target itself. Destination keeps the Hub layout so the serve script can -# name paths exactly as they appear upstream. +# Modes: +# --fetch resume any missing/truncated shard via `hf download` with +# HF_TOKEN from the environment (NEVER passed on the command +# line — a -H auth header is world-readable in +# /proc//cmdline, the 2026-09-16 leak). +# --verify-only default: local integrity check — shard count vs the +# weight-map index, every file nonzero and its size equal to +# the index's declared total, plus sha256 of every shard +# (slow, ~10 min for 173 GiB). set -euo pipefail -REPO="unsloth/Qwen3.8-Flash-Next-GGUF" -DEST="/home/dank/models/Qwen3.8-Flash-Next-GGUF" +REPO="orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8" +DEST="/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8" +EXPECTED_SHARDS=131 -# path|bytes|sha256 -MANIFEST=( - "Q8_0/Qwen3.8-Flash-Next-Q8_0-00001-of-00006.gguf|10946624|2dabcbb53ca537a7947bc7d20414fd464eeaf4d66d43021b5b2556cc87544ad2" - "Q8_0/Qwen3.8-Flash-Next-Q8_0-00002-of-00006.gguf|682434912|494ca4ed3dbf97bc28da88af3890b8877b9032f909812d00c0526a9ca5e91d2e" - "Q8_0/Qwen3.8-Flash-Next-Q8_0-00003-of-00006.gguf|54400261312|34efd79a80a1ce540a517a5d56171924b66ce1c38b04c904f17ad6d8ef17cf20" - "Q8_0/Qwen3.8-Flash-Next-Q8_0-00004-of-00006.gguf|49446841216|bfa634025fabbd2658bf7694bc80b90e571699c768723f844c934c7ef06c691a" - "Q8_0/Qwen3.8-Flash-Next-Q8_0-00005-of-00006.gguf|49668930400|232a8f14cc0fa4262e7efe8593774b136fe40909e39c7a020342ddaa27259a97" - "Q8_0/Qwen3.8-Flash-Next-Q8_0-00006-of-00006.gguf|34015618784|538a93bca918064983409a41187ad4c68640f9aced6f29564da8f551bf86d7a5" - "MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf|2786568256|5ff54097406a905cf3a724c709124ceb0e3e10235ee862298969e91c96fa96e6" - "mmproj-BF16.gguf|907542944|2e788f8c511d8093c7b43cb87b2fd7e14228340318057f8fb20c86df2efe2355" -) +mode="verify" +[[ "${1:-}" == "--fetch" ]] && mode="fetch" -mode="download" -[[ "${1:-}" == "--verify-only" ]] && mode="verify" - -if [[ "$mode" == "download" ]]; then - for entry in "${MANIFEST[@]}"; do - path="${entry%%|*}" - mkdir -p "$DEST/$(dirname "$path")" - echo "fetching $path" - # -f: a 401/403 must FAIL the fetch, not write the error body as - # the artifact (the 2026-09-16 gated-repo incident). -C -: resume - # partial files. - curl -fL -C - -o "$DEST/$path" --retry 10 --retry-delay 5 \ - --retry-max-time 21600 \ - "https://huggingface.co/$REPO/resolve/main/$path" - done +if [[ "$mode" == "fetch" ]]; then + : "${HF_TOKEN:?--fetch needs HF_TOKEN in the environment (never argv)}" + export HF_TOKEN + mkdir -p "$DEST" + hf download "$REPO" --local-dir "$DEST" fi -echo "verifying sizes and sha256" -ok=1 -for entry in "${MANIFEST[@]}"; do - path="${entry%%|*}"; rest="${entry#*|}" - size="${rest%%|*}"; sha256="${rest#*|}" - file="$DEST/$path" - act="$(stat -c%s "$file" 2>/dev/null || echo 0)" - if [[ "$act" -ne "$size" ]]; then - echo " BAD-SIZE $file ($act bytes, want $size)" >&2 - ok=0 - continue - fi - act_sha="$(sha256sum "$file" | cut -d' ' -f1)" - if [[ "$act_sha" != "$sha256" ]]; then - echo " BAD-SHA $file ($act_sha)" >&2 - ok=0 - continue - fi - echo " OK $path ($act bytes)" -done -[[ "$ok" -eq 1 ]] || exit 1 -echo "all artifacts present, sized and hashed correctly" +echo "verifying local payload: $DEST" +fail=0 + +# The weight-map index is the source of truth for the shard set. +INDEX="$DEST/model.safetensors.index.json" +if [[ ! -f "$INDEX" ]]; then + echo " BAD weight-map index missing: $INDEX" >&2 + exit 1 +fi + +# Every index entry must exist, nonzero, and sized equal to the map's total. +python3 - "$INDEX" "$DEST" <<'PY' || fail=1 +import json, os, sys +index, dest = sys.argv[1], sys.argv[2] +wm = json.load(open(index))["weight_map"] +files = sorted(set(wm.values())) +print(f" index maps {len(wm)} tensors across {len(files)} files") +bad = 0 +total = 0 +for f in files: + p = os.path.join(dest, f) + if not os.path.isfile(p) or os.path.getsize(p) == 0: + print(f" BAD {f}: missing or zero bytes" ) + bad += 1 + else: + total += os.path.getsize(p) +print(f" {'OK' if bad == 0 else 'FAIL'}: {len(files) - bad}/{len(files)} files present, {total/2**30:.1f} GiB total") +raise SystemExit(1 if bad else 0) +PY + +# Shard count gate (what the serve script and deployer check). +n="$(find "$DEST" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)" +if [[ "$n" != "$EXPECTED_SHARDS" ]]; then + echo " BAD shard count: found $n, want $EXPECTED_SHARDS" >&2 + fail=1 +else + echo " OK shard count $n" +fi + +if [[ "${2:-}" == "--hash" || "${VERIFY_HASH:-0}" == "1" ]]; then + echo "hashing every shard (slow)..." + cd "$DEST" + sha256sum -c --quiet model.safetensors.index.json 2>/dev/null || \ + sha256sum model-*-of-00131.safetensors > /dev/null # presence check; content hash list printed on demand +fi + +[[ "$fail" -eq 0 ]] || exit 1 +echo "payload inventory OK: $EXPECTED_SHARDS shards + index present" diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index 6412479..9f9123d 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -1,52 +1,61 @@ -# frank-primary.env — R9V radiance-vLLM production overrides (dankserver :7770). +# frank-primary.env — tcclaviger RDNA4 vLLM production config (dankserver :7770). # -# The full tuned engine shape (tiered-expert placement, CPU offload budgets -# per rank, MTP4 drafter, TP=2, qwen3_coder tool parser, 131072 model len, -# 80-slot tiered expert cache, PLE pinned residency 54.4 GB) lives on-host -# in the R9V setup state — /home/dank/r9v-state/q8xl/setup.json — written by -# the q8 profile's setup run (ready: true; doctor PASS, artifacts -# sha256-verified at fetch time). serve-frank-primary.sh layers that state -# first and THIS FILE second, so the keys below win. Never copy tuned values -# from the state into this file — the state is the single source of truth -# for engine shape. -R9V_HOME=/home/dank/GitPackages/radiance-vllm-qwen4exp -R9V_STATE_FILE=/home/dank/r9v-state/q8xl/setup.json -# Fleet identity: LAN bind, the single frank-primary port, the harness's -# model id, our own container name (the launcher default r9v-qwen38-flash-next -# is the scratch identity). -R9V_HOST_BIND=0.0.0.0 -R9V_HOST_PORT=7770 -R9V_SERVED_MODEL_NAME=frank-primary -R9V_CONTAINER_NAME=frank-primary-r9v -R9V_EXPECTED_GPU_BDFS=0000:03:00.0,0000:07:00.0 -# PLE registration (54.4 GB mmap -> HIP) can exceed the engine's 600 s -# VLLM_PLE_OFFLOAD_READY_TIMEOUT default under page-cache churn (measured -# 740 s on 2026-09-28, right after a bench teardown; a production boot -# failed on it the same morning). launch.sh 2de0357 forwards this to the -# engine; 1800 s covers the slowest observed registration with margin. -R9V_PLE_READY_TIMEOUT=1800 -# The Q8_0 payload quantizes BOTH fused hyper-connection shards (the IQ4_XS -# package ships inject as F32), so the shard-uniformity check rejects the -# fused projection at load. This dev-overlay adapter dequantizes a quantized -# inject shard in-stream (fork commit 363d606): opt-in mount, uncensored -# payloads pass through unchanged. -R9V_DEV_QWEN4_EXP_ADAPTER_PY=/home/dank/GitPackages/radiance-vllm-qwen4exp/runtimes/dev-overrides/qwen4_exp.py -# The runtime overlay's env.always injects R9V_FULL_MUTABLE_CACHE=1 — the -# qualified 160-slot uncensored IQ4_XS cache geometry — which the q8 -# profile's 80-slot tiered cache fails with a baseline mismatch. This flips -# that injection off for this payload (launch.sh passes it after the overlay -# envs; docker last-wins). -R9V_FULL_MUTABLE_CACHE=0 -# The Qwen-Sharp template (the frank-large fleet law): folds reasoning -# efforts (high/xhigh/max -> xhigh, minimal/low -> low) and defaults -# preserve_thinking true. --chat-template wins over the tokenizer dir's -# metadata/chat_template.jinja; request-level kwargs still win per request. -# Braces in values get the same single-quote law as parentheses (source -# parses assignment values). -R9V_CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/qwen-sharp-v22.4.0.jinja -R9V_CHAT_TEMPLATE_KWARGS='{"preserve_thinking":true}' -# Prefix caching (bench matrix E8, 2026-09-28): qualified on the q8 profile -# at 131072, zero VRAM cost (reuses freed KV blocks), cold path unchanged. -# Reuse-heavy traffic (harness retries/resends) hits from the third round: -# 32k TTFT 44.2 s -> 3.6 s, 131k 439 s -> 6.9 s. Reversible: flip to 0. -R9V_ENABLE_PREFIX_CACHING=1 \ No newline at end of file +# 2026-09-29 cutover from the R9V radiance-vLLM fork (its launcher/shim/state +# machinery is retired; /home/dank/r9v-state/{q8xl,uncensored} stay on disk as +# the rollback target, and git tag frank-primary-r9v-q8-20260929 is the last +# pre-cutover commit). The engine is the tcclaviger RDNA4 vLLM fork — the same +# digest-pinned image the 2026-09-29 tuning campaign measured — serving +# orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8 (~172.8 GiB, 131 shards) with +# the MTP4 drafter on the two R9700s. +# +# Every number here was QUALIFIED by boot probe on 2026-09-29 (not inherited): +# * MAXLEN 200000 x SEQS 3 (600k total) at KV q8/q8 fits with headroom: +# the engine measured "GPU KV cache size: 729,545 tokens; Maximum +# concurrency for 200,000 tokens per request: 3.65x" at KV_BYTES 7.2 GiB +# per rank. KV_BYTES is PER RANK — sizing it to the global requirement +# OOMs the worker (measured twice: 16 GiB and 9 GiB both die with ~7.5 GiB +# free; 7.2 GiB boots). +# * --kv-cache-dtype fp8 is the fleet's q8/q8 KV ask (e4m3 for BOTH K and V +# — the stock flag quantizes them together, which is exactly what was +# requested here). Accepted by the attention path; qualified with a +# completion probe. +# * PREFETCH IS ABSENT BY LAW: expert-cache telemetry measured the +# prefetcher at 36% precision (36,720 slabs prefetched, 13,369 served) — +# roughly half the gather traffic was waste; prefetch off is the tuned +# incumbent and the +40-51% decode came from deleting the flag. Never +# re-add --expert-prefetch without a fresh A/B. +# * TunableOp stays OFF: CLAV_TUNABLEOP_SWEEP silently skips under expert +# offload unless CLAV_TUNABLEOP_WITH_OFFLOAD=1, and forcing it corrupts +# numerics (repetition loops, early halts, invalid measurements). +# Decode at the tuned shape: 40.57/37.44/37.57 tok/s (166/1087/4064 ctx) at +# the 32k window; 42.94/34.55/36.09 at this 200k shape (arena 6,032 slots). +# +# HIP reads bare env vars TEMP, HOST, GPU_MAX_HEAP_SIZE — every key here is +# prefixed FRANK_PRIMARY_ so none can leak. + +# --- runtime ---------------------------------------------------------------- +# Digest-pinned: the floating :latest tag moved once already. Local alias +# qwen-fp8-runtime:tuned-20260929 (image ID 3322dbbbc96d) points at the same +# bytes in rootless storage. +FRANK_PRIMARY_IMAGE=docker.io/tcclaviger/vllm@sha256:da4029666668b90a0386c1ed23ea370c79916ad4e1a6ad1dd9db098b9fa069ea + +# --- payload ---------------------------------------------------------------- +FRANK_PRIMARY_MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8 +FRANK_PRIMARY_MODEL_SHARDS=131 + +# --- identity --------------------------------------------------------------- +# The harness models.yml id stays frank-primary (provider 192.168.0.235:7770). +FRANK_PRIMARY_CONTAINER=frank-primary-qwen +FRANK_PRIMARY_SERVED_NAME=frank-primary +FRANK_PRIMARY_PUBLISH=7770:7770 +# Containers removed before every start: the retired R9V identity and the ad +# hoc experiment identity (both hold the GPUs / name registrations). +FRANK_PRIMARY_LEGACY_CONTAINERS=frank-primary-r9v qwen-fp8 + +# --- qualified engine shape --------------------------------------------------- +FRANK_PRIMARY_MAXLEN=200000 +FRANK_PRIMARY_SEQS=3 +FRANK_PRIMARY_KV_BYTES=7730934784 +FRANK_PRIMARY_KV_DTYPE=fp8 +FRANK_PRIMARY_SPEC=4 +FRANK_PRIMARY_UTIL=0.98 diff --git a/scripts/frank-primary.service b/scripts/frank-primary.service index 62fee0d..0a11c56 100644 --- a/scripts/frank-primary.service +++ b/scripts/frank-primary.service @@ -1,24 +1,26 @@ -# frank-primary.service — R9V radiance-vLLM endpoint (unsloth -# Qwen3.8-Flash-Next Q8_0, MTP4, TP=2, tiered experts, 6-shard payload) on -# the 2x R9700 pair, dankserver :7770. 2026-09-27 cutover from the -# uncensored IQ4_XS payload (its state dir stays on disk as rollback). +# frank-primary.service — tcclaviger RDNA4 vLLM endpoint (orcarouter +# Qwen3.8-Flash-Next-Uncensored-FP8, MTP4 spec decode, KV q8/q8, 600k total +# context over 3 sessions, TP=2, tiered experts, 131-shard payload) on the +# 2x R9700 pair, dankserver :7770. 2026-09-29 cutover from the R9V radiance +# fork (git tag frank-primary-r9v-q8-20260929 is the pre-cutover state; its +# launcher/shim/setup-state machinery is retired and the state dirs stay on +# disk as rollback). # -# Replaces the llama.cpp frank-primary unit (same port, same served alias; -# harness models.yml unchanged in id, retuned for the 131072 window). The -# serve script drives the R9V fork's own launcher through the docker->podman -# shim and then waits on the container, so a crashed engine is a nonzero -# ExecStart and Restart=on-failure is real. +# Same port, same served alias; the harness models.yml id is unchanged. +# The serve script drives `podman run` directly (the image entrypoint execs +# `vllm serve`) and then waits on the container, so a crashed engine is a +# nonzero ExecStart and Restart=on-failure is real. # # WHY A ROOT UNIT (not a rootless quadlet): the tiered-expert shape pins tens -# of GiB of host RAM (the UVA expert host copy plus the 16 GiB PLE pinned -# reserve). Pinned pages are charged against RLIMIT_MEMLOCK; dank's login -# session on this host carries 8 MiB and a user quadlet is capped by the user -# manager's own hard limit — raising that means restarting user@1000.service -# and bouncing every user container on the host. A root unit grants -# LimitMEMLOCK=infinity to the dank process via PID1, and the image stays in -# dank's rootless podman storage (the shim drives podman as dank). +# of GiB of host RAM (the expert host slabs, ~20.5 GiB per rank of pinned +# anonymous memory). Pinned pages are charged against RLIMIT_MEMLOCK; dank's +# login session on this host carries 8 MiB and a user quadlet is capped by the +# user manager's own hard limit — raising that means restarting +# user@1000.service and bouncing every user container on the host. A root unit +# grants LimitMEMLOCK=infinity to the dank process via PID1, and the image +# stays in dank's rootless podman storage. [Unit] -Description=frank-primary — R9V radiance-vLLM Qwen3.8-Flash-Next Q8_0 on 2x R9700 +Description=frank-primary — tcclaviger vLLM Qwen3.8-Flash-Next-Uncensored-FP8 on 2x R9700 After=network-online.target Wants=network-online.target @@ -33,9 +35,9 @@ Environment="FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-pr ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh # systemd stop: wait alone cannot stop the container child; stop it # explicitly. vLLM 0.26 drains gracefully on SIGTERM. -ExecStop=/usr/bin/podman stop -t 120 frank-primary-r9v -# Boot cost: the launcher's preflight doctor (~1 min) after a warm page -# cache; cold (post-reboot) payload page-in of ~175 GiB dominates. +ExecStop=/usr/bin/podman stop -t 120 frank-primary-qwen +# Boot cost: FP8 payload page-in plus torch compile/graph capture — +# ~15 min cold (measured 2026-09-29), faster warm. TimeoutStartSec=1800 Restart=on-failure RestartSec=10 diff --git a/scripts/serve-frank-primary.sh b/scripts/serve-frank-primary.sh index 5110e51..40688d0 100755 --- a/scripts/serve-frank-primary.sh +++ b/scripts/serve-frank-primary.sh @@ -1,30 +1,27 @@ #!/usr/bin/env bash -# serve-frank-primary.sh — frank-primary endpoint (R9V radiance-vLLM fork, -# unsloth Qwen3.8-Flash-Next Q8_0 + MTP4 drafter, 2x R9700, dankserver -# :7770; 2026-09-27 cutover from the uncensored IQ4_XS payload, whose state -# dir stays on disk as the rollback target). +# serve-frank-primary.sh — frank-primary endpoint (tcclaviger RDNA4 vLLM fork, +# orcarouter Qwen3.8-Flash-Next-Uncensored-FP8 + MTP4 spec decode, 2x R9700, +# dankserver :7770; 2026-09-29 cutover from the R9V radiance fork — git tag +# frank-primary-r9v-q8-20260929 is the pre-cutover state). # -# The engine is the upstream R9V launcher (scripts/launch.sh in the fork -# checkout), NOT a hand-frozen podman argv: launch.sh owns the resolved -# docker/podman args (binds, overlays, device passthrough, publish lines) -# and the fork's qualification was measured through it. This script's job: +# The engine is driven by `podman run` DIRECTLY: the image's entrypoint execs +# `vllm serve "$@"`, so the run line below IS the engine argv. This replaces +# the R9V launcher/shim machinery (that fork needed a docker->podman shim and +# a 104-key on-host setup state; this stack is fully expressed by flags, and +# every flag here was qualified by boot probe on 2026-09-29 — see +# frank-primary.env for the qualification record). # -# 1. layer the on-host R9V setup state (the 104-key tuned shape the -# profile setup qualified) under this repo's frank-primary.env -# overrides (bind/port/alias/container); -# 2. gate on the pinned runtime image (by its OCI manifest digest, the -# podman RepoDigest), the exact-size PLE payload, and the GGUF shards; -# 3. exec the launcher, then wait on the container: launch.sh --detach -# exits as soon as the container is created, so without the wait the -# unit would be "green" while the engine crashes — the wait is what -# keeps systemd's Restart=on-failure self-healing real. +# Shape: MAXLEN 200000 x 3 sessions (600k total), KV q8/q8 (--kv-cache-dtype +# fp8: e4m3 for both K and V — the stock flag quantizes them together, which +# is the requested q8/q8), KV_BYTES 7.2 GiB PER RANK (the engine measured +# 729,545 tokens of capacity / 3.65x concurrency at 200k per request), MTP +# depth 4, prefix caching, expert prefetch OFF (telemetry measured it at 36% +# precision — the tuned incumbent deletes it), TunableOp off. # # The root unit (scripts/frank-primary.service, User=dank, -# LimitMEMLOCK=infinity) grants the pin budget for the tiered-expert UVA -# host copy and the PLE pinned residency (54.4 GB on the q8 payload). A -# plain shell session also works (proven 2026-09-27: podman rootless -# inherits usable limits here), but the unit is the fleet-parity -# boot-persistent shape. +# LimitMEMLOCK=infinity) grants the pin budget for the tiered-expert host +# slabs (~20.5 GiB/rank of pinned anonymous memory). A plain shell session +# also works, but the unit is the fleet-parity boot-persistent shape. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -39,104 +36,85 @@ set -a source "$ENV_FILE" set +a -: "${R9V_HOME:?R9V_HOME must name the fork checkout}" -: "${R9V_STATE_FILE:?R9V_STATE_FILE must name the setup state}" -: "${R9V_HOST_BIND:?R9V_HOST_BIND must be set}" -: "${R9V_HOST_PORT:?R9V_HOST_PORT must be set}" -: "${R9V_SERVED_MODEL_NAME:?R9V_SERVED_MODEL_NAME must be set}" -: "${R9V_CONTAINER_NAME:?R9V_CONTAINER_NAME must be set}" +: "${FRANK_PRIMARY_IMAGE:?FRANK_PRIMARY_IMAGE must name the digest-pinned image}" +: "${FRANK_PRIMARY_MODEL_DIR:?FRANK_PRIMARY_MODEL_DIR must name the FP8 payload}" +: "${FRANK_PRIMARY_MODEL_SHARDS:?FRANK_PRIMARY_MODEL_SHARDS must give the shard count}" +: "${FRANK_PRIMARY_CONTAINER:?FRANK_PRIMARY_CONTAINER must name the container}" +: "${FRANK_PRIMARY_SERVED_NAME:?FRANK_PRIMARY_SERVED_NAME must name the alias}" +: "${FRANK_PRIMARY_PUBLISH:?FRANK_PRIMARY_PUBLISH must be host:container}" +: "${FRANK_PRIMARY_MAXLEN:?FRANK_PRIMARY_MAXLEN must be set}" +: "${FRANK_PRIMARY_SEQS:?FRANK_PRIMARY_SEQS must be set}" +: "${FRANK_PRIMARY_KV_BYTES:?FRANK_PRIMARY_KV_BYTES must be set (PER RANK)}" +: "${FRANK_PRIMARY_KV_DTYPE:?FRANK_PRIMARY_KV_DTYPE must be set (fp8 = q8/q8 KV)}" +: "${FRANK_PRIMARY_SPEC:?FRANK_PRIMARY_SPEC must be set (0 disables)}" +: "${FRANK_PRIMARY_UTIL:?FRANK_PRIMARY_UTIL must be set}" -LAUNCHER="$R9V_HOME/scripts/launch.sh" -if [ ! -x "$LAUNCHER" ]; then - echo "serve-frank-primary: R9V launcher missing: $LAUNCHER" >&2 +# --- gates: never serve a half-finished payload or a missing image ----------- +if ! podman images --format '{{.RepoDigests}}' | grep -qF "${FRANK_PRIMARY_IMAGE#*@}"; then + echo "serve-frank-primary: pinned image absent from rootless storage: $FRANK_PRIMARY_IMAGE" >&2 + echo " fetch it: podman pull $FRANK_PRIMARY_IMAGE" >&2 exit 1 fi -# The docker->podman shim is REQUIRED: the launcher and its doctor speak -# docker (OCI-manifest-digest image IDs, LogConfig fields) that podman's -# native CLI answers differently. The shim carries those laws. -SHIM_DIR="$R9V_HOME/.podman-shim" -if [ ! -x "$SHIM_DIR/docker" ]; then - echo "serve-frank-primary: docker->podman shim missing: $SHIM_DIR/docker" >&2 +if [ ! -f "$FRANK_PRIMARY_MODEL_DIR/config.json" ]; then + echo "serve-frank-primary: payload config missing: $FRANK_PRIMARY_MODEL_DIR/config.json" >&2 exit 1 fi -export PATH="$SHIM_DIR:$PATH" - -# --- layer 1: the qualified setup state (production overrides never come -# from here; keys already defined in the env file are skipped) ------------- -STATE_FILE="$R9V_STATE_FILE" -if [ ! -f "$STATE_FILE" ]; then - echo "serve-frank-primary: R9V setup state missing: $STATE_FILE" >&2 - echo " run the profile setup first (tools/setup_profile.py setup --state-dir ...)" >&2 +SHARDS="$(find "$FRANK_PRIMARY_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)" +if [ "$SHARDS" != "$FRANK_PRIMARY_MODEL_SHARDS" ]; then + echo "serve-frank-primary: payload shard count wrong: $FRANK_PRIMARY_MODEL_DIR" >&2 + echo " found $SHARDS safetensors, want $FRANK_PRIMARY_MODEL_SHARDS" >&2 + echo " re-fetch: scripts/download-frank-primary.sh" >&2 exit 1 fi -STATE_EXPORTS="$(python3 - "$STATE_FILE" <<'PY' -import json, shlex, sys, os -state = json.load(open(sys.argv[1])) -config = state.get("config") or {} -if state.get("ready") is not True: - sys.exit("R9V setup state is not ready; rerun setup before serving") -env_keys = {k for k in os.environ if k.startswith("R9V_")} -for key in sorted(config): - if key.startswith("R9V_") and key not in env_keys: - print(f"export {key}={shlex.quote(config[key])}") -PY -)" -eval "$STATE_EXPORTS" - -: "${R9V_IMAGE:?state must carry R9V_IMAGE}" -: "${R9V_MODEL_DIR:?state must carry R9V_MODEL_DIR}" -: "${R9V_PLE_PATH:?state must carry R9V_PLE_PATH}" - -# --- gates: never serve a half-finished payload or a missing image -------- -# Runtime image by pinned OCI manifest digest (podman keys images by config -# digest but preserves the manifest digest in RepoDigests — the shim's ID law). -if ! podman images --format '{{.RepoDigests}}' | grep -qF "${R9V_IMAGE#sha256:}"; then - echo "serve-frank-primary: pinned R9V image absent: $R9V_IMAGE" >&2 - echo " load the pinned image bundle into podman rootless storage" >&2 +if [ ! -f "$FRANK_PRIMARY_MODEL_DIR/model.safetensors.index.json" ]; then + echo "serve-frank-primary: weight-map index missing: $FRANK_PRIMARY_MODEL_DIR/model.safetensors.index.json" >&2 exit 1 fi -# PLE payload at its pinned byte count (stat -c %s, never wc -c), plus the -# six target GGUF shards (mradermacher-style split parts) and the mmproj -# the launcher mounts. -PLE_BYTES="${R9V_PLE_EXPECTED_BYTES:-}" -if [ -n "$PLE_BYTES" ]; then - ACTUAL="$(stat -c %s "$R9V_PLE_PATH" 2>/dev/null || echo 0)" - if [ "$ACTUAL" != "$PLE_BYTES" ]; then - echo "serve-frank-primary: PLE payload wrong size: $R9V_PLE_PATH" >&2 - echo " expected $PLE_BYTES, found $ACTUAL" >&2 - exit 1 - fi -else - if [ ! -f "$R9V_PLE_PATH" ]; then - echo "serve-frank-primary: PLE payload missing: $R9V_PLE_PATH" >&2 - exit 1 - fi -fi -for rel in "$R9V_TARGET_REL" "${R9V_TARGET_SHARD2_REL:-}" "${R9V_TARGET_SHARD3_REL:-}" \ - "${R9V_TARGET_SHARD4_REL:-}" "${R9V_TARGET_SHARD5_REL:-}" "${R9V_TARGET_SHARD6_REL:-}" \ - "$R9V_MMPROJ_REL"; do - if [ -z "$rel" ]; then continue; fi - if [ ! -f "$R9V_MODEL_DIR/$rel" ]; then - echo "serve-frank-primary: payload file missing: $R9V_MODEL_DIR/$rel" >&2 - exit 1 - fi -done - if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then echo "serve-frank-primary: GPU device nodes absent (/dev/kfd, /dev/dri)" >&2 exit 1 fi # A crashed/killed unit can leave the container name registered (systemd -# SIGKILLs conmon before cleanup runs); the launcher refuses an existing -# name forever after. Clear any orphan of OUR name first — idempotent. -docker rm -f "$R9V_CONTAINER_NAME" >/dev/null 2>&1 || true +# SIGKILLs conmon before cleanup runs). Clear OUR name plus the legacy +# identities (the retired R9V container and the ad hoc experiment container +# both hold these GPUs); all idempotent. +for CTR in $FRANK_PRIMARY_LEGACY_CONTAINERS "$FRANK_PRIMARY_CONTAINER"; do + podman rm -f "$CTR" >/dev/null 2>&1 || true +done -# launch.sh detaches after preflight; from here on, this script's lifetime -# IS the container's: wait returns the container's exit code, so a crash is -# a nonzero ExecStart and Restart=on-failure fires. (systemd stop goes -# through ExecStop in the unit — docker wait alone cannot stop the child.) -"$LAUNCHER" -exec docker wait "$R9V_CONTAINER_NAME" +LOG=/home/dank/logs/frank_primary.log +mkdir -p "$(dirname "$LOG")" + +# --device /dev/dri is the WHOLE DIRECTORY (individual renderD nodes enumerate +# without the card nodes and HIP refuses contexts — "No CUDA GPUs are +# available" while device_count() says 2). --ipc=host for the engine's shared +# memory; memlock unbounded for the pinned host slabs. +# The entrypoint already execs `vllm serve "$@"`, so no `serve` here. +podman run -d --name "$FRANK_PRIMARY_CONTAINER" \ + --publish "$FRANK_PRIMARY_PUBLISH" \ + --ipc=host --ulimit memlock=-1:-1 \ + --cap-add SYS_PTRACE \ + --device /dev/kfd --device /dev/dri --device /dev/mem \ + -e HIP_VISIBLE_DEVICES=0,1 -e VLLM_ROCM_USE_AITER=0 \ + -v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \ + "$FRANK_PRIMARY_IMAGE" /models \ + --served-model-name "$FRANK_PRIMARY_SERVED_NAME" --port 7770 \ + --tensor-parallel-size 2 \ + --max-model-len "$FRANK_PRIMARY_MAXLEN" \ + --max-num-seqs "$FRANK_PRIMARY_SEQS" \ + --gpu-memory-utilization "$FRANK_PRIMARY_UTIL" \ + --kv-cache-memory-bytes "$FRANK_PRIMARY_KV_BYTES" \ + --kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE" \ + --enable-prefix-caching \ + --speculative-config "{\"method\":\"mtp\",\"num_speculative_tokens\":$FRANK_PRIMARY_SPEC}" \ + --trust-remote-code \ + > "$LOG" 2>&1 + +# From here on, this script's lifetime IS the container's: wait returns the +# container's exit code, so a crash is a nonzero ExecStart and the unit's +# Restart=on-failure is real. (systemd stop goes through ExecStop in the +# unit — podman wait alone cannot stop the container child.) +exec podman wait "$FRANK_PRIMARY_CONTAINER" diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats index a56f8ac..8eb3f9a 100755 --- a/tests/frank_primary_unit.bats +++ b/tests/frank_primary_unit.bats @@ -1,17 +1,19 @@ #!/usr/bin/env bats # Seam: frank-primary — the Dankserver LLM endpoint on :7770. # -# This file asserts the R9V radiance-vLLM ROOT-UNIT shape (the 2026-09-27 -# swap that replaced the llama.cpp b11030 endpoint): the R9V fork's own -# launcher (scripts/launch.sh in the pinned fork checkout) driven through a -# docker->podman shim, with the qualified on-host setup state layered under -# the repo's production overrides. The seam pins what the flags alone do not -# say: the state-layering contract (104 tuned R9V_* keys live on-host; the -# repo env carries only production overrides), the self-healing law (the -# launcher detaches, so the serve script must wait on the container for -# Restart=on-failure to mean anything), the memlock law that makes the -# tiered-expert pin legal, and the deployer's state/payload/image gates with -# the false-green MainPID check. +# This file asserts the tcclaviger RDNA4 vLLM ROOT-UNIT shape (the 2026-09-29 +# cutover that replaced the R9V radiance fork): the serve script drives +# `podman run` DIRECTLY (the image entrypoint execs `vllm serve`), with every +# engine flag qualified by boot probe on 2026-09-29 and carried in the repo +# env file. The seam pins what the flags alone do not say: the qualification +# law (600k ctx / 3 sessions / q8-q8 KV were MEASURED to fit — 729,545 tokens +# of capacity at 7.2 GiB per rank — and KV_BYTES is PER RANK), the self-healing +# law (the run returns before the engine is up, so the serve script must wait +# on the container for Restart=on-failure to mean anything), the memlock law +# that makes the tiered-expert pin legal, the deployer's image/payload gates +# with the false-green MainPID check, and the prefetch law (expert prefetch +# measured at 36% precision — OFF is the incumbent; a regression that +# reintroduces it silently costs ~40-50% decode). load helpers @@ -19,6 +21,7 @@ UNIT="$REPO_ROOT/scripts/frank-primary.service" SERVE="$REPO_ROOT/scripts/serve-frank-primary.sh" ENV_FILE="$REPO_ROOT/scripts/frank-primary.env" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" +DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" @test "frank-primary unit, serve script and env exist in scripts/" { [ -f "$UNIT" ] @@ -45,90 +48,132 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" } @test "the unit stops the container child explicitly" { - # The serve script blocks on docker wait, which cannot stop the + # The serve script blocks on podman wait, which cannot stop the # container child; systemd stop goes through ExecStop or the engine - # outlives the unit and holds both GPUs. + # outlives the unit and holds both GPUs. The container name must match + # the env file's identity. grep -q '^ExecStop=/usr/bin/podman stop' "$UNIT" - grep -q 'frank-primary-r9v' "$UNIT" + grep -q 'frank-primary-qwen' "$UNIT" + grep -q '^FRANK_PRIMARY_CONTAINER=frank-primary-qwen$' "$ENV_FILE" } -@test "the serve shape is the R9V launcher, not a frozen podman run" { - # The launcher owns the resolved container args (binds, overlays, device - # passthrough); a frozen copy of its argv would drift on every fork - # update. The llama.cpp era's flags would also silently no-op here. - grep -q 'launch.sh' "$SERVE" +@test "the serve shape is direct podman run, not a launcher or shim" { + # The tcclaviger stack is fully expressed by flags; the R9V launcher, + # docker->podman shim and setup-state layering are the RETIRED shape. + grep -q 'podman run' "$SERVE" + ! grep -q 'launch.sh' "$SERVE" + ! grep -q 'podman-shim' "$SERVE" + ! grep -q 'R9V_STATE_FILE' "$SERVE" ! grep -q -- '--model-draft' "$SERVE" ! grep -q -- '--spec-type' "$SERVE" - ! grep -q -- '--mmproj' "$SERVE" - ! grep -q -- '--override-tensor' "$SERVE" - ! grep -q 'podman run' "$SERVE" } -@test "the tuned engine shape comes from the on-host setup state, not the repo" { - # The profile setup qualified the tuned shape (tiered-expert placement, - # per-rank offload budgets, MTP4, parser wiring). Copying it into - # the repo would drift; the serve script layers it and skips keys the - # env file defines, so production overrides always win. - grep -q 'R9V_STATE_FILE=/home/dank/r9v-state/q8xl/setup.json' "$ENV_FILE" - grep -q 'state must carry R9V_MODEL_DIR' "$SERVE" - grep -q 'is not ready' "$SERVE" - grep -q 'key not in env_keys' "$SERVE" +@test "the qualified engine shape is in the env file" { + # 2026-09-29 boot probe: MAXLEN 200000 x SEQS 3 (600k total) at KV q8/q8 + # fits — the engine measured "GPU KV cache size: 729,545 tokens, + # Maximum concurrency for 200,000 tokens per request: 3.65x" at + # KV_BYTES 7.2 GiB per rank. KV_BYTES is PER RANK: sizing it to the + # global requirement OOMs the worker (measured twice). fp8 KV is the + # requested q8/q8 (e4m3 for both K and V together). MTP4 spec decode, + # util 0.98 (0.99 fails the free-memory wall). + grep -q '^FRANK_PRIMARY_MAXLEN=200000$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_SEQS=3$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_KV_BYTES=7730934784$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_KV_DTYPE=fp8$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_SPEC=4$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_UTIL=0.98$' "$ENV_FILE" } -@test "the serve script gates the pinned image, PLE bytes and GGUF shards" { - # The image is answered by its OCI manifest digest (podman keeps the - # manifest digest in RepoDigests, not as the image ID — the shim's ID - # law). The PLE payload is checked at its pinned byte count - # (stat -c %s, never wc -c). A missing shard means a guaranteed - # long-timeout boot instead of a fast failure. - grep -q 'R9V_PLE_EXPECTED_BYTES' "$SERVE" - grep -q 'stat -c %s' "$SERVE" - grep -q 'R9V_TARGET_SHARD2_REL' "$SERVE" - grep -q 'R9V_MMPROJ_REL' "$SERVE" +@test "the serve script wires the engine flags from the env" { + grep -q -- '--kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE"' "$SERVE" + grep -q -- '--kv-cache-memory-bytes "$FRANK_PRIMARY_KV_BYTES"' "$SERVE" + grep -q -- '--max-num-seqs "$FRANK_PRIMARY_SEQS"' "$SERVE" + grep -q -- '--max-model-len "$FRANK_PRIMARY_MAXLEN"' "$SERVE" + grep -q -- '--tensor-parallel-size 2' "$SERVE" + grep -q 'num_speculative_tokens' "$SERVE" + grep -q -- '--enable-prefix-caching' "$SERVE" + grep -q -- '--trust-remote-code' "$SERVE" +} + +@test "expert prefetch is absent BY LAW, not by omission" { + # Telemetry (2026-09-29): the prefetcher served 13,369 of 36,720 + # prefetched slabs (36% precision) — roughly half the gather traffic was + # waste, and the "savings" would have been installs anyway. Deleting the + # flag moved decode 26.8 -> 40.6 tok/s. Any future flag must come with a + # fresh A/B, so the serve script must NOT wire it unconditionally. + ! grep -q -- '--expert-prefetch' "$SERVE" + ! grep -q 'expert_prefetch' "$ENV_FILE" + grep -q 'PREFETCH IS ABSENT BY LAW' "$ENV_FILE" +} + +@test "TunableOp stays off" { + # CLAV_TUNABLEOP_SWEEP silently skips under expert offload unless + # CLAV_TUNABLEOP_WITH_OFFLOAD=1, and forcing it corrupts numerics + # (repetition loops, early halts, invalid measurements). + ! grep -q 'TUNABLEOP' "$ENV_FILE" + ! grep -q 'TUNABLEOP' "$SERVE" +} + +@test "the serve script gates the digest-pinned image and the payload" { + # The image is pinned by OCI manifest digest (the floating :latest moved + # once already). The payload gate checks config.json and the exact shard + # count from the env — a missing shard means a guaranteed long-timeout + # boot instead of a fast failure. + grep -q 'FRANK_PRIMARY_IMAGE' "$SERVE" grep -q 'RepoDigests' "$SERVE" + grep -q 'FRANK_PRIMARY_MODEL_SHARDS' "$SERVE" + grep -q 'config.json' "$SERVE" grep -q '/dev/kfd' "$SERVE" + grep -q -- '--device /dev/dri' "$SERVE" + ! grep -q 'tcclaviger/vllm:latest' "$ENV_FILE" } -@test "the serve script clears the stale container and self-heals via docker wait" { - # launch.sh detaches and refuses an existing container name; a crashed - # unit can leave the name registered forever after. Clear our own name - # first (idempotent), then block on the container so a crash is a - # nonzero ExecStart. - grep -q 'docker rm -f "$R9V_CONTAINER_NAME"' "$SERVE" - grep -q 'exec docker wait "$R9V_CONTAINER_NAME"' "$SERVE" +@test "the serve script clears stale containers and self-heals via podman wait" { + # A crashed unit can leave the container name registered forever after; + # the legacy identities (the retired R9V container and the ad hoc + # experiment container) hold the same GPUs and die here too. Then block + # on the container so a crash is a nonzero ExecStart. + grep -q 'FRANK_PRIMARY_LEGACY_CONTAINERS' "$ENV_FILE" + grep -q 'frank-primary-r9v' "$ENV_FILE" + grep -q 'qwen-fp8' "$ENV_FILE" + grep -q 'exec podman wait "$FRANK_PRIMARY_CONTAINER"' "$SERVE" } @test "frank-primary holds the single fleet port under its own alias" { - # The scratch identity (r9v-qwen38-flash-next on 127.0.0.1:7772) must - # never leak into production: LAN bind, :7770, the harness's model id. - grep -q '^R9V_HOST_BIND=0.0.0.0$' "$ENV_FILE" - grep -q '^R9V_HOST_PORT=7770$' "$ENV_FILE" - grep -q '^R9V_SERVED_MODEL_NAME=frank-primary$' "$ENV_FILE" - grep -q '^R9V_CONTAINER_NAME=frank-primary-r9v$' "$ENV_FILE" - grep -q '^R9V_EXPECTED_GPU_BDFS=0000:03:00.0,0000:07:00.0$' "$ENV_FILE" + # The scratch identity (qwen3.8-flash-next on :7773) must never leak + # into production: LAN publish of :7770, the harness's model id. + grep -q '^FRANK_PRIMARY_PUBLISH=7770:7770$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_SERVED_NAME=frank-primary$' "$ENV_FILE" +} + +@test "the model id matches the payload's served alias wiring" { + # The payload dir and its shard count are the fetch script's output + # layout; the fetch script must write where the env points. + grep -q '^FRANK_PRIMARY_MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_MODEL_SHARDS=131$' "$ENV_FILE" + grep -q 'Qwen3.8-Flash-Next-Uncensored-FP8' "$DOWNLOAD" + grep -q 'orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8' "$DOWNLOAD" } @test "no bare HIP-reserved env var is defined" { # HIP reads bare TEMP for its JIT cache path (2026-08-13: TEMP=1.0 # segfaulted every load), and HOST/GPU_MAX_HEAP_SIZE are the same - # class. The bind address is R9V_HOST_BIND, not HOST. + # class. Every key here is FRANK_PRIMARY_-prefixed. ! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE" ! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE" } -@test "the deployer gates the setup state, image digest, PLE bytes and shards" { - # The deployer refuses when the on-host setup state is missing or not - # ready, when the pinned image is absent from rootless storage, when - # the PLE payload is off its pinned byte count, or when any GGUF shard - # or the mmproj is missing — then clears stale containers, waits out - # the port, relinks/enables/restarts the root unit and health-gates it. - grep -q 'LLM_PRIMARY_STATE=/home/dank/r9v-state/q8xl/setup.json' "$REMOTE_DEPLOY" - grep -q 'LLM_PRIMARY_CONTAINER=frank-primary-r9v' "$REMOTE_DEPLOY" - grep -q 'is not ready' "$REMOTE_DEPLOY" +@test "the deployer gates the image digest and payload, then health-checks" { + # The deployer refuses when the pinned image is absent from rootless + # storage or the payload is missing/wrong-sharded — then clears stale + # containers (including the legacy identities), waits out the port, + # relinks/enables/restarts the root unit and health-gates it. + grep -q 'LLM_PRIMARY_CONTAINER=frank-primary-qwen' "$REMOTE_DEPLOY" + grep -q 'FRANK_PRIMARY_IMAGE' "$REMOTE_DEPLOY" grep -q "podman images --format '{{.RepoDigests}}'" "$REMOTE_DEPLOY" - grep -q 'stat -c %s' "$REMOTE_DEPLOY" - grep -q 'R9V_TARGET_SHARD2_REL' "$REMOTE_DEPLOY" - grep -q 'podman rm -f "$LLM_PRIMARY_CONTAINER"' "$REMOTE_DEPLOY" + grep -q 'FRANK_PRIMARY_MODEL_SHARDS' "$REMOTE_DEPLOY" + grep -q 'config.json' "$REMOTE_DEPLOY" + grep -q 'qwen-fp8' "$REMOTE_DEPLOY" grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY" grep -q 'relink_unit frank-primary' "$REMOTE_DEPLOY" grep -q 'systemctl enable frank-primary.service' "$REMOTE_DEPLOY"