diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index a28f8a5..303189d 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1183,98 +1183,73 @@ deploy_llm_small() { wait_for_health frank-small http://127.0.0.1:7771/health 900 } -# deploy_llm_primary — frank-primary is unsloth llama.cpp (the b11030-mix -# prebuilt, ROCm gfx120X, bundled runtime) serving Qwen3.8-Flash-Next at -# Q8_0 on the two R9700s, as a ROOT unit — the same shape as frank-large on -# Dankclaw: a host binary, no container image, no generated unit. -# relink_unit points /etc/systemd/system/frank-primary.service at the repo -# file and boot persistence is that unit's [Install] section. +# deploy_llm_primary — frank-primary is vLLM 0.30 (locally built ROCm image, +# gfx1201) serving Qwen3.8-Flash-Next-FP8 on the two R9700s as a ROOT unit +# whose serve script execs a rootless-podman container as dank. Same port and +# alias as the llama.cpp endpoint it replaces (2026-09-27 cutover); the +# harness models.yml and homepage cards are unchanged. # -# The 2026-09-26 swap that replaced the Lucebox DS4.1 Quadlet on this port. -# The payload is six Q8_0 split parts plus the shared MTP head and the -# projector, so the gate walks every artifact at its declared byte count (a -# partial download must never be served — and stat -c %s, never wc -c, which -# read the whole 377 GiB lucebox file for minutes on 2026-09-26). -LLM_PRIMARY_RUNTIME=/home/dank/GitPackages/unsloth-llama-b11030/llama-server -LLM_PRIMARY_ROOT=/home/dank/models/Qwen3.8-Flash-Next-GGUF -LLM_PRIMARY_PART_SIZES=( - 10946624 - 682434912 - 54400261312 - 49446841216 - 49668930400 - 34015618784 -) -LLM_PRIMARY_DRAFT="$LLM_PRIMARY_ROOT/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" -LLM_PRIMARY_DRAFT_SIZE=2786568256 -LLM_PRIMARY_MMPROJ="$LLM_PRIMARY_ROOT/mmproj-BF16.gguf" -LLM_PRIMARY_MMPROJ_SIZE=907542944 - -# The lucebox quadlet held this port until the 2026-09-26 cutover. Its -# generated user unit is host state, not repo state: stop it and move the -# installed .container into dated home trash (the same retire pattern as -# retire_halogen_large_quadlet). The image and the 377 GiB payload stay on -# disk as the rollback target. -retire_lucebox_primary_quadlet() { - local trash="$HOME/.AGENTTRASH/20260926-llamacpp-cutover/quadlet" - systemctl --user stop frank-primary.service 2>/dev/null || true - if [[ -e "$quadlet_unit_dir/frank-primary.container" ]]; then - mkdir -p "$trash" - mv -- "$quadlet_unit_dir/frank-primary.container" "$trash/frank-primary.container" - echo "retired quadlet frank-primary.container -> $trash" - fi - systemctl --user daemon-reload -} +# Gates: the image must exist in dank's rootless storage, and the payload +# must be complete — every shard named by the model's own +# model.safetensors.index.json present on disk (131 shards, 172.78 GiB; +# per-shard sha256 + byte-count verification is download-frank-primary-fp8.sh's +# job at fetch time, never re-hashed here). +LLM_PRIMARY_IMAGE=localhost/frank-primary:vllm030 +LLM_PRIMARY_ROOT=/home/dank/models/Qwen3.8-Flash-Next-FP8 +LLM_PRIMARY_CONTAINER=frank-primary-vllm deploy_llm_primary() { - # Refuse rather than restart-loop. A unit whose runtime or payload is + # Refuse rather than restart-loop: a unit whose image or payload is # missing never becomes healthy, and a green deploy would leave the - # endpoint dark with nothing in the deploy log to say why. - local missing=0 i part want_size got_size - if [[ ! -x "$LLM_PRIMARY_RUNTIME" ]]; then - echo "ERROR: runtime absent at $LLM_PRIMARY_RUNTIME" >&2 - echo "frank-primary: fetch the unsloth b11030-mix gfx120X prebuilt (see the runbook)" >&2 - missing=1 - fi - for i in 1 2 3 4 5 6; do - part="$LLM_PRIMARY_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-0000$i-of-00006.gguf" - want_size="${LLM_PRIMARY_PART_SIZES[$((i-1))]}" - got_size="$(stat -c %s "$part" 2>/dev/null || echo 0)" - if [[ "$got_size" != "$want_size" ]]; then - echo "ERROR: part $i absent or incomplete at $part (have $got_size, want $want_size)" >&2 - missing=1 - fi - done - for part in "$LLM_PRIMARY_DRAFT:$LLM_PRIMARY_DRAFT_SIZE" \ - "$LLM_PRIMARY_MMPROJ:$LLM_PRIMARY_MMPROJ_SIZE"; do - local file="${part%%:*}" want="${part##*:}" - got_size="$(stat -c %s "$file" 2>/dev/null || echo 0)" - if [[ "$got_size" != "$want" ]]; then - echo "ERROR: artifact absent or incomplete at $file (have $got_size, want $want)" >&2 - missing=1 - fi - done - [[ "$missing" -eq 0 ]] || { - echo "frank-primary: run scripts/download-frank-primary.sh on this host" >&2 + # endpoint dark with nothing in the log to say why. + if ! podman image inspect "$LLM_PRIMARY_IMAGE" >/dev/null 2>&1; then + echo "ERROR: image absent: $LLM_PRIMARY_IMAGE" >&2 + echo "frank-primary: build the fp030 image first (see the runbook)" >&2 return 1 - } + fi + local want_shards have_shards + want_shards="$(python3 - "$LLM_PRIMARY_ROOT/model.safetensors.index.json" <<'PY' +import json, sys +print(len(set(json.load(open(sys.argv[1]))["weight_map"].values()))) +PY +)" || return 1 + have_shards="$(find "$LLM_PRIMARY_ROOT" -maxdepth 1 -name '*.safetensors' | wc -l)" + if [[ -z "$want_shards" || "$have_shards" != "$want_shards" ]]; then + echo "ERROR: payload incomplete at $LLM_PRIMARY_ROOT (have $have_shards shards, index names $want_shards)" >&2 + echo "frank-primary: run scripts/download-frank-primary-fp8.sh on this host" >&2 + return 1 + fi - retire_lucebox_primary_quadlet - # On a re-deploy the predecessor is THIS unit: stop it before waiting on - # the port, or the wait deadlocks on the server it just stopped serving. + # A stale container with the same name makes `podman run --name` fail at + # start; remove any leftover from a failed previous attempt. + podman rm -f "$LLM_PRIMARY_CONTAINER" >/dev/null 2>&1 || true + # On a re-deploy the predecessor is THIS unit (the llama.cpp unit until + # the 2026-09-27 cutover, the vLLM unit after it): stop it before + # waiting on the port, or the wait deadlocks on the server it just + # stopped serving. run_privileged systemctl stop frank-primary.service 2>/dev/null || true - # Nothing starts until the port is released: the lucebox cutover on - # 2026-09-26 and the frank-large cutover on 2026-09-16 both failed + # Nothing starts until the port is released: the lucebox cutover + # (2026-09-26) and the frank-large cutover (2026-09-16) both failed # exactly here when the next engine bound 7770 a second too early. wait_port_free 7770 || return 1 relink_unit frank-primary run_privileged systemctl daemon-reload run_privileged systemctl enable frank-primary.service run_privileged systemctl restart frank-primary.service - # Cold start page-ins 175 GiB of expert pages and builds the graphs; - # 1800s parity with the lucebox HealthStartPeriod this port has always - # needed. - wait_for_health frank-primary http://127.0.0.1:7770/health 1800 + # Cold start: triton/inductor compile on first boot plus reading+pinning + # ~173 GiB of weights. Parity with every predecessor on this port. + wait_for_health frank-primary http://127.0.0.1:7770/health 1800 || return 1 + # False-green hardening (the 2026-09-26 lesson): a bare curl against the + # port cannot tell WHO answers — a stray probe holding :7770 satisfies + # it while the unit is stopped. The deploy is only green if the unit + # itself is active with a live MainPID behind the health answer. + local main_pid + main_pid="$(run_privileged systemctl show -p MainPID --value frank-primary.service)" + if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then + echo "FAIL: :7770 answered but frank-primary.service has no MainPID — a stray process holds the port" >&2 + return 1 + fi + echo "frank-primary: health answered by unit MainPID $main_pid" } wait_for_health() { diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index 67bf1cc..0bb9d23 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -1,113 +1,96 @@ -# frank-primary — unsloth llama.cpp (b11030-mix prebuilt, ROCm gfx120X) -# serving unsloth/Qwen3.8-Flash-Next-GGUF at Q8_0 on the two Radeon AI PRO -# R9700s (gfx1201), with the shared Q8_0 MTP drafter head, the BF16 vision -# projector, and the draft-mtp + ngram-mod spec chain. +# frank-primary.env — tunables for the vLLM 0.30 frank-primary endpoint +# (Qwen3.8-Flash-Next-FP8, 2x R9700 gfx1201, dankserver :7770). # -# These are the tunables read by scripts/serve-frank-primary.sh. The flag -# list is owned by the serve script; this file only sets fleet values. +# Paths and numbers only — NEVER credentials. Do not name a tunable TEMP, HOST +# or GPU_MAX_HEAP_SIZE: the HIP runtime reads those bare names (2026-08-13). # -# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP -# runtime reads those bare (the 2026-08-13 TEMP incident). The bind address -# is BIND_HOST, not HOST — frank-large's env predates that lesson and gets -# away with it; do not copy the shape. +# Container paths: the model tree is mounted at /models (host +# /home/dank/models); the compile caches live under /cache (host +# /home/dank/.cache/frank-primary-vllm, keyed on model + stack + graph knobs — +# never share it across configurations). -# The payload (scripts/download-frank-primary.sh): the serve script refuses -# to start without all six Q8_0 parts, the MTP head and the projector. -MODEL_ROOT=/home/dank/models/Qwen3.8-Flash-Next-GGUF - -# The runtime: the unsloth b11030-mix prebuilt for gfx120X (the same -# b11030-mix build frank-large runs on gfx1151 — it bundles its own ROCm -# runtime, no host SDK gate). -RUNTIME_DIR=/home/dank/GitPackages/unsloth-llama-b11030 - -# Fleet endpoint: the port and alias the lucebox DS4.1 quadlet held before -# the 2026-09-26 swap, so no client, probe or models.yml entry changes -# address. +# --- identity --------------------------------------------------------------- +SERVED_NAME=frank-primary BIND_HOST=0.0.0.0 PORT=7770 -# Four conversations, each the model's full native 262144 window. -# --parallel divides --ctx-size into slots — the pool is 4x262144 cells -# (~23 GiB at the V-quantized law below, split across both cards). Slots -# DIVIDE the pool; they never multiply it. Per-slot window stays the -# full native 262144, so no client or models.yml change is implied. -CONTEXT_SIZE=1048576 -PARALLEL_SLOTS=4 +# --- model ------------------------------------------------------------------ +# Host-side dir: /home/dank/models/Qwen3.8-Flash-Next-FP8 (131 safetensors +# shards, 172.78 GiB, verified against model.safetensors.index.json). +MODEL_DIR=/models/Qwen3.8-Flash-Next-FP8 -# Spec chain: draft-mtp (the shared Q8_0 head) at depth 2, with the -# draftless ngram-mod matcher chained behind it. Depth 2 is the measured -# optimum class for this pair at long contexts (the retired Dankclaw -# Flash-Next deployment ran exactly this head at depth 2 with 0.83-0.92 -# acceptance); each extra verified position re-batches the CPU expert -# reads, so depth buys less than bandwidth here. -SPEC_DRAFT_N_MAX=2 +# --- placement -------------------------------------------------------------- +# Fixed 2-card tensor parallel (the R9700 pair; the Raphael iGPU is index 2 +# and excluded by the visibility pins). +ROCR_VISIBLE_DEVICES=0,1 +HIP_VISIBLE_DEVICES=0,1 +TP=2 +# 0.97 of 31.86 GiB/card (34,208,743,424 B) — leaves ~1 GiB for the ROCm +# baseline. Raise only with a measured reason. +GPU_UTIL=0.97 +# The model's native window. +CTX_SIZE=262144 +# Sequences the scheduler may hold. 8 is the proven batch shape on this host +# (radiance era); revisit with a bench after bring-up. +MAX_SEQS=8 -# Device placement. -ngl 999 puts every layer's non-overridden tensors on -# the GPUs; the big overrides then move the bandwidth-huge tensors to host -# RAM (computed there — the x4-pair measurement: streaming over PCIe loses, -# computing in RAM does not care). OT_OVERRIDES is ONE comma-separated -# = list passed as a single --override-tensor -# flag (b11030 logs DEPRECATED for repeated flags and wants the joined -# form), and the loader applies the FIRST matching pattern per tensor — -# so the block-specific patterns MUST precede the broad exps=CPU sweep. -# -# Hot expert blocks stay GPU-resident (2026-09-26): -ot granularity is the -# whole per-block expert tensor (512 experts share one tensor), so "critical -# experts" = blocks co-located on the card that holds their layers. -# Measured cost: 2.42 GiB/block on ROCm0, 2.49 GiB/block on ROCm1. -# Blocks 0-4 on ROCm0 + 45-47 on ROCm1: card0 runs at ~2.5 GiB free at -# boot / ~2.1 GB under load (vision graph included) — the operator -# directed the fill PAST the 3 GB checkpoint floor on 2026-09-26 -# ("shift a few more experts"); measured, battery-verified. Card1 keeps -# ~3.4 GB. Card1 CANNOT take another block: 3.31 GiB free − 2.49 GiB = -# 0.82 GiB, and the vision graph + boot drift alone exceed that. -# * exps=CPU — the remaining routed experts (blk.*.ffn_*_exps.weight, -# ~100 GiB at Q8_0 even with 7 blocks GPU-side), read once per token; -# the CPU backend computes them from page cache. -# * per_layer_token_embd.weight=CPU — the 51B-element PLE n-gram table -# (dims [160, 320001536], Q8_0, 54.4 GiB — the whole of part 3): -# sparse row lookups, tiny PCIe traffic from CPU RAM. It doubles as -# the shared MTP head's borrow tensor — GPU-resident, the draft load -# crashes with "invalid vector subscript" (llama.cpp PR #28243) — so -# this override is REQUIRED, not just a placement choice. NEVER give -# it a GPU target (the 23 -> 13.4 t/s regression) and never --no-mmap -# (the table must stay page-cache-backed; anonymous memory -# cgroup-kills silently). -NGL=999 -OT_OVERRIDES='blk\.[0-4]\.ffn_(up|gate|down)_exps=ROCm0,blk\.4[5-7]\.ffn_(up|gate|down)_exps=ROCm1,exps=CPU,per_layer_token_embd.weight=CPU' -TENSOR_SPLIT=1,1 +# --- weight offloading (the UVA shape) --------------------------------------- +# --offload-backend uva: matched weights live in PINNED host RAM and are read +# in place over PCIe by the kernels (zero-copy), instead of being resident +# on the cards. is_uva_available() = is_pin_memory_available() on ROCm; the +# pin needs the unit's LimitMEMLOCK=infinity (a root unit grants it via +# PID1 — the dankserver user session defaults to 8 MiB and cannot raise it). +# If pinning is unavailable the offloader falls back to a copy path — the +# load log MUST be checked for the UVA path, not just /health. +# --cpu-offload-gb is PER GPU (GiB). 2x R9700 total pinned budget ≈ 160 GiB of +# the 246 GiB host RAM; the rest covers the OS, llama.cpp-era page cache +# drain and the pinned staging buffers. Do not raise without checking +# MemAvailable at load time. +# --cpu-offload-params: name SEGMENTS (dot-delimited components) of tensors +# to offload. Measured from the payload headers (2026-09-26): `ple` matches +# 138 tensors = 131 n-gram table shards + 7 per-layer conv/proj weights, +# 47.75 GiB; `experts` matches 150,528 tensors, 114.86 GiB. Combined +# 162.60 GiB — with TP=2 each rank needs ~81.3 GiB of pinned host RAM, +# hence OFFLOAD_GB=82 per GPU (164 GiB total; 80/card left 2.6 GiB of +# experts resident, inflating the on-card weight share). Resident after +# the full offload: 10.16 GiB of dense/attention/MTP/ViT/lm_head weights. +# Matching is greedy-up-to-budget in traversal order; unmatched params are +# never offloaded. +OFFLOAD_BACKEND=uva +OFFLOAD_GB=82 +OFFLOAD_PARAMS=ple,experts -# The drafter MUST sit on the SECOND R9700 (ROCm1): with -ts 1,1 the main -# model's output head lands on the last-layers card, and the shared MTP head -# BORROWS it. --override-tensor sets has_tensor_overrides(), which pins -# n_copies==1 in the draft's scheduler — a borrowed output.weight sitting in -# a ROCm1 buffer then aborts the draft graph with ggml-backend.cpp:941 -# "pre-allocated tensor (output.weight) in a buffer (ROCm1) that cannot run -# the operation" when the draft is elsewhere (verified live 2026-09-26: the -# ROCm0 placement crash-looped 4x at ~12 s, ROCm1 loaded and served). -SPEC_DRAFT_DEVICE=ROCm1 +# --- engine ----------------------------------------------------------------- +# No aiter wheel in this image (gfx9-only upstream); the AMD qwen4_exp path +# runs its in-tree triton kernels. Deterministic rather than relying on the +# availability probe. +VLLM_ROCM_USE_AITER=0 +# Compile/inductor/triton caches (container /cache, TRITON_CACHE_DIR etc.) +CACHE_ROOT=/cache -# Batch geometry: logical batch 8192 with ubatch 512 — the measured optimum -# class on this fleet's ROCm stack (a bigger logical batch is free; compute -# buffers reserve at max graph size). ubatch 512 balances prefill -# throughput against the asymmetric compute-buffer cost on the card that -# holds the output layers and the drafter. -BATCH_SIZE=8192 -UBATCH_SIZE=512 +# --- parsers / template (the harness wire contract) --------------------------- +# vLLM 0.30 emits the thinking channel as `reasoning` (the deprecated +# `reasoning_content` is renamed server-side, chat_completion/protocol.py:554). +# The qwen3 reasoning parser extracts the template's think-block tags into +# that channel; the qwen3_xml tool parser structures tool-call XML output into +# tool_calls (auto-tool-choice enables the structured path for API clients). +# The family precedent is the host's own Ornith-era serve scripts +# (~/vllm/failedAttempts/OrnithFP8.sh); the FP8 tokenizer's template +# consumes enable_thinking + preserve_thinking (verified in tokenizer_config). +TOOL_PARSER=qwen3_xml +REASONING_PARSER=qwen3 +CHAT_TEMPLATE_KWARGS='{"enable_thinking": true, "preserve_thinking": true}' -# KV cache types: V quantized to q8_0, K held at the f16 default. The V -# half is what quantizes on this build: main KV drops 24,576 -> 18,816 -# B/token (12 of 48 layers carry KV; the 36 SSM layers are constant-state -# and never scale with ctx). NOTE the shared MTP drafter's KV ignores the -# V type (measured: draft pool 2,048 MiB = the f16 law at 1,048,576 -# cells) — one layer's worth, so the miss costs ~0.5 GiB on ROCm1. -CACHE_TYPE_K=f16 -CACHE_TYPE_V=q8_0 +# --- speculative decode ------------------------------------------------------ +# The hybrid stack (GatedDeltaNet + QSA short-conv) uses the v1 engine's +# mamba-capable attention backends, negotiated by the model, not by env. +# MTP drafting comes from the model's own mtp.* weights via +# --speculative-config, only when SPEC_CONFIG is non-empty. Bring-up runs +# with it empty on purpose: measure the base shape clean, then A/B; never +# change two knobs at once. +SPEC_CONFIG= -# Prompt-cache budget: 16 GiB. The V-quantized KV pool at four full -# windows is ~23 GiB spread across both cards (the hybrid DeltaNet/QSA -# pattern keeps even 4x262144 a fraction of a full-ctx x all-layers -# pool); the prompt cache is the RAM-side spend that actually moves -# prefill on repeat traffic. Host RAM budget: ~90 GiB of expert pages + -# this cache + the server's own arenas inside 246 GiB. -CACHE_RAM=16384 + +# --- container --------------------------------------------------------------- +IMAGE=localhost/frank-primary:vllm030 +CONTAINER_NAME=frank-primary-vllm diff --git a/scripts/frank-primary.service b/scripts/frank-primary.service index 4e4aa0a..85d2e2a 100644 --- a/scripts/frank-primary.service +++ b/scripts/frank-primary.service @@ -1,19 +1,35 @@ +# frank-primary.service — vLLM 0.30 endpoint (Qwen3.8-Flash-Next-FP8, +# UVA offload shape) on the 2x R9700 pair, dankserver :7770. +# +# Replaces the llama.cpp frank-primary unit of 2026-09-26 (same port, same +# served alias; harness models.yml unchanged). The serve script execs a +# podman container (image localhost/frank-primary:vllm030) as dank. +# +# WHY A ROOT UNIT (not a rootless quadlet): the UVA offload pins ~160 GiB of +# host RAM (PLE n-gram table + routed experts). Pinned pages are charged +# against RLIMIT_MEMLOCK; dank's login session on this host carries 8 MiB and +# a user quadlet is capped by the user manager's own hard limit — raising +# that means restarting user@1000.service and bouncing every user container +# on the host. A root unit grants LimitMEMLOCK=infinity to the dank process +# via PID1, no user-manager restart, and the image stays in dank's rootless +# podman storage (podman run as dank reads ~/.local/share/containers). [Unit] -Description=unsloth llama.cpp (b11030-mix, ROCm gfx120X) Qwen3.8-Flash-Next Q8_0, shared-Q8_0 MTP + ngram-mod spec chain, BF16 vision, 262144 ctx / 1 slot, root systemd unit (dankserver, frank-primary, 2x Radeon AI PRO R9700) +Description=frank-primary — vLLM 0.30 Qwen3.8-Flash-Next-FP8 (UVA offload) on 2x R9700 After=network-online.target Wants=network-online.target [Service] -Type=simple User=dank Group=dank -# The model is mmap'd (175 GiB of expert pages + the 51B-param n-gram -# table live in page cache); an RLIMIT_MEMLOCK smaller than what the -# runtime pins would fail the load mid-start. Parity with the other LLM -# units on this fleet. +# The UVA offloader pins the offloaded weights in host RAM; an RLIMIT_MEMLOCK +# smaller than the pinned set fails the load mid-start. PID1 grants infinity +# here — parity with the fleet's other LLM units. LimitMEMLOCK=infinity Environment="FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env" ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh +# Cold start: Triton/inductor compile (several minutes on first boot) plus +# reading+pinning ~173 GiB of weights. Parity with the radiance-era timeout. +TimeoutStartSec=1800 Restart=on-failure RestartSec=10 diff --git a/scripts/serve-frank-primary.sh b/scripts/serve-frank-primary.sh index 031522f..8513053 100755 --- a/scripts/serve-frank-primary.sh +++ b/scripts/serve-frank-primary.sh @@ -1,140 +1,123 @@ #!/usr/bin/env bash -# Serve unsloth/Qwen3.8-Flash-Next-GGUF at Q8_0 as frank-primary on -# Dankserver's two Radeon AI PRO R9700s (gfx1201), using the unsloth -# b11030-mix prebuilt (ROCm gfx120X, bundled runtime). +# serve-frank-primary.sh — frank-primary endpoint (vLLM 0.30, +# Qwen3.8-Flash-Next-FP8 on the 2x R9700 pair, dankserver :7770). # -# Shape: one stream, the model's full 262144 context, all non-overridden -# tensors on the GPUs (--n-gpu-layers 999 across both cards), with the -# routed experts and the 51B-param n-gram embedding table computed in host -# RAM via --override-tensor (the x4-pair law: streaming over PCIe loses, -# computing in RAM does not care). mmap stays ON (the PLE table must stay -# page-cache-backed; --no-mmap forces it into anonymous memory and the -# cgroup kill is silent). +# Validates the image, the model payload and the env tunables, then execs the +# container. The unit (scripts/frank-primary.service, root systemd, User=dank, +# LimitMEMLOCK=infinity) is what makes the UVA pin budget legal — a plain +# shell here carries the 8 MiB session limit and the pinned offload fails. # -# Spec chain: draft-mtp (the shared Q8_0 head, loaded via -md) chained with -# the draftless ngram-mod matcher. The shared head borrows the target's -# embedding tensors, which is why per_layer_token_embd.weight must be -# CPU-resident (see frank-primary.env). depth 2. +# OFFLOAD SHAPE (the only legal fit for a ~125B-param FP8 model on 2x 32 GiB +# cards): --offload-backend uva pins the PLE n-gram table (51.2B params) and +# the routed experts in host RAM and reads them in place over PCIe. The +# first start MUST be watched for the UVA engagement log (not just /health): +# a silent fallback to the copy path turns the endpoint into a slideshow. # -# Vision: the BF16 projector, loaded explicitly via --mmproj (this build -# never autoloads projectors — the frank-medium 2026-09-17 trap: a server -# started without it comes up healthy and text-only, and answers image -# requests with 500). -# -# Template: the GGUF's embedded chat template via --jinja. The Qwen3.8 -# family template gates reasoning_effort to {xhigh, medium, low} — the -# harness folds its levels onto that set in models.yml -# (reasoningEffortMap). If the embedded template ever falls back to -# peg-native and bloats prompts (~73,600 tokens for a ~250-token request), -# switch to --chat-template-file scripts/qwen-sharp-v22.4.0.jinja the way -# frank-large does. -# -# Tunables come from frank-primary.env unless FRANK_PRIMARY_ENV_FILE -# overrides it; anything not set there falls through to the defaults below. +# NO --model-draft, no draft GGUF: vLLM's MTP drafting comes from the model's +# own mtp.* weights via --speculative-config, and only when SPEC_CONFIG is +# non-empty. Bring-up runs with it empty on purpose. set -euo pipefail -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ENV_FILE="${FRANK_PRIMARY_ENV_FILE:-$SCRIPT_DIR/frank-primary.env}" -if [[ -f "$ENV_FILE" ]]; then - set -a - source "$ENV_FILE" - set +a -fi -# --- defaults (used when the env file does not set them) ------------------ -: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}" -: "${MODEL_ROOT:=/home/dank/models/Qwen3.8-Flash-Next-GGUF}" -: "${BIND_HOST:=0.0.0.0}" -: "${PORT:=7770}" -: "${CONTEXT_SIZE:=262144}" -: "${PARALLEL_SLOTS:=1}" -: "${SPEC_DRAFT_N_MAX:=2}" -: "${NGL:=999}" -: "${OT_OVERRIDES:=exps=CPU,per_layer_token_embd.weight=CPU}" -: "${TENSOR_SPLIT:=1,1}" -: "${SPEC_DRAFT_DEVICE:=ROCm0}" -: "${BATCH_SIZE:=8192}" -: "${UBATCH_SIZE:=512}" -: "${CACHE_TYPE_V:=f16}" -: "${CACHE_TYPE_K:=f16}" -: "${VERBOSITY:=4}" - -SERVER_BIN="$RUNTIME_DIR/llama-server" -MODEL_PARTS=( - "$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00001-of-00006.gguf" - "$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00002-of-00006.gguf" - "$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00003-of-00006.gguf" - "$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00004-of-00006.gguf" - "$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00005-of-00006.gguf" - "$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00006-of-00006.gguf" -) -DRAFTER_FILE="$MODEL_ROOT/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" -MMPROJ_FILE="$MODEL_ROOT/mmproj-BF16.gguf" - -if [[ ! -x "$SERVER_BIN" ]]; then - echo "runtime not found at $SERVER_BIN" >&2 - echo " fetch the unsloth b11030-mix gfx120X prebuilt (see the runbook)" >&2 +if [ ! -f "$ENV_FILE" ]; then + echo "serve-frank-primary: env file missing: $ENV_FILE" >&2 exit 1 fi -# Every part is checked: part 1 is the -m target and parts 2-6 are followed -# by the split-file loader, so a missing tail part fails at LOAD time with -# a confusing partial-mmap error unless it is caught here. -for f in "${MODEL_PARTS[@]}" "$DRAFTER_FILE" "$MMPROJ_FILE"; do - if [[ ! -f "$f" ]]; then - echo "missing asset: $f" >&2 - echo " run scripts/download-frank-primary.sh on this host" >&2 - exit 1 - fi -done +set -a +# shellcheck disable=SC1090 +source "$ENV_FILE" +set +a -# --- env (the unsloth build's bundled ROCm runtime) ----------------------- -# Both R9700s (gfx1201, HIP devices 0 and 1); the host's Raphael iGPU is -# device 2 and must stay hidden from the enumeration. -export HIP_VISIBLE_DEVICES=0,1 -export LD_LIBRARY_PATH="$RUNTIME_DIR:${LD_LIBRARY_PATH:-}" +: "${SERVED_NAME:?SERVED_NAME must be set}" +: "${BIND_HOST:?BIND_HOST must be set}" +: "${PORT:?PORT must be set}" +: "${MODEL_DIR:?MODEL_DIR must be set}" +: "${TP:?TP must be set}" +: "${GPU_UTIL:?GPU_UTIL must be set}" +: "${CTX_SIZE:?CTX_SIZE must be set}" +: "${MAX_SEQS:?MAX_SEQS must be set}" +: "${OFFLOAD_BACKEND:?OFFLOAD_BACKEND must be set}" +: "${OFFLOAD_GB:?OFFLOAD_GB must be set}" +: "${OFFLOAD_PARAMS:?OFFLOAD_PARAMS must be set}" +: "${VLLM_ROCM_USE_AITER:?VLLM_ROCM_USE_AITER must be set}" +: "${IMAGE:?IMAGE must be set}" +: "${CONTAINER_NAME:?CONTAINER_NAME must be set}" +: "${ROCR_VISIBLE_DEVICES:?ROCR_VISIBLE_DEVICES must be set}" +: "${TOOL_PARSER:?TOOL_PARSER must be set}" +: "${REASONING_PARSER:?REASONING_PARSER must be set}" +: "${CHAT_TEMPLATE_KWARGS:?CHAT_TEMPLATE_KWARGS must be set}" +: "${CACHE_ROOT:?CACHE_ROOT must be set}" -# --- the serve command --------------------------------------------------- -ARGS=( +if [ "$OFFLOAD_BACKEND" != "uva" ]; then + echo "serve-frank-primary: OFFLOAD_BACKEND=$OFFLOAD_BACKEND — this deployment is only qualified for uva" >&2 + exit 1 +fi + +if ! podman image inspect "$IMAGE" > /dev/null 2>&1; then + echo "serve-frank-primary: image missing: $IMAGE — build it before serving" >&2 + exit 1 +fi + +# The model payload is verified by the fetch/verify step (131 shards, index +# cross-checked). The serve-time check is the load-bearing minimum: the +# wrapper config and the shard index must exist, or the container start is a +# guaranteed 20-minute timeout instead of a fast failure. +HOST_MODEL_DIR="/home/dank/models/${MODEL_DIR#/models/}" +if [ ! -f "$HOST_MODEL_DIR/config.json" ] || [ ! -f "$HOST_MODEL_DIR/model.safetensors.index.json" ]; then + echo "serve-frank-primary: model payload incomplete at $HOST_MODEL_DIR (config.json or index missing)" >&2 + echo "serve-frank-primary: run scripts/download-frank-primary-fp8.sh on this host" >&2 + exit 1 +fi + +if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then + echo "serve-frank-primary: /dev/kfd or /dev/dri missing — GPU stack not up" >&2 + exit 1 +fi + +mkdir -p /home/dank/.cache/frank-primary-vllm + +SERVE_ARGS=(serve "$MODEL_DIR" + --served-model-name "$SERVED_NAME" --host "$BIND_HOST" --port "$PORT" - --model "${MODEL_PARTS[0]}" - --alias frank-primary - --ctx-size "$CONTEXT_SIZE" - --parallel "$PARALLEL_SLOTS" - -lv "$VERBOSITY" - --metrics - --n-gpu-layers "$NGL" - --tensor-split "$TENSOR_SPLIT" - --jinja - --reasoning auto - --reasoning-format deepseek - --reasoning-budget -1 - --cache-type-k "$CACHE_TYPE_K" - --cache-type-v "$CACHE_TYPE_V" - --batch-size "$BATCH_SIZE" - --ubatch-size "$UBATCH_SIZE" - --cache-ram "$CACHE_RAM" - # Vision: explicit, never autoloaded by this build - --mmproj "$MMPROJ_FILE" - # Spec chain: the shared MTP head + the draftless ngram matcher - --spec-type draft-mtp,ngram-mod - --model-draft "$DRAFTER_FILE" - --spec-draft-n-max "$SPEC_DRAFT_N_MAX" - --spec-draft-device "$SPEC_DRAFT_DEVICE" - # Greedy server default: deterministic e2e, and the harness overrides - # sampling per-request (models.yml compat.extraBody carries Qwen's - # recommended temp 1.0 / top_p 0.95 / top_k 20). - --temp 0 - --top-p 1.0 - --top-k 0 - --min-p 0.0 + --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_UTIL" + --max-model-len "$CTX_SIZE" + --max-num-seqs "$MAX_SEQS" + --offload-backend "$OFFLOAD_BACKEND" + --cpu-offload-gb "$OFFLOAD_GB" + --cpu-offload-params "$OFFLOAD_PARAMS" + --enable-auto-tool-choice --tool-call-parser "$TOOL_PARSER" + --reasoning-parser "$REASONING_PARSER" + --default-chat-template-kwargs "$CHAT_TEMPLATE_KWARGS" ) -# Comma-separated = pairs, ONE flag: b11030 logs -# DEPRECATED for repeated --override-tensor and wants the joined list. -# The loader applies the FIRST matching pattern per tensor, so the -# specific block patterns must precede the broad exps=CPU sweep in -# frank-primary.env. -ARGS+=(--override-tensor "$OT_OVERRIDES") +# SPEC_CONFIG (optional, JSON) is how MTP drafting is enabled after the +# bring-up measurement: --speculative-config '{"method":"mtp",...}'. +if [ -n "${SPEC_CONFIG:-}" ]; then + SERVE_ARGS+=(--speculative-config "$SPEC_CONFIG") +fi -exec "$SERVER_BIN" "${ARGS[@]}" +exec podman run --rm \ + --name "$CONTAINER_NAME" \ + --userns=keep-id \ + --network=host \ + --ipc=host \ + --security-opt seccomp=unconfined \ + --cap-add SYS_PTRACE \ + --device /dev/kfd \ + --device /dev/dri \ + --group-add keep-groups \ + -e ROCR_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" \ + -e HIP_VISIBLE_DEVICES="$HIP_VISIBLE_DEVICES" \ + -e VLLM_ROCM_USE_AITER="$VLLM_ROCM_USE_AITER" \ + -e TRITON_CACHE_DIR="$CACHE_ROOT/triton" \ + -e TORCHINDUCTOR_CACHE_DIR="$CACHE_ROOT/inductor" \ + -e VLLM_CACHE_DIR="$CACHE_ROOT/vllm" \ + -e TRITON_ALLOW_NON_CONST_GLOBALS=1 \ + --volume /home/dank/models:/models \ + --volume /home/dank/.cache/frank-primary-vllm:"$CACHE_ROOT" \ + "$IMAGE" \ + vllm "${SERVE_ARGS[@]}" diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats index 9dd7e13..2f5536e 100755 --- a/tests/frank_primary_unit.bats +++ b/tests/frank_primary_unit.bats @@ -1,16 +1,17 @@ #!/usr/bin/env bats # Seam: frank-primary — the Dankserver LLM endpoint on :7770. # -# This file asserts the llama.cpp ROOT-UNIT shape (the 2026-09-26 swap that -# replaced the Lucebox DS4.1 Quadlet): the unsloth b11030-mix prebuilt on -# the two R9700s (gfx1201) serving unsloth/Qwen3.8-Flash-Next-GGUF at Q8_0 -# with the shared Q8_0 MTP head, the BF16 projector, and the -# draft-mtp + ngram-mod chain. The seam pins what the flags alone do not -# say: the CPU-override placement law (experts and the 51B n-gram table -# are COMPUTED in host RAM, never streamed), mmap staying on, the -# shared-head borrow requirement (per_layer_token_embd on CPU), the -# explicit projector (never autoloaded by this build), HIP env hygiene, -# and the deployer's byte-count gates over a six-part payload. +# This file asserts the vLLM 0.30 ROOT-UNIT shape (the 2026-09-27 swap that +# replaced the llama.cpp b11030 endpoint): a locally built ROCm gfx1201 +# image (localhost/frank-primary:vllm030) serving Qwen3.8-Flash-Next-FP8 +# (Qwen4ExpForConditionalGeneration, 131 shards, 172.78 GiB) on the two +# R9700s with the UVA offload shape. The seam pins what the flags alone do +# not say: the offload budget arithmetic (the bandwidth-huge tensors — the +# 47.75 GiB PLE family and the 114.86 GiB routed experts — are pinned in +# host RAM and read in place, never streamed), the memlock law that makes +# the pin legal, the parser contract that keeps the harness wire formats +# (tool_calls + the reasoning channel) working, HIP env hygiene, and the +# deployer's image/payload gates with the false-green MainPID check. load helpers @@ -27,8 +28,10 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" @test "frank-primary is boot-persistent, self-healing and unprivileged" { # Root unit parity with frank-large/frank-small: enable is the [Install] - # section, the restart policy survives a crash, and the mmap'd payload - # needs an unbounded memlock. + # section, the restart policy survives a crash, and the ~164 GiB UVA pin + # needs an unbounded memlock (a root unit grants it via PID1; the dank + # session's own hard limit is 8 MiB and cannot be raised without + # restarting user@1000.service). grep -q '^WantedBy=multi-user.target$' "$UNIT" grep -q '^Restart=on-failure$' "$UNIT" grep -q '^RestartSec=' "$UNIT" @@ -41,99 +44,96 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" grep -q 'FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env' "$UNIT" } -@test "frank-primary drafts with the shared MTP head chained behind ngram-mod" { - # The chain is draft-mtp (context-backed) + ngram-mod (draftless): the - # only legal way to layer two spec implementations in one process. The - # shared Q8_0 head borrows the target's embedding tensors — which is - # why per_layer_token_embd.weight must be CPU-resident (a GPU-resident - # borrow crashes the draft load with "invalid vector subscript"). - grep -q -- '--spec-type draft-mtp,ngram-mod' "$SERVE" - grep -q '^SPEC_DRAFT_N_MAX=2$' "$ENV_FILE" - grep -q -- '--spec-draft-n-max "$SPEC_DRAFT_N_MAX"' "$SERVE" - grep -q 'DRAFTER_FILE="$MODEL_ROOT/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"' "$SERVE" - grep -q -- '--spec-draft-device "$SPEC_DRAFT_DEVICE"' "$SERVE" +@test "the serve shape is vLLM: no llama.cpp-ism survives the cutover" { + # Clean cutover: the GGUF era's flags would silently no-op or crash a vLLM + # server, so their absence is part of the contract. + ! grep -q -- '--model-draft' "$SERVE" + ! grep -q -- '--spec-type' "$SERVE" + ! grep -q -- '--mmproj' "$SERVE" + ! grep -q -- '--override-tensor' "$SERVE" + ! grep -q '\.gguf' "$SERVE" + ! grep -q '\.gguf' "$ENV_FILE" } -@test "the bandwidth-huge tensors are computed in host RAM, not streamed" { - # The x4-pair law: streaming experts over PCIe loses (3.7 tok/s) while - # computing them in RAM does not care (20+ tok/s on the same model - # shape). exps=CPU plus the n-gram embedding table on CPU; the PLE - # table must NEVER carry a GPU target (the 23 -> 13.4 t/s regression). - # ONE joined flag (b11030 deprecates repeated --override-tensor), and - # first-match-wins means the block-specific patterns must precede the - # broad exps=CPU sweep — the exact value pins the whole law. - grep -q -- '--override-tensor "$OT_OVERRIDES"' "$SERVE" - ! grep -q -- '--no-mmap' "$SERVE" - ! grep -q '^MMPROJ_ON_GPU' "$ENV_FILE" - ot="$(sed -nE "s/^OT_OVERRIDES='(.*)'$/\1/p" "$ENV_FILE")" - [ "$ot" = 'blk\.[0-4]\.ffn_(up|gate|down)_exps=ROCm0,blk\.4[5-7]\.ffn_(up|gate|down)_exps=ROCm1,exps=CPU,per_layer_token_embd.weight=CPU' ] +@test "the bandwidth-huge tensors are pinned in host RAM, not streamed" { + # The UVA shape (the x4-pair law: streaming experts over PCIe loses, + # computing them in host RAM does not care): --offload-backend uva pins + # the matched weights with zero-copy reads. Measured from the payload + # headers (2026-09-26): `ple` matches 138 tensors (131 n-gram table + # shards + 7 per-layer conv/proj weights) = 47.75 GiB; `experts` matches + # 150,528 tensors = 114.86 GiB; combined 162.60 GiB. cpu-offload-gb is + # PER GPU: under TP=2 each rank needs ~81.3 GiB, hence 82 — 80/card + # would strand 2.6 GiB of experts on the cards. + grep -q '^OFFLOAD_BACKEND=uva$' "$ENV_FILE" + grep -q '^OFFLOAD_GB=82$' "$ENV_FILE" + grep -q '^OFFLOAD_PARAMS=ple,experts$' "$ENV_FILE" + grep -q -- '--offload-backend "$OFFLOAD_BACKEND"' "$SERVE" + grep -q -- '--cpu-offload-gb "$OFFLOAD_GB"' "$SERVE" + grep -q -- '--cpu-offload-params "$OFFLOAD_PARAMS"' "$SERVE" + # auto/prefetch are the copy paths: this deployment is only qualified + # for uva, so the serve script refuses anything else. + grep -q 'only qualified for uva' "$SERVE" } -@test "the KV pool is V-quantized, K held at f16" { - # V q8_0 halves nothing (it is 1/4 of the per-token law: 18,816 of the - # 24,576 B/token f16/f16 cost) but it is the deploy's stated shape; the - # flags must be env-wired tunables so the quant can be reverted - # without touching the serve script. - grep -q '^CACHE_TYPE_K=f16$' "$ENV_FILE" - grep -q '^CACHE_TYPE_V=q8_0$' "$ENV_FILE" - grep -q -- '--cache-type-k "$CACHE_TYPE_K"' "$SERVE" - grep -q -- '--cache-type-v "$CACHE_TYPE_V"' "$SERVE" +@test "the engine shape: TP=2 over both R9700s, full native window, batch 8" { + grep -q '^TP=2$' "$ENV_FILE" + grep -q '^GPU_UTIL=0.97$' "$ENV_FILE" + grep -q '^CTX_SIZE=262144$' "$ENV_FILE" + grep -q '^MAX_SEQS=8$' "$ENV_FILE" + grep -q -- '--tensor-parallel-size "$TP"' "$SERVE" + grep -q -- '--gpu-memory-utilization "$GPU_UTIL"' "$SERVE" + grep -q -- '--max-model-len "$CTX_SIZE"' "$SERVE" + grep -q -- '--max-num-seqs "$MAX_SEQS"' "$SERVE" } -@test "the projector is explicit, never autoloaded" { - # This build never autoloads mmproj from the model directory: without - # the flag the server comes up healthy and text-only, answering image - # requests with 500 (the frank-medium 2026-09-17 trap). - grep -q -- '--mmproj "$MMPROJ_FILE"' "$SERVE" - grep -q 'MMPROJ_FILE="$MODEL_ROOT/mmproj-BF16.gguf"' "$SERVE" +@test "the payload is the FP8 tree, vision served from the model's own config" { + # Qwen4ExpForConditionalGeneration: vLLM auto-loads the ViT and the + # mtp.* weights from the payload config — no explicit projector flag + # exists in this runtime (the llama.cpp --mmproj trap does not apply). + grep -q '^MODEL_DIR=/models/Qwen3.8-Flash-Next-FP8$' "$ENV_FILE" + grep -q '^IMAGE=localhost/frank-primary:vllm030$' "$ENV_FILE" } -@test "frank-primary is vision-serving but holds the single fleet port" { +@test "frank-primary holds the single fleet port under its own alias" { grep -q '^PORT=7770$' "$ENV_FILE" grep -q '^BIND_HOST=0.0.0.0$' "$ENV_FILE" - grep -q -- '--alias frank-primary' "$SERVE" + grep -q '^SERVED_NAME=frank-primary$' "$ENV_FILE" + grep -q -- '--served-model-name "$SERVED_NAME"' "$SERVE" grep -q -- '--host "$BIND_HOST"' "$SERVE" grep -q -- '--port "$PORT"' "$SERVE" } -@test "four streams, each the model's full native window" { - # --parallel divides --ctx-size into slots; the contract is four - # conversations each holding the FULL native 262144 window, so the - # per-slot window (ctx / parallel) equals the native size. - local ctx parallel - ctx="$(sed -nE 's/^CONTEXT_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" - parallel="$(sed -nE 's/^PARALLEL_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")" - [ -n "$ctx" ] && [ -n "$parallel" ] - [ "$parallel" -eq 4 ] - [ "$((ctx / parallel))" -eq 262144 ] - grep -q -- '--ctx-size "$CONTEXT_SIZE"' "$SERVE" - grep -q -- '--parallel "$PARALLEL_SLOTS"' "$SERVE" +@test "tool_calls and the reasoning channel are wired for the harness" { + # The wire contract: vLLM 0.30 emits the thinking channel as `reasoning` + # (the deprecated `reasoning_content` is renamed server-side), the qwen3 + # parser extracts the template's think block into it, and qwen3_xml + + # auto-tool-choice structure tool calls. Precedent: the host's own + # Ornith-era scripts (~/vllm/failedAttempts/OrnithFP8.sh); the FP8 + # template consumes enable_thinking/preserve_thinking (verified in + # tokenizer_config.json). + grep -q '^TOOL_PARSER=qwen3_xml$' "$ENV_FILE" + grep -q '^REASONING_PARSER=qwen3$' "$ENV_FILE" + grep -q -- '--enable-auto-tool-choice --tool-call-parser "$TOOL_PARSER"' "$SERVE" + grep -q -- '--reasoning-parser "$REASONING_PARSER"' "$SERVE" + # The template kwargs are JSON: the env value must be single-quoted or + # bash splits it and CHAT_TEMPLATE_KWARGS arrives empty (the -ot lesson). + grep -q "^CHAT_TEMPLATE_KWARGS='{\"enable_thinking\": true, \"preserve_thinking\": true}'$" "$ENV_FILE" + grep -q -- '--default-chat-template-kwargs "$CHAT_TEMPLATE_KWARGS"' "$SERVE" } -@test "reasoning is extracted to reasoning_content and tools are legal" { - # The harness contract: llama.cpp endpoints emit reasoning_content - # (the deepseek format is the one that fills it), and tools require - # --jinja (the ik_llama 500 lesson). The Qwen3.8 template gates - # reasoning_effort to {xhigh, medium, low} — folded in models.yml. - grep -q -- '--jinja' "$SERVE" - grep -q -- '--reasoning auto' "$SERVE" - grep -q -- '--reasoning-format deepseek' "$SERVE" - grep -q -- '--reasoning-budget -1' "$SERVE" -} - -@test "the greedy server default is explicit" { - # Deterministic e2e probes; the harness overrides sampling per request. - grep -q -- '--temp 0' "$SERVE" - grep -q -- '--top-p 1.0' "$SERVE" - grep -q -- '--top-k 0' "$SERVE" - grep -q -- '--min-p 0.0' "$SERVE" +@test "speculative decode is env-wired and OFF at bring-up" { + # MTP drafting comes from the model's own mtp.* weights via + # --speculative-config; bring-up measures the base shape clean, so + # SPEC_CONFIG is empty and the flag is only added when non-empty. + grep -q '^SPEC_CONFIG=$' "$ENV_FILE" + grep -q -- '--speculative-config "$SPEC_CONFIG"' "$SERVE" } @test "both R9700s are visible and the Raphael iGPU is not" { - # The host has a third (Raphael) GPU the server must never place - # tensors on; the bundled ROCm runtime is the only one linked. - grep -q 'export HIP_VISIBLE_DEVICES=0,1$' "$SERVE" - grep -q 'export LD_LIBRARY_PATH="$RUNTIME_DIR' "$SERVE" + grep -q '^ROCR_VISIBLE_DEVICES=0,1$' "$ENV_FILE" + grep -q '^HIP_VISIBLE_DEVICES=0,1$' "$ENV_FILE" + grep -q -- '-e ROCR_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"' "$SERVE" + grep -q -- '-e HIP_VISIBLE_DEVICES="$HIP_VISIBLE_DEVICES"' "$SERVE" } @test "no bare HIP-reserved env var is defined" { @@ -145,40 +145,36 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" ! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE" } -@test "the serve script refuses to start on a half-finished download" { - # Six split parts plus the head and the projector: a missing tail part - # fails at LOAD time with a confusing partial-mmap error, so every - # artifact is checked before the server execs. - grep -q 'missing asset' "$SERVE" - grep -q 'download-frank-primary.sh' "$SERVE" +@test "the serve script refuses to start on a half-finished payload or missing image" { + # A payload missing its wrapper config or shard index turns the + # container start into a guaranteed 20-minute timeout; the checks fail + # fast with a pointer at the fetch script instead. + grep -q 'config.json or index missing' "$SERVE" + grep -q 'download-frank-primary-fp8.sh' "$SERVE" + grep -q 'podman image inspect' "$SERVE" } -@test "the prompt-cache cap is set below the RAM the host can spare" { - grep -q '^CACHE_RAM=16384$' "$ENV_FILE" - grep -q -- '--cache-ram "$CACHE_RAM"' "$SERVE" -} - -@test "the deployer retires the lucebox quadlet, gates every part and waits out the port" { - # The cutover order: the user-scope quadlet is stopped and its - # generated .container moved to dated home trash BEFORE the root unit - # binds 7770 (nothing starts until the port is released — the 2026-09-26 - # and 2026-09-16 cutover failures were both here). - grep -q 'retire_lucebox_primary_quadlet' "$REMOTE_DEPLOY" - grep -q 'systemctl --user stop frank-primary.service' "$REMOTE_DEPLOY" +@test "the deployer gates the image and the full shard set before starting" { + # The deployer refuses when the image or any index-named shard is + # absent (never serve a partial download), clears any stale container + # that would break `podman run --name`, and waits out the port before + # the new engine binds it. + grep -q 'LLM_PRIMARY_IMAGE=localhost/frank-primary:vllm030' "$REMOTE_DEPLOY" + grep -q 'LLM_PRIMARY_ROOT=/home/dank/models/Qwen3.8-Flash-Next-FP8' "$REMOTE_DEPLOY" + grep -q 'weight_map' "$REMOTE_DEPLOY" + grep -q 'download-frank-primary-fp8.sh' "$REMOTE_DEPLOY" + grep -q "podman rm -f \"\$LLM_PRIMARY_CONTAINER\"" "$REMOTE_DEPLOY" + grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY" grep -q 'relink_unit frank-primary' "$REMOTE_DEPLOY" grep -q 'systemctl enable frank-primary.service' "$REMOTE_DEPLOY" grep -q 'systemctl restart frank-primary.service' "$REMOTE_DEPLOY" - grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY" - grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY" -} - -@test "the deployer gates the six-part payload at declared byte counts" { - # stat -c %s, never wc -c — wc read the whole 377 GiB lucebox file for - # minutes on 2026-09-26 before it could reach the cutover. A partial - # download must never be served. - grep -q 'LLM_PRIMARY_PART_SIZES' "$REMOTE_DEPLOY" - grep -q 'stat -c %s' "$REMOTE_DEPLOY" - grep -q '2786568256' "$REMOTE_DEPLOY" - grep -q '907542944' "$REMOTE_DEPLOY" grep -q 'wait_for_health frank-primary http://127.0.0.1:7770/health 1800' "$REMOTE_DEPLOY" } + +@test "the deployer will not report green on a stray port-holder" { + # The 2026-09-26 false green: a leftover probe held :7770 and satisfied + # the health curl while the real unit was stopped. The deploy is only + # green when the unit's own MainPID is live behind the health answer. + grep -q 'systemctl show -p MainPID --value frank-primary.service' "$REMOTE_DEPLOY" + grep -q 'stray process holds the port' "$REMOTE_DEPLOY" +}