From bd5c28c4e919b4d78e4d4b9973085de669a3b348 Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 5 Oct 2026 19:17:09 -0700 Subject: [PATCH] =?UTF-8?q?frank-primary:=20FreeToken=20(PR587=20ROCm)=20c?= =?UTF-8?q?utover=20=E2=80=94=20trio,=20seam,=20deployer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the tcclaviger vLLM fork with FreeToken at PR #587 head 77a7201 built on dankserver into localhost/frank-primary-freetoken:pr587-77a7201 (scripts/docker/frank-primary-freetoken.Dockerfile: digest-pinned rocm/pytorch rocm7.14 torch-2.11.0 base, libhiprtc dev symlink, curated --no-deps install per docs/install_amd.md). Shape (first-boot conservative): ONE R9700 (second card idle; --tensor-parallel-size exists for future 2-card tuning), KV pool --num-tokens 300000 (7.2 GiB at 24 KiB/token; clears the 177,356-token harness worst case and one native 262,144 conversation), MoE LRU --moe-cache-rate 0.10, --max-running-requests 2, --max-output-tokens 36864. Parsers AUTO: thinking -> reasoning_content natively (models.yml flips reasoningContentField on the harness side). Experts + PLE pin ~170 GiB host RAM — root unit keeps LimitMEMLOCK=infinity. Container identity renamed to the unit name (Frank's call): frank-primary; vLLM-era frank-primary-qwen joins the legacy cleanup list. Deployer image gate becomes podman image exists (a local build carries no RepoDigests; the old digest grep refused exactly this shape). Rollback tag: frank-primary-vllm-shapeG-20261003. --- scripts/deploy-remote.sh | 54 ++- .../docker/frank-primary-freetoken.Dockerfile | 5 +- scripts/frank-primary.env | 426 +++------------- scripts/frank-primary.service | 82 ++-- scripts/serve-frank-primary.sh | 194 +++----- tests/frank_primary_unit.bats | 455 ++++++------------ 6 files changed, 347 insertions(+), 869 deletions(-) diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index e389a3f..8f7ad46 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1250,24 +1250,25 @@ deploy_llm_small() { echo "frank-small: health answered by unit MainPID $main_pid" } -# deploy_llm_primary — frank-primary is the tcclaviger RDNA4 vLLM fork -# (digest-pinned image, gfx1201) serving the orcarouter -# Qwen3.8-Flash-Next-Uncensored-FP8 payload (131 safetensors shards) with -# MTP4 spec decode and q8/q8 KV on the two R9700s as a ROOT unit whose serve -# script drives `podman run` directly (the image entrypoint execs -# `vllm serve`). 2026-09-29 cutover from the R9V radiance fork, whose -# launcher/shim/setup-state machinery is retired (state dirs stay on disk as -# rollback; pre-cutover tag frank-primary-r9v-q8-20260929). Same port and -# alias as the endpoints it replaced; the harness models.yml id is unchanged. +# deploy_llm_primary — frank-primary is the FreeToken engine (PR #587 ROCm +# build, gfx1201) serving the orcarouter Qwen3.8-Flash-Next-Uncensored-FP8 +# payload (131 safetensors shards) on ONE R9700 as a ROOT unit whose serve +# script drives `podman run` directly (the image carries `ft serve`). +# 2026-10-05 cutover from the tcclaviger vLLM fork (pre-cutover tag +# frank-primary-vllm-shapeG-20261003); before that, 2026-09-29 cutover from +# the R9V radiance fork, whose launcher/shim/setup-state machinery is +# retired. Same port and alias as the endpoints it replaced; the harness +# models.yml id is unchanged. # -# Gates: the pinned runtime image must be loadable from dank's rootless -# storage by its OCI manifest digest (the env file's FRANK_PRIMARY_IMAGE), -# the payload dir must carry config.json and the expected shard count -# (FRANK_PRIMARY_MODEL_SHARDS), and the unit env must be complete. Per-file -# sha256 verification was the fetch script's job at download time, never -# re-hashed here. -LLM_PRIMARY_CONTAINER=frank-primary-qwen -LLM_PRIMARY_LEGACY_CONTAINERS="frank-primary-r9v qwen-fp8 glm-probe" +# Gates: the runtime image must exist in dank's rootless storage +# (FRANK_PRIMARY_IMAGE — a LOCAL tag: the image is built on the host from +# scripts/docker/frank-primary-freetoken.Dockerfile, which pins the +# FreeToken PR-head SHA and the base image), the payload dir must carry +# config.json and the expected shard count (FRANK_PRIMARY_MODEL_SHARDS), +# and the unit env must be complete. Per-file sha256 verification was the +# fetch script's job at download time, never re-hashed here. +LLM_PRIMARY_CONTAINER=frank-primary +LLM_PRIMARY_LEGACY_CONTAINERS="frank-primary-qwen frank-primary-r9v qwen-fp8 glm-probe" deploy_llm_primary() { # Refuse rather than restart-loop: a unit whose image or payload is @@ -1286,15 +1287,16 @@ deploy_llm_primary() { echo "ERROR: frank-primary.env missing FRANK_PRIMARY_IMAGE/MODEL_DIR/MODEL_SHARDS" >&2 return 1 fi - # Capture podman's output BEFORE grepping: under set -o pipefail, the - # classic `podman images | grep -q` races (grep exits on match, podman - # dies SIGPIPE 141 writing the rest, pipefail calls it no-match — the - # 2026-09-30 deploy refusal). - local digests - digests="$(podman images --format '{{.RepoDigests}}')" - if ! grep -qF "${image#*@}" <<<"$digests"; then - echo "ERROR: pinned runtime image absent from rootless storage: $image" >&2 - echo "frank-primary: fetch it (podman pull $image)" >&2 + # The image gate is EXISTENCE IN STORAGE, not a registry digest: the + # FreeToken image is built ON THE HOST (a local tag carries NO + # RepoDigests entry — the old grep refused exactly this shape), and + # pinning lives in the Dockerfile (pinned PR-head SHA + digest-pinned + # base image). `podman image exists` accepts a local tag and a + # name@digest ref alike, in one command — no pipefail/grep race (the + # 2026-09-30 lesson) for either shape. + if ! podman image exists "$image"; then + echo "ERROR: runtime image absent from rootless storage: $image" >&2 + echo "frank-primary: build it (scripts/docker/frank-primary-freetoken.Dockerfile) or pull it" >&2 return 1 fi if [[ ! -f "$model_dir/config.json" ]]; then diff --git a/scripts/docker/frank-primary-freetoken.Dockerfile b/scripts/docker/frank-primary-freetoken.Dockerfile index 941af92..2d9e4b1 100644 --- a/scripts/docker/frank-primary-freetoken.Dockerfile +++ b/scripts/docker/frank-primary-freetoken.Dockerfile @@ -31,8 +31,9 @@ # Triton kernels on ROCm. # # Base is digest-pinned (2026-10-05): sha256:a223aee1… is the manifest this -# build was qualified against; a re-pushed tag cannot change the stack. -FROM rocm/pytorch@sha256:a223aee17aef5d21c3b9f63436dd19d27d1c665ec8b2f40011c9546cabae2a80 +# build was qualified against; a re-pushed tag cannot change the stack. The +# readable tag rides along for humans (podman verifies the digest matches). +FROM rocm/pytorch:rocm7.14_ubuntu24.04_py3.12_pytorch_release_2.11.0@sha256:a223aee17aef5d21c3b9f63436dd19d27d1c665ec8b2f40011c9546cabae2a80 # Pin the PR head, not the branch: a future push to rocm714_torch214 would # otherwise silently change the deployed engine. diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index 4bb5e97..e458def 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -1,362 +1,84 @@ -# frank-primary.env — tcclaviger RDNA4 vLLM production config (dankserver :7770). +# frank-primary.env — FreeToken (PR #587 ROCm) serving the orcarouter +# Qwen3.8-Flash-Next-Uncensored-FP8 payload on dankserver :7770. # -# 2026-10-03 shape G (as shipped, after the 1.00 revert): the CUDA-graph plan -# term dropped from the analytic 0.55 GiB to the measured 0.07 GiB (0 base + -# 12 MiB/graph), and UTIL at the 0.982 law. MEASURED on the live deploy: arena -# 17.77 GiB / 7,762 slots, KV 401,408 tokens / 2.00x concurrency (the first -# shape holding two full-length conversations with real spare), driver used -# 30.97 GiB of 31.86 — 0.89 GiB of real free VRAM per card, held through a -# live 4,096-token generation. UTIL stays 0.982 ALWAYS: 1.00 is unbootable on -# this fork (worker init refuses budget >= device total) and 0.13-GiB-margin -# boots die under load. -# `budget - non-expert - KV reserve - graphs - overhead` and the KV reserve is a -# registry constant, so those two terms are the whole flag surface. -# 2026-10-03 shape F (absorbed): MNBT 2048 + the graph term pinned. Measured on -# the live deploy: arena 17.52 GiB / 7,653 slots, budget 31.29, graphs 0.32. -# 2026-10-02 shape E: fp8 KV, MAXLEN 200704 x SEQS 3, pool 3.15e9/rank, and -# --expert-precise-memory ON. The window shrink from the native 262144 and the -# --max-num-seqs 3 lock both come from shape D (SEQS=2 CANNOT boot this model — -# the measured 2x2 is below); shape E adds the precise-overhead plan, which -# trades the default 7.60 GiB compile headroom for the fitted 5.25 GiB and puts -# the 2.35 GiB difference into the expert arena: 6,496 -> 7,522 slots (+16%). -# The pool is deliberately NOT cut (option A): the pool is not an arena lever -# (measured), so 3.15e9 keeps shape D's 1.99x concurrency while the arena grows. -# Same digest-pinned tcclaviger RDNA4 fork. Rollback for the fp8+MTP4 era is git -# tag frank-primary-fp8-mtp-20261001. +# 2026-10-05 cutover from the tcclaviger RDNA4 vLLM fork (git tag +# frank-primary-vllm-shapeG-20261003 is the pre-cutover state; the 362-line +# vLLM-era env is in that tag). The engine is FreeToken at PR head +# 77a7201f98f00d93f618c6cf8b81ffe8360e0899 (branch rocm714_torch214), built +# on the host into a local image by +# scripts/docker/frank-primary-freetoken.Dockerfile — the PR's own +# docs/install_amd.md stack (ROCm 7.14 base image, torch 2.11.0+rocm7.14, +# triton 3.7) for gfx1201. # -# HIP reads bare env vars TEMP, HOST, GPU_MAX_HEAP_SIZE — every key here is -# prefixed FRANK_PRIMARY_ so none can leak. The one bare name we WANT inside the -# container (GPU_MAX_HW_QUEUES) is held here under the prefix and passed on the -# run line as `-e GPU_MAX_HW_QUEUES=…`. +# WHY FREETOKEN: the vLLM fork's expert offload planned VRAM/host tiers per +# boot from an estimator (util 0.982 law, arena 17.77 GiB / 7,762 slots, +# ~40k-token GDN state faults) and 1.0 was unbootable. FreeToken keeps ALL +# expert weights in pinned HOST RAM (~120 GiB) with an LRU GPU cache, sizes +# KV from an explicit token pool, and serves on ONE R9700 (32 GB) with the +# second card idle. RAM fit: 246 GiB total, service stopped frees the vLLM +# container's ~170 GiB; weights ~120 GiB + PLE 47.7 GiB + overhead ≈ 190 GiB. # +# PAYLOAD: unchanged — /home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8, +# 131 safetensors shards. Quant is standard modelopt block-FP8: 75,264 +# weight_scale_inv tensors (block 128x128), zero input_scale (FreeToken #462 +# covers exactly this export shape), one weight_scale on the PLE ngram +# embedding. FreeToken loads HF safetensors directly; no FTW conversion. # -# --- runtime ---------------------------------------------------------------- -# Digest-pinned: the floating :latest tag moved once already. Local alias -# qwen-fp8-runtime:tuned-20260929 (image ID 3322dbbbc96d) points at the same -# bytes in rootless storage. -FRANK_PRIMARY_IMAGE=docker.io/tcclaviger/vllm@sha256:da4029666668b90a0386c1ed23ea370c79916ad4e1a6ad1dd9db098b9fa069ea +# SERVING FACTS (docs/cli.md, verified against main+PR587): +# - `ft serve` loads the checkpoint directly; `--served-model-name` keeps the +# harness id; reasoning/tool parsers are AUTO (reasoning lands in +# reasoning_content — models.yml flips reasoningContentField to match). +# - KV: --page-size must stay 1 for GDN/SWA models (default); --cache-type +# radix auto-picks the GDN-aware variant. +# - MoE: --moe-strategy auto -> offload (fp8 experts CANNOT use +# --moe-cpu-layers; only bf16/nvfp4/mxfp4 have a CPU executor). +# --moe-cache-rate holds the GPU LRU fraction of all experts. +# - `ft ctl cache --kv --moe` grows the pools LIVE on a running server +# (shrinking needs a restart); `ft ctl stats` reports pool occupancy. +# - KV MATH: 12 full-attention layers x 2 KV heads x 256 head_dim x 2 (K+V) +# x 2 B = 24 KiB/token (f16 KV; the 36 GDN layers use fixed per-seq state, +# not per-token KV). FRANK_PRIMARY_FT_NUM_TOKENS=300000 -> ~7.2 GiB. +# Harness worst case = 140,492-token prompt + 36,864 max_tokens = 177,356 +# < 300,000; one NATIVE 262,144-token conversation + full budget also fits. +# - PLE: the ~47.7 GiB ngram-embedding table pins in HOST RAM by default +# (--ple-backend disk is the alternative); the unit's LimitMEMLOCK=infinity +# makes the pin legal. TENS of GiB of pinned pages are why this stays a +# ROOT unit — see frank-primary.service. -# --- payload ---------------------------------------------------------------- +# --- Identity (deploy + serve contract) -------------------------------- +FRANK_PRIMARY_IMAGE=localhost/frank-primary-freetoken:pr587-77a7201 FRANK_PRIMARY_MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8 FRANK_PRIMARY_MODEL_SHARDS=131 - -# --- identity --------------------------------------------------------------- -# The harness models.yml id stays frank-primary (provider 192.168.0.235:7770). -FRANK_PRIMARY_CONTAINER=frank-primary-qwen +# Container identity is the SERVICE name (Frank's call, 2026-10-05): one +# name across systemctl and podman ps. The vLLM-era name frank-primary-qwen +# moved into the legacy list so every deploy cleans it up. +FRANK_PRIMARY_CONTAINER=frank-primary +FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-qwen frank-primary-r9v qwen-fp8 glm-probe" FRANK_PRIMARY_SERVED_NAME=frank-primary FRANK_PRIMARY_PUBLISH=7770:7770 -# Containers removed before every start: the retired R9V identity and the ad -# hoc experiment identity (both hold the GPUs / name registrations). -FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-r9v qwen-fp8" -# --- qualified engine shape (2026-10-02, shape E) --------------------------- -# WHAT THE SLOT COUNT DOES AND DOES NOT DO. The endpoint advertises 3 slots -# (SEQS 3), but the POOL, not the slot count, decides how many full-length -# conversations are resident. MEASURED LIVE 2026-10-02 at this shape: -# GPU KV cache size: 398,658 tokens, -# Maximum concurrency for 200,704 tokens per request: 1.99x -# (3,146,129,408 B reserved — 7,892 B/token). The per-token price is -# SHAPE-DEPENDENT, not a constant: the 262144 shape priced at 7,701 B/token -# (6,100,000,000 B -> 792,069 tokens), ~2.5% cheaper, so never extrapolate a -# concurrency figure across geometries. Shorter requests use the spare blocks, -# and a third LONG request queues rather than being refused. Verified: three -# concurrent 67-token requests answered in parallel, ~4.1 s each. -# -# WHY NOT --max-num-seqs 2 (measured 2026-10-02, single-variable boot probes -# against the then-deployed known-good shape, production stopped for each): -# SEQS 2 + MNBT 1024 -> Engine core initialization failed -# SEQS 2 + MNBT 2048 -> Engine core initialization failed -# SEQS 3 + MNBT 1024 -> healthy (792,069 tokens, 3.02x, arena 6,248) -# SEQS 3 + MNBT 2048 -> healthy (the deployed incumbent) -# The failure is torch.fx ConstraintViolationError: query_start_loc.size()[0] -# "marked dynamic but your code specialized it to be a constant (3)", raised -# from AOT build_guards(strict_error=True) during profile_run -> _dummy_run. -# model_runner.py sizes that input buffer max_num_reqs + 1 and _dummy_run sets -# num_reqs = min(max_num_batched_tokens, max_num_reqs), so the width is -# SEQS + 1 for ANY MNBT >= SEQS: 3 at SEQS 2, 4 at SEQS 3. MNBT only moves the -# compile RANGE ((1,1024) vs (1,2048)) and cannot dodge it — the 2x2 above is -# why MNBT is not a lever here. --enforce-eager bypasses the path but loses -# CUDA graphs. Same failure class killed the 2026-10-01 bf16 2 x 262144 shape. -# -# THE ARENA LEVER IS max_model_len, NOT KV_BYTES (measured 2026-10-02). The -# planner sizes expert cache as budget - non-expert - KV reserve - graphs - -# overhead, and the KV reserve is registry-priced at 9190 B/token x max_model_len: -# MAXLEN 262144 -> reserve 2.48 GiB -> arena 14.30 GiB / 6,248 slots -# MAXLEN 200704 -> reserve 1.91 GiB -> arena 14.87 GiB / 6,496 slots -# d(reserve) 0.57 GiB = d(arena) 0.57 GiB = +248 slots. The 6.1e9 -> 3.15e9 -# pool cut buys ZERO arena; it buys runtime HEADROOM (the shape it replaced -# died with 0 bytes free on a real 995-token request). To grow the arena, -# lower MAXLEN — not KV_BYTES. The OTHER arena lever is the overhead term: -# --expert-precise-memory (shape E) takes it from 7.60 to 5.25 GiB and lands -# those 2.35 GiB in the arena — 14.87 -> 17.22 GiB / 6,496 -> 7,522 slots. -# -# GATHER 32x64 stays: R4D_LRU_GATHER_CHUNKS=32 / R4D_LRU_GATHER_LANES=64 -# measured +18.7% decode vs default geometry and the win grows with depth. -# Do not drop it. -# CHUNKED PREFILL: the fork exposes the standard scheduler flags. --max-num-batched-tokens -# (MNBT) is the lever; --enable-chunked-prefill is an auto-flag (True in SchedulerConfig) -# and --prefill-schedule-interval is the gap between iterations. The running shape carries -# --max-num-batched-tokens 2048 with --max-model-len 200704, so chunked prefill is ON and -# already active: a 200704-token prompt is prefilled in ~98 iterations of 2048 tokens. -# MEASURED 2026-09-29 on the ad hoc qwen-fp8 container (same fork, same payload, TP=2): -# MNBT 8192 vs the default 2048 is NEUTRAL on decode at every depth, and small values -# re-chunk prefill only — they hurt TTFT and never touch decode, because a decode step is -# max_num_seqs x (spec+1) tokens, independent of MNBT. There is NO --ubatch-size -# equivalent in vLLM: that is a llama.cpp compute-buffer concept (the frank-small pair -# doubled on 2026-10-05 has no counterpart here). MNBT is therefore NOT a lever on this -# shape — the deployed 2048 is the fork's default — and every change re-enters the -# compile RANGE path, so FRANK_PRIMARY_CACHE_DIR pays one recompilation. -# MNBT 2048 (Frank's call, 2026-10-03; the lock was 1024). The measured law -# still holds: MNBT 512 doubles TTFT at 4064 ctx (2.46 s vs 1.36 s), 1024-4096 -# are interchangeable on decode, 12288 costs 2.5% plus arena. 2048 is ARENA-FREE -# by construction: the registry's overhead model is -# base + 0.22 GiB x max(0, (nbt - 4096)/1024) with the anchor at nbt 4096, so -# every rung up to and including 4096 prices the SAME 5.25 GiB overhead — and -# the fitted 5.25 was fitted at nbt 4096, i.e. our 2048 sits UNDER the anchor, -# so the activation term is covered by a figure measured at a bigger batch. -# SEQS 3 boots at both 1024 and 2048 (measured 2026-10-02; the SEQS=2 -# ConstraintViolationError is a SEQS property, not an MNBT one). Costs exactly -# one recompilation: the compile RANGE is part of the cache key, so -# FRANK_PRIMARY_CACHE_DIR pays for whichever rung is chosen once. -# PREFETCH IS ABSENT BY LAW: expert-cache telemetry measured the prefetcher at -# 36% precision (36,720 slabs prefetched, 13,369 served) — roughly half the -# gather traffic was waste. Never re-add --expert-prefetch without a fresh A/B. -# TunableOp stays OFF: CLAV_TUNABLEOP_SWEEP silently skips under expert offload -# unless CLAV_TUNABLEOP_WITH_OFFLOAD=1, and forcing it corrupts numerics. -# --expert-precise-memory is ON (shape E). It selects the fitted overhead -# (5.25 GiB) over the default (7.60 GiB = fitted + compile/graph-capture -# headroom); the 2.35 GiB difference lands in the expert arena. MEASURED -# 2026-10-02 by a boot probe on the shape-D base (production stopped, candidate -# booted AS the service, plan read off the engine's own log): -# runtime overhead 7.60 -> 5.25 GiB (registry Qwen4Exp…, precise -# gfx1201 TP2 nbt4096/seqs16/len262144) -# expert cache 14.87 -> 17.22 GiB -# slots 6,496 -> 7,522 (+1,026 = +16%, 2.344 MiB each) -# swap region 13.7 GiB / 5,986 slots, 133-134 per layer -# boot healthy in 420 s, no OOM, no knife-edge -# The flag was called a non-option after it crash-looped the 2 x 262144 bf16 -# boot (0.13 GiB free of 31.86 GiB, NRestarts=3 at ~6.7 min each) — that shape -# was already at the ceiling, so precise-memory was the last 2.35 GiB over, not -# the cause. At 200704 x 3 it fits with room; the run line also reports -# 'expert_precise_memory': True in non-default args, so a silent argparse drop -# would be visible in the boot log. -FRANK_PRIMARY_PRECISE_MEMORY=1 -# GPU_MAX_HW_QUEUES caps the HSA hardware queues per GPU (AMD's own tuning -# guide recommends 2 for this class of workload; the runtime default on Linux -# is 4). Verified a REAL knob, not a no-op: the string is present in -# /opt/rocm/core-10.0/lib/libhsa-runtime64.so. NOT A/B MEASURED — Frank's call, -# shipped as a deliberate CPU-side tune alongside the 32x64 gather geometry. -FRANK_PRIMARY_GPU_MAX_HW_QUEUES=2 -# Persistent compile cache. MEASURED 2026-10-02: the image ships NO -# /root/.cache at all (`ls: cannot access '/root/.cache'` in a fresh -# container), and after our boot it holds /root/.cache/vllm at 1,154,714,333 B -# in 4,665 files plus /root/.cache/comgr at 229 MB — both inside the -# container's ephemeral layer, so every `podman rm -f` (every deploy, restart -# and reboot) throws them away and re-pays the build. The engine's own log -# names the loss: `torch.compile took 181.20 s in total`, of which 'Compiling a -# graph for compile range (1, 1024) takes 173.19 s', followed by 'saved AOT -# compiled function to /root/.cache/vllm/torch_compile_cache/…'. Mounting a -# host dir at /root/.cache persists the vLLM AOT/torch_compile tree AND the -# comgr HIP-kernel JIT cache, and hides nothing (the image has no such dir). -# PROVEN LIVE 2026-10-02, two boots of this exact shape: -# cold `Compiling a graph for compile range (1, 1024) takes 179.24 s` -# `torch.compile took 188.26 s in total` -# warm `Directly load the compiled graph(s) for compile range (1, 1024) -# from the cache, took 2.143 s` -# `torch.compile took 3.97 s in total` -# `init engine (profile, create kv cache, warmup model) took 41.13 s -# (compilation: 3.98 s)` -# (Total time to /health is dominated by the 172.8 GiB payload load, not the -# compile — the saving is the ~184 s of inductor/AOT build.) -# Safe to grow/prune: entries are keyed by version + config hash, so a stale -# entry is a miss, never a wrong answer. -FRANK_PRIMARY_CACHE_DIR=/home/dank/.cache/frank-primary -FRANK_PRIMARY_MAXLEN=200704 -FRANK_PRIMARY_SEQS=3 -# The POOL is deliberately kept at shape D's 3.15e9 (option A): because the pool -# is not an arena lever, raising it back from a cut buys concurrency only, and -# 3.15e9 preserves the measured 1.99x while precise-memory grows the arena. A -# 2.2e9 pool was probed in the same window (277,686 tokens / 1.38x) and rejected -# as the shipped value; it remains the conservative arm if free VRAM ever binds. -# KV forced to 400,700 tokens (Frank, 2026-10-03). The engine prices this pool -# at 7,891.8 B/token at this shape and loses ~3.87 MB to block alignment, so -# 3,166,500,000 B is the ask that lands at/above 400,700 (verify the reported -# "GPU KV cache size" line on every deploy; it is not a constant ratio across -# geometries — 7,701 B/token at 262144). -FRANK_PRIMARY_KV_BYTES=3166500000 -FRANK_PRIMARY_KV_DTYPE=fp8 -FRANK_PRIMARY_MNBT=2048 -FRANK_PRIMARY_GATHER_CHUNKS=32 -FRANK_PRIMARY_GATHER_LANES=64 -# PREFIX CACHING: ON. An OFF experiment ran 2026-10-05 (below) and FAILED -# to stop the corruption, so the historical flag is back. -# MEASURED 2026-10-04, 18:46:07 PDT (5.6 h into the live boot, one running -# request, prefix hit rate 48.1%): 48 gdn_hip state faults in one 0.6 s -# burst, then the process-lifetime silence cap: -# [gdn_hip] ERROR gdn_decode_r: state out of ssm-cache range: max|S|=0 -# with non-finite values (slot=42 head=3 limit=3.40282e+38) fault #1-23 -# [gdn_hip] ERROR gdn_decode_r: ... (slot=42 head=8 ...) fault #24-46 -# [gdn_hip] ERROR: state fault report limit (32) reached; further faults -# are counted silently -# One physical SSM slot, two heads, BOTH ranks (heads 3 and 8), 23 layers -# each. The guard's own message in this build: "known Qwen3.8-Flash-Next -# (QFN) flaw under prefix caching and is being actively worked; output -# quality for this request may degrade. Mitigation: -# --no-enable-prefix-caching". -# MECHANISM: with prefix caching on, this fork puts QFN on -# mamba_cache_mode 'align' (the default-for-this-arch branch in -# config/vllm.py; 'all' is deprecated and 'none' is the no-prefix-cache -# mode, so align is the only sound mode WITH the flag). Align mode -# checkpoints each sequence's GDN (Gated DeltaNet) recurrent state into a -# mamba page every block, where block = 3136 tokens (the attention block -# size is raised to 3136 so the page sizes match exactly — the boot log's -# "Padding mamba page size by 0.13% ..."), and RE-SEEDS the state from that -# page whenever a request resumes from a cached prefix. The corrupt state -# landed at the 13th boundary: 13 x 3136 = 40,768 tokens — the "~40k" mark -# where the request's output turned repetitive/corrupted. The NaN stayed -# in the mamba page, so every later turn of that conversation re-seeded the -# same non-finite state for those two heads (silent: the fault-report cap -# was already spent), while fresh short sessions looked fine. -# RAM WAS "RULED OUT" THE SAME DAY — AND THAT WAS WRONG (RESOLVED below). -# What the evidence actually supported: zero MCE/EDAC records is the -# EXPECTED silence of consumer non-ECC memory (a silent flip raises no -# machine check); an 8 GiB pattern check over 60 s covers ~0.5 % of -# 246 GiB — no verdict on a speed setting; "4800 is the JEDEC floor on -# 5600-rated DIMMs" is a rating, not a stability proof at four-DIMM -# loading. Only the two-head NaN signature argument was load-bearing, -# and the revert test outvoted it. -# COST ACCEPTED: no prefix reuse, so each turn re-prefills the whole -# history instead of at most 3135 tail tokens (a full 40k context costs -# tens of seconds of TTFT per turn at this shape's prefill rate). A silently -# corrupted conversation is worse than a slower first token. -# RESULT (2026-10-05, later that day): THE MITIGATION DID NOT WORK. With -# --no-enable-prefix-caching deployed and mechanically verified (zero -# gdn_hip faults, a 40,771-token single-request boundary probe coherent), -# live long conversations STILL degraded at ~40k on the running endpoint -# (Frank, direct). The container serving that window logged ZERO faults -# while the corruption reproduced, so the align-mode page re-seed is not -# the (only) mechanism — see the runbook's RESULT note for the open -# hypotheses (re-fed poisoned transcript vs a fault-free long-context -# path such as fp8 KV at depth). Cost bought nothing: caching is back ON. -# The knob stays commented for a future fork build with a fixed GDN state -# path: set 1, redeploy, re-verify past 40,768 tokens (one conversation -# coherent AND zero gdn_hip lines). -# RESOLVED 2026-10-05 EVENING (Frank): IT WAS THE RAM. 4800 MT/s (in -# effect since the 2026-10-04 17:17 boot) was marginal; 4600 MT/s (since -# the 2026-10-05 12:21 boot) and the endpoint is clean — zero gdn_hip -# ERROR lines, zero MCE, and the live agent session on it past 140k -# context with no degradation. Both failure eras sit under the RAM: the -# 2026-10-04 NaN burst (corrupted weight/activation traffic poisoning a -# GDN state write — the guard-visible path) and the fault-free ~40k -# degradation (silent flips in KV/expert data, which no guard covers on -# non-ECC). CONSEQUENCE FOR THE KNOB: the OFF experiment was CONFOUNDED -# — both arms ran on the marginal speed — so it neither proved nor -# disproved the fork's align-mode flaw. Prefix caching stays ON (zero -# gdn_hip lines under heavy re-send traffic since the speed drop); flip -# the knob only if a fault recurs on stable RAM or the fork ships a fix. -# FRANK_PRIMARY_NO_PREFIX_CACHING=1 -# UTIL is an ARENA lever: the fork plans `budget = device_total x util`, and at -# 31.86 GiB/card every +0.01 is +0.32 GiB of expert cache (~+139 slots), and -# 0.982 is Frank's standing law: NEVER 1.00. The 1.00 attempt (2026-10-03, -# shape G v1) died in worker init BEFORE loading a byte of weights — the fork -# refuses budget >= device total with -# ValueError: Free memory on device cuda:N (31.29/31.86 GiB ...) -# on BOTH ranks at 09:53 and 09:54, NRestarts=2 — so 1.0 is not even a runtime -# knife-edge here, it is unbootable. The two historical ceiling deaths (the -# 2x262144 bf16 boot at 0.13 GiB free; the 5x262144 boot at "0 bytes free" on a -# real 995-token request) remain the reason the margin exists. Rollback note -# for the other direction: if a request ever dies with -# `Tried to allocate ... 0 bytes free`, the first knob back is THIS number -# down a notch, not the window. -FRANK_PRIMARY_UTIL=0.982 -# CUDA-graph plan term, priced at the measured capture. expert_plan.py charges -# estimate_cudagraph_memory() = BASE_MB + PER_GRAPH_MB x n_graphs (defaults -# 256/52 = 0.55 GiB for our 6-graph plan), and the estimator's own docstring -# says to tune these against the post-capture line. MEASURED twice on this -# shape: shape E reported "Graph capturing finished ... took 0.31 GiB" (TP0) and -# 0.17 GiB (TP1); shape F, with the 0.32 GiB we then charged, reported -# "took -0.06 GiB" — a NEGATIVE delta, i.e. capture fits inside memory the -# profiler had already counted. So the base pool (256 MiB) is the over-charge: -# charge 0 base + 12 MiB/graph = 0.07 GiB for the shipped no-spec 6-graph plan, -# which returns 0.25 GiB more to the arena while still paying per graph, so a -# future spec-decode shape (18 graphs) is charged 0.21 GiB rather than zero. -# Verify every deploy against the "Estimated CUDA graph memory" vs "Graph -# capturing finished ... took X GiB" lines; if capture ever goes positive and -# exceeds the charge, raise FRANK_PRIMARY_CUDAGRAPH_PER_GRAPH_MB back toward 52. -# A THIRD env var exists and is the only way to state the graph term outright: -# VLLM_CUDAGRAPH_MEM_ESTIMATE_MB pins the WHOLE term to an exact MiB total in -# one knob — estimate_cudagraph_memory() checks it FIRST and returns base/ -# per-graph untouched when set (verified in the shipped image, -# v1/worker/gpu/cudagraph_mem.py). We ship the BASE/PER_GRAPH pair instead so -# a future spec-decode shape re-prices automatically; switch to ESTIMATE_MB -# only if the drift check below ever shows the pair cannot track capture. -FRANK_PRIMARY_CUDAGRAPH_BASE_MB=0 -FRANK_PRIMARY_CUDAGRAPH_PER_GRAPH_MB=12 - -# HOST-region cap — the last arena-plan term that still autofit per boot. With -# no cap the planner's host bound is MemAvailable AT BOOT minus PLE 47.68 - -# 2 x 7.0 process - 5.0 api/engine - 5% margin (measured boot 2026-10-05: -# MemAvailable 237.97 -> 159.39 GiB usable across 2 ranks; the demand is 105.48 -# = 52.74/rank x 2 ranks, 23,040 swap-target experts). The same host reads 75.4 -# GiB available while running — the bound tracks whatever the co-tenants hold at -# boot, and a boot where that drops near the demand silently pins next-hottest -# experts VRAM-only instead (smaller swap region; the pinned_extra warning is -# the only trace). Pin 120 GiB: measured demand + 14% headroom, on a host with -# 246.1 GiB total. If a boot ever logs "expert-offload-mem … exceeds -# MemAvailable at plan time", THAT is the contract being broken — look at the -# co-tenant RAM before touching this number. Plan semantics verified in the -# shipped image (offloader/expert_plan.py): the cap binds as the bound, an -# exceedance only warns. -FRANK_PRIMARY_EXPERT_OFFLOAD_MEM=120 - -# --- optional engine knobs (unset == the flag is absent) --------------------- -# FRANK_PRIMARY_SPEC holds the WHOLE --speculative-config JSON; setting it adds -# the flag. THE RETIREMENT VERDICT WAS INVALID: c1248d4 (2026-10-02) retired MTP -# by comparing fp8+MTP4 (37.4-40.6 tok/s, 54% acceptance, 09-29) against the -# bf16 no-spec boot (36.4-39.4) — and bf16 then proved unbootable, so fp8 came -# back without MTP and fp8+spec was NEVER compared against fp8+no-spec on the -# same shape. The no-spec incumbent measures ~30 tok/s single-stream (below the -# fleet's 35 tok/s target); every vendor flash-next recipe ships MTP3; and our -# MTP4 run used the fork's default GREEDY draft sampling, while the recipes pass -# draft_sample_method=probabilistic (greedy drafts are rejected more often under -# temp>0 — the plausible cause of the 54%). MTP also re-adds drafter workspace: -# the old note says the 7.2 GiB/rank KV ceiling WAS the MTP4 workspace, and -# --expert-precise-memory spends that same budget, so an MTP arm must be tested -# with precise memory BOTH on and off. -# MTP3: '{"method":"mtp","num_speculative_tokens":3,"draft_sample_method":"probabilistic"}' -FRANK_PRIMARY_SPEC= -# FRANK_PRIMARY_COMPCONFIG holds the whole --compilation-config JSON. The -# capture-ladder law: the piecewise cudagraph ladder must contain -# num_speculative_tokens + 1 (MTP3 -> 4 -> ladder 4,8,12; MTP4 -> 5,10,15; -# MTP5 -> 6,12,18,24), because every draft step proposes n+1 tokens. The default -# ladder is multiples of 1+4 up to 512 then powers of two, so an MTP3 shape can -# miss its own step size. Pinning the ladder is ALSO the candidate fix for the -# SEQS>3 wall: query_start_loc is specialised to SEQS+1 and that constant must -# land on a captured graph size (3 is not in the default ladder, 4 is — -# measured 2026-10-02, and the vendor runs SEQS 16 on this model/TP2). -# combo_kernels false/true are BOTH in the shipped recipes, so it is not a law. -# MTP3 at SEQS 3: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[1,4,8,12],"max_cudagraph_capture_size":12}' -FRANK_PRIMARY_COMPCONFIG= -# FRANK_PRIMARY_GEN_CONFIG holds the whole --override-generation-config JSON. -# The payload's HF generation_config.json sets temperature 1.0 / top_p 0.95 / -# top_k 20 and the engine WARNS about it on every boot, so a client that sends -# no sampling fields gets creativity it never asked for. This is the one knob -# that changes ANSWERS, not speed. -# '{"max_tokens":65536,"temperature":0.7,"top_p":0.95,"top_k":20}' -FRANK_PRIMARY_GEN_CONFIG= - -# --- client contract: reasoning split, tools, preserve_thinking ---------------- -# Without --reasoning-parser the engine leaves the thinking text INLINE in -# `content` with a dangling `` and a null `reasoning` field (measured -# 2026-10-01: content = 'We need answer...Brief.\n\n391') — the -# malformed thinking block clients choke on. The qwen3 parser splits it -# properly. The GLM rollback (d91379d) had dropped all three of these; the -# working values are the ones ecaad36 shipped for this model. -FRANK_PRIMARY_REASONING_PARSER=qwen3 -# Tools: --enable-auto-tool-choice + the qwen3_xml parser is what makes -# tool_calls work here (without it, tool_choice auto returns 400). -FRANK_PRIMARY_TOOL_PARSER=qwen3_xml -# preserve_thinking keeps prior thinking blocks in the rendered prompt across -# turns; enable_thinking makes thinking the server-side default. Verified the -# payload's tokenizer_config template consumes both keys. -FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS='{"enable_thinking": true, "preserve_thinking": true}' +# --- FreeToken engine knobs (first-boot conservative, resize live) ----- +# Single GPU: the R9700 at index 0. HIP_VISIBLE_DEVICES=0 hides card 1 so +# the in-container index 0 IS the right card; --gpu 0 selects it. +FRANK_PRIMARY_FT_GPU=0 +# KV pool in TOKENS (not bytes — vLLM-era FRANK_PRIMARY_KV_BYTES is gone). +# 300,000 tokens x 24 KiB = 7.2 GiB; clears the harness worst case 177,356. +# Grows live with `ft ctl cache --kv `; shrink needs a restart. +FRANK_PRIMARY_FT_NUM_TOKENS=300000 +# Concurrent decode slots. 2 is the first-boot conservative value; the +# radix pool admits ~2 full-length 200,704-token conversations. Raise with +# --max-running-requests only after a real-traffic look at slot occupancy. +FRANK_PRIMARY_FT_MAX_RUNNING=2 +# Per-request output budget ceiling: the harness's maxTokens (36,864). +# Requests that omit max_tokens get this default instead of the engine's +# 32,768. +FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS=36864 +# GPU expert LRU cache as a FRACTION of all experts (~120 GiB fp8 total -> +# ~12 GiB VRAM at 0.10). First-boot conservative; resize live with +# `ft ctl cache --moe` after reading the boot ledger + ft ctl stats. +FRANK_PRIMARY_FT_MOE_RATE=0.10 +# Escape hatch for boot-ladder experiments (YaRN, --max-prefill-length, +# --moe-cpu-threads, --sampling-defaults none, ...). Unset -> no extra +# argv. Keep multi-flag values as one string; the serve script expands it +# unquoted on purpose. +FRANK_PRIMARY_FT_EXTRA_ARGS= diff --git a/scripts/frank-primary.service b/scripts/frank-primary.service index 512746c..583afb7 100644 --- a/scripts/frank-primary.service +++ b/scripts/frank-primary.service @@ -1,29 +1,26 @@ -# frank-primary.service — tcclaviger RDNA4 vLLM endpoint (orcarouter -# Qwen3.8-Flash-Next-Uncensored-FP8, fp8 KV, NO spec decode, 3 slots x 200704 -# context with the pool sized to ~2 full-length conversations, TP=2, tiered -# experts, 131-shard payload) on the 2x R9700 pair, dankserver :7770. -# 2026-09-29 cutover from the R9V radiance fork (git tag -# frank-primary-r9v-q8-20260929 is the pre-cutover state; its -# launcher/shim/setup-state machinery is retired and the state dirs stay on -# disk as rollback). +# frank-primary.service — FreeToken (PR #587 ROCm) endpoint (orcarouter +# Qwen3.8-Flash-Next-Uncensored-FP8, 131-shard payload) on ONE R9700 +# (gfx1201), dankserver :7770. 2026-10-05 cutover from the tcclaviger vLLM +# fork (git tag frank-primary-vllm-shapeG-20261003 is the pre-cutover +# state; the retired unit/env/serve trio is in that tag). # -# Same port, same served alias; the harness models.yml id is unchanged. -# The serve script drives `podman run` directly (the image entrypoint execs -# `vllm serve`) and then waits on the container. NOTE: that wait can return -# 0 even when the engine died mid-request (vLLM's API server drains and -# exits clean on a worker fault), which is why the restart policy is -# `always` — see the [Service] note. +# Same port, same served alias, same container identity as the unit name; +# the harness models.yml id is unchanged. The serve script drives +# `podman run` directly (the image carries `ft serve`) and then waits on +# the container. The image is built ON THE HOST from +# scripts/docker/frank-primary-freetoken.Dockerfile (FreeToken PR head +# 77a7201 in the PR's own ROCm 7.14 base image). # -# WHY A ROOT UNIT (not a rootless quadlet): the tiered-expert shape pins tens -# of GiB of host RAM (the expert host slabs, ~20.5 GiB per rank of pinned -# anonymous memory). Pinned pages are charged against RLIMIT_MEMLOCK; dank's -# login session on this host carries 8 MiB and a user quadlet is capped by the -# user manager's own hard limit — raising that means restarting -# user@1000.service and bouncing every user container on the host. A root unit -# grants LimitMEMLOCK=infinity to the dank process via PID1, and the image -# stays in dank's rootless podman storage. +# WHY A ROOT UNIT (not a rootless quadlet): FreeToken pins ~120 GiB of +# expert weights plus the ~47.7 GiB PLE ngram table in HOST RAM. Pinned +# pages are charged against RLIMIT_MEMLOCK; dank's login session on this +# host carries 8 MiB and a user quadlet is capped by the user manager's own +# hard limit — raising that means restarting user@1000.service and bouncing +# every user container on the host. A root unit grants +# LimitMEMLOCK=infinity to the dank process via PID1, and the image stays +# in dank's rootless podman storage. [Unit] -Description=frank-primary — tcclaviger vLLM Qwen3.8-Flash-Next-Uncensored-FP8 on 2x R9700 +Description=frank-primary — FreeToken (PR587) Qwen3.8-Flash-Next-Uncensored-FP8 on 1x R9700 After=network-online.target Wants=network-online.target # Never let the start-rate limiter give up after a crash loop (a failed LLM @@ -33,34 +30,35 @@ StartLimitIntervalSec=0 [Service] User=dank Group=dank -# The tiered-expert offloader pins host RAM; an RLIMIT_MEMLOCK smaller than -# the pinned set fails the load mid-start. PID1 grants infinity here — +# The expert LRU + PLE pin ~170 GiB of host RAM; an RLIMIT_MEMLOCK smaller +# than the pinned set fails the load mid-start. PID1 grants infinity here — # parity with the fleet's other LLM units. LimitMEMLOCK=infinity Environment="FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env" ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh # systemd stop: wait alone cannot stop the container child; stop it -# explicitly. vLLM 0.26 drains gracefully on SIGTERM. -ExecStop=/usr/bin/podman stop -t 120 frank-primary-qwen -# Boot cost: FP8 payload page-in plus torch compile/graph capture — -# ~15 min cold (measured 2026-09-29), faster warm. +# explicitly. ft serve drops connections on SIGTERM; no graceful drain +# contract is claimed (the OpenAI front end holds no queue to drain — +# in-flight decodes are lost, which a stop during a bench window accepts). +ExecStop=/usr/bin/podman stop -t 120 frank-primary +# Boot cost: 131-shard FP8 page-in + PLE pin + expert upload. vLLM-era +# budget (1800 s) carried forward; the FreeToken first boot measured here +# once it ran (see the runbook for the real number). TimeoutStartSec=1800 -# Restart=always, NOT on-failure (measured 2026-10-01): a runtime -# torch.OutOfMemoryError kills the engine while vLLM's API server still -# exits 0, so ExecStart returned clean, systemd logged "Deactivated -# successfully", and on-failure NEVER fired — the endpoint stayed down until -# a human noticed. `always` restarts on any container death; an explicit -# `systemctl stop` still wins (systemd does not restart a unit the operator -# stopped), so bench windows and deploys are unaffected. +# Restart=always, NOT on-failure (the 2026-10-01 vLLM lesson carries over: +# a clean API-server exit after a worker fault reads as success to +# systemd). `always` restarts on any container death; an explicit +# `systemctl stop` still wins, so bench windows and deploys are unaffected. Restart=always RestartSec=10 -# ExecStop allows the engine 120 s to drain (`podman stop -t 120`); without an -# explicit TimeoutStopSec systemd's 90 s default cuts the stop short, kills it, -# and records 'Failed with result timeout' (measured 2026-10-01 13:27:17). -# Must exceed the podman grace with margin for the engine's own shutdown. +# ExecStop allows the engine 120 s to drain (`podman stop -t 120`); without +# an explicit TimeoutStopSec systemd's 90 s default cuts the stop short, +# kills it, and records 'Failed with result timeout' (measured 2026-10-01 +# 13:27:17). Must exceed the podman grace with margin. TimeoutStopSec=180 -# The engine pins tens of GiB of host RAM (expert host slabs); it is the -# last process on this host that should be chosen by the kernel OOM killer. +# The engine pins tens of GiB of host RAM (expert rows + PLE table); it is +# the last process on this host that should be chosen by the kernel OOM +# killer. OOMScoreAdjust=-500 [Install] diff --git a/scripts/serve-frank-primary.sh b/scripts/serve-frank-primary.sh index 2d8387a..22697c6 100755 --- a/scripts/serve-frank-primary.sh +++ b/scripts/serve-frank-primary.sh @@ -1,58 +1,31 @@ #!/usr/bin/env bash -# serve-frank-primary.sh — frank-primary endpoint (tcclaviger RDNA4 vLLM fork, -# orcarouter Qwen3.8-Flash-Next-Uncensored-FP8, 2x R9700, -# dankserver :7770; 2026-09-29 cutover from the R9V radiance fork — git tag -# frank-primary-r9v-q8-20260929 is the pre-cutover state). -# -# The engine is driven by `podman run` DIRECTLY: the image's entrypoint execs -# `vllm serve "$@"`, so the run line below IS the engine argv. This replaces -# the R9V launcher/shim machinery (that fork needed a docker->podman shim and -# a 104-key on-host setup state; this stack is fully expressed by flags, and -# every flag here was qualified by boot probe on 2026-09-29 — see -# frank-primary.env for the qualification record). -# -# Shape (2026-10-03 shape F, on the 2026-10-02 shape E base): MAXLEN 200704 x 3 -# slots, fp8 KV, KV_BYTES -# 3,150,000,000 PER RANK — the POOL, not the slot count, caps resident -# conversations at ~2 full-length (measured: 398,658 tokens = 1.99x); -# shorter requests use the spare blocks. --max-num-seqs stays 3 because SEQS=2 -# CANNOT boot this model -# (torch.fx ConstraintViolationError on query_start_loc, measured at both -# MNBT 1024 and 2048 — the 2x2 lives in frank-primary.env). Speculative decode -# is OFF today by ENV KNOB, not by measurement: FRANK_PRIMARY_SPEC / _COMPCONFIG -# / _GEN_CONFIG hold the whole JSON value and unset means no flag. The 2026-10-02 -# retirement of MTP compared the wrong pair — see frank-primary.env. -# --max-num-batched-tokens 2048 (Frank's call 2026-10-03; arena-free because the -# registry prices overhead flat up to its nbt 4096 anchor), MoE gather geometry -# R4D_LRU_GATHER_CHUNKS=32 / R4D_LRU_GATHER_LANES=64 (+18.7% decode, grows -# with depth), prefix caching OFF 2026-10-05 (the fork's QFN align-mode GDN -# state fault — the evidence block is in frank-primary.env), expert prefetch -# OFF (36% precision measured), -# TunableOp off. Shape E adds --expert-precise-memory (fitted 5.25 GiB overhead -# instead of the 7.60 GiB default -> expert arena 17.22 GiB / 7,522 slots, +16%) -# and GPU_MAX_HW_QUEUES=2; shape F pins the plan's CUDA-graph term to the -# measured capture cost (VLLM_CUDAGRAPH_MEM_BASE_MB / _PER_GRAPH_MB, container -# env — the analytic default charges 0.55 GiB where capture took 0.31 GiB, so -# the 0.24 GiB difference goes to the arena) and raises util 0.98 -> 0.982, -# Frank's cap. The native-262144 and fp8+MTP4 shapes are retired to -# git tag frank-primary-fp8-mtp-20261001. -# -# THE COMPILE CACHE IS MOUNTED, and that is a boot-cost fix, not a tuning knob: -# the image ships no /root/.cache at all, our first boot writes 1.1 GB of vLLM -# AOT/torch_compile artifacts plus 229 MB of comgr JIT kernels into the -# container's ephemeral layer, and `podman rm -f` deletes them — so every -# deploy/restart/reboot re-pays `torch.compile took 181.20 s in total` (173.19 s -# of it one graph). Mounting FRANK_PRIMARY_CACHE_DIR at /root/.cache makes all -# later boots load the compiled artifact instead, and hides nothing. PROVEN -# LIVE 2026-10-02 on consecutive boots of this shape: cold `torch.compile took -# 188.26 s`, warm `Directly load the compiled graph(s) for compile range -# (1, 1024) from the cache, took 2.143 s` / `torch.compile took 3.97 s in -# total` / `init engine … took 41.13 s (compilation: 3.98 s)`. +# serve-frank-primary.sh — frank-primary endpoint (FreeToken PR #587 ROCm +# build, orcarouter Qwen3.8-Flash-Next-Uncensored-FP8, ONE R9700 gfx1201, +# dankserver :7770; 2026-10-05 cutover from the tcclaviger vLLM fork — git +# tag frank-primary-vllm-shapeG-20261003 is the pre-cutover state). # +# The engine is driven by `podman run` DIRECTLY and the run line below IS +# the engine argv: the image carries FreeToken installed at pinned PR head +# 77a7201 (scripts/docker/frank-primary-freetoken.Dockerfile builds it from +# the PR's own docs/install_amd.md stack). Unlike the vLLM fork there is no +# per-boot planner: experts live in pinned host RAM with an LRU GPU cache, +# KV is an explicit token pool, and the boot ledger in the log names +# Shape: 1 GPU (first-boot conservative — the second R9700 sits idle; +# --tensor-parallel-size EXISTS in FreeToken, so a 2-card TP shape is future +# tuning once this single-card shape has a decode baseline), KV pool 300,000 +# tokens (24 KiB/token f16 across the 12 full-attention layers = 7.2 GiB; +# clears the harness worst case of 177,356 tokens and admits one NATIVE +# 262,144-token conversation), MoE LRU at 10% of all experts (~12 GiB), +# 2 decode slots. First-boot conservative; `ft ctl cache --kv/--moe` grows +# pools live, and the env comment block carries the resize law. Flag +# surface verified against docs/cli.md of the pinned head AND against +# `ft serve --help` inside the built image; every value here is env-driven +# (frank-primary.env). + # The root unit (scripts/frank-primary.service, User=dank, -# LimitMEMLOCK=infinity) grants the pin budget for the tiered-expert host -# slabs (~20.5 GiB/rank of pinned anonymous memory). A plain shell session -# also works, but the unit is the fleet-parity boot-persistent shape. +# LimitMEMLOCK=infinity) grants the pin budget: ~120 GiB of expert weights +# plus the ~47.7 GiB PLE table pin in host RAM. A plain shell session also +# works, but the unit is the fleet-parity boot-persistent shape. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -67,27 +40,30 @@ set -a source "$ENV_FILE" set +a -: "${FRANK_PRIMARY_IMAGE:?FRANK_PRIMARY_IMAGE must name the digest-pinned image}" +: "${FRANK_PRIMARY_IMAGE:?FRANK_PRIMARY_IMAGE must name the FreeToken image}" : "${FRANK_PRIMARY_MODEL_DIR:?FRANK_PRIMARY_MODEL_DIR must name the FP8 payload}" : "${FRANK_PRIMARY_MODEL_SHARDS:?FRANK_PRIMARY_MODEL_SHARDS must give the shard count}" : "${FRANK_PRIMARY_CONTAINER:?FRANK_PRIMARY_CONTAINER must name the container}" : "${FRANK_PRIMARY_SERVED_NAME:?FRANK_PRIMARY_SERVED_NAME must name the alias}" : "${FRANK_PRIMARY_PUBLISH:?FRANK_PRIMARY_PUBLISH must be host:container}" -: "${FRANK_PRIMARY_MAXLEN:?FRANK_PRIMARY_MAXLEN must be set}" -: "${FRANK_PRIMARY_SEQS:?FRANK_PRIMARY_SEQS must be set}" -: "${FRANK_PRIMARY_KV_BYTES:?FRANK_PRIMARY_KV_BYTES must be set (PER RANK)}" -: "${FRANK_PRIMARY_KV_DTYPE:?FRANK_PRIMARY_KV_DTYPE must be set (auto = bf16 KV)}" -: "${FRANK_PRIMARY_REASONING_PARSER:?FRANK_PRIMARY_REASONING_PARSER must be set (qwen3)}" -: "${FRANK_PRIMARY_TOOL_PARSER:?FRANK_PRIMARY_TOOL_PARSER must be set (qwen3_xml)}" -: "${FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS:?FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS must be set}" -# Capture podman's output BEFORE grepping: under `set -o pipefail`, the -# classic `podman images | grep -q` pattern races — grep exits on its first -# match, podman dies SIGPIPE (141) writing the remaining lines, and pipefail -# turns the 141 into "no match" (the 2026-09-30 deploy refusal). -DIGESTS="$(podman images --format '{{.RepoDigests}}')" -if ! grep -qF "${FRANK_PRIMARY_IMAGE#*@}" <<<"$DIGESTS"; then - echo "serve-frank-primary: pinned image absent from rootless storage: $FRANK_PRIMARY_IMAGE" >&2 - echo " fetch it: podman pull $FRANK_PRIMARY_IMAGE" >&2 +: "${FRANK_PRIMARY_FT_GPU:?FRANK_PRIMARY_FT_GPU must name the GPU index}" +: "${FRANK_PRIMARY_FT_NUM_TOKENS:?FRANK_PRIMARY_FT_NUM_TOKENS must size the KV pool}" +: "${FRANK_PRIMARY_FT_MAX_RUNNING:?FRANK_PRIMARY_FT_MAX_RUNNING must be set}" +: "${FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS:?FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS must be set}" +: "${FRANK_PRIMARY_FT_MOE_RATE:?FRANK_PRIMARY_FT_MOE_RATE must be set}" + +# The image gate is EXISTENCE IN STORAGE, not a registry digest: the tag is +# built ON THE HOST (podman build from +# scripts/docker/frank-primary-freetoken.Dockerfile) and local builds carry +# no RepoDigests entry — the 362-line vLLM-era RepoDigests grep refused +# exactly this shape. Pinning lives in the Dockerfile instead: the FreeToken +# source is checked out at a hardcoded PR-head SHA and the base image is +# digest-pinned there. `podman image exists` accepts both a local tag and a +# name@digest ref, and it is one command — no pipefail/grep race (the +# 2026-09-30 lesson) for either shape. +if ! podman image exists "$FRANK_PRIMARY_IMAGE"; then + echo "serve-frank-primary: image absent from rootless storage: $FRANK_PRIMARY_IMAGE" >&2 + echo " build it: podman build -t $FRANK_PRIMARY_IMAGE -f scripts/docker/frank-primary-freetoken.Dockerfile scripts/docker/" >&2 exit 1 fi @@ -114,8 +90,8 @@ fi # A crashed/killed unit can leave the container name registered (systemd # SIGKILLs conmon before cleanup runs). Clear OUR name plus the legacy -# identities (the retired R9V container and the ad hoc experiment container -# both hold these GPUs); all idempotent. +# identities (the vLLM-era frank-primary-qwen, the retired R9V container and +# the ad hoc experiment containers all hold this GPU); all idempotent. for CTR in $FRANK_PRIMARY_LEGACY_CONTAINERS "$FRANK_PRIMARY_CONTAINER"; do podman rm -f "$CTR" >/dev/null 2>&1 || true done @@ -123,74 +99,34 @@ done LOG=/home/dank/logs/frank_primary.log mkdir -p "$(dirname "$LOG")" -# Persistent compile cache (see the header). Create it before the mount so even -# the first, cold boot has a target; unset -> no mount, exactly the old -# ephemeral behaviour. It carries /root/.cache/vllm (AOT + torch_compile) and -# /root/.cache/comgr (HIP kernel JIT). -CACHE_DIR="${FRANK_PRIMARY_CACHE_DIR:-}" -if [ -n "$CACHE_DIR" ]; then - mkdir -p "$CACHE_DIR" -fi - # --device /dev/dri is the WHOLE DIRECTORY (individual renderD nodes enumerate -# without the card nodes and HIP refuses contexts — "No CUDA GPUs are -# available" while device_count() says 2). --ipc=host for the engine's shared -# memory; memlock unbounded for the pinned host slabs. -# The entrypoint already execs `vllm serve "$@"`, so no `serve` here. -# Prefix caching: ON — the shipped contract. FRANK_PRIMARY_NO_PREFIX_- -# CACHING=1 flips to the fork-named mitigation (--no-enable-prefix-caching) -# for the QFN align-mode GDN state fault; it was tried 2026-10-05 and -# REVERTED the same day (live conversations still degraded at ~40k with -# caching off and zero faults — see the evidence block in -# frank-primary.env). NO comment lines inside the backslash chain below — -# a comment there terminates the continuation and the next line runs as -# its own command (the 2026-10-05 exit-127 crash loop). -# The ${VAR:-word} / ${VAR:+word} double-expansion for the two directions is -# AMBIGUOUS bash (${VAR:--flag} parses as a substring offset when set — -# measured 2026-10-05: rendered "--no-enable-prefix-caching1"), so the flag -# is branched explicitly: exactly one of the two reaches the engine. -if [ -n "${FRANK_PRIMARY_NO_PREFIX_CACHING:-}" ]; then - FRANK_PRIMARY_PREFIX_CACHE_FLAG=--no-enable-prefix-caching -else - FRANK_PRIMARY_PREFIX_CACHE_FLAG=--enable-prefix-caching -fi +# without the card nodes and HIP refuses contexts). --ipc=host for the +# engine's shared memory; memlock unbounded for the ~120 GiB pinned expert +# rows + the 47.7 GiB PLE pin (the unit's LimitMEMLOCK=infinity is the other +# half of that law). +# HIP_VISIBLE_DEVICES pins visibility to ONE card so in-container index 0 is +# the right R9700; --gpu 0 selects it inside the visible set. podman run -d --name "$FRANK_PRIMARY_CONTAINER" \ --publish "$FRANK_PRIMARY_PUBLISH" \ --ipc=host --ulimit memlock=-1:-1 \ --cap-add SYS_PTRACE \ --device /dev/kfd --device /dev/dri --device /dev/mem \ - -e HIP_VISIBLE_DEVICES=0,1 -e VLLM_ROCM_USE_AITER=0 \ - ${FRANK_PRIMARY_GPU_MAX_HW_QUEUES:+-e GPU_MAX_HW_QUEUES="$FRANK_PRIMARY_GPU_MAX_HW_QUEUES"} \ - ${FRANK_PRIMARY_GATHER_CHUNKS:+-e R4D_LRU_GATHER_CHUNKS="$FRANK_PRIMARY_GATHER_CHUNKS"} \ - ${FRANK_PRIMARY_GATHER_LANES:+-e R4D_LRU_GATHER_LANES="$FRANK_PRIMARY_GATHER_LANES"} \ - ${FRANK_PRIMARY_CUDAGRAPH_BASE_MB:+-e VLLM_CUDAGRAPH_MEM_BASE_MB="$FRANK_PRIMARY_CUDAGRAPH_BASE_MB"} \ - ${FRANK_PRIMARY_CUDAGRAPH_PER_GRAPH_MB:+-e VLLM_CUDAGRAPH_MEM_PER_GRAPH_MB="$FRANK_PRIMARY_CUDAGRAPH_PER_GRAPH_MB"} \ - ${CACHE_DIR:+-v "$CACHE_DIR":/root/.cache} \ + -e HIP_VISIBLE_DEVICES="$FRANK_PRIMARY_FT_GPU" \ -v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \ - "$FRANK_PRIMARY_IMAGE" /models \ - --served-model-name "$FRANK_PRIMARY_SERVED_NAME" --port 7770 \ - --tensor-parallel-size 2 \ - ${FRANK_PRIMARY_DCP:+--decode-context-parallel-size "$FRANK_PRIMARY_DCP"} \ - --max-model-len "$FRANK_PRIMARY_MAXLEN" \ - --max-num-seqs "$FRANK_PRIMARY_SEQS" \ - ${FRANK_PRIMARY_MNBT:+--max-num-batched-tokens "$FRANK_PRIMARY_MNBT"} \ - ${FRANK_PRIMARY_PRECISE_MEMORY:+--expert-precise-memory} \ - ${FRANK_PRIMARY_EXPERT_OFFLOAD_MEM:+--expert-offload-mem "$FRANK_PRIMARY_EXPERT_OFFLOAD_MEM"} \ - --gpu-memory-utilization "$FRANK_PRIMARY_UTIL" \ - --kv-cache-memory-bytes "$FRANK_PRIMARY_KV_BYTES" \ - --kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE" \ - "$FRANK_PRIMARY_PREFIX_CACHE_FLAG" \ - --enable-auto-tool-choice --tool-call-parser "$FRANK_PRIMARY_TOOL_PARSER" \ - --reasoning-parser "$FRANK_PRIMARY_REASONING_PARSER" \ - --default-chat-template-kwargs "$FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS" \ - ${FRANK_PRIMARY_SPEC:+--speculative-config "$FRANK_PRIMARY_SPEC"} \ - ${FRANK_PRIMARY_COMPCONFIG:+--compilation-config "$FRANK_PRIMARY_COMPCONFIG"} \ - ${FRANK_PRIMARY_GEN_CONFIG:+--override-generation-config "$FRANK_PRIMARY_GEN_CONFIG"} \ - --trust-remote-code \ + "$FRANK_PRIMARY_IMAGE" \ + ft serve --model /models \ + --served-model-name "$FRANK_PRIMARY_SERVED_NAME" \ + --host 0.0.0.0 --port 7770 \ + --gpu "$FRANK_PRIMARY_FT_GPU" \ + --num-tokens "$FRANK_PRIMARY_FT_NUM_TOKENS" \ + --max-running-requests "$FRANK_PRIMARY_FT_MAX_RUNNING" \ + --max-output-tokens "$FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS" \ + --moe-cache-rate "$FRANK_PRIMARY_FT_MOE_RATE" \ + ${FRANK_PRIMARY_FT_EXTRA_ARGS:+$FRANK_PRIMARY_FT_EXTRA_ARGS} \ > "$LOG" 2>&1 # From here on, this script's lifetime IS the container's: wait returns the # container's exit code, so a crash is a nonzero ExecStart and the unit's -# Restart=on-failure is real. (systemd stop goes through ExecStop in the -# unit — podman wait alone cannot stop the container child.) +# Restart=always is real. (systemd stop goes through ExecStop in the unit — +# podman wait alone cannot stop the container child.) exec podman wait "$FRANK_PRIMARY_CONTAINER" diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats index 771fd24..3a77496 100755 --- a/tests/frank_primary_unit.bats +++ b/tests/frank_primary_unit.bats @@ -1,26 +1,25 @@ #!/usr/bin/env bats # Seam: frank-primary — the Dankserver LLM endpoint on :7770. # -# This file asserts the tcclaviger RDNA4 vLLM ROOT-UNIT shape (the -# 2026-10-01 BF16 cutover, the overnight redo campaign's verdict): the serve -# script drives `podman run` DIRECTLY (the image entrypoint execs -# `vllm serve`), with every engine flag qualified by boot probe and carried -# in the repo env file. The seam pins what the flags alone do not say: the -# KV qualification law (the night ladder: 262144/245760 OOM at graph -# capture; 241664 boots at 732,359 tokens / 3.03x, util 0.98; -# the bf16-KV trade (double the attention-KV bytes/token, ~14% decode cost, -# clean cache + 262k windows), the automatic-offload law (the fork plans -# the tiered-expert offload itself; no flag exists to set), the self-healing -# law (the run returns before the engine is up, so the serve script must -# wait on the container for Restart=on-failure to mean anything), the -# memlock law that makes the pinned host rows legal, the deployer's -# image/payload gates with the false-green MainPID check, and the measured -# laws: expert prefetch ABSENT (36% precision), NO speculative decode (the -# no-spec family measured fastest once the prefetch tax was removed; MTP -# re-measures pending), gather geometry 32x64 PRESENT (+18.7% decode), -# MNBT 2048, the CUDA-graph plan term pinned to the measured capture cost, and -# the client contract (reasoning parser qwen3, -# tool parser qwen3_xml, preserve_thinking on). +# This file asserts the FreeToken (PR #587 ROCm) ROOT-UNIT shape (the +# 2026-10-05 cutover from the tcclaviger RDNA4 vLLM fork, git tag +# frank-primary-vllm-shapeG-20261003): the serve script drives `podman run` +# DIRECTLY (the image carries `ft serve`, built on the host from +# scripts/docker/frank-primary-freetoken.Dockerfile), one R9700, with every +# engine value carried in the repo env file. The seam pins what flags alone +# do not say: the KV-pool math (300k tokens at 24 KiB/token clears the +# harness worst case of 177,356), the deployed single-GPU shape +# (--tensor-parallel-size EXISTS in FreeToken — a 2-card TP shape is future +# tuning; HIP_VISIBLE_DEVICES pins THIS shape to one card), the host-RAM +# law (experts + PLE pin ~170 GiB — the memlock law that makes the pin +# legal), the self-healing law (the run returns before the engine is up, so +# the serve script must wait on the container for Restart=always to mean +# anything), the image-existence gate (a local build carries no +# RepoDigests — the old digest grep refused exactly this shape; pinning +# lives in the Dockerfile), the deployer's payload gates with the +# false-green MainPID check, and the client contract (parsers AUTO: +# reasoning_content emission is the engine's native behavior — models.yml +# maps reasoningContentField on the harness side, not here). load helpers @@ -29,29 +28,30 @@ SERVE="$REPO_ROOT/scripts/serve-frank-primary.sh" ENV_FILE="$REPO_ROOT/scripts/frank-primary.env" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" +DOCKERFILE="$REPO_ROOT/scripts/docker/frank-primary-freetoken.Dockerfile" -@test "frank-primary unit, serve script and env exist in scripts/" { +@test "frank-primary unit, serve script, env and Dockerfile exist" { [ -f "$UNIT" ] [ -x "$SERVE" ] [ -f "$ENV_FILE" ] + [ -f "$DOCKERFILE" ] } -@test "frank-primary is boot-persistent, self-healing and unprivileged" { +@test "frank-primary unit is boot-persistent, self-healing and unprivileged" { # Root unit parity with frank-large/frank-small: enable is the [Install] - # section, the restart policy survives a crash, and the tiered-expert - # pin needs an unbounded memlock (a root unit grants it via PID1; the - # dank session's own hard limit is 8 MiB and cannot be raised without - # restarting user@1000.service). + # section, the restart policy survives a crash, and the pinned host RAM + # (expert rows + PLE table) needs an unbounded memlock (a root unit + # grants it via PID1; the dank session's own hard limit is 8 MiB). grep -q '^WantedBy=multi-user.target$' "$UNIT" - # Restart=always, not on-failure: a runtime worker OOM makes vLLM's API - # server exit CLEAN, so ExecStart returns 0 and on-failure never fires - # (measured 2026-10-01 — the endpoint stayed down until a human noticed). + # Restart=always, not on-failure: the vLLM-era lesson carries over — a + # clean front-end exit after a worker fault reads as success to systemd, + # so on-failure never fires. grep -q '^Restart=always$' "$UNIT" ! grep -q '^Restart=on-failure$' "$UNIT" - # Exiting the engine takes as long as vLLM's drain: ExecStop runs - # `podman stop -t 120`, so the stop timeout MUST exceed 120 s or systemd - # kills the stop and records 'Failed with result timeout' (measured - # 2026-10-01). Guard the relationship, not just the literal. + # ExecStop runs `podman stop -t 120`, so the stop timeout MUST exceed + # 120 s or systemd kills the stop and records 'Failed with result + # timeout' (measured 2026-10-01). Guard the relationship, not the + # literal. stop_t=$(awk -F= '/^TimeoutStopSec=/{print $2}' "$UNIT") [ -n "$stop_t" ] && [ "$stop_t" -gt 120 ] grep -q '^ExecStop=.*podman stop -t 120' "$UNIT" @@ -69,320 +69,117 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" grep -q 'FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env' "$UNIT" } -@test "the unit stops the container child explicitly" { +@test "the unit stops the container child explicitly — container identity is the unit name" { # The serve script blocks on podman wait, which cannot stop the # container child; systemd stop goes through ExecStop or the engine - # outlives the unit and holds both GPUs. The container name must match - # the env file's identity. + # outlives the unit and holds the GPU. Frank's 2026-10-05 call: the + # container name IS the service name (frank-primary), and the vLLM-era + # name frank-primary-qwen moved into the legacy cleanup list. grep -q '^ExecStop=/usr/bin/podman stop' "$UNIT" - grep -q 'frank-primary-qwen' "$UNIT" - grep -q '^FRANK_PRIMARY_CONTAINER=frank-primary-qwen$' "$ENV_FILE" + grep -q 'podman stop -t 120 frank-primary$' "$UNIT" + grep -q '^FRANK_PRIMARY_CONTAINER=frank-primary$' "$ENV_FILE" + grep -q 'frank-primary-qwen' "$ENV_FILE" } -@test "the serve shape is direct podman run, not a launcher or shim" { - # The tcclaviger stack is fully expressed by flags; the R9V launcher, - # docker->podman shim and setup-state layering are the RETIRED shape. +@test "the serve shape is direct podman run, not a launcher, shim, or the vLLM fork" { grep -q 'podman run' "$SERVE" ! grep -q 'launch.sh' "$SERVE" ! grep -q 'podman-shim' "$SERVE" ! grep -q 'R9V_STATE_FILE' "$SERVE" - ! grep -q -- '--model-draft' "$SERVE" - ! grep -q -- '--spec-type' "$SERVE" + # The tcclaviger vLLM fork is RETIRED — its image name and its + # fork-specific flags must not survive in the FreeToken run line + # (TP=2 is gone with the fork; FreeToken serves ONE GPU). + ! grep -q 'tcclaviger' "$SERVE" + ! grep -q 'tcclaviger' "$ENV_FILE" + ! grep -q -- '--tensor-parallel-size' "$SERVE" + ! grep -q -- '--kv-cache-dtype' "$SERVE" + ! grep -q -- '--max-num-seqs' "$SERVE" } -@test "the qualified engine shape is in the env file" { - # 2026-10-02 shape D. History: the 2026-10-01 night ladder put SEQS=3 - # bf16 at 241664 (KV 732,359 / 3.03x); SEQS=6 fp8 does NOT fit (6 x - # 241664 asked 10.70 GiB against 9.18 GiB free); SEQS=5 fp8 x 262144 - # BOOTED (1,350,182 / 5.15x) but left ~0 bytes free and a real 995-token - # request killed the worker with `Tried to allocate 50.00 MiB ... 0 bytes - # free`. UTIL is a REAL arena lever (the fork plans budget = total x util) - # but a weak one, and Frank caps it at 0.982 — never 1.00 (see - # FRANK_PRIMARY_UTIL for the two ceiling deaths that justify the margin). - # SEQS=2 is IMPOSSIBLE on this model, measured 2026-10-02 by - # single-variable boot probe against the then-deployed shape: SEQS 2 fails - # at BOTH MNBT 1024 and 2048 with torch.fx ConstraintViolationError - # (query_start_loc.size()[0] marked dynamic but specialized to a constant - # (3)); SEQS 3 passes at both. model_runner sizes that buffer - # max_num_reqs + 1, so the width is SEQS + 1 and MNBT only moves the - # compile RANGE. - # So the shape is 3 slots with a pool that admits only ~2 full-length - # conversations: measured live, KV_BYTES 3.15e9/rank reports 398,658 - # tokens = 1.99x for a 200704-token request (7,892 B/token at THIS shape; - # the per-token price is shape-dependent, 7,701 at 262144). Shorter - # requests use the spare blocks; a third LONG request queues. - # The arena is sized by max_model_len, NOT by KV_BYTES: the planner - # reserves registry 9190 B/token x max_model_len (2.48 GiB at 262144 -> - # arena 6,248 slots; 1.91 GiB at 200704 -> 6,496). The pool cut buys - # runtime headroom, not arena. - grep -q '^FRANK_PRIMARY_MAXLEN=200704$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_SEQS=3$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_UTIL=0.982$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_KV_BYTES=3166500000$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_MNBT=2048$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_KV_DTYPE=fp8$' "$ENV_FILE" - # --expert-precise-memory IS set (shape E, 2026-10-02). It was refused - # while the 2 x 262144 bf16 boot sat 0.13 GiB under the 31.86 GiB ceiling - # (arena 16.65 device-resident, NRestarts=3 at ~6.7 min each) — there it was - # the last 2.35 GiB over the line, not the cause. Probed on the shape-D base - # with production stopped: boots healthy in 420 s, plan reports runtime - # overhead 5.25 GiB (not 7.60), expert cache 17.22 GiB / 7,522 slots - # (+1,026 / +16% over 6,496). The pool is NOT cut with it (option A): the - # pool is not an arena lever, so 3.15e9 keeps the measured 1.99x. - grep -q '^FRANK_PRIMARY_PRECISE_MEMORY=1$' "$ENV_FILE" - # GPU_MAX_HW_QUEUES caps HSA hw queues per GPU; real knob (the literal is in - # libhsa-runtime64.so), passed inside the container as a bare name — the env - # file keeps only the prefixed form. - grep -q '^FRANK_PRIMARY_GPU_MAX_HW_QUEUES=2$' "$ENV_FILE" - # The compile cache is a BOOT-COST fix: the image ships no /root/.cache at - # all, our boot writes 1.1 GB of vLLM AOT artifacts (4,665 files) plus - # 229 MB of comgr JIT kernels into the ephemeral layer, and every - # `podman rm -f` throws them away — re-paying `torch.compile took 181.20 s` - # (173.19 s of it one graph) on every deploy, restart and reboot. - grep -q '^FRANK_PRIMARY_CACHE_DIR=/home/dank/.cache/frank-primary$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_GATHER_CHUNKS=32$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_GATHER_LANES=64$' "$ENV_FILE" - # Prefix caching is ON (reverted 2026-10-05 the same day it was - # disabled): the fork's QFN align-mode GDN state fault (48 gdn_hip - # state faults, slot 42 heads 3/8, at the 13th 3136-token boundary = - # 40,768 — the "~40k corruption" report) was mitigated with - # --no-enable-prefix-caching, but live long conversations STILL - # degraded at ~40k with caching off and zero faults logged — cost - # with no fix, so the historical flag returned. The knob stays - # commented in the env file for a future fixed fork build; flip it to - # 1 only together with a re-verification past 40,768 tokens. - grep -qE '^# FRANK_PRIMARY_NO_PREFIX_CACHING=1$' "$ENV_FILE" +@test "the FreeToken engine shape is in the env file" { + # KV pool: TOKENS, not bytes. 24 KiB/token (12 full-attention layers x + # 2 KV heads x 256 head_dim x K+V x f16) — the 36 GDN layers use fixed + # per-seq state, not per-token KV. 300,000 tokens = 7.2 GiB and clears + # the harness worst case (140,492-token prompt + 36,864 max_tokens = + # 177,356); one NATIVE 262,144-token conversation + full budget also + # fits. The arithmetic guard keeps future resizes honest. + grep -q '^FRANK_PRIMARY_FT_NUM_TOKENS=300000$' "$ENV_FILE" + nt=$(awk -F= '/^FRANK_PRIMARY_FT_NUM_TOKENS=/{print $2}' "$ENV_FILE") + [ "$nt" -ge 177356 ] + # First-boot conservative shape: 2 decode slots, 10% expert LRU + # (~12 GiB of the ~120 GiB fp8 expert set), output ceiling = the + # harness maxTokens, single R9700 at index 0. + grep -q '^FRANK_PRIMARY_FT_MAX_RUNNING=2$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_FT_MOE_RATE=0.10$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS=36864$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_FT_GPU=0$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_MODEL_SHARDS=131$' "$ENV_FILE" } @test "the serve script wires the engine flags from the env" { - grep -q -- '--max-num-seqs "$FRANK_PRIMARY_SEQS"' "$SERVE" - grep -q -- '--max-model-len "$FRANK_PRIMARY_MAXLEN"' "$SERVE" - grep -q -- '--kv-cache-memory-bytes "$FRANK_PRIMARY_KV_BYTES"' "$SERVE" - grep -q -- '--kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE"' "$SERVE" - grep -q -- '--max-num-batched-tokens "$FRANK_PRIMARY_MNBT"' "$SERVE" - grep -q -- '-e R4D_LRU_GATHER_CHUNKS="$FRANK_PRIMARY_GATHER_CHUNKS"' "$SERVE" - grep -q -- '-e R4D_LRU_GATHER_LANES="$FRANK_PRIMARY_GATHER_LANES"' "$SERVE" - # Speculative decode, the cudagraph capture ladder and the sampling - # defaults are all ENV-DRIVEN knobs (unset -> the flag is absent, which is - # the shipped shape today). JSON values carry no shell-special characters - # that this quoting cannot carry. - grep -q -- '${FRANK_PRIMARY_SPEC:+--speculative-config "$FRANK_PRIMARY_SPEC"}' "$SERVE" - grep -q -- '${FRANK_PRIMARY_COMPCONFIG:+--compilation-config "$FRANK_PRIMARY_COMPCONFIG"}' "$SERVE" - grep -q -- '${FRANK_PRIMARY_GEN_CONFIG:+--override-generation-config "$FRANK_PRIMARY_GEN_CONFIG"}' "$SERVE" - grep -q -- '--tensor-parallel-size 2' "$SERVE" - # Prefix caching renders exactly ONE flag from the knob, branched - # explicitly in the serve script (the ${VAR:--flag} double-expansion is - # ambiguous bash — when set it parsed as a substring offset, measured - # 2026-10-05): set -> --no-enable-prefix-caching (the fork-named - # mitigation, tried and REVERTED the same day — live corruption - # persisted at ~40k with zero faults), unset -> the shipped - # --enable-prefix-caching. Both directions are spelled out so a fork - # default change cannot silently flip the contract. - grep -qF 'FRANK_PRIMARY_PREFIX_CACHE_FLAG=--no-enable-prefix-caching' "$SERVE" - grep -qF 'FRANK_PRIMARY_PREFIX_CACHE_FLAG=--enable-prefix-caching' "$SERVE" - grep -qF '"$FRANK_PRIMARY_PREFIX_CACHE_FLAG"' "$SERVE" - # The observable contract: the exact flag word the engine receives, - # rendered from the shipped env through the serve script's branch (the - # same render-assert pattern the spec knobs use above). - run bash -c ' - set -a; . "$1" >/dev/null; set +a - if [ -n "${FRANK_PRIMARY_NO_PREFIX_CACHING:-}" ]; then - FRANK_PRIMARY_PREFIX_CACHE_FLAG=--no-enable-prefix-caching - else - FRANK_PRIMARY_PREFIX_CACHE_FLAG=--enable-prefix-caching - fi - printf %s "$FRANK_PRIMARY_PREFIX_CACHE_FLAG"' _ "$ENV_FILE" - [ "$output" = "--enable-prefix-caching" ] - grep -q -- '--trust-remote-code' "$SERVE" - # DCP is opt-in: the flag is passed only when FRANK_PRIMARY_DCP is set - # (DCP reuses the TP ranks when PCP is off, so TP=2 + DCP=2 fits two GPUs). - grep -q -- '--decode-context-parallel-size "$FRANK_PRIMARY_DCP"' "$SERVE" - ! grep -q '^FRANK_PRIMARY_DCP=' "$ENV_FILE" - # --expert-precise-memory selects the fitted overhead (5.25 GiB) over the - # default (7.60 GiB = fitted + compile/graph-capture headroom), which only - # moves the plan's `room` term and therefore the expert arena: ~1,025 - # more slots at 2.344 MiB each. - grep -q -- '${FRANK_PRIMARY_PRECISE_MEMORY:+--expert-precise-memory}' "$SERVE" - # Shape E host/CPU wiring: the cache dir is created before it is mounted - # (a cold boot must have a target) and the mount lands on /root/.cache, - # whose vllm/ child is the path VLLM_CACHE_ROOT defaults to — the engine log - # names it as `Using cache directory: /root/.cache/vllm/torch_compile_cache/…`. - grep -q -- 'mkdir -p "$CACHE_DIR"' "$SERVE" - grep -q -- '${CACHE_DIR:+-v "$CACHE_DIR":/root/.cache}' "$SERVE" - grep -q -- '-e GPU_MAX_HW_QUEUES="$FRANK_PRIMARY_GPU_MAX_HW_QUEUES"' "$SERVE" - # Shape F/G: the CUDA-graph plan term is carried as CONTAINER env (the fork - # reads it at call time inside estimate_cudagraph_memory), and both must be - # present — the graph term and UTIL are the ONLY plan terms this repo can - # move, and shrinking them is what grows the arena. The base must be shipped - # explicitly: unset, the estimator would fall back to its 256 MiB default. - grep -q -- '-e VLLM_CUDAGRAPH_MEM_BASE_MB="$FRANK_PRIMARY_CUDAGRAPH_BASE_MB"' "$SERVE" - grep -q -- '-e VLLM_CUDAGRAPH_MEM_PER_GRAPH_MB="$FRANK_PRIMARY_CUDAGRAPH_PER_GRAPH_MB"' "$SERVE" - grep -q '^FRANK_PRIMARY_CUDAGRAPH_BASE_MB=0$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_CUDAGRAPH_PER_GRAPH_MB=12$' "$ENV_FILE" - # The host region is the ONE plan term that autofits per-boot: with no cap - # the bound is MemAvailable at that moment (boot 2026-10-05: 237.97 GiB -> - # 159.39 GiB usable, while the running host later reads 75.4 GiB available). - # An explicit --expert-offload-mem cap makes the plan deterministic whatever - # the co-tenant RAM state is at boot; 120 GiB covers the measured 105.48 - # GiB of swappable experts (52.74/rank x 2) with 14% headroom. - grep -q -- '${FRANK_PRIMARY_EXPERT_OFFLOAD_MEM:+--expert-offload-mem "$FRANK_PRIMARY_EXPERT_OFFLOAD_MEM"}' "$SERVE" - grep -q '^FRANK_PRIMARY_EXPERT_OFFLOAD_MEM=120$' "$ENV_FILE" + grep -q 'ft serve --model /models' "$SERVE" + grep -q -- '--served-model-name "$FRANK_PRIMARY_SERVED_NAME"' "$SERVE" + grep -q -- '--host 0.0.0.0 --port 7770' "$SERVE" + grep -q -- '--gpu "$FRANK_PRIMARY_FT_GPU"' "$SERVE" + grep -q -- '--num-tokens "$FRANK_PRIMARY_FT_NUM_TOKENS"' "$SERVE" + grep -q -- '--max-running-requests "$FRANK_PRIMARY_FT_MAX_RUNNING"' "$SERVE" + grep -q -- '--max-output-tokens "$FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS"' "$SERVE" + grep -q -- '--moe-cache-rate "$FRANK_PRIMARY_FT_MOE_RATE"' "$SERVE" + # Single-GPU law: visibility is pinned to one card so in-container + # index 0 is the right R9700 (mirrors the env's FT_GPU). + grep -q -- '-e HIP_VISIBLE_DEVICES="$FRANK_PRIMARY_FT_GPU"' "$SERVE" + # The boot-ladder escape hatch is opt-in: unset renders NO extra argv. + grep -q -- '${FRANK_PRIMARY_FT_EXTRA_ARGS:+$FRANK_PRIMARY_FT_EXTRA_ARGS}' "$SERVE" + ! grep -q '^FRANK_PRIMARY_FT_EXTRA_ARGS=.' "$ENV_FILE" } -@test "tools and reasoning are wired — the client contract is served" { - # 2026-10-01: the rollback-era trio served NO tool dialect and left - # thinking inline in `content` with a dangling `` and a null - # `reasoning` field (measured: content 'We need answer...Brief.\n - # \n\n391') — the malformed thinking block clients choke on. The working - # values are the ones ecaad36 shipped for this model and this fork's - # registries confirm: reasoning parser `qwen3`, tool parser `qwen3_xml`. - # preserve_thinking keeps prior thinking blocks across turns. - grep -q '^FRANK_PRIMARY_REASONING_PARSER=qwen3$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_TOOL_PARSER=qwen3_xml$' "$ENV_FILE" - grep -q -- '--reasoning-parser "$FRANK_PRIMARY_REASONING_PARSER"' "$SERVE" - grep -q -- '--enable-auto-tool-choice' "$SERVE" - grep -q -- '--tool-call-parser "$FRANK_PRIMARY_TOOL_PARSER"' "$SERVE" - grep -q -- '--default-chat-template-kwargs "$FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS"' "$SERVE" - grep -q 'preserve_thinking' "$ENV_FILE" - grep -q 'enable_thinking' "$ENV_FILE" +@test "parsers are AUTO — the engine infers them natively, the harness maps the field" { + # FreeToken infers the qwen3.8 reasoning/tool dialects from the model + # family and emits thinking in `reasoning_content` natively — NO + # --reasoning-parser/--tool-call-parser override flags. The vLLM-era + # client-contract wiring (qwen3 parser, qwen3_xml tool parser, + # default-chat-template-kwargs) is gone; models.yml's + # reasoningContentField flip is the harness-side contract and lives in + # the operator's home, not this repo. + ! grep -q -- '--reasoning-parser' "$SERVE" + ! grep -q -- '--tool-call-parser' "$SERVE" + ! grep -q -- '--enable-auto-tool-choice' "$SERVE" + ! grep -q -- '--default-chat-template-kwargs' "$SERVE" + ! grep -q 'FRANK_PRIMARY_REASONING_PARSER' "$ENV_FILE" + ! grep -q 'FRANK_PRIMARY_TOOL_PARSER' "$ENV_FILE" } -@test "expert offload is AUTOMATIC — the payload cannot fit VRAM and no flag sets it" { - # ~86 GiB/rank of FP8 trunk against 32 GiB of VRAM: the fork's tiered- - # expert planner moves the overflow to pinned host rows by itself (the - # arena 6,574 slots x 2.344 MiB plus host rows). There is no - # --enable-expert-offload flag in this arm; the legality of the pin is - # the unit's unbounded LimitMEMLOCK. - ! grep -q -- '--enable-expert-offload' "$SERVE" - grep -q '^LimitMEMLOCK=infinity$' "$UNIT" - grep -q -- '--ipc=host --ulimit memlock=-1:-1' "$SERVE" -} - -@test "speculative decode is an ENV KNOB — the old retirement verdict was invalid" { - # The 2026-10-02 c1248d4 retirement of MTP compared the WRONG PAIR: - # fp8+MTP4 (37.4-40.6 tok/s, 54% acceptance) against the bf16 no-spec boot - # (36.4-39.4), then bf16 proved unbootable and fp8 came back WITHOUT MTP — - # so fp8+spec was never compared against fp8+no-spec on the same shape. The - # no-spec incumbent runs ~30 tok/s, under the fleet's 35 tok/s target, while - # EVERY vendor flash-next recipe ships MTP3, and our MTP4 run used the - # default GREEDY draft sampling (probabilistic is the other fork option and - # is what the recipes use under temp>0). So spec is a knob, not a law: the - # shipped value is whatever a matched-pair re-measurement wins. - grep -q 'THE RETIREMENT VERDICT WAS INVALID' "$ENV_FILE" - # The knobs are empty in the shipped env, so the `${VAR:+…}` wiring renders - # NO spec/compilation flag — asserted by rendering, not by grepping the - # assignment (shape E today has neither flag in its argv). - run bash -c 'set -a; . "$1" >/dev/null; set +a - printf %s "${FRANK_PRIMARY_SPEC:+--speculative-config $FRANK_PRIMARY_SPEC}${FRANK_PRIMARY_COMPCONFIG:+--compilation-config $FRANK_PRIMARY_COMPCONFIG}"' _ "$ENV_FILE" - [ -z "$output" ] - # The capture-ladder law: the piecewise graph ladder must contain - # num_speculative_tokens + 1, so an MTP3 shape captures [4, 8, 12] and the - # 2026-10-02 SEQS=2 failure (query_start_loc specialized to the constant 3) - # is why SEQS stays a multiple-compatible value. - grep -q 'num_speculative_tokens + 1' "$ENV_FILE" -} - -@test "KV dtype is fp8 — the incumbent, and the only one that boots" { - # fp8 costs ~7,701 B/token at 262144 (7,892 at 200704 — shape-dependent), - # decomposed as 6,138 attention + 1,563 mamba/side (dtype-independent); - # bf16 costs 13,839 (12,276 + 1,563). - # bf16 was abandoned 2026-10-01: its 2 x 262144 boot died with 0 bytes - # free on a real request, and the SEQS=2 slot count it was designed - # around cannot compile this model at all (see the shape test above). - grep -q '^FRANK_PRIMARY_KV_DTYPE=fp8$' "$ENV_FILE" - grep -q -- '--kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE"' "$SERVE" -} - -@test "gather geometry 32x64 is PRESENT BY MEASUREMENT" { - # R4D_LRU_GATHER_CHUNKS=32 / R4D_LRU_GATHER_LANES=64: the overnight - # sweep's biggest lever, +18.7% decode vs default geometry, and the win - # GROWS with depth (39.5@166 -> 54.1@1087 -> 49.3@4064 in the fp8 era). - # g64x128 is past the peak; g8x16 mild. They are container ENV (the - # fork's knobs), not engine flags — passed through with -e. - grep -q '^FRANK_PRIMARY_GATHER_CHUNKS=32$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_GATHER_LANES=64$' "$ENV_FILE" - grep -q -- '-e R4D_LRU_GATHER_CHUNKS="$FRANK_PRIMARY_GATHER_CHUNKS"' "$SERVE" -} - -@test "the arena sits just under the ceiling and the KV ask is honoured" { - # The planner sizes expert cache as budget - non-expert - KV reserve - - # graphs - overhead, and two of those five terms are reachable from a - # deploy: budget (= device_total x util) and graphs (VLLM_CUDAGRAPH_MEM_*). - # UTIL has a HARD LAW, not a field cap: 0.982, never 1.00. The 1.00 attempt - # is UNBOOTABLE on this fork — worker init raises - # `ValueError: Free memory on device cuda:N (31.29/31.86 GiB ...)` - # on both ranks because the worker demands free > budget and budget == - # total — and even if it booted, 0.13-GiB-margin shapes have died under - # real load. So the arena tops out at 0.982 budget (31.29 GiB). - # Graphs: charge strictly below the fork's analytic 563 MiB default while - # still paying per graph (the measured post-capture delta is -0.06 GiB, i.e. - # the base pool is the over-charge, but a future spec-decode shape captures - # 18 graphs and must still be charged). - # MNBT stays 2048: 512 doubles TTFT at 4064 ctx, 1024-4096 are decode- - # interchangeable AND arena-identical (the registry prices overhead flat up - # to its nbt 4096 anchor), 12288 costs 2.5% plus arena. - # The KV ask: the pool is priced at the engine's measured 7,892 B/token at - # this shape, and a served conversation needs 200,704 tokens, so the pool - # must clear 400,700 tokens to keep two full-length resident. - grep -q '^FRANK_PRIMARY_MNBT=2048$' "$ENV_FILE" - grep -q -- '--max-num-batched-tokens "$FRANK_PRIMARY_MNBT"' "$SERVE" - grep -q '^FRANK_PRIMARY_UTIL=0.982$' "$ENV_FILE" - base=$(awk -F= '/^FRANK_PRIMARY_CUDAGRAPH_BASE_MB=/{print $2}' "$ENV_FILE") - per_graph=$(awk -F= '/^FRANK_PRIMARY_CUDAGRAPH_PER_GRAPH_MB=/{print $2}' "$ENV_FILE") - [ "$base" -eq 0 ] && [ "$per_graph" -gt 0 ] - [ $((base + 6 * per_graph)) -lt 563 ] - kv=$(awk -F= '/^FRANK_PRIMARY_KV_BYTES=/{print $2}' "$ENV_FILE") - [ $((kv / 7892)) -ge 400700 ] -} - -@test "expert prefetch is absent BY LAW, not by omission" { - # Telemetry (2026-09-29): the prefetcher served 13,369 of 36,720 - # prefetched slabs (36% precision) — roughly half the gather traffic was - # waste, and the "savings" would have been installs anyway. Deleting the - # flag moved decode 26.8 -> 40.6 tok/s. Any future flag must come with a - # fresh A/B, so the serve script must NOT wire it unconditionally. - ! grep -q -- '--expert-prefetch' "$SERVE" - ! grep -q 'expert_prefetch' "$ENV_FILE" - grep -q 'PREFETCH IS ABSENT BY LAW' "$ENV_FILE" -} - -@test "TunableOp stays off" { - # CLAV_TUNABLEOP_SWEEP silently skips under expert offload unless - # CLAV_TUNABLEOP_WITH_OFFLOAD=1, and forcing it corrupts numerics - # (repetition loops, early halts, invalid measurements). - ! grep -q 'TUNABLEOP' "$ENV_FILE" - ! grep -q 'TUNABLEOP' "$SERVE" -} - -@test "the serve script gates the digest-pinned image and the payload" { - # The image is pinned by OCI manifest digest (the floating :latest moved - # once already). The payload gate checks config.json and the exact shard - # count from the env — a missing shard means a guaranteed long-timeout - # boot instead of a fast failure. +@test "the serve script gates the image, the payload and the GPU nodes" { + # The image gate is EXISTENCE (podman image exists): the FreeToken + # image is built ON THE HOST — a local tag carries no RepoDigests — and + # pinning lives in the Dockerfile (pinned PR-head SHA, base image). + # The payload gate checks config.json and the exact shard count from + # the env — a missing shard means a guaranteed long-timeout boot + # instead of a fast failure. grep -q 'FRANK_PRIMARY_IMAGE' "$SERVE" - grep -q 'RepoDigests' "$SERVE" + grep -q 'podman image exists' "$SERVE" grep -q 'FRANK_PRIMARY_MODEL_SHARDS' "$SERVE" grep -q 'config.json' "$SERVE" + grep -q 'model.safetensors.index.json' "$SERVE" grep -q '/dev/kfd' "$SERVE" grep -q -- '--device /dev/dri' "$SERVE" - ! grep -q 'tcclaviger/vllm:latest' "$ENV_FILE" } @test "the serve script clears stale containers and self-heals via podman wait" { # A crashed unit can leave the container name registered forever after; - # the legacy identities (the retired R9V container and the ad hoc - # experiment container) hold the same GPUs and die here too. Then block - # on the container so a crash is a nonzero ExecStart. + # the legacy identities (the vLLM-era frank-primary-qwen container, the + # retired R9V container and the ad hoc experiment containers) hold the + # same GPU and die here too. Then block on the container so a crash is + # a nonzero ExecStart. grep -q 'FRANK_PRIMARY_LEGACY_CONTAINERS' "$ENV_FILE" + grep -q 'frank-primary-qwen' "$ENV_FILE" grep -q 'frank-primary-r9v' "$ENV_FILE" grep -q 'qwen-fp8' "$ENV_FILE" grep -q 'exec podman wait "$FRANK_PRIMARY_CONTAINER"' "$SERVE" } @test "frank-primary holds the single fleet port under its own alias" { - # The scratch identity (qwen3.8-flash-next on :7773) must never leak - # into production: LAN publish of :7770, the harness's model id. + # LAN publish of :7770, the harness's model id. grep -q '^FRANK_PRIMARY_PUBLISH=7770:7770$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_SERVED_NAME=frank-primary$' "$ENV_FILE" } @@ -404,16 +201,38 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" ! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE" } -@test "the deployer gates the image digest and payload, then health-checks" { - # The deployer refuses when the pinned image is absent from rootless - # storage or the payload is missing/wrong-sharded — then clears stale - # containers (including the legacy identities), waits out the port, +@test "the Dockerfile pins the FreeToken source and follows the ROCm install law" { + # The engine is the PR #587 head, NOT a moving branch: a future push to + # rocm714_torch214 would otherwise silently change the deployed engine. + grep -q '77a7201f98f00d93f618c6cf8b81ffe8360e0899' "$DOCKERFILE" + grep -q 'pull/587/head' "$DOCKERFILE" + # docs/install_amd.md's build law: --no-build-isolation --no-deps (a + # plain `pip install -e .` on ROCm can REPLACE the ROCm torch with a + # CUDA build via its NVIDIA-authored dependency set), curated portable + # deps, flashlib --no-deps. + grep -q -- '--no-build-isolation --no-deps -e .' "$DOCKERFILE" + grep -q 'flashlib==0.3.0' "$DOCKERFILE" + # The CUDA-only backends have no ROCm builds and are intentionally + # absent (the PR's 12 pytest failures are exactly those backends; + # FreeToken falls back to its Triton kernels). + ! grep -q 'flashinfer' "$DOCKERFILE" + ! grep -q 'sgl_kernel' "$DOCKERFILE" + ! grep -q 'nvidia-cutlass-dsl' "$DOCKERFILE" + # The base image is digest-pinned (the registry tag is not trusted). + grep -qE '^FROM rocm/pytorch:.*@sha256:[0-9a-f]{64}$' "$DOCKERFILE" +} + +@test "the deployer gates the image, payload and containers, then health-checks" { + # The deployer refuses when the image is absent from rootless storage + # or the payload is missing/wrong-sharded — then clears stale containers + # (including the legacy identities), waits out the port, # relinks/enables/restarts the root unit and health-gates it. - grep -q 'LLM_PRIMARY_CONTAINER=frank-primary-qwen' "$REMOTE_DEPLOY" + grep -q 'LLM_PRIMARY_CONTAINER=frank-primary$' "$REMOTE_DEPLOY" grep -q 'FRANK_PRIMARY_IMAGE' "$REMOTE_DEPLOY" - grep -q "podman images --format '{{.RepoDigests}}'" "$REMOTE_DEPLOY" + grep -q 'podman image exists' "$REMOTE_DEPLOY" grep -q 'FRANK_PRIMARY_MODEL_SHARDS' "$REMOTE_DEPLOY" grep -q 'config.json' "$REMOTE_DEPLOY" + grep -q 'frank-primary-qwen' "$REMOTE_DEPLOY" grep -q 'qwen-fp8' "$REMOTE_DEPLOY" grep -q 'glm-probe' "$REMOTE_DEPLOY" grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"