frank-primary: FreeToken (PR587 ROCm) cutover — trio, seam, deployer
Replaces the tcclaviger vLLM fork with FreeToken at PR #587 head 77a7201 built on dankserver into localhost/frank-primary-freetoken:pr587-77a7201 (scripts/docker/frank-primary-freetoken.Dockerfile: digest-pinned rocm/pytorch rocm7.14 torch-2.11.0 base, libhiprtc dev symlink, curated --no-deps install per docs/install_amd.md). Shape (first-boot conservative): ONE R9700 (second card idle; --tensor-parallel-size exists for future 2-card tuning), KV pool --num-tokens 300000 (7.2 GiB at 24 KiB/token; clears the 177,356-token harness worst case and one native 262,144 conversation), MoE LRU --moe-cache-rate 0.10, --max-running-requests 2, --max-output-tokens 36864. Parsers AUTO: thinking -> reasoning_content natively (models.yml flips reasoningContentField on the harness side). Experts + PLE pin ~170 GiB host RAM — root unit keeps LimitMEMLOCK=infinity. Container identity renamed to the unit name (Frank's call): frank-primary; vLLM-era frank-primary-qwen joins the legacy cleanup list. Deployer image gate becomes podman image exists (a local build carries no RepoDigests; the old digest grep refused exactly this shape). Rollback tag: frank-primary-vllm-shapeG-20261003.
This commit is contained in:
Frank
+28
-26
@@ -1250,24 +1250,25 @@ deploy_llm_small() {
|
||||
echo "frank-small: health answered by unit MainPID $main_pid"
|
||||
}
|
||||
|
||||
# deploy_llm_primary — frank-primary is the tcclaviger RDNA4 vLLM fork
|
||||
# (digest-pinned image, gfx1201) serving the orcarouter
|
||||
# Qwen3.8-Flash-Next-Uncensored-FP8 payload (131 safetensors shards) with
|
||||
# MTP4 spec decode and q8/q8 KV on the two R9700s as a ROOT unit whose serve
|
||||
# script drives `podman run` directly (the image entrypoint execs
|
||||
# `vllm serve`). 2026-09-29 cutover from the R9V radiance fork, whose
|
||||
# launcher/shim/setup-state machinery is retired (state dirs stay on disk as
|
||||
# rollback; pre-cutover tag frank-primary-r9v-q8-20260929). Same port and
|
||||
# alias as the endpoints it replaced; the harness models.yml id is unchanged.
|
||||
# deploy_llm_primary — frank-primary is the FreeToken engine (PR #587 ROCm
|
||||
# build, gfx1201) serving the orcarouter Qwen3.8-Flash-Next-Uncensored-FP8
|
||||
# payload (131 safetensors shards) on ONE R9700 as a ROOT unit whose serve
|
||||
# script drives `podman run` directly (the image carries `ft serve`).
|
||||
# 2026-10-05 cutover from the tcclaviger vLLM fork (pre-cutover tag
|
||||
# frank-primary-vllm-shapeG-20261003); before that, 2026-09-29 cutover from
|
||||
# the R9V radiance fork, whose launcher/shim/setup-state machinery is
|
||||
# retired. Same port and alias as the endpoints it replaced; the harness
|
||||
# models.yml id is unchanged.
|
||||
#
|
||||
# Gates: the pinned runtime image must be loadable from dank's rootless
|
||||
# storage by its OCI manifest digest (the env file's FRANK_PRIMARY_IMAGE),
|
||||
# the payload dir must carry config.json and the expected shard count
|
||||
# (FRANK_PRIMARY_MODEL_SHARDS), and the unit env must be complete. Per-file
|
||||
# sha256 verification was the fetch script's job at download time, never
|
||||
# re-hashed here.
|
||||
LLM_PRIMARY_CONTAINER=frank-primary-qwen
|
||||
LLM_PRIMARY_LEGACY_CONTAINERS="frank-primary-r9v qwen-fp8 glm-probe"
|
||||
# Gates: the runtime image must exist in dank's rootless storage
|
||||
# (FRANK_PRIMARY_IMAGE — a LOCAL tag: the image is built on the host from
|
||||
# scripts/docker/frank-primary-freetoken.Dockerfile, which pins the
|
||||
# FreeToken PR-head SHA and the base image), the payload dir must carry
|
||||
# config.json and the expected shard count (FRANK_PRIMARY_MODEL_SHARDS),
|
||||
# and the unit env must be complete. Per-file sha256 verification was the
|
||||
# fetch script's job at download time, never re-hashed here.
|
||||
LLM_PRIMARY_CONTAINER=frank-primary
|
||||
LLM_PRIMARY_LEGACY_CONTAINERS="frank-primary-qwen frank-primary-r9v qwen-fp8 glm-probe"
|
||||
|
||||
deploy_llm_primary() {
|
||||
# Refuse rather than restart-loop: a unit whose image or payload is
|
||||
@@ -1286,15 +1287,16 @@ deploy_llm_primary() {
|
||||
echo "ERROR: frank-primary.env missing FRANK_PRIMARY_IMAGE/MODEL_DIR/MODEL_SHARDS" >&2
|
||||
return 1
|
||||
fi
|
||||
# Capture podman's output BEFORE grepping: under set -o pipefail, the
|
||||
# classic `podman images | grep -q` races (grep exits on match, podman
|
||||
# dies SIGPIPE 141 writing the rest, pipefail calls it no-match — the
|
||||
# 2026-09-30 deploy refusal).
|
||||
local digests
|
||||
digests="$(podman images --format '{{.RepoDigests}}')"
|
||||
if ! grep -qF "${image#*@}" <<<"$digests"; then
|
||||
echo "ERROR: pinned runtime image absent from rootless storage: $image" >&2
|
||||
echo "frank-primary: fetch it (podman pull $image)" >&2
|
||||
# The image gate is EXISTENCE IN STORAGE, not a registry digest: the
|
||||
# FreeToken image is built ON THE HOST (a local tag carries NO
|
||||
# RepoDigests entry — the old grep refused exactly this shape), and
|
||||
# pinning lives in the Dockerfile (pinned PR-head SHA + digest-pinned
|
||||
# base image). `podman image exists` accepts a local tag and a
|
||||
# name@digest ref alike, in one command — no pipefail/grep race (the
|
||||
# 2026-09-30 lesson) for either shape.
|
||||
if ! podman image exists "$image"; then
|
||||
echo "ERROR: runtime image absent from rootless storage: $image" >&2
|
||||
echo "frank-primary: build it (scripts/docker/frank-primary-freetoken.Dockerfile) or pull it" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$model_dir/config.json" ]]; then
|
||||
|
||||
@@ -31,8 +31,9 @@
|
||||
# Triton kernels on ROCm.
|
||||
#
|
||||
# Base is digest-pinned (2026-10-05): sha256:a223aee1… is the manifest this
|
||||
# build was qualified against; a re-pushed tag cannot change the stack.
|
||||
FROM rocm/pytorch@sha256:a223aee17aef5d21c3b9f63436dd19d27d1c665ec8b2f40011c9546cabae2a80
|
||||
# build was qualified against; a re-pushed tag cannot change the stack. The
|
||||
# readable tag rides along for humans (podman verifies the digest matches).
|
||||
FROM rocm/pytorch:rocm7.14_ubuntu24.04_py3.12_pytorch_release_2.11.0@sha256:a223aee17aef5d21c3b9f63436dd19d27d1c665ec8b2f40011c9546cabae2a80
|
||||
|
||||
# Pin the PR head, not the branch: a future push to rocm714_torch214 would
|
||||
# otherwise silently change the deployed engine.
|
||||
|
||||
+74
-352
@@ -1,362 +1,84 @@
|
||||
# frank-primary.env — tcclaviger RDNA4 vLLM production config (dankserver :7770).
|
||||
# frank-primary.env — FreeToken (PR #587 ROCm) serving the orcarouter
|
||||
# Qwen3.8-Flash-Next-Uncensored-FP8 payload on dankserver :7770.
|
||||
#
|
||||
# 2026-10-03 shape G (as shipped, after the 1.00 revert): the CUDA-graph plan
|
||||
# term dropped from the analytic 0.55 GiB to the measured 0.07 GiB (0 base +
|
||||
# 12 MiB/graph), and UTIL at the 0.982 law. MEASURED on the live deploy: arena
|
||||
# 17.77 GiB / 7,762 slots, KV 401,408 tokens / 2.00x concurrency (the first
|
||||
# shape holding two full-length conversations with real spare), driver used
|
||||
# 30.97 GiB of 31.86 — 0.89 GiB of real free VRAM per card, held through a
|
||||
# live 4,096-token generation. UTIL stays 0.982 ALWAYS: 1.00 is unbootable on
|
||||
# this fork (worker init refuses budget >= device total) and 0.13-GiB-margin
|
||||
# boots die under load.
|
||||
# `budget - non-expert - KV reserve - graphs - overhead` and the KV reserve is a
|
||||
# registry constant, so those two terms are the whole flag surface.
|
||||
# 2026-10-03 shape F (absorbed): MNBT 2048 + the graph term pinned. Measured on
|
||||
# the live deploy: arena 17.52 GiB / 7,653 slots, budget 31.29, graphs 0.32.
|
||||
# 2026-10-02 shape E: fp8 KV, MAXLEN 200704 x SEQS 3, pool 3.15e9/rank, and
|
||||
# --expert-precise-memory ON. The window shrink from the native 262144 and the
|
||||
# --max-num-seqs 3 lock both come from shape D (SEQS=2 CANNOT boot this model —
|
||||
# the measured 2x2 is below); shape E adds the precise-overhead plan, which
|
||||
# trades the default 7.60 GiB compile headroom for the fitted 5.25 GiB and puts
|
||||
# the 2.35 GiB difference into the expert arena: 6,496 -> 7,522 slots (+16%).
|
||||
# The pool is deliberately NOT cut (option A): the pool is not an arena lever
|
||||
# (measured), so 3.15e9 keeps shape D's 1.99x concurrency while the arena grows.
|
||||
# Same digest-pinned tcclaviger RDNA4 fork. Rollback for the fp8+MTP4 era is git
|
||||
# tag frank-primary-fp8-mtp-20261001.
|
||||
# 2026-10-05 cutover from the tcclaviger RDNA4 vLLM fork (git tag
|
||||
# frank-primary-vllm-shapeG-20261003 is the pre-cutover state; the 362-line
|
||||
# vLLM-era env is in that tag). The engine is FreeToken at PR head
|
||||
# 77a7201f98f00d93f618c6cf8b81ffe8360e0899 (branch rocm714_torch214), built
|
||||
# on the host into a local image by
|
||||
# scripts/docker/frank-primary-freetoken.Dockerfile — the PR's own
|
||||
# docs/install_amd.md stack (ROCm 7.14 base image, torch 2.11.0+rocm7.14,
|
||||
# triton 3.7) for gfx1201.
|
||||
#
|
||||
# HIP reads bare env vars TEMP, HOST, GPU_MAX_HEAP_SIZE — every key here is
|
||||
# prefixed FRANK_PRIMARY_ so none can leak. The one bare name we WANT inside the
|
||||
# container (GPU_MAX_HW_QUEUES) is held here under the prefix and passed on the
|
||||
# run line as `-e GPU_MAX_HW_QUEUES=…`.
|
||||
# WHY FREETOKEN: the vLLM fork's expert offload planned VRAM/host tiers per
|
||||
# boot from an estimator (util 0.982 law, arena 17.77 GiB / 7,762 slots,
|
||||
# ~40k-token GDN state faults) and 1.0 was unbootable. FreeToken keeps ALL
|
||||
# expert weights in pinned HOST RAM (~120 GiB) with an LRU GPU cache, sizes
|
||||
# KV from an explicit token pool, and serves on ONE R9700 (32 GB) with the
|
||||
# second card idle. RAM fit: 246 GiB total, service stopped frees the vLLM
|
||||
# container's ~170 GiB; weights ~120 GiB + PLE 47.7 GiB + overhead ≈ 190 GiB.
|
||||
#
|
||||
# PAYLOAD: unchanged — /home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8,
|
||||
# 131 safetensors shards. Quant is standard modelopt block-FP8: 75,264
|
||||
# weight_scale_inv tensors (block 128x128), zero input_scale (FreeToken #462
|
||||
# covers exactly this export shape), one weight_scale on the PLE ngram
|
||||
# embedding. FreeToken loads HF safetensors directly; no FTW conversion.
|
||||
#
|
||||
# --- runtime ----------------------------------------------------------------
|
||||
# Digest-pinned: the floating :latest tag moved once already. Local alias
|
||||
# qwen-fp8-runtime:tuned-20260929 (image ID 3322dbbbc96d) points at the same
|
||||
# bytes in rootless storage.
|
||||
FRANK_PRIMARY_IMAGE=docker.io/tcclaviger/vllm@sha256:da4029666668b90a0386c1ed23ea370c79916ad4e1a6ad1dd9db098b9fa069ea
|
||||
# SERVING FACTS (docs/cli.md, verified against main+PR587):
|
||||
# - `ft serve` loads the checkpoint directly; `--served-model-name` keeps the
|
||||
# harness id; reasoning/tool parsers are AUTO (reasoning lands in
|
||||
# reasoning_content — models.yml flips reasoningContentField to match).
|
||||
# - KV: --page-size must stay 1 for GDN/SWA models (default); --cache-type
|
||||
# radix auto-picks the GDN-aware variant.
|
||||
# - MoE: --moe-strategy auto -> offload (fp8 experts CANNOT use
|
||||
# --moe-cpu-layers; only bf16/nvfp4/mxfp4 have a CPU executor).
|
||||
# --moe-cache-rate holds the GPU LRU fraction of all experts.
|
||||
# - `ft ctl cache --kv --moe` grows the pools LIVE on a running server
|
||||
# (shrinking needs a restart); `ft ctl stats` reports pool occupancy.
|
||||
# - KV MATH: 12 full-attention layers x 2 KV heads x 256 head_dim x 2 (K+V)
|
||||
# x 2 B = 24 KiB/token (f16 KV; the 36 GDN layers use fixed per-seq state,
|
||||
# not per-token KV). FRANK_PRIMARY_FT_NUM_TOKENS=300000 -> ~7.2 GiB.
|
||||
# Harness worst case = 140,492-token prompt + 36,864 max_tokens = 177,356
|
||||
# < 300,000; one NATIVE 262,144-token conversation + full budget also fits.
|
||||
# - PLE: the ~47.7 GiB ngram-embedding table pins in HOST RAM by default
|
||||
# (--ple-backend disk is the alternative); the unit's LimitMEMLOCK=infinity
|
||||
# makes the pin legal. TENS of GiB of pinned pages are why this stays a
|
||||
# ROOT unit — see frank-primary.service.
|
||||
|
||||
# --- payload ----------------------------------------------------------------
|
||||
# --- Identity (deploy + serve contract) --------------------------------
|
||||
FRANK_PRIMARY_IMAGE=localhost/frank-primary-freetoken:pr587-77a7201
|
||||
FRANK_PRIMARY_MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8
|
||||
FRANK_PRIMARY_MODEL_SHARDS=131
|
||||
|
||||
# --- identity ---------------------------------------------------------------
|
||||
# The harness models.yml id stays frank-primary (provider 192.168.0.235:7770).
|
||||
FRANK_PRIMARY_CONTAINER=frank-primary-qwen
|
||||
# Container identity is the SERVICE name (Frank's call, 2026-10-05): one
|
||||
# name across systemctl and podman ps. The vLLM-era name frank-primary-qwen
|
||||
# moved into the legacy list so every deploy cleans it up.
|
||||
FRANK_PRIMARY_CONTAINER=frank-primary
|
||||
FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-qwen frank-primary-r9v qwen-fp8 glm-probe"
|
||||
FRANK_PRIMARY_SERVED_NAME=frank-primary
|
||||
FRANK_PRIMARY_PUBLISH=7770:7770
|
||||
# Containers removed before every start: the retired R9V identity and the ad
|
||||
# hoc experiment identity (both hold the GPUs / name registrations).
|
||||
FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-r9v qwen-fp8"
|
||||
|
||||
# --- qualified engine shape (2026-10-02, shape E) ---------------------------
|
||||
# WHAT THE SLOT COUNT DOES AND DOES NOT DO. The endpoint advertises 3 slots
|
||||
# (SEQS 3), but the POOL, not the slot count, decides how many full-length
|
||||
# conversations are resident. MEASURED LIVE 2026-10-02 at this shape:
|
||||
# GPU KV cache size: 398,658 tokens,
|
||||
# Maximum concurrency for 200,704 tokens per request: 1.99x
|
||||
# (3,146,129,408 B reserved — 7,892 B/token). The per-token price is
|
||||
# SHAPE-DEPENDENT, not a constant: the 262144 shape priced at 7,701 B/token
|
||||
# (6,100,000,000 B -> 792,069 tokens), ~2.5% cheaper, so never extrapolate a
|
||||
# concurrency figure across geometries. Shorter requests use the spare blocks,
|
||||
# and a third LONG request queues rather than being refused. Verified: three
|
||||
# concurrent 67-token requests answered in parallel, ~4.1 s each.
|
||||
#
|
||||
# WHY NOT --max-num-seqs 2 (measured 2026-10-02, single-variable boot probes
|
||||
# against the then-deployed known-good shape, production stopped for each):
|
||||
# SEQS 2 + MNBT 1024 -> Engine core initialization failed
|
||||
# SEQS 2 + MNBT 2048 -> Engine core initialization failed
|
||||
# SEQS 3 + MNBT 1024 -> healthy (792,069 tokens, 3.02x, arena 6,248)
|
||||
# SEQS 3 + MNBT 2048 -> healthy (the deployed incumbent)
|
||||
# The failure is torch.fx ConstraintViolationError: query_start_loc.size()[0]
|
||||
# "marked dynamic but your code specialized it to be a constant (3)", raised
|
||||
# from AOT build_guards(strict_error=True) during profile_run -> _dummy_run.
|
||||
# model_runner.py sizes that input buffer max_num_reqs + 1 and _dummy_run sets
|
||||
# num_reqs = min(max_num_batched_tokens, max_num_reqs), so the width is
|
||||
# SEQS + 1 for ANY MNBT >= SEQS: 3 at SEQS 2, 4 at SEQS 3. MNBT only moves the
|
||||
# compile RANGE ((1,1024) vs (1,2048)) and cannot dodge it — the 2x2 above is
|
||||
# why MNBT is not a lever here. --enforce-eager bypasses the path but loses
|
||||
# CUDA graphs. Same failure class killed the 2026-10-01 bf16 2 x 262144 shape.
|
||||
#
|
||||
# THE ARENA LEVER IS max_model_len, NOT KV_BYTES (measured 2026-10-02). The
|
||||
# planner sizes expert cache as budget - non-expert - KV reserve - graphs -
|
||||
# overhead, and the KV reserve is registry-priced at 9190 B/token x max_model_len:
|
||||
# MAXLEN 262144 -> reserve 2.48 GiB -> arena 14.30 GiB / 6,248 slots
|
||||
# MAXLEN 200704 -> reserve 1.91 GiB -> arena 14.87 GiB / 6,496 slots
|
||||
# d(reserve) 0.57 GiB = d(arena) 0.57 GiB = +248 slots. The 6.1e9 -> 3.15e9
|
||||
# pool cut buys ZERO arena; it buys runtime HEADROOM (the shape it replaced
|
||||
# died with 0 bytes free on a real 995-token request). To grow the arena,
|
||||
# lower MAXLEN — not KV_BYTES. The OTHER arena lever is the overhead term:
|
||||
# --expert-precise-memory (shape E) takes it from 7.60 to 5.25 GiB and lands
|
||||
# those 2.35 GiB in the arena — 14.87 -> 17.22 GiB / 6,496 -> 7,522 slots.
|
||||
#
|
||||
# GATHER 32x64 stays: R4D_LRU_GATHER_CHUNKS=32 / R4D_LRU_GATHER_LANES=64
|
||||
# measured +18.7% decode vs default geometry and the win grows with depth.
|
||||
# Do not drop it.
|
||||
# CHUNKED PREFILL: the fork exposes the standard scheduler flags. --max-num-batched-tokens
|
||||
# (MNBT) is the lever; --enable-chunked-prefill is an auto-flag (True in SchedulerConfig)
|
||||
# and --prefill-schedule-interval is the gap between iterations. The running shape carries
|
||||
# --max-num-batched-tokens 2048 with --max-model-len 200704, so chunked prefill is ON and
|
||||
# already active: a 200704-token prompt is prefilled in ~98 iterations of 2048 tokens.
|
||||
# MEASURED 2026-09-29 on the ad hoc qwen-fp8 container (same fork, same payload, TP=2):
|
||||
# MNBT 8192 vs the default 2048 is NEUTRAL on decode at every depth, and small values
|
||||
# re-chunk prefill only — they hurt TTFT and never touch decode, because a decode step is
|
||||
# max_num_seqs x (spec+1) tokens, independent of MNBT. There is NO --ubatch-size
|
||||
# equivalent in vLLM: that is a llama.cpp compute-buffer concept (the frank-small pair
|
||||
# doubled on 2026-10-05 has no counterpart here). MNBT is therefore NOT a lever on this
|
||||
# shape — the deployed 2048 is the fork's default — and every change re-enters the
|
||||
# compile RANGE path, so FRANK_PRIMARY_CACHE_DIR pays one recompilation.
|
||||
# MNBT 2048 (Frank's call, 2026-10-03; the lock was 1024). The measured law
|
||||
# still holds: MNBT 512 doubles TTFT at 4064 ctx (2.46 s vs 1.36 s), 1024-4096
|
||||
# are interchangeable on decode, 12288 costs 2.5% plus arena. 2048 is ARENA-FREE
|
||||
# by construction: the registry's overhead model is
|
||||
# base + 0.22 GiB x max(0, (nbt - 4096)/1024) with the anchor at nbt 4096, so
|
||||
# every rung up to and including 4096 prices the SAME 5.25 GiB overhead — and
|
||||
# the fitted 5.25 was fitted at nbt 4096, i.e. our 2048 sits UNDER the anchor,
|
||||
# so the activation term is covered by a figure measured at a bigger batch.
|
||||
# SEQS 3 boots at both 1024 and 2048 (measured 2026-10-02; the SEQS=2
|
||||
# ConstraintViolationError is a SEQS property, not an MNBT one). Costs exactly
|
||||
# one recompilation: the compile RANGE is part of the cache key, so
|
||||
# FRANK_PRIMARY_CACHE_DIR pays for whichever rung is chosen once.
|
||||
# PREFETCH IS ABSENT BY LAW: expert-cache telemetry measured the prefetcher at
|
||||
# 36% precision (36,720 slabs prefetched, 13,369 served) — roughly half the
|
||||
# gather traffic was waste. Never re-add --expert-prefetch without a fresh A/B.
|
||||
# TunableOp stays OFF: CLAV_TUNABLEOP_SWEEP silently skips under expert offload
|
||||
# unless CLAV_TUNABLEOP_WITH_OFFLOAD=1, and forcing it corrupts numerics.
|
||||
# --expert-precise-memory is ON (shape E). It selects the fitted overhead
|
||||
# (5.25 GiB) over the default (7.60 GiB = fitted + compile/graph-capture
|
||||
# headroom); the 2.35 GiB difference lands in the expert arena. MEASURED
|
||||
# 2026-10-02 by a boot probe on the shape-D base (production stopped, candidate
|
||||
# booted AS the service, plan read off the engine's own log):
|
||||
# runtime overhead 7.60 -> 5.25 GiB (registry Qwen4Exp…, precise
|
||||
# gfx1201 TP2 nbt4096/seqs16/len262144)
|
||||
# expert cache 14.87 -> 17.22 GiB
|
||||
# slots 6,496 -> 7,522 (+1,026 = +16%, 2.344 MiB each)
|
||||
# swap region 13.7 GiB / 5,986 slots, 133-134 per layer
|
||||
# boot healthy in 420 s, no OOM, no knife-edge
|
||||
# The flag was called a non-option after it crash-looped the 2 x 262144 bf16
|
||||
# boot (0.13 GiB free of 31.86 GiB, NRestarts=3 at ~6.7 min each) — that shape
|
||||
# was already at the ceiling, so precise-memory was the last 2.35 GiB over, not
|
||||
# the cause. At 200704 x 3 it fits with room; the run line also reports
|
||||
# 'expert_precise_memory': True in non-default args, so a silent argparse drop
|
||||
# would be visible in the boot log.
|
||||
FRANK_PRIMARY_PRECISE_MEMORY=1
|
||||
# GPU_MAX_HW_QUEUES caps the HSA hardware queues per GPU (AMD's own tuning
|
||||
# guide recommends 2 for this class of workload; the runtime default on Linux
|
||||
# is 4). Verified a REAL knob, not a no-op: the string is present in
|
||||
# /opt/rocm/core-10.0/lib/libhsa-runtime64.so. NOT A/B MEASURED — Frank's call,
|
||||
# shipped as a deliberate CPU-side tune alongside the 32x64 gather geometry.
|
||||
FRANK_PRIMARY_GPU_MAX_HW_QUEUES=2
|
||||
# Persistent compile cache. MEASURED 2026-10-02: the image ships NO
|
||||
# /root/.cache at all (`ls: cannot access '/root/.cache'` in a fresh
|
||||
# container), and after our boot it holds /root/.cache/vllm at 1,154,714,333 B
|
||||
# in 4,665 files plus /root/.cache/comgr at 229 MB — both inside the
|
||||
# container's ephemeral layer, so every `podman rm -f` (every deploy, restart
|
||||
# and reboot) throws them away and re-pays the build. The engine's own log
|
||||
# names the loss: `torch.compile took 181.20 s in total`, of which 'Compiling a
|
||||
# graph for compile range (1, 1024) takes 173.19 s', followed by 'saved AOT
|
||||
# compiled function to /root/.cache/vllm/torch_compile_cache/…'. Mounting a
|
||||
# host dir at /root/.cache persists the vLLM AOT/torch_compile tree AND the
|
||||
# comgr HIP-kernel JIT cache, and hides nothing (the image has no such dir).
|
||||
# PROVEN LIVE 2026-10-02, two boots of this exact shape:
|
||||
# cold `Compiling a graph for compile range (1, 1024) takes 179.24 s`
|
||||
# `torch.compile took 188.26 s in total`
|
||||
# warm `Directly load the compiled graph(s) for compile range (1, 1024)
|
||||
# from the cache, took 2.143 s`
|
||||
# `torch.compile took 3.97 s in total`
|
||||
# `init engine (profile, create kv cache, warmup model) took 41.13 s
|
||||
# (compilation: 3.98 s)`
|
||||
# (Total time to /health is dominated by the 172.8 GiB payload load, not the
|
||||
# compile — the saving is the ~184 s of inductor/AOT build.)
|
||||
# Safe to grow/prune: entries are keyed by version + config hash, so a stale
|
||||
# entry is a miss, never a wrong answer.
|
||||
FRANK_PRIMARY_CACHE_DIR=/home/dank/.cache/frank-primary
|
||||
FRANK_PRIMARY_MAXLEN=200704
|
||||
FRANK_PRIMARY_SEQS=3
|
||||
# The POOL is deliberately kept at shape D's 3.15e9 (option A): because the pool
|
||||
# is not an arena lever, raising it back from a cut buys concurrency only, and
|
||||
# 3.15e9 preserves the measured 1.99x while precise-memory grows the arena. A
|
||||
# 2.2e9 pool was probed in the same window (277,686 tokens / 1.38x) and rejected
|
||||
# as the shipped value; it remains the conservative arm if free VRAM ever binds.
|
||||
# KV forced to 400,700 tokens (Frank, 2026-10-03). The engine prices this pool
|
||||
# at 7,891.8 B/token at this shape and loses ~3.87 MB to block alignment, so
|
||||
# 3,166,500,000 B is the ask that lands at/above 400,700 (verify the reported
|
||||
# "GPU KV cache size" line on every deploy; it is not a constant ratio across
|
||||
# geometries — 7,701 B/token at 262144).
|
||||
FRANK_PRIMARY_KV_BYTES=3166500000
|
||||
FRANK_PRIMARY_KV_DTYPE=fp8
|
||||
FRANK_PRIMARY_MNBT=2048
|
||||
FRANK_PRIMARY_GATHER_CHUNKS=32
|
||||
FRANK_PRIMARY_GATHER_LANES=64
|
||||
# PREFIX CACHING: ON. An OFF experiment ran 2026-10-05 (below) and FAILED
|
||||
# to stop the corruption, so the historical flag is back.
|
||||
# MEASURED 2026-10-04, 18:46:07 PDT (5.6 h into the live boot, one running
|
||||
# request, prefix hit rate 48.1%): 48 gdn_hip state faults in one 0.6 s
|
||||
# burst, then the process-lifetime silence cap:
|
||||
# [gdn_hip] ERROR gdn_decode_r: state out of ssm-cache range: max|S|=0
|
||||
# with non-finite values (slot=42 head=3 limit=3.40282e+38) fault #1-23
|
||||
# [gdn_hip] ERROR gdn_decode_r: ... (slot=42 head=8 ...) fault #24-46
|
||||
# [gdn_hip] ERROR: state fault report limit (32) reached; further faults
|
||||
# are counted silently
|
||||
# One physical SSM slot, two heads, BOTH ranks (heads 3 and 8), 23 layers
|
||||
# each. The guard's own message in this build: "known Qwen3.8-Flash-Next
|
||||
# (QFN) flaw under prefix caching and is being actively worked; output
|
||||
# quality for this request may degrade. Mitigation:
|
||||
# --no-enable-prefix-caching".
|
||||
# MECHANISM: with prefix caching on, this fork puts QFN on
|
||||
# mamba_cache_mode 'align' (the default-for-this-arch branch in
|
||||
# config/vllm.py; 'all' is deprecated and 'none' is the no-prefix-cache
|
||||
# mode, so align is the only sound mode WITH the flag). Align mode
|
||||
# checkpoints each sequence's GDN (Gated DeltaNet) recurrent state into a
|
||||
# mamba page every block, where block = 3136 tokens (the attention block
|
||||
# size is raised to 3136 so the page sizes match exactly — the boot log's
|
||||
# "Padding mamba page size by 0.13% ..."), and RE-SEEDS the state from that
|
||||
# page whenever a request resumes from a cached prefix. The corrupt state
|
||||
# landed at the 13th boundary: 13 x 3136 = 40,768 tokens — the "~40k" mark
|
||||
# where the request's output turned repetitive/corrupted. The NaN stayed
|
||||
# in the mamba page, so every later turn of that conversation re-seeded the
|
||||
# same non-finite state for those two heads (silent: the fault-report cap
|
||||
# was already spent), while fresh short sessions looked fine.
|
||||
# RAM WAS "RULED OUT" THE SAME DAY — AND THAT WAS WRONG (RESOLVED below).
|
||||
# What the evidence actually supported: zero MCE/EDAC records is the
|
||||
# EXPECTED silence of consumer non-ECC memory (a silent flip raises no
|
||||
# machine check); an 8 GiB pattern check over 60 s covers ~0.5 % of
|
||||
# 246 GiB — no verdict on a speed setting; "4800 is the JEDEC floor on
|
||||
# 5600-rated DIMMs" is a rating, not a stability proof at four-DIMM
|
||||
# loading. Only the two-head NaN signature argument was load-bearing,
|
||||
# and the revert test outvoted it.
|
||||
# COST ACCEPTED: no prefix reuse, so each turn re-prefills the whole
|
||||
# history instead of at most 3135 tail tokens (a full 40k context costs
|
||||
# tens of seconds of TTFT per turn at this shape's prefill rate). A silently
|
||||
# corrupted conversation is worse than a slower first token.
|
||||
# RESULT (2026-10-05, later that day): THE MITIGATION DID NOT WORK. With
|
||||
# --no-enable-prefix-caching deployed and mechanically verified (zero
|
||||
# gdn_hip faults, a 40,771-token single-request boundary probe coherent),
|
||||
# live long conversations STILL degraded at ~40k on the running endpoint
|
||||
# (Frank, direct). The container serving that window logged ZERO faults
|
||||
# while the corruption reproduced, so the align-mode page re-seed is not
|
||||
# the (only) mechanism — see the runbook's RESULT note for the open
|
||||
# hypotheses (re-fed poisoned transcript vs a fault-free long-context
|
||||
# path such as fp8 KV at depth). Cost bought nothing: caching is back ON.
|
||||
# The knob stays commented for a future fork build with a fixed GDN state
|
||||
# path: set 1, redeploy, re-verify past 40,768 tokens (one conversation
|
||||
# coherent AND zero gdn_hip lines).
|
||||
# RESOLVED 2026-10-05 EVENING (Frank): IT WAS THE RAM. 4800 MT/s (in
|
||||
# effect since the 2026-10-04 17:17 boot) was marginal; 4600 MT/s (since
|
||||
# the 2026-10-05 12:21 boot) and the endpoint is clean — zero gdn_hip
|
||||
# ERROR lines, zero MCE, and the live agent session on it past 140k
|
||||
# context with no degradation. Both failure eras sit under the RAM: the
|
||||
# 2026-10-04 NaN burst (corrupted weight/activation traffic poisoning a
|
||||
# GDN state write — the guard-visible path) and the fault-free ~40k
|
||||
# degradation (silent flips in KV/expert data, which no guard covers on
|
||||
# non-ECC). CONSEQUENCE FOR THE KNOB: the OFF experiment was CONFOUNDED
|
||||
# — both arms ran on the marginal speed — so it neither proved nor
|
||||
# disproved the fork's align-mode flaw. Prefix caching stays ON (zero
|
||||
# gdn_hip lines under heavy re-send traffic since the speed drop); flip
|
||||
# the knob only if a fault recurs on stable RAM or the fork ships a fix.
|
||||
# FRANK_PRIMARY_NO_PREFIX_CACHING=1
|
||||
# UTIL is an ARENA lever: the fork plans `budget = device_total x util`, and at
|
||||
# 31.86 GiB/card every +0.01 is +0.32 GiB of expert cache (~+139 slots), and
|
||||
# 0.982 is Frank's standing law: NEVER 1.00. The 1.00 attempt (2026-10-03,
|
||||
# shape G v1) died in worker init BEFORE loading a byte of weights — the fork
|
||||
# refuses budget >= device total with
|
||||
# ValueError: Free memory on device cuda:N (31.29/31.86 GiB ...)
|
||||
# on BOTH ranks at 09:53 and 09:54, NRestarts=2 — so 1.0 is not even a runtime
|
||||
# knife-edge here, it is unbootable. The two historical ceiling deaths (the
|
||||
# 2x262144 bf16 boot at 0.13 GiB free; the 5x262144 boot at "0 bytes free" on a
|
||||
# real 995-token request) remain the reason the margin exists. Rollback note
|
||||
# for the other direction: if a request ever dies with
|
||||
# `Tried to allocate ... 0 bytes free`, the first knob back is THIS number
|
||||
# down a notch, not the window.
|
||||
FRANK_PRIMARY_UTIL=0.982
|
||||
# CUDA-graph plan term, priced at the measured capture. expert_plan.py charges
|
||||
# estimate_cudagraph_memory() = BASE_MB + PER_GRAPH_MB x n_graphs (defaults
|
||||
# 256/52 = 0.55 GiB for our 6-graph plan), and the estimator's own docstring
|
||||
# says to tune these against the post-capture line. MEASURED twice on this
|
||||
# shape: shape E reported "Graph capturing finished ... took 0.31 GiB" (TP0) and
|
||||
# 0.17 GiB (TP1); shape F, with the 0.32 GiB we then charged, reported
|
||||
# "took -0.06 GiB" — a NEGATIVE delta, i.e. capture fits inside memory the
|
||||
# profiler had already counted. So the base pool (256 MiB) is the over-charge:
|
||||
# charge 0 base + 12 MiB/graph = 0.07 GiB for the shipped no-spec 6-graph plan,
|
||||
# which returns 0.25 GiB more to the arena while still paying per graph, so a
|
||||
# future spec-decode shape (18 graphs) is charged 0.21 GiB rather than zero.
|
||||
# Verify every deploy against the "Estimated CUDA graph memory" vs "Graph
|
||||
# capturing finished ... took X GiB" lines; if capture ever goes positive and
|
||||
# exceeds the charge, raise FRANK_PRIMARY_CUDAGRAPH_PER_GRAPH_MB back toward 52.
|
||||
# A THIRD env var exists and is the only way to state the graph term outright:
|
||||
# VLLM_CUDAGRAPH_MEM_ESTIMATE_MB pins the WHOLE term to an exact MiB total in
|
||||
# one knob — estimate_cudagraph_memory() checks it FIRST and returns base/
|
||||
# per-graph untouched when set (verified in the shipped image,
|
||||
# v1/worker/gpu/cudagraph_mem.py). We ship the BASE/PER_GRAPH pair instead so
|
||||
# a future spec-decode shape re-prices automatically; switch to ESTIMATE_MB
|
||||
# only if the drift check below ever shows the pair cannot track capture.
|
||||
FRANK_PRIMARY_CUDAGRAPH_BASE_MB=0
|
||||
FRANK_PRIMARY_CUDAGRAPH_PER_GRAPH_MB=12
|
||||
|
||||
# HOST-region cap — the last arena-plan term that still autofit per boot. With
|
||||
# no cap the planner's host bound is MemAvailable AT BOOT minus PLE 47.68 -
|
||||
# 2 x 7.0 process - 5.0 api/engine - 5% margin (measured boot 2026-10-05:
|
||||
# MemAvailable 237.97 -> 159.39 GiB usable across 2 ranks; the demand is 105.48
|
||||
# = 52.74/rank x 2 ranks, 23,040 swap-target experts). The same host reads 75.4
|
||||
# GiB available while running — the bound tracks whatever the co-tenants hold at
|
||||
# boot, and a boot where that drops near the demand silently pins next-hottest
|
||||
# experts VRAM-only instead (smaller swap region; the pinned_extra warning is
|
||||
# the only trace). Pin 120 GiB: measured demand + 14% headroom, on a host with
|
||||
# 246.1 GiB total. If a boot ever logs "expert-offload-mem … exceeds
|
||||
# MemAvailable at plan time", THAT is the contract being broken — look at the
|
||||
# co-tenant RAM before touching this number. Plan semantics verified in the
|
||||
# shipped image (offloader/expert_plan.py): the cap binds as the bound, an
|
||||
# exceedance only warns.
|
||||
FRANK_PRIMARY_EXPERT_OFFLOAD_MEM=120
|
||||
|
||||
# --- optional engine knobs (unset == the flag is absent) ---------------------
|
||||
# FRANK_PRIMARY_SPEC holds the WHOLE --speculative-config JSON; setting it adds
|
||||
# the flag. THE RETIREMENT VERDICT WAS INVALID: c1248d4 (2026-10-02) retired MTP
|
||||
# by comparing fp8+MTP4 (37.4-40.6 tok/s, 54% acceptance, 09-29) against the
|
||||
# bf16 no-spec boot (36.4-39.4) — and bf16 then proved unbootable, so fp8 came
|
||||
# back without MTP and fp8+spec was NEVER compared against fp8+no-spec on the
|
||||
# same shape. The no-spec incumbent measures ~30 tok/s single-stream (below the
|
||||
# fleet's 35 tok/s target); every vendor flash-next recipe ships MTP3; and our
|
||||
# MTP4 run used the fork's default GREEDY draft sampling, while the recipes pass
|
||||
# draft_sample_method=probabilistic (greedy drafts are rejected more often under
|
||||
# temp>0 — the plausible cause of the 54%). MTP also re-adds drafter workspace:
|
||||
# the old note says the 7.2 GiB/rank KV ceiling WAS the MTP4 workspace, and
|
||||
# --expert-precise-memory spends that same budget, so an MTP arm must be tested
|
||||
# with precise memory BOTH on and off.
|
||||
# MTP3: '{"method":"mtp","num_speculative_tokens":3,"draft_sample_method":"probabilistic"}'
|
||||
FRANK_PRIMARY_SPEC=
|
||||
# FRANK_PRIMARY_COMPCONFIG holds the whole --compilation-config JSON. The
|
||||
# capture-ladder law: the piecewise cudagraph ladder must contain
|
||||
# num_speculative_tokens + 1 (MTP3 -> 4 -> ladder 4,8,12; MTP4 -> 5,10,15;
|
||||
# MTP5 -> 6,12,18,24), because every draft step proposes n+1 tokens. The default
|
||||
# ladder is multiples of 1+4 up to 512 then powers of two, so an MTP3 shape can
|
||||
# miss its own step size. Pinning the ladder is ALSO the candidate fix for the
|
||||
# SEQS>3 wall: query_start_loc is specialised to SEQS+1 and that constant must
|
||||
# land on a captured graph size (3 is not in the default ladder, 4 is —
|
||||
# measured 2026-10-02, and the vendor runs SEQS 16 on this model/TP2).
|
||||
# combo_kernels false/true are BOTH in the shipped recipes, so it is not a law.
|
||||
# MTP3 at SEQS 3: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[1,4,8,12],"max_cudagraph_capture_size":12}'
|
||||
FRANK_PRIMARY_COMPCONFIG=
|
||||
# FRANK_PRIMARY_GEN_CONFIG holds the whole --override-generation-config JSON.
|
||||
# The payload's HF generation_config.json sets temperature 1.0 / top_p 0.95 /
|
||||
# top_k 20 and the engine WARNS about it on every boot, so a client that sends
|
||||
# no sampling fields gets creativity it never asked for. This is the one knob
|
||||
# that changes ANSWERS, not speed.
|
||||
# '{"max_tokens":65536,"temperature":0.7,"top_p":0.95,"top_k":20}'
|
||||
FRANK_PRIMARY_GEN_CONFIG=
|
||||
|
||||
# --- client contract: reasoning split, tools, preserve_thinking ----------------
|
||||
# Without --reasoning-parser the engine leaves the thinking text INLINE in
|
||||
# `content` with a dangling `</think>` and a null `reasoning` field (measured
|
||||
# 2026-10-01: content = 'We need answer...Brief.\n\n391') — the
|
||||
# malformed thinking block clients choke on. The qwen3 parser splits it
|
||||
# properly. The GLM rollback (d91379d) had dropped all three of these; the
|
||||
# working values are the ones ecaad36 shipped for this model.
|
||||
FRANK_PRIMARY_REASONING_PARSER=qwen3
|
||||
# Tools: --enable-auto-tool-choice + the qwen3_xml parser is what makes
|
||||
# tool_calls work here (without it, tool_choice auto returns 400).
|
||||
FRANK_PRIMARY_TOOL_PARSER=qwen3_xml
|
||||
# preserve_thinking keeps prior thinking blocks in the rendered prompt across
|
||||
# turns; enable_thinking makes thinking the server-side default. Verified the
|
||||
# payload's tokenizer_config template consumes both keys.
|
||||
FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS='{"enable_thinking": true, "preserve_thinking": true}'
|
||||
# --- FreeToken engine knobs (first-boot conservative, resize live) -----
|
||||
# Single GPU: the R9700 at index 0. HIP_VISIBLE_DEVICES=0 hides card 1 so
|
||||
# the in-container index 0 IS the right card; --gpu 0 selects it.
|
||||
FRANK_PRIMARY_FT_GPU=0
|
||||
# KV pool in TOKENS (not bytes — vLLM-era FRANK_PRIMARY_KV_BYTES is gone).
|
||||
# 300,000 tokens x 24 KiB = 7.2 GiB; clears the harness worst case 177,356.
|
||||
# Grows live with `ft ctl cache --kv <n>`; shrink needs a restart.
|
||||
FRANK_PRIMARY_FT_NUM_TOKENS=300000
|
||||
# Concurrent decode slots. 2 is the first-boot conservative value; the
|
||||
# radix pool admits ~2 full-length 200,704-token conversations. Raise with
|
||||
# --max-running-requests only after a real-traffic look at slot occupancy.
|
||||
FRANK_PRIMARY_FT_MAX_RUNNING=2
|
||||
# Per-request output budget ceiling: the harness's maxTokens (36,864).
|
||||
# Requests that omit max_tokens get this default instead of the engine's
|
||||
# 32,768.
|
||||
FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS=36864
|
||||
# GPU expert LRU cache as a FRACTION of all experts (~120 GiB fp8 total ->
|
||||
# ~12 GiB VRAM at 0.10). First-boot conservative; resize live with
|
||||
# `ft ctl cache --moe` after reading the boot ledger + ft ctl stats.
|
||||
FRANK_PRIMARY_FT_MOE_RATE=0.10
|
||||
# Escape hatch for boot-ladder experiments (YaRN, --max-prefill-length,
|
||||
# --moe-cpu-threads, --sampling-defaults none, ...). Unset -> no extra
|
||||
# argv. Keep multi-flag values as one string; the serve script expands it
|
||||
# unquoted on purpose.
|
||||
FRANK_PRIMARY_FT_EXTRA_ARGS=
|
||||
|
||||
@@ -1,29 +1,26 @@
|
||||
# frank-primary.service — tcclaviger RDNA4 vLLM endpoint (orcarouter
|
||||
# Qwen3.8-Flash-Next-Uncensored-FP8, fp8 KV, NO spec decode, 3 slots x 200704
|
||||
# context with the pool sized to ~2 full-length conversations, TP=2, tiered
|
||||
# experts, 131-shard payload) on the 2x R9700 pair, dankserver :7770.
|
||||
# 2026-09-29 cutover from the R9V radiance fork (git tag
|
||||
# frank-primary-r9v-q8-20260929 is the pre-cutover state; its
|
||||
# launcher/shim/setup-state machinery is retired and the state dirs stay on
|
||||
# disk as rollback).
|
||||
# frank-primary.service — FreeToken (PR #587 ROCm) endpoint (orcarouter
|
||||
# Qwen3.8-Flash-Next-Uncensored-FP8, 131-shard payload) on ONE R9700
|
||||
# (gfx1201), dankserver :7770. 2026-10-05 cutover from the tcclaviger vLLM
|
||||
# fork (git tag frank-primary-vllm-shapeG-20261003 is the pre-cutover
|
||||
# state; the retired unit/env/serve trio is in that tag).
|
||||
#
|
||||
# Same port, same served alias; the harness models.yml id is unchanged.
|
||||
# The serve script drives `podman run` directly (the image entrypoint execs
|
||||
# `vllm serve`) and then waits on the container. NOTE: that wait can return
|
||||
# 0 even when the engine died mid-request (vLLM's API server drains and
|
||||
# exits clean on a worker fault), which is why the restart policy is
|
||||
# `always` — see the [Service] note.
|
||||
# Same port, same served alias, same container identity as the unit name;
|
||||
# the harness models.yml id is unchanged. The serve script drives
|
||||
# `podman run` directly (the image carries `ft serve`) and then waits on
|
||||
# the container. The image is built ON THE HOST from
|
||||
# scripts/docker/frank-primary-freetoken.Dockerfile (FreeToken PR head
|
||||
# 77a7201 in the PR's own ROCm 7.14 base image).
|
||||
#
|
||||
# WHY A ROOT UNIT (not a rootless quadlet): the tiered-expert shape pins tens
|
||||
# of GiB of host RAM (the expert host slabs, ~20.5 GiB per rank of pinned
|
||||
# anonymous memory). Pinned pages are charged against RLIMIT_MEMLOCK; dank's
|
||||
# login session on this host carries 8 MiB and a user quadlet is capped by the
|
||||
# user manager's own hard limit — raising that means restarting
|
||||
# user@1000.service and bouncing every user container on the host. A root unit
|
||||
# grants LimitMEMLOCK=infinity to the dank process via PID1, and the image
|
||||
# stays in dank's rootless podman storage.
|
||||
# WHY A ROOT UNIT (not a rootless quadlet): FreeToken pins ~120 GiB of
|
||||
# expert weights plus the ~47.7 GiB PLE ngram table in HOST RAM. Pinned
|
||||
# pages are charged against RLIMIT_MEMLOCK; dank's login session on this
|
||||
# host carries 8 MiB and a user quadlet is capped by the user manager's own
|
||||
# hard limit — raising that means restarting user@1000.service and bouncing
|
||||
# every user container on the host. A root unit grants
|
||||
# LimitMEMLOCK=infinity to the dank process via PID1, and the image stays
|
||||
# in dank's rootless podman storage.
|
||||
[Unit]
|
||||
Description=frank-primary — tcclaviger vLLM Qwen3.8-Flash-Next-Uncensored-FP8 on 2x R9700
|
||||
Description=frank-primary — FreeToken (PR587) Qwen3.8-Flash-Next-Uncensored-FP8 on 1x R9700
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
# Never let the start-rate limiter give up after a crash loop (a failed LLM
|
||||
@@ -33,34 +30,35 @@ StartLimitIntervalSec=0
|
||||
[Service]
|
||||
User=dank
|
||||
Group=dank
|
||||
# The tiered-expert offloader pins host RAM; an RLIMIT_MEMLOCK smaller than
|
||||
# the pinned set fails the load mid-start. PID1 grants infinity here —
|
||||
# The expert LRU + PLE pin ~170 GiB of host RAM; an RLIMIT_MEMLOCK smaller
|
||||
# than the pinned set fails the load mid-start. PID1 grants infinity here —
|
||||
# parity with the fleet's other LLM units.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh
|
||||
# systemd stop: wait alone cannot stop the container child; stop it
|
||||
# explicitly. vLLM 0.26 drains gracefully on SIGTERM.
|
||||
ExecStop=/usr/bin/podman stop -t 120 frank-primary-qwen
|
||||
# Boot cost: FP8 payload page-in plus torch compile/graph capture —
|
||||
# ~15 min cold (measured 2026-09-29), faster warm.
|
||||
# explicitly. ft serve drops connections on SIGTERM; no graceful drain
|
||||
# contract is claimed (the OpenAI front end holds no queue to drain —
|
||||
# in-flight decodes are lost, which a stop during a bench window accepts).
|
||||
ExecStop=/usr/bin/podman stop -t 120 frank-primary
|
||||
# Boot cost: 131-shard FP8 page-in + PLE pin + expert upload. vLLM-era
|
||||
# budget (1800 s) carried forward; the FreeToken first boot measured here
|
||||
# once it ran (see the runbook for the real number).
|
||||
TimeoutStartSec=1800
|
||||
# Restart=always, NOT on-failure (measured 2026-10-01): a runtime
|
||||
# torch.OutOfMemoryError kills the engine while vLLM's API server still
|
||||
# exits 0, so ExecStart returned clean, systemd logged "Deactivated
|
||||
# successfully", and on-failure NEVER fired — the endpoint stayed down until
|
||||
# a human noticed. `always` restarts on any container death; an explicit
|
||||
# `systemctl stop` still wins (systemd does not restart a unit the operator
|
||||
# stopped), so bench windows and deploys are unaffected.
|
||||
# Restart=always, NOT on-failure (the 2026-10-01 vLLM lesson carries over:
|
||||
# a clean API-server exit after a worker fault reads as success to
|
||||
# systemd). `always` restarts on any container death; an explicit
|
||||
# `systemctl stop` still wins, so bench windows and deploys are unaffected.
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
# ExecStop allows the engine 120 s to drain (`podman stop -t 120`); without an
|
||||
# explicit TimeoutStopSec systemd's 90 s default cuts the stop short, kills it,
|
||||
# and records 'Failed with result timeout' (measured 2026-10-01 13:27:17).
|
||||
# Must exceed the podman grace with margin for the engine's own shutdown.
|
||||
# ExecStop allows the engine 120 s to drain (`podman stop -t 120`); without
|
||||
# an explicit TimeoutStopSec systemd's 90 s default cuts the stop short,
|
||||
# kills it, and records 'Failed with result timeout' (measured 2026-10-01
|
||||
# 13:27:17). Must exceed the podman grace with margin.
|
||||
TimeoutStopSec=180
|
||||
# The engine pins tens of GiB of host RAM (expert host slabs); it is the
|
||||
# last process on this host that should be chosen by the kernel OOM killer.
|
||||
# The engine pins tens of GiB of host RAM (expert rows + PLE table); it is
|
||||
# the last process on this host that should be chosen by the kernel OOM
|
||||
# killer.
|
||||
OOMScoreAdjust=-500
|
||||
|
||||
[Install]
|
||||
|
||||
+65
-129
@@ -1,58 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# serve-frank-primary.sh — frank-primary endpoint (tcclaviger RDNA4 vLLM fork,
|
||||
# orcarouter Qwen3.8-Flash-Next-Uncensored-FP8, 2x R9700,
|
||||
# dankserver :7770; 2026-09-29 cutover from the R9V radiance fork — git tag
|
||||
# frank-primary-r9v-q8-20260929 is the pre-cutover state).
|
||||
#
|
||||
# The engine is driven by `podman run` DIRECTLY: the image's entrypoint execs
|
||||
# `vllm serve "$@"`, so the run line below IS the engine argv. This replaces
|
||||
# the R9V launcher/shim machinery (that fork needed a docker->podman shim and
|
||||
# a 104-key on-host setup state; this stack is fully expressed by flags, and
|
||||
# every flag here was qualified by boot probe on 2026-09-29 — see
|
||||
# frank-primary.env for the qualification record).
|
||||
#
|
||||
# Shape (2026-10-03 shape F, on the 2026-10-02 shape E base): MAXLEN 200704 x 3
|
||||
# slots, fp8 KV, KV_BYTES
|
||||
# 3,150,000,000 PER RANK — the POOL, not the slot count, caps resident
|
||||
# conversations at ~2 full-length (measured: 398,658 tokens = 1.99x);
|
||||
# shorter requests use the spare blocks. --max-num-seqs stays 3 because SEQS=2
|
||||
# CANNOT boot this model
|
||||
# (torch.fx ConstraintViolationError on query_start_loc, measured at both
|
||||
# MNBT 1024 and 2048 — the 2x2 lives in frank-primary.env). Speculative decode
|
||||
# is OFF today by ENV KNOB, not by measurement: FRANK_PRIMARY_SPEC / _COMPCONFIG
|
||||
# / _GEN_CONFIG hold the whole JSON value and unset means no flag. The 2026-10-02
|
||||
# retirement of MTP compared the wrong pair — see frank-primary.env.
|
||||
# --max-num-batched-tokens 2048 (Frank's call 2026-10-03; arena-free because the
|
||||
# registry prices overhead flat up to its nbt 4096 anchor), MoE gather geometry
|
||||
# R4D_LRU_GATHER_CHUNKS=32 / R4D_LRU_GATHER_LANES=64 (+18.7% decode, grows
|
||||
# with depth), prefix caching OFF 2026-10-05 (the fork's QFN align-mode GDN
|
||||
# state fault — the evidence block is in frank-primary.env), expert prefetch
|
||||
# OFF (36% precision measured),
|
||||
# TunableOp off. Shape E adds --expert-precise-memory (fitted 5.25 GiB overhead
|
||||
# instead of the 7.60 GiB default -> expert arena 17.22 GiB / 7,522 slots, +16%)
|
||||
# and GPU_MAX_HW_QUEUES=2; shape F pins the plan's CUDA-graph term to the
|
||||
# measured capture cost (VLLM_CUDAGRAPH_MEM_BASE_MB / _PER_GRAPH_MB, container
|
||||
# env — the analytic default charges 0.55 GiB where capture took 0.31 GiB, so
|
||||
# the 0.24 GiB difference goes to the arena) and raises util 0.98 -> 0.982,
|
||||
# Frank's cap. The native-262144 and fp8+MTP4 shapes are retired to
|
||||
# git tag frank-primary-fp8-mtp-20261001.
|
||||
#
|
||||
# THE COMPILE CACHE IS MOUNTED, and that is a boot-cost fix, not a tuning knob:
|
||||
# the image ships no /root/.cache at all, our first boot writes 1.1 GB of vLLM
|
||||
# AOT/torch_compile artifacts plus 229 MB of comgr JIT kernels into the
|
||||
# container's ephemeral layer, and `podman rm -f` deletes them — so every
|
||||
# deploy/restart/reboot re-pays `torch.compile took 181.20 s in total` (173.19 s
|
||||
# of it one graph). Mounting FRANK_PRIMARY_CACHE_DIR at /root/.cache makes all
|
||||
# later boots load the compiled artifact instead, and hides nothing. PROVEN
|
||||
# LIVE 2026-10-02 on consecutive boots of this shape: cold `torch.compile took
|
||||
# 188.26 s`, warm `Directly load the compiled graph(s) for compile range
|
||||
# (1, 1024) from the cache, took 2.143 s` / `torch.compile took 3.97 s in
|
||||
# total` / `init engine … took 41.13 s (compilation: 3.98 s)`.
|
||||
# serve-frank-primary.sh — frank-primary endpoint (FreeToken PR #587 ROCm
|
||||
# build, orcarouter Qwen3.8-Flash-Next-Uncensored-FP8, ONE R9700 gfx1201,
|
||||
# dankserver :7770; 2026-10-05 cutover from the tcclaviger vLLM fork — git
|
||||
# tag frank-primary-vllm-shapeG-20261003 is the pre-cutover state).
|
||||
#
|
||||
# The engine is driven by `podman run` DIRECTLY and the run line below IS
|
||||
# the engine argv: the image carries FreeToken installed at pinned PR head
|
||||
# 77a7201 (scripts/docker/frank-primary-freetoken.Dockerfile builds it from
|
||||
# the PR's own docs/install_amd.md stack). Unlike the vLLM fork there is no
|
||||
# per-boot planner: experts live in pinned host RAM with an LRU GPU cache,
|
||||
# KV is an explicit token pool, and the boot ledger in the log names
|
||||
# Shape: 1 GPU (first-boot conservative — the second R9700 sits idle;
|
||||
# --tensor-parallel-size EXISTS in FreeToken, so a 2-card TP shape is future
|
||||
# tuning once this single-card shape has a decode baseline), KV pool 300,000
|
||||
# tokens (24 KiB/token f16 across the 12 full-attention layers = 7.2 GiB;
|
||||
# clears the harness worst case of 177,356 tokens and admits one NATIVE
|
||||
# 262,144-token conversation), MoE LRU at 10% of all experts (~12 GiB),
|
||||
# 2 decode slots. First-boot conservative; `ft ctl cache --kv/--moe` grows
|
||||
# pools live, and the env comment block carries the resize law. Flag
|
||||
# surface verified against docs/cli.md of the pinned head AND against
|
||||
# `ft serve --help` inside the built image; every value here is env-driven
|
||||
# (frank-primary.env).
|
||||
|
||||
# The root unit (scripts/frank-primary.service, User=dank,
|
||||
# LimitMEMLOCK=infinity) grants the pin budget for the tiered-expert host
|
||||
# slabs (~20.5 GiB/rank of pinned anonymous memory). A plain shell session
|
||||
# also works, but the unit is the fleet-parity boot-persistent shape.
|
||||
# LimitMEMLOCK=infinity) grants the pin budget: ~120 GiB of expert weights
|
||||
# plus the ~47.7 GiB PLE table pin in host RAM. A plain shell session also
|
||||
# works, but the unit is the fleet-parity boot-persistent shape.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -67,27 +40,30 @@ set -a
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
|
||||
: "${FRANK_PRIMARY_IMAGE:?FRANK_PRIMARY_IMAGE must name the digest-pinned image}"
|
||||
: "${FRANK_PRIMARY_IMAGE:?FRANK_PRIMARY_IMAGE must name the FreeToken image}"
|
||||
: "${FRANK_PRIMARY_MODEL_DIR:?FRANK_PRIMARY_MODEL_DIR must name the FP8 payload}"
|
||||
: "${FRANK_PRIMARY_MODEL_SHARDS:?FRANK_PRIMARY_MODEL_SHARDS must give the shard count}"
|
||||
: "${FRANK_PRIMARY_CONTAINER:?FRANK_PRIMARY_CONTAINER must name the container}"
|
||||
: "${FRANK_PRIMARY_SERVED_NAME:?FRANK_PRIMARY_SERVED_NAME must name the alias}"
|
||||
: "${FRANK_PRIMARY_PUBLISH:?FRANK_PRIMARY_PUBLISH must be host:container}"
|
||||
: "${FRANK_PRIMARY_MAXLEN:?FRANK_PRIMARY_MAXLEN must be set}"
|
||||
: "${FRANK_PRIMARY_SEQS:?FRANK_PRIMARY_SEQS must be set}"
|
||||
: "${FRANK_PRIMARY_KV_BYTES:?FRANK_PRIMARY_KV_BYTES must be set (PER RANK)}"
|
||||
: "${FRANK_PRIMARY_KV_DTYPE:?FRANK_PRIMARY_KV_DTYPE must be set (auto = bf16 KV)}"
|
||||
: "${FRANK_PRIMARY_REASONING_PARSER:?FRANK_PRIMARY_REASONING_PARSER must be set (qwen3)}"
|
||||
: "${FRANK_PRIMARY_TOOL_PARSER:?FRANK_PRIMARY_TOOL_PARSER must be set (qwen3_xml)}"
|
||||
: "${FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS:?FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS must be set}"
|
||||
# Capture podman's output BEFORE grepping: under `set -o pipefail`, the
|
||||
# classic `podman images | grep -q` pattern races — grep exits on its first
|
||||
# match, podman dies SIGPIPE (141) writing the remaining lines, and pipefail
|
||||
# turns the 141 into "no match" (the 2026-09-30 deploy refusal).
|
||||
DIGESTS="$(podman images --format '{{.RepoDigests}}')"
|
||||
if ! grep -qF "${FRANK_PRIMARY_IMAGE#*@}" <<<"$DIGESTS"; then
|
||||
echo "serve-frank-primary: pinned image absent from rootless storage: $FRANK_PRIMARY_IMAGE" >&2
|
||||
echo " fetch it: podman pull $FRANK_PRIMARY_IMAGE" >&2
|
||||
: "${FRANK_PRIMARY_FT_GPU:?FRANK_PRIMARY_FT_GPU must name the GPU index}"
|
||||
: "${FRANK_PRIMARY_FT_NUM_TOKENS:?FRANK_PRIMARY_FT_NUM_TOKENS must size the KV pool}"
|
||||
: "${FRANK_PRIMARY_FT_MAX_RUNNING:?FRANK_PRIMARY_FT_MAX_RUNNING must be set}"
|
||||
: "${FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS:?FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS must be set}"
|
||||
: "${FRANK_PRIMARY_FT_MOE_RATE:?FRANK_PRIMARY_FT_MOE_RATE must be set}"
|
||||
|
||||
# The image gate is EXISTENCE IN STORAGE, not a registry digest: the tag is
|
||||
# built ON THE HOST (podman build from
|
||||
# scripts/docker/frank-primary-freetoken.Dockerfile) and local builds carry
|
||||
# no RepoDigests entry — the 362-line vLLM-era RepoDigests grep refused
|
||||
# exactly this shape. Pinning lives in the Dockerfile instead: the FreeToken
|
||||
# source is checked out at a hardcoded PR-head SHA and the base image is
|
||||
# digest-pinned there. `podman image exists` accepts both a local tag and a
|
||||
# name@digest ref, and it is one command — no pipefail/grep race (the
|
||||
# 2026-09-30 lesson) for either shape.
|
||||
if ! podman image exists "$FRANK_PRIMARY_IMAGE"; then
|
||||
echo "serve-frank-primary: image absent from rootless storage: $FRANK_PRIMARY_IMAGE" >&2
|
||||
echo " build it: podman build -t $FRANK_PRIMARY_IMAGE -f scripts/docker/frank-primary-freetoken.Dockerfile scripts/docker/" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -114,8 +90,8 @@ fi
|
||||
|
||||
# A crashed/killed unit can leave the container name registered (systemd
|
||||
# SIGKILLs conmon before cleanup runs). Clear OUR name plus the legacy
|
||||
# identities (the retired R9V container and the ad hoc experiment container
|
||||
# both hold these GPUs); all idempotent.
|
||||
# identities (the vLLM-era frank-primary-qwen, the retired R9V container and
|
||||
# the ad hoc experiment containers all hold this GPU); all idempotent.
|
||||
for CTR in $FRANK_PRIMARY_LEGACY_CONTAINERS "$FRANK_PRIMARY_CONTAINER"; do
|
||||
podman rm -f "$CTR" >/dev/null 2>&1 || true
|
||||
done
|
||||
@@ -123,74 +99,34 @@ done
|
||||
LOG=/home/dank/logs/frank_primary.log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
# Persistent compile cache (see the header). Create it before the mount so even
|
||||
# the first, cold boot has a target; unset -> no mount, exactly the old
|
||||
# ephemeral behaviour. It carries /root/.cache/vllm (AOT + torch_compile) and
|
||||
# /root/.cache/comgr (HIP kernel JIT).
|
||||
CACHE_DIR="${FRANK_PRIMARY_CACHE_DIR:-}"
|
||||
if [ -n "$CACHE_DIR" ]; then
|
||||
mkdir -p "$CACHE_DIR"
|
||||
fi
|
||||
|
||||
# --device /dev/dri is the WHOLE DIRECTORY (individual renderD nodes enumerate
|
||||
# without the card nodes and HIP refuses contexts — "No CUDA GPUs are
|
||||
# available" while device_count() says 2). --ipc=host for the engine's shared
|
||||
# memory; memlock unbounded for the pinned host slabs.
|
||||
# The entrypoint already execs `vllm serve "$@"`, so no `serve` here.
|
||||
# Prefix caching: ON — the shipped contract. FRANK_PRIMARY_NO_PREFIX_-
|
||||
# CACHING=1 flips to the fork-named mitigation (--no-enable-prefix-caching)
|
||||
# for the QFN align-mode GDN state fault; it was tried 2026-10-05 and
|
||||
# REVERTED the same day (live conversations still degraded at ~40k with
|
||||
# caching off and zero faults — see the evidence block in
|
||||
# frank-primary.env). NO comment lines inside the backslash chain below —
|
||||
# a comment there terminates the continuation and the next line runs as
|
||||
# its own command (the 2026-10-05 exit-127 crash loop).
|
||||
# The ${VAR:-word} / ${VAR:+word} double-expansion for the two directions is
|
||||
# AMBIGUOUS bash (${VAR:--flag} parses as a substring offset when set —
|
||||
# measured 2026-10-05: rendered "--no-enable-prefix-caching1"), so the flag
|
||||
# is branched explicitly: exactly one of the two reaches the engine.
|
||||
if [ -n "${FRANK_PRIMARY_NO_PREFIX_CACHING:-}" ]; then
|
||||
FRANK_PRIMARY_PREFIX_CACHE_FLAG=--no-enable-prefix-caching
|
||||
else
|
||||
FRANK_PRIMARY_PREFIX_CACHE_FLAG=--enable-prefix-caching
|
||||
fi
|
||||
# without the card nodes and HIP refuses contexts). --ipc=host for the
|
||||
# engine's shared memory; memlock unbounded for the ~120 GiB pinned expert
|
||||
# rows + the 47.7 GiB PLE pin (the unit's LimitMEMLOCK=infinity is the other
|
||||
# half of that law).
|
||||
# HIP_VISIBLE_DEVICES pins visibility to ONE card so in-container index 0 is
|
||||
# the right R9700; --gpu 0 selects it inside the visible set.
|
||||
podman run -d --name "$FRANK_PRIMARY_CONTAINER" \
|
||||
--publish "$FRANK_PRIMARY_PUBLISH" \
|
||||
--ipc=host --ulimit memlock=-1:-1 \
|
||||
--cap-add SYS_PTRACE \
|
||||
--device /dev/kfd --device /dev/dri --device /dev/mem \
|
||||
-e HIP_VISIBLE_DEVICES=0,1 -e VLLM_ROCM_USE_AITER=0 \
|
||||
${FRANK_PRIMARY_GPU_MAX_HW_QUEUES:+-e GPU_MAX_HW_QUEUES="$FRANK_PRIMARY_GPU_MAX_HW_QUEUES"} \
|
||||
${FRANK_PRIMARY_GATHER_CHUNKS:+-e R4D_LRU_GATHER_CHUNKS="$FRANK_PRIMARY_GATHER_CHUNKS"} \
|
||||
${FRANK_PRIMARY_GATHER_LANES:+-e R4D_LRU_GATHER_LANES="$FRANK_PRIMARY_GATHER_LANES"} \
|
||||
${FRANK_PRIMARY_CUDAGRAPH_BASE_MB:+-e VLLM_CUDAGRAPH_MEM_BASE_MB="$FRANK_PRIMARY_CUDAGRAPH_BASE_MB"} \
|
||||
${FRANK_PRIMARY_CUDAGRAPH_PER_GRAPH_MB:+-e VLLM_CUDAGRAPH_MEM_PER_GRAPH_MB="$FRANK_PRIMARY_CUDAGRAPH_PER_GRAPH_MB"} \
|
||||
${CACHE_DIR:+-v "$CACHE_DIR":/root/.cache} \
|
||||
-e HIP_VISIBLE_DEVICES="$FRANK_PRIMARY_FT_GPU" \
|
||||
-v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \
|
||||
"$FRANK_PRIMARY_IMAGE" /models \
|
||||
--served-model-name "$FRANK_PRIMARY_SERVED_NAME" --port 7770 \
|
||||
--tensor-parallel-size 2 \
|
||||
${FRANK_PRIMARY_DCP:+--decode-context-parallel-size "$FRANK_PRIMARY_DCP"} \
|
||||
--max-model-len "$FRANK_PRIMARY_MAXLEN" \
|
||||
--max-num-seqs "$FRANK_PRIMARY_SEQS" \
|
||||
${FRANK_PRIMARY_MNBT:+--max-num-batched-tokens "$FRANK_PRIMARY_MNBT"} \
|
||||
${FRANK_PRIMARY_PRECISE_MEMORY:+--expert-precise-memory} \
|
||||
${FRANK_PRIMARY_EXPERT_OFFLOAD_MEM:+--expert-offload-mem "$FRANK_PRIMARY_EXPERT_OFFLOAD_MEM"} \
|
||||
--gpu-memory-utilization "$FRANK_PRIMARY_UTIL" \
|
||||
--kv-cache-memory-bytes "$FRANK_PRIMARY_KV_BYTES" \
|
||||
--kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE" \
|
||||
"$FRANK_PRIMARY_PREFIX_CACHE_FLAG" \
|
||||
--enable-auto-tool-choice --tool-call-parser "$FRANK_PRIMARY_TOOL_PARSER" \
|
||||
--reasoning-parser "$FRANK_PRIMARY_REASONING_PARSER" \
|
||||
--default-chat-template-kwargs "$FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS" \
|
||||
${FRANK_PRIMARY_SPEC:+--speculative-config "$FRANK_PRIMARY_SPEC"} \
|
||||
${FRANK_PRIMARY_COMPCONFIG:+--compilation-config "$FRANK_PRIMARY_COMPCONFIG"} \
|
||||
${FRANK_PRIMARY_GEN_CONFIG:+--override-generation-config "$FRANK_PRIMARY_GEN_CONFIG"} \
|
||||
--trust-remote-code \
|
||||
"$FRANK_PRIMARY_IMAGE" \
|
||||
ft serve --model /models \
|
||||
--served-model-name "$FRANK_PRIMARY_SERVED_NAME" \
|
||||
--host 0.0.0.0 --port 7770 \
|
||||
--gpu "$FRANK_PRIMARY_FT_GPU" \
|
||||
--num-tokens "$FRANK_PRIMARY_FT_NUM_TOKENS" \
|
||||
--max-running-requests "$FRANK_PRIMARY_FT_MAX_RUNNING" \
|
||||
--max-output-tokens "$FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS" \
|
||||
--moe-cache-rate "$FRANK_PRIMARY_FT_MOE_RATE" \
|
||||
${FRANK_PRIMARY_FT_EXTRA_ARGS:+$FRANK_PRIMARY_FT_EXTRA_ARGS} \
|
||||
> "$LOG" 2>&1
|
||||
|
||||
# From here on, this script's lifetime IS the container's: wait returns the
|
||||
# container's exit code, so a crash is a nonzero ExecStart and the unit's
|
||||
# Restart=on-failure is real. (systemd stop goes through ExecStop in the
|
||||
# unit — podman wait alone cannot stop the container child.)
|
||||
# Restart=always is real. (systemd stop goes through ExecStop in the unit —
|
||||
# podman wait alone cannot stop the container child.)
|
||||
exec podman wait "$FRANK_PRIMARY_CONTAINER"
|
||||
|
||||
Reference in New Issue
Block a user