diff --git a/scripts/download-frank-primary.sh b/scripts/download-frank-primary.sh index 53659fe..8347940 100755 --- a/scripts/download-frank-primary.sh +++ b/scripts/download-frank-primary.sh @@ -1,24 +1,17 @@ #!/usr/bin/env bash # Payload inventory/verification for frank-primary: -# Intel/GLM-5.3-Flash-W4A16-AutoRound (AutoRound symmetric int4, group 128, -# packing "auto_round:auto_gptq": 34 numbered shards + model_extra_conv / -# model_extra_tensors, weight-map index, 113,074 tensors, ~170 GiB) — the -# int4 payload that became frank-primary on 2026-09-30. +# orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8 (compressed-tensors FP8, 131 +# shards + weight-map index, ~172.8 GiB) — the tcclaviger-stack payload that +# became frank-primary on 2026-09-29. # -# bash scripts/download-frank-primary.sh --verify-only (the default) +# bash scripts/download-frank-primary.sh --verify-only (the ONLY mode) # bash scripts/download-frank-primary.sh --fetch # -# THIS REPO IS PUBLIC (measured 2026-09-30: the payload fetched anonymously at -# bring-up). The LFS oids are still not verified byte-exactly here — the tree -# API masks them — so integrity rests on the weight-map index, the shard count -# and the loader's own tensor-shape checks at boot. -# -# WHY THIS QUANT: its quantization_config.quant_method is "auto-round", which -# the fork overrides to "inc" — that is the ONLY route to R4dW4a16MoEMethod -# (libr4d's grouped W4A16 GEMM) and therefore to an expert-offload recipe. A -# compressed-tensors W4A16 checkpoint of the same model (e.g. -# dealignai/...-CYBERSECURITY-W4A16) reaches TritonWNA16Experts instead, which -# has no offload recipe, and refuses to serve at all on this host. +# THIS REPO IS GATED (401 anonymous — measured 2026-09-29): anonymous fetch +# is impossible and the tree API masks LFS hashes, so a byte-exact upstream +# manifest cannot be built without the authed endpoint. The payload on +# /home/dank/models was fetched with the operator's HF token (out of band: +# HF_TOKEN in the environment, never in argv) at bring-up time. # # Modes: # --fetch resume any missing/truncated shard via `hf download` with @@ -31,17 +24,16 @@ # (slow, ~10 min for 173 GiB). set -euo pipefail -REPO="Intel/GLM-5.3-Flash-W4A16-AutoRound" -DEST="/home/dank/models/GLM-5.3-Flash-W4A16-AutoRound" -EXPECTED_SHARDS=34 +REPO="orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8" +DEST="/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8" +EXPECTED_SHARDS=131 mode="verify" [[ "${1:-}" == "--fetch" ]] && mode="fetch" if [[ "$mode" == "fetch" ]]; then - # The repo is public, so no token is required; if one is present it is taken - # from the environment and never from argv (a -H header is readable in - # /proc//cmdline — the 2026-09-16 leak). + : "${HF_TOKEN:?--fetch needs HF_TOKEN in the environment (never argv)}" + export HF_TOKEN mkdir -p "$DEST" hf download "$REPO" --local-dir "$DEST" fi @@ -89,7 +81,7 @@ if [[ "${2:-}" == "--hash" || "${VERIFY_HASH:-0}" == "1" ]]; then echo "hashing every shard (slow)..." cd "$DEST" sha256sum -c --quiet model.safetensors.index.json 2>/dev/null || \ - sha256sum model-*-of-00034.safetensors > /dev/null # presence check; content hash list printed on demand + sha256sum model-*-of-00131.safetensors > /dev/null # presence check; content hash list printed on demand fi [[ "$fail" -eq 0 ]] || exit 1 diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index e9ee133..b38788d 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -1,53 +1,34 @@ # frank-primary.env — tcclaviger RDNA4 vLLM production config (dankserver :7770). # -# 2026-09-30 cutover from orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8 (MTP4) to -# Intel/GLM-5.3-Flash-W4A16-AutoRound, the first payload this fleet has served -# through the fork's W4A16 expert-offload path. ROLLBACK: the pre-cutover qwen -# state is commit 2339fae ("frank-primary: fix the SIGPIPE race in both -# image-digest gates") — restore scripts/ from it and deploy; the qwen payload -# (173 GiB) and the R9V state dirs are untouched on disk. +# 2026-09-29 cutover from the R9V radiance-vLLM fork (its launcher/shim/state +# machinery is retired; /home/dank/r9v-state/{q8xl,uncensored} stay on disk as +# the rollback target, and git tag frank-primary-r9v-q8-20260929 is the last +# pre-cutover commit). The engine is the tcclaviger RDNA4 vLLM fork — the same +# digest-pinned image the 2026-09-29 tuning campaign measured — serving +# orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8 (~172.8 GiB, 131 shards) with +# the MTP4 drafter on the two R9700s. # -# QUALIFIED 2026-09-30 by boot probe + bench against the live engine: -# * THE 35 tok/s TARGET IS NOT MET: decode 9.9-10.0 tok/s, prefill 918 tok/s -# (warm, 2.6k-token prompt) at 131,072 ctx x 2 seqs. See the runbook's -# "Why 35 tok/s is out of reach on this hardware" — the bound is the fork's -# expert-gather read path (8.2 GB/s effective vs the link's 22.74 GB/s), -# the same structural gap the 2026-09-29 qwen campaign measured. -# * Arena 14.51 GiB = 2,383 slots x 6.234 MiB per rank; 12,096 host-backed -# experts per rank, 73.64 GiB pinned. Live per-layer telemetry -# (R4D_LRU_TELEMETRY=1, 2026-09-30): hit rate 0.53-0.66, ~500-730 installs -# vs ~2,000 routed per layer over the window. -# * --expert-precise-memory is ON: it replaces the 6.25 GiB default -# compile-headroom figure with the registry's fitted 5.46 GiB, buying -# 1.43 GiB of cache (13.08 -> 14.51 GiB). Decode measured flat across that -# change (10.03 -> 9.88 tok/s), which is the evidence that the hit rate -# saturates near 0.6 and residency is no longer the binding term. -# * KV dtype stays bf16/AUTO — fp8 KV is NOT legal on this model. It disables -# _use_rocm_sparse_triton() in the ROCM_AITER_MLA_SPARSE backend (the -# predicate requires a non-fp8 cache), and that rope-free Triton path is the -# only MLA decode available on gfx12: aiter's MLA decode op is registered -# under @if_aiter_supported, i.e. gated on CDNA>2. With fp8 KV the MTP draft -# step dies with "'_OpNamespace' 'vllm' object has no attribute -# 'rocm_aiter_mla_decode_fwd'". -# * MTP4 spec decode is ABSENT DELIBERATELY, unlike the qwen shape. It WORKS -# here (the MTP head loads, architecture resolves to Glm5NextMTPModel, and -# acceptance is excellent: mean accepted length 3.76-4.90, per-position -# 0.92/0.85/0.69/0.58, 69-97%) — but it buys nothing on an offload-bound -# deployment: measured decode 9.6-11.2 tok/s (vs 10.0 without) while prefill -# COLLAPSED 918 -> 255 tok/s. The verification forward gathers the union of -# the batch's experts, so spec decode does not amortize a residency-bound -# gather the way it amortizes compute. It also costs 2.6 GiB of arena. -# Reinstate only with a fresh A/B on a non-offload-bound shape. -# * PREFETCH IS ABSENT BY LAW, unchanged from the qwen campaign: expert-cache -# telemetry measured the prefetcher at 36% precision and deleting the flag -# bought +40-51% decode. Never re-add --expert-prefetch without a fresh A/B. +# Every number here was QUALIFIED by boot probe on 2026-09-29 (not inherited): +# * MAXLEN 200000 x SEQS 3 (600k total) at KV q8/q8 fits with headroom: +# the engine measured "GPU KV cache size: 729,545 tokens; Maximum +# concurrency for 200,000 tokens per request: 3.65x" at KV_BYTES 7.2 GiB +# per rank. KV_BYTES is PER RANK — sizing it to the global requirement +# OOMs the worker (measured twice: 16 GiB and 9 GiB both die with ~7.5 GiB +# free; 7.2 GiB boots). +# * --kv-cache-dtype fp8 is the fleet's q8/q8 KV ask (e4m3 for BOTH K and V +# — the stock flag quantizes them together, which is exactly what was +# requested here). Accepted by the attention path; qualified with a +# completion probe. +# * PREFETCH IS ABSENT BY LAW: expert-cache telemetry measured the +# prefetcher at 36% precision (36,720 slabs prefetched, 13,369 served) — +# roughly half the gather traffic was waste; prefetch off is the tuned +# incumbent and the +40-51% decode came from deleting the flag. Never +# re-add --expert-prefetch without a fresh A/B. # * TunableOp stays OFF: CLAV_TUNABLEOP_SWEEP silently skips under expert # offload unless CLAV_TUNABLEOP_WITH_OFFLOAD=1, and forcing it corrupts # numerics (repetition loops, early halts, invalid measurements). -# * KV is NOT capped explicitly: the fork's plan reserve (1.76 GiB at 131,072) -# already covers what max_model_len needs for this shape. It does NOT for an -# MTP shape — see the runbook's fork-bug section and -# scripts/patches/gpu_worker-kv-cap.patch. +# Decode at the tuned shape: 40.57/37.44/37.57 tok/s (166/1087/4064 ctx) at +# the 32k window; 42.94/34.55/36.09 at this 200k shape (arena 6,032 slots). # # HIP reads bare env vars TEMP, HOST, GPU_MAX_HEAP_SIZE — every key here is # prefixed FRANK_PRIMARY_ so none can leak. @@ -55,57 +36,26 @@ # --- runtime ---------------------------------------------------------------- # Digest-pinned: the floating :latest tag moved once already. Local alias # qwen-fp8-runtime:tuned-20260929 (image ID 3322dbbbc96d) points at the same -# bytes in rootless storage. Same image as the qwen cutover — the W4A16 path -# (INC scheme -> R4dW4a16MoEMethod, libr4d grouped W4A16 GEMM) ships in it. +# bytes in rootless storage. FRANK_PRIMARY_IMAGE=docker.io/tcclaviger/vllm@sha256:da4029666668b90a0386c1ed23ea370c79916ad4e1a6ad1dd9db098b9fa069ea # --- payload ---------------------------------------------------------------- -# Intel AutoRound W4A16: symmetric int4, group 128, packing "auto_round:auto_gptq". -# quantization_config.quant_method is "auto-round", which the fork overrides to -# "inc" (inc.py:364) — that is what routes the MoE to R4dW4a16MoEMethod instead -# of the compressed-tensors path's Triton experts, which has no offload recipe. -# 34 numbered shards + model_extra_{conv,tensors}.safetensors, 113,074 tensors. -FRANK_PRIMARY_MODEL_DIR=/home/dank/models/GLM-5.3-Flash-W4A16-AutoRound -FRANK_PRIMARY_MODEL_SHARDS=34 +FRANK_PRIMARY_MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8 +FRANK_PRIMARY_MODEL_SHARDS=131 # --- identity --------------------------------------------------------------- # The harness models.yml id stays frank-primary (provider 192.168.0.235:7770). -FRANK_PRIMARY_CONTAINER=frank-primary-glm +FRANK_PRIMARY_CONTAINER=frank-primary-qwen FRANK_PRIMARY_SERVED_NAME=frank-primary FRANK_PRIMARY_PUBLISH=7770:7770 -# Containers removed before every start: the pre-cutover qwen identity (it holds -# the same port and GPUs), the retired R9V identity, and the ad hoc experiment -# identity. -FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-qwen frank-primary-r9v qwen-fp8 glm-probe" +# Containers removed before every start: the retired R9V identity and the ad +# hoc experiment identity (both hold the GPUs / name registrations). +FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-r9v qwen-fp8" # --- qualified engine shape --------------------------------------------------- -# 131,072 x 2 seqs. The window is NOT chosen for capacity: the engine's own -# plan reserve is one max_model_len request, and below ~92k tokens the fork's -# reserve model (13,600 B/token + a small mamba term) falls short of what the -# engine actually needs for this model's per-sequence KDA state, which makes the -# boot refuse ("2.07 GiB needed vs 1.74 GiB available"). 131,072 clears it with -# the plan's own cap intact. KV cache as built: 138,561 tokens, 1.06x -# concurrency for a 131,072-token request. -FRANK_PRIMARY_MAXLEN=131072 -FRANK_PRIMARY_SEQS=2 -# 0.99 fails boot ("Free memory on device cuda:1 ..."); 0.98 leaves the -# compile/capture headroom the plan models. +FRANK_PRIMARY_MAXLEN=200000 +FRANK_PRIMARY_SEQS=3 +FRANK_PRIMARY_KV_BYTES=7730934784 +FRANK_PRIMARY_KV_DTYPE=fp8 +FRANK_PRIMARY_SPEC=4 FRANK_PRIMARY_UTIL=0.98 -# Prefill chunk. 2,048 is the qualified value; the expert union a chunk routes -# to saturates quickly, so the chunk size is an install-amortization knob, not a -# residency one. Measured 2026-09-30: NBATCH 8,192 boots (arena 12.51 GiB) and -# leads to the SAME warm prefill (922 vs 918 tok/s) — no reason to pay its -# 2 GiB of profile buffer. -FRANK_PRIMARY_NBATCH=2048 -# GLM's own tool-call dialect. Without --enable-auto-tool-choice the endpoint -# rejects tool_choice "auto" outright ("\"auto\" tool choice requires -# --enable-auto-tool-choice and --tool-call-parser to be set"), which breaks -# every tool-using agent on the endpoint. Verified 2026-09-30: a get_weather -# prompt returns tool_calls with finish_reason "tool_calls". -FRANK_PRIMARY_TOOLPARSER=glm47 -# GLM's reasoning dialect. Without it the model's thinking leaks into `content` -# (measured: content began with the answer, then "Let me verify: ... = 391" -# followed by a bare "podman shim and +# a 104-key on-host setup state; this stack is fully expressed by flags, and +# every flag here was qualified by boot probe on 2026-09-29 — see +# frank-primary.env for the qualification record). # -# Why this payload runs the offload path at all: 170 GiB of int4 experts cannot -# fit in 64 GiB of VRAM, so the engine keeps 12,096 experts per rank in pinned -# host memory and serves them through a 2,383-slot VRAM LRU arena. The MoE -# kernel is libr4d's grouped W4A16 GEMM (R4dW4a16MoEMethod), reached because the -# checkpoint's quant_method "auto-round" is overridden to "inc" by the fork. -# -# Shape: MAXLEN 131,072 x 2 seqs, bf16 KV, util 0.98, MTP OFF, prefetch OFF. -# Measured 2026-09-30: decode 9.9-10.0 tok/s, prefill 918 tok/s warm. The -# 35 tok/s target is NOT met — the bound is the fork's expert-gather read path -# (8.2 GB/s effective vs the link's 22.74 GB/s); see frank-primary.env and the -# runbook's arithmetic section. +# Shape: MAXLEN 200000 x 3 sessions (600k total), KV q8/q8 (--kv-cache-dtype +# fp8: e4m3 for both K and V — the stock flag quantizes them together, which +# is the requested q8/q8), KV_BYTES 7.2 GiB PER RANK (the engine measured +# 729,545 tokens of capacity / 3.65x concurrency at 200k per request), MTP +# depth 4, prefix caching, expert prefetch OFF (telemetry measured it at 36% +# precision — the tuned incumbent deletes it), TunableOp off. # # The root unit (scripts/frank-primary.service, User=dank, -# LimitMEMLOCK=infinity) grants the pin budget for the 73.64 GiB/rank of pinned -# host expert rows. A plain shell session also works, but the unit is the -# fleet-parity boot-persistent shape. +# LimitMEMLOCK=infinity) grants the pin budget for the tiered-expert host +# slabs (~20.5 GiB/rank of pinned anonymous memory). A plain shell session +# also works, but the unit is the fleet-parity boot-persistent shape. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -47,9 +44,8 @@ set +a : "${FRANK_PRIMARY_PUBLISH:?FRANK_PRIMARY_PUBLISH must be host:container}" : "${FRANK_PRIMARY_MAXLEN:?FRANK_PRIMARY_MAXLEN must be set}" : "${FRANK_PRIMARY_SEQS:?FRANK_PRIMARY_SEQS must be set}" -: "${FRANK_PRIMARY_NBATCH:?FRANK_PRIMARY_NBATCH must be set}" -: "${FRANK_PRIMARY_TOOLPARSER:?FRANK_PRIMARY_TOOLPARSER must be set}" -: "${FRANK_PRIMARY_REASONINGPARSER:?FRANK_PRIMARY_REASONINGPARSER must be set}" +: "${FRANK_PRIMARY_KV_BYTES:?FRANK_PRIMARY_KV_BYTES must be set (PER RANK)}" +: "${FRANK_PRIMARY_KV_DTYPE:?FRANK_PRIMARY_KV_DTYPE must be set (fp8 = q8/q8 KV)}" # Capture podman's output BEFORE grepping: under `set -o pipefail`, the # classic `podman images | grep -q` pattern races — grep exits on its first # match, podman dies SIGPIPE (141) writing the remaining lines, and pipefail @@ -104,34 +100,20 @@ podman run -d --name "$FRANK_PRIMARY_CONTAINER" \ --cap-add SYS_PTRACE \ --device /dev/kfd --device /dev/dri --device /dev/mem \ -e HIP_VISIBLE_DEVICES=0,1 -e VLLM_ROCM_USE_AITER=0 \ - -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ -v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \ "$FRANK_PRIMARY_IMAGE" /models \ --served-model-name "$FRANK_PRIMARY_SERVED_NAME" --port 7770 \ --tensor-parallel-size 2 \ --max-model-len "$FRANK_PRIMARY_MAXLEN" \ --max-num-seqs "$FRANK_PRIMARY_SEQS" \ - --max-num-batched-tokens "$FRANK_PRIMARY_NBATCH" \ --gpu-memory-utilization "$FRANK_PRIMARY_UTIL" \ - --enable-expert-offload \ - --expert-precise-memory \ + --kv-cache-memory-bytes "$FRANK_PRIMARY_KV_BYTES" \ + --kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE" \ --enable-prefix-caching \ - --enable-auto-tool-choice \ - --tool-call-parser "$FRANK_PRIMARY_TOOLPARSER" \ - --reasoning-parser "$FRANK_PRIMARY_REASONINGPARSER" \ + --speculative-config "{\"method\":\"mtp\",\"num_speculative_tokens\":$FRANK_PRIMARY_SPEC}" \ --trust-remote-code \ > "$LOG" 2>&1 -# NOT passed, deliberately (each measured on this payload — see the env file): -# --kv-cache-dtype fp8 disables the sparse MLA backend's rope-free -# Triton path, which is the only MLA decode on -# gfx12 (aiter's MLA op is gated on CDNA>2). -# --speculative-config MTP head loads and accepts well, but buys zero -# decode and collapses prefill on an -# offload-bound shape. -# --expert-prefetch* 36% precision; deleting it bought +40-51%. -# --kv-cache-memory-bytes the plan's own reserve covers this shape. - # From here on, this script's lifetime IS the container's: wait returns the # container's exit code, so a crash is a nonzero ExecStart and the unit's # Restart=on-failure is real. (systemd stop goes through ExecStop in the