diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index c1216ac..15e1cf1 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1175,9 +1175,9 @@ deploy_llm_small() { # PCIe, once per forward pass. That is the "pipeline parallelism" the endpoint # wanted, with no collective anywhere in it. LLM_MEDIUM_BIN="${LLM_MEDIUM_BIN:-/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server}" -LLM_MEDIUM_MODEL="${LLM_MEDIUM_MODEL:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}" -LLM_MEDIUM_DRAFT="${LLM_MEDIUM_DRAFT:-/home/dank/models/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q8_0.gguf}" -LLM_MEDIUM_MMPROJ="${LLM_MEDIUM_MMPROJ:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/mmproj-BF16.gguf}" +LLM_MEDIUM_MODEL="${LLM_MEDIUM_MODEL:-/home/dank/models/Qwen3.8-27B-Uncensored-GGUF/Qwen3.8-27B-Uncensored-Q8_0.gguf}" +LLM_MEDIUM_DRAFT="${LLM_MEDIUM_DRAFT:-/home/dank/models/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}" +LLM_MEDIUM_MMPROJ="${LLM_MEDIUM_MMPROJ:-/home/dank/models/Qwen3.8-27B-Uncensored-GGUF/mmproj-Qwen3.8-27B-Uncensored-f16.gguf}" # The three vLLM user units are retired HERE, not left to rot: the proxy # published 7770 on the host network, so the llama-server below cannot bind diff --git a/scripts/download-frank-medium-gguf.sh b/scripts/download-frank-medium-gguf.sh index d599922..d7840a4 100755 --- a/scripts/download-frank-medium-gguf.sh +++ b/scripts/download-frank-medium-gguf.sh @@ -1,84 +1,109 @@ #!/usr/bin/env bash -# the target GGUF, the BF16 vision projector and the DFlash2 draft model. +# Fetch the frank-medium payload: the target GGUF, the f16 vision projector and +# the DFlash2 draft model. # # bash scripts/download-frank-medium-gguf.sh # bash scripts/download-frank-medium-gguf.sh --verify-only # -# Both repositories are public (anonymous download; no token needed). If a -# token exists at ~/.cache/huggingface/token it is used, which keeps a 31 GB -# transfer clear of the anonymous rate limit. The script is resumable -# (curl -C -) and checks every artifact's exact byte count afterwards, so a -# truncated file fails here instead of load-crashing the server. +# The target repo (orcarouter) is GATED: anonymous requests get a 401, so a +# token is REQUIRED for it. The script reads ~/.cache/huggingface/token +# (mode 600) or the HF_TOKEN environment variable, and passes it to curl +# out-of-band via a mode-600 -K file (a -H header is world-readable in +# /proc//cmdline -- the 2026-09-16 HF-token leak). The drafter repo +# (z-lab) is public; the same token is used for it when present. # -# WHY THESE TWO FILES: -# * Target: ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF at -# UD-Q8_K_XL -- Unsloth Dynamic 3.0 quant (per-tensor types from Unsloth's -# own quant of the same size, on Unsloth's importance matrix). The file -# carries the model's MTP layer (`blk.64.nextn.*`), so llama.cpp can also -# draft from the model itself with no second copy in VRAM. -# * Draft: z-lab/Qwen3.8-27B-DFlash2-GGUF at Q8_0 -- the block-diffusion +# WHY THESE FILES: +# * Target: orcarouter/Qwen3.8-27B-Uncensored-GGUF at Q8_0 -- the +# operator's requested quant (2026-09-19). Arch qwen35, 262144 trained +# window, and the file carries the model's MTP layer (blk.64.nextn.*), so +# llama.cpp can also draft from the model itself with no second copy in +# VRAM. +# * Draft: z-lab/Qwen3.8-27B-DFlash2-GGUF at Q4_K_M -- the block-diffusion # drafter (5 layers, hidden 5120, block_size 8, a 2048-token sliding -# window). Q8_0 is 1.92 GiB against BF16's 3.6 GiB and the drafter's job is -# throughput, not output quality (the target verifies every drafted token). +# window). Q4_K_M is 1.06 GiB against Q8_0's 1.92 GiB and the drafter's +# job is throughput, not output quality (the target verifies every drafted +# token). +# +# The script is resumable (curl -C -) and checks every artifact's exact byte +# count AND sha256, so a truncated or mis-fetched file fails here instead of +# load-crashing the server. set -euo pipefail -MODEL_REPO="ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF" -MODEL_DIR="/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF" -MODEL_FILE="Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf" -MODEL_SIZE=31457990112 -# The BF16 vision projector from the same repo; llama-server autoloads any -# mmproj*.gguf beside the target, so it must land in MODEL_DIR. -MM_PROJ_FILE="mmproj-BF16.gguf" -MM_PROJ_SIZE=931145888 +MODEL_REPO="orcarouter/Qwen3.8-27B-Uncensored-GGUF" +MODEL_DIR="/home/dank/models/Qwen3.8-27B-Uncensored-GGUF" +MODEL_FILE="Qwen3.8-27B-Uncensored-Q8_0.gguf" +MODEL_SIZE=29047084416 +MODEL_SHA256=31756fca94beca71ea4b8706d6fdc896dab2a3c6376ab0c1863b98512a24f8d6 +# The f16 vision projector from the same repo; the serve script names it via +# --mmproj (no autoload in this build), so it must land in MODEL_DIR. +MM_PROJ_FILE="mmproj-Qwen3.8-27B-Uncensored-f16.gguf" +MM_PROJ_SIZE=931145984 +MM_PROJ_SHA256=add205b7bfdb3f71f6da36b0a82aa20928dd829a920878c602628cdfbebc5288 DRAFT_REPO="z-lab/Qwen3.8-27B-DFlash2-GGUF" DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DFlash2-GGUF" -DRAFT_FILE="Qwen3.8-27B-DFlash2-Q8_0.gguf" -DRAFT_SIZE=2056414816 +DRAFT_FILE="Qwen3.8-27B-DFlash2-Q4_K_M.gguf" +DRAFT_SIZE=1143006816 +DRAFT_SHA256=1a25c56858e1ebe93f2718ac1d49d1151f9323325c1bbfd6209370f4db131ebd mode="download" [[ "${1:-}" == "--verify-only" ]] && mode="verify" -token="" -auth=() -if [[ -r "$HOME/.cache/huggingface/token" ]]; then +token="${HF_TOKEN:-}" +if [[ -z "$token" && -r "$HOME/.cache/huggingface/token" ]]; then token="$(cat "$HOME/.cache/huggingface/token")" - auth=(-H "Authorization: Bearer $token") -else - echo "no HF token at ~/.cache/huggingface/token; downloading anonymously" fi +auth_file="" +cleanup() { [[ -n "$auth_file" ]] && rm -f "$auth_file"; } +trap cleanup EXIT + dl() { # dl mkdir -p "$2" echo " fetching $3" - curl -fL -C - -o "$2/$3" --retry 10 --retry-delay 5 --retry-max-time 21600 \ - "${auth[@]}" "https://huggingface.co/$1/resolve/main/$3" + local args=(-fL -C - -o "$2/$3" --retry 10 --retry-delay 5 --retry-max-time 21600) + [[ -n "$auth_file" ]] && args+=(-K "$auth_file") + curl "${args[@]}" "https://huggingface.co/$1/resolve/main/$3" } if [[ "$mode" == "download" ]]; then - echo "target: $MODEL_REPO" + if [[ -z "$token" ]]; then + echo "ERROR: the target repo is gated and no HF token is available" >&2 + echo " (set HF_TOKEN or write it to ~/.cache/huggingface/token, mode 600)" >&2 + exit 1 + fi + auth_file="$(mktemp)" + printf 'header = "Authorization: Bearer %s"\n' "$token" > "$auth_file" + chmod 600 "$auth_file" + + echo "target: $MODEL_REPO (gated)" dl "$MODEL_REPO" "$MODEL_DIR" "$MODEL_FILE" - echo "projector: $MODEL_REPO" + echo "projector: $MODEL_REPO (gated)" dl "$MODEL_REPO" "$MODEL_DIR" "$MM_PROJ_FILE" echo "draft: $DRAFT_REPO" dl "$DRAFT_REPO" "$DRAFT_DIR" "$DRAFT_FILE" fi -echo "verifying sizes" +echo "verifying sizes and sha256" ok=1 -check() { # check - local act +check() { # check + local act act_sha act="$(stat -c%s "$1" 2>/dev/null || echo 0)" - if [[ "$act" -eq "$2" ]]; then - echo " OK $1 ($act bytes)" - else - echo " BAD $1 ($act bytes, want $2)" >&2 + if [[ "$act" -ne "$2" ]]; then + echo " BAD-SIZE $1 ($act bytes, want $2)" >&2 ok=0 + return fi + act_sha="$(sha256sum "$1" | cut -d' ' -f1)" + if [[ "$act_sha" != "$3" ]]; then + echo " BAD-SHA $1 ($act_sha)" >&2 + ok=0 + return + fi + echo " OK $1 ($act bytes)" } -check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE" -check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE" -check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE" -[[ -z "$token" ]] && echo "note: no token used; the byte counts above are the integrity check" +check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE" "$MODEL_SHA256" +check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE" "$MM_PROJ_SHA256" +check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE" "$DRAFT_SHA256" [[ "$ok" -eq 1 ]] || exit 1 -echo "all artifacts present and sized correctly" +echo "all artifacts present, sized and hashed correctly" diff --git a/scripts/frank-medium.env b/scripts/frank-medium.env index cfcdc14..f41afe8 100644 --- a/scripts/frank-medium.env +++ b/scripts/frank-medium.env @@ -1,7 +1,7 @@ -# Tunables for frank-medium (ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP -# UD-Q8_K_XL + z-lab/Qwen3.8-27B-DFlash2-GGUF Q8_0 drafter), mainline llama.cpp -# on Dankserver, 2x AMD Radeon AI PRO R9700 (gfx1201). Since 2026-09-16 this -# is the SOLE LLM runtime on the box: the vllm-radiance replica pair and its +# Tunables for frank-medium (orcarouter/Qwen3.8-27B-Uncensored-GGUF Q8_0 + +# z-lab/Qwen3.8-27B-DFlash2-GGUF Q4_K_M drafter), mainline llama.cpp on +# Dankserver, 2x AMD Radeon AI PRO R9700 (gfx1201). Since 2026-09-16 this is +# the SOLE LLM runtime on the box: the vllm-radiance replica pair and its # routing proxy are retired. # # Hardware: two R9700 (Navi 48, RDNA4, 32,624 MiB each), no XGMI, different @@ -31,17 +31,17 @@ # The ref is pinned by TWO features: DFlash2 (ggml-org/llama.cpp#27342 -- the # drafter is detected from its GGUF) and qwen35 MTP. Do not float it. # -# Model: ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF @ UD-Q8_K_XL, -# 29.29 GiB, arch qwen35, Unsloth Dynamic 3.0 (per-tensor types from Unsloth's -# own quant of the same size, on Unsloth's importance matrix). Abliteration is -# orcarouter's single refusal direction projected out of 131 tensors; the vision -# tower loads from mmproj-BF16.gguf (see NO_MMPROJ). The file also -# carries the model's MTP head (blk.64.nextn.*), so a file-level switch to -# --spec-type draft-mtp is available without a second copy in VRAM. +# Model: orcarouter/Qwen3.8-27B-Uncensored-GGUF @ Q8_0, 27.0 GiB, arch qwen35 +# (verified from the GGUF header 2026-09-19: 262144 trained window, 24 heads / +# 4 KV heads, nextn_predict_layers=1). The file ALSO carries the model's MTP +# head (blk.64.nextn.*), so --spec-type draft-mtp is available without a second +# copy in VRAM -- one SPEC_TYPE change, the drafter below stays on disk. # -# Drafter: z-lab/Qwen3.8-27B-DFlash2-GGUF @ Q8_0, 1.92 GiB: 5 layers, hidden +# Drafter: z-lab/Qwen3.8-27B-DFlash2-GGUF @ Q4_K_M, 1.06 GiB (operator call +# 2026-09-19: the Q4 rung -- the drafter's job is throughput, not output +# quality, and the target verifies every drafted token). 5 layers, hidden # 5120, a block-diffusion proposal (block_size 8) under a 2048-token sliding -# window. Decoding stays lossless -- the target verifies every drafted token. +# window (dflash.attention.sliding_window=2048, verified from the header). # The window is what makes it cheap HERE: its KV is bounded at 2048 positions # (~40 MiB) instead of spanning the context (see SIZING). # @@ -52,10 +52,14 @@ # in the attention layers only, plus ~0.16 GiB per sequence of f32 DeltaNet # state at any depth. # -# SIZING, measured (rocm-smi, 2026-09-16). Weights split across the two cards; -# the drafter AND its KV ride the second. The loader's compute buffers are part -# of what is seen below -- they are allocated at load, not per request, and they -# scale with --ubatch-size: +# SIZING, the deployed shape (32:33 / ctx 327680 / 6144x512 / two slots / +# DFlash2 Q4_K_M), measured 2026-09-19 after the model swap (rocm-smi): +# +# card 0 (x16) 26.9 GiB used, 5.7 GiB free +# card 1 (x1) 27.8 GiB used, 4.8 GiB free +# +# The predecessor's measured table (rocm-smi, 2026-09-16, UD-Q8_K_XL target) +# stays as the calibration history: # # CTX split batch slots drafter GPU 0 free GPU 1 free note (ubatch 1536) # 262144 1,1 2048 1 DFlash2 8.64 GiB 4.43 GiB day's start @@ -71,7 +75,7 @@ # batch block below): # # 262144 36,29 6144 512 2 DFlash2 6.01 GiB 7.32 GiB ubatch only -# 262144 35,30 6144 512 2 DFlash2 6.44 GiB 6.90 GiB <- deployed +# 262144 35,30 6144 512 2 DFlash2 6.44 GiB 6.90 GiB # # ONE -ts STEP IS ONE LAYER, AND LAYERS ARE NOT THE SAME SIZE. This is a hybrid # stack -- 16 x (3 Gated DeltaNet + 1 Gated Attention) blocks -- so moving a @@ -80,36 +84,27 @@ # were measured, which is why the balance is found by measurement rather than by # interpolating between splits. # -# 262144 is the window the model ships (n_ctx_train) and it fits with room, so -# the fleet gets the whole thing; the arithmetic that used to cap this endpoint -# at 114688 was the vLLM predecessor's, not this runtime's. If a future change -# adds VRAM (a longer drafter, quantized KV, more slots) re-read the numbers -# rather than assuming, and prefer CTX_SIZE over --gpu-layers. +# WHY THE SPLIT IS 32:33 (operator call, 2026-09-19). The operator asked for +# the tensors BALANCED across the two cards, which for 65 layers is 32:33 -- +# the most even split that exists. The predecessor's 35:30 skew existed to lean +# work onto the x16 card (ROCm0), because ROCm1's Gen4 x1 chipset path made it +# the 98 % utilization bottleneck; the measured trade of the balanced point is +# that the x1 card runs hotter while the x16 card idles a little more, and the +# operator accepted that for symmetry. Do not re-skew without asking. # -# WHY THE SPLIT IS 35:30. Three effects decide it, and all three are measured: -# * THE DRAFTER AND ITS OUTPUT LAYERS RIDE ROCm1. It carries the drafter's -# weights, the draft graph and the last target layers, so it takes fewer -# than half the layer count. -# * THE UBATCH BUFFERS COST ROCm1 MORE THAN ROCm0, and shrinking them pays -# ROCm1 back more: 1536 -> 512 freed 1.30 GiB on ROCm0 but 2.41 GiB on -# ROCm1 at this shape. Read the other way, that is exactly why the smaller -# batch moved the balance toward ROCm1 -- and cost ROCm0 one layer. -# * THE LAYER GRANULARITY IS THE FLOOR ON HOW WELL ANY SPLIT CAN BALANCE. One -# -ts step is one layer -- ~0.43 GiB DeltaNet, ~1.43 GiB attention -- so the -# reachable minimum gap is comparable to one step rather than to zero. 35:30 -# sits at 0.46 GiB, i.e. one DeltaNet layer's worth, against 1.31 GiB at -# 36:29. -# It also leaves the most room: 6.44 / 6.90 GiB free clears the operator's 1.5 -# GiB-per-card floor by ~5 GiB, so no headroom has to be bought out of CTX_SIZE -# and the full window stays. +# THE TWO CARDS' PATHS ARE NOT SYMMETRIC (measured, 2026-09-17): ROCm0 +# (03:00.0) hangs off CPU root port 00:01.1 at Gen4 x16 end to end (~31.5 GB/s); +# ROCm1 (0b:00.0) rides the B650 chipset through bridges that each negotiate +# Gen4 x1 (~1.97 GB/s). The layer-split boundary itself is not the cost +# (~5.2 MB of activation per 512-token ubatch rides the wire); the x1 card is +# COMPUTE-bound, not PCIe-bound. # # A FAULT TO WATCH. A 39:26 split, at 1x batch / 1 slot, died on its eighth # request: the amdgpu driver logged SDMA0 UTCL2 page faults and the engine # surfaced `ROCm error: an illegal memory access was encountered` -> ggml_abort, # i.e. a fault in the cross-card boundary copy, this layer split's hot path. It -# has not reproduced (36:29 served this file's verification battery and the e2e -# gate, and 35:30 is the deployed shape), but the signature is worth recognising -# -- it is NOT an OOM, and no amount of headroom prevents it. +# has not reproduced since, but the signature is worth recognising -- it is NOT +# an OOM, and no amount of headroom prevents it. # # WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (HIP reads # those bare vars; 2026-08-13 incident). llama.cpp takes --host/--port/--temp @@ -119,15 +114,15 @@ LLAMA_BIN=/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server # Payload, fetched with scripts/download-frank-medium-gguf.sh. -MODEL_DIR=/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF -MODEL_FILE=Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf -# The BF16 vision projector, same repo as the target. llama-server autoloads -# any mmproj*.gguf beside the model; the serve script refuses to start without -# it when vision is enabled, so a missing file is a deploy error, not a -# text-only server that pretends to be multimodal. -MM_PROJ_FILE=mmproj-BF16.gguf +MODEL_DIR=/home/dank/models/Qwen3.8-27B-Uncensored-GGUF +MODEL_FILE=Qwen3.8-27B-Uncensored-Q8_0.gguf +# The f16 vision projector, same repo as the target. llama-server loads it via +# the explicit --mmproj flag (no autoload in this build); the serve script +# refuses to start without it when vision is enabled, so a missing file is a +# deploy error, not a text-only server that pretends to be multimodal. +MM_PROJ_FILE=mmproj-Qwen3.8-27B-Uncensored-f16.gguf DRAFT_MODEL_DIR=/home/dank/models/Qwen3.8-27B-DFlash2-GGUF -DRAFT_MODEL_FILE=Qwen3.8-27B-DFlash2-Q8_0.gguf +DRAFT_MODEL_FILE=Qwen3.8-27B-DFlash2-Q4_K_M.gguf # Fleet identity. 7770 is the fleet's dankserver endpoint; the alias is the id # at /v1/models and what every client asks for. @@ -141,64 +136,23 @@ HIP_VISIBLE_DEVICES=0,1 DEVICES=ROCm0,ROCm1 SPLIT_MODE=layer -# 35:30 -- operator call 2026-09-17: lean the work onto the x16 card. The -# two R9700s do NOT have symmetric PCIe paths. ROCm0 (03:00.0) hangs off -# CPU root port 00:01.1 at Gen4 x16 end to end (~31.5 GB/s); ROCm1 -# (0b:00.0) rides the B650 chipset through bridges that each negotiate -# Gen4 x1 (~1.97 GB/s). Live rocm-smi showed the x1 card at 98% busy -# against 40% on the x16 card: the MTP drafter, the output layers and the -# heavier compute buffers are structurally pinned to the card holding the -# LAST layers (the draft context shares the target's weights there), so -# the bottlenecked card was doing most of the work. Moving one layer to -# ROCm0 shifts ~0.43 GiB of DeltaNet weights (layer 34 is a DeltaNet -- -# the 4k..4k+2 position of block 8) off the x1 card and gives the healthy -# card the bigger share. The layer-split boundary itself is not the cost -# (~5.2 MB of activation per 512-token ubatch rides the wire), but the -# card behind the x1 link should not be the saturated one while its -# x16-sibling idles. -# -# WHY NOT FURTHER. One -ts step is one LAYER: ~0.43 GiB (DeltaNet) or -# ~1.43 GiB (attention). The next step, 36:29, would cross the -# full-attention layer at 35: the seam fit puts card 0 at 31.41 GiB -# (free 0.45 GiB, under the 1.0 GiB floor) with a 2.22 GiB gap (over the -# 1.483 GiB slack) -- both gates fail. 35:30 (fit: 30.59 / 30.01, gap -# 0.58) is the farthest safe shift toward the healthy card at this pool. -# Do not move it without re-measuring. - -# MEASURED 2026-09-18 (isolated :7790, 2k-token decode, AVG rocm-smi util over -# the request -- peak util lies, a layer split touches both cards every token): -# -# split draft_dev decode AVG util0(x16) AVG util1(x1) -# 35,30 ROCm1 31.52 41 % 98 % <- deployed -# 36,29 ROCm1 31.49 43 % 98 % -# 35,30 ROCm0 31.52 42 % 98 % -# 36,29 ROCm0 31.48 43 % 98 % -# -# The x1 card stays the 98 % bottleneck in EVERY configuration. The lever that -# was tried is moving the MTP drafter to the x16 card (SPEC_DRAFT_DEVICE=ROCm0): -# it does NOT flip the balance, because the drafter is a small nextn head and the -# per-step cost is dominated by the VERIFY phase -- both cards run the full target -# -- plus the LOGITS, which are pinned to the card holding the last layers. A -# target-layer move (36:29) buys 2 points on the x16 card and erodes its VRAM -# margin (fit: free 0.45 GiB) for nothing. The x1 card is COMPUTE-bound, not -# PCIe-bound (~5.2 MB of activation per 512-token ubatch crosses the link); -# saturating it costs no tokens beyond what any R9700 would. 35:30 is the -# balance; do not chase the x16 card past it. -TENSOR_SPLIT=35,30 +# 32:33 -- balanced (operator call 2026-09-19): 65 layers split as evenly as +# they can be, ROCm1 (the last-layers card, which also carries the drafter and +# the output layers) holding the extra one. See the WHY THE SPLIT IS 32:33 +# block above for the trade against the predecessor's 35:30 x16-lean. +TENSOR_SPLIT=32,33 NGL=999 # Auto-fit has no business resizing a context we set explicitly. FIT=off FLASH_ATTN=on -# 393216 positions over TWO slots, so each conversation gets 196608 -- +# 327680 positions over TWO slots, so each conversation gets 163840 -- # --parallel DIVIDES --ctx-size, so more slots would shrink every -# conversation rather than grow the cache. The checkpoint's trained window is -# 262144 and the GGUF carries NO rope scaling (freq_base 1e7, no YaRN keys), -# so positions beyond 262144 extrapolate on the base rope: it runs, and 1.5x -# is a modest ask of a rope, but the top 131k of the pool is untrained -# territory. -CTX_SIZE=393216 +# conversation rather than grow the cache. 163840 is inside the checkpoint's +# trained 262144 window, so the whole pool is trained territory (no rope +# extrapolation, unlike the predecessor's 393216). +CTX_SIZE=327680 PARALLEL=2 # MEASURED 2026-09-16 with scripts/bench-batch-ubatch.sh on this box: llama-bench, @@ -258,20 +212,13 @@ CTX_CHECKPOINTS=0 # (server_prompt_cache, tools/server/server-task.cpp): update() evicts # oldest-first past the cap, and alloc() REFUSES outright any single state # bigger than the whole cap. A state is the serialized KV of a whole sequence -# plus its DeltaNet state plus the MTP draft context's own KV, so at this shape -# it costs 68 KiB per token -- +# plus its DeltaNet state plus the draft context's own KV, so at this shape it +# costs ~68 KiB per token -- # 16 full-attention layers x 4 kv heads x 256 dim x 2 B x 2 (K,V) = 64 KiB -# plus 4096 B for the draft context (one nextn layer) -# -- which makes the cap a TOKEN budget in disguise: 8192 MiB held ~120k -# tokens, 16384 MiB holds ~240k. -# -# DOUBLED 8192 -> 16384 on 2026-09-18 because the old cap was BINDING, not -# theoretical: alloc() refused 137 states in the two days from 2026-09-16 with -# - prompt state size 11388.990 MiB exceeds cache size limit 8192.000 MiB, skipping -# i.e. 11.0-11.4 GiB states -- conversations of ~170k tokens at the law above -- -# so every deep conversation was silently uncacheable and re-prefilled from -# scratch on its next turn. At 16384 a FULL 196608-token slot (~12.8 GiB) fits -# with ~3 GiB of the budget spare for a second entry. +# plus the draft context's window-bounded KV (a ~40 MiB constant for DFlash2) +# -- which makes the cap a TOKEN budget in disguise: 16384 MiB holds ~240k +# tokens, i.e. a FULL 163840-token slot (~10.6 GiB) with ~5 GiB of the budget +# spare for a second entry. # # The cost is host RAM, allocated on demand and never pinned (plain heap # vectors), so no VRAM number above moves. dankserver has 62 GiB beside the @@ -300,24 +247,21 @@ REASONING_BUDGET=-1 CHAT_TEMPLATE_FILE= CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}' -# Speculative decoding: the model's OWN MTP head -- the blk..nextn.* -# tensors inside the target file. No sidecar file, no second copy of the -# weights: the draft context runs on the main model (shares_model), so the -# cost is one nextn layer's KV (4096 B/position) plus small buffers on the -# card that holds the last layers. This is what makes the 393216 pool fit: -# the DFlash2 sidecar's 1.92 GiB file and its buffers are off ROCm1. -# n_max: 3 is the measured MTP winner (see the table below -- n=7 accepts 33 % -# of 147, n=3 accepts 58 % of 72). The drafter rides ROCm1. -SPEC_TYPE=draft-mtp -SPEC_DRAFT_N_MAX=3 +# Speculative decoding: the DFlash2 sidecar (operator call 2026-09-19). The +# target file also carries its own MTP head (nextn_predict_layers=1), so +# draft-mtp is one SPEC_TYPE change away -- the drafter file stays on disk. +# n_max 7 is the measured DFlash2 winner (see the table below); the drafter +# rides ROCm1, the card that holds the last layers. +SPEC_TYPE=draft-dflash +SPEC_DRAFT_N_MAX=7 SPEC_DRAFT_NGL=999 SPEC_DRAFT_DEVICE=ROCm1 SPEC_DRAFT_TYPE_K=f16 SPEC_DRAFT_TYPE_V=f16 -# MTP (the model's own head) vs the DFlash2 drafter -- MEASURED 2026-09-16 -# (temperature 0, 1k prompt / 512 out; the autoregressive baseline and both -# MTP depths were run in one session, same binary, same prompts): +# DFlash2 vs the model's own MTP head -- MEASURED 2026-09-16 on the predecessor +# payload (temperature 0, 1k prompt / 512 out; the autoregressive baseline and +# both MTP depths were run in one session, same binary, same prompts): # # none 17.7 tok/s card1 26.07 GB # draft-mtp n=3 32.4 tok/s 58 % accepted card1 29.35 GB @@ -326,11 +270,10 @@ SPEC_DRAFT_TYPE_V=f16 # # All four produced IDENTICAL greedy text, so verification is lossless # whichever proposer runs. DFlash2 is the faster proposer (2.46x over -# autoregressive, 34 % over the best MTP); the operator switched to MTP on -# 2026-09-17 for the VRAM, not the speed: the MTP head adds ~3.3 GB to the -# card that holds it, against 2.06 GB for the drafter's file, and it is the -# only drafter that fits the 393216 pool. Do not switch this without -# re-measuring. +# autoregressive, 34 % over the best MTP). The MTP head adds ~3.3 GB to the +# card that holds it, against 1.06 GB for the Q4_K_M drafter's file -- at the +# 327680 pool the drafter's window-bounded KV (~40 MiB) leaves headroom on +# both cards, so the speed wins. # # THE TWO CANNOT BE CHAINED. `--spec-type draft-mtp,draft-dflash` dies at load: # context type MTP requested but model doesn't contain MTP layers @@ -344,7 +287,7 @@ SPEC_DRAFT_TYPE_V=f16 # ngram-mod: the draftless n-gram matcher (CPU hash table in host RAM, no VRAM, # no model file, no draft context). It joins the chain unconditionally -- # unlike MTP, which needs a draft context and therefore cannot share one with -# DFlash2. NOT CURRENTLY ACTIVE: SPEC_TYPE is draft-mtp alone, and the serve +# DFlash2. NOT CURRENTLY ACTIVE: SPEC_TYPE is draft-dflash alone, and the serve # script only passes --spec-ngram-mod-* when the type contains "ngram". The # window below is retained because the DFlash2 + ngram chain was measured free # on prose and 2.6x on prompt-echo -- one SPEC_TYPE change brings it back. @@ -352,7 +295,7 @@ SPEC_DRAFT_TYPE_V=f16 # WINDOW MEASURED 2026-09-16 (chain at each window, decode tok/s): # # n_min/n_max prose 1k/512 prompt-echo 1.5k/400 -# 48/64 43.9 200.6 <- deployed +# 48/64 43.9 200.6 # 16/32 36.0 161.0 # 8/16 36.2 113.8 # @@ -369,9 +312,7 @@ SPEC_NGRAM_N_MATCH=24 SPEC_NGRAM_N_MIN=48 SPEC_NGRAM_N_MAX=64 -# Vision enabled: llama-server autoloads mmproj-BF16.gguf from MODEL_DIR -# (888 MiB, lands on ROCm0 -- the fit above charges it there). frank-small -# (Dankclaw, RTX 3090) remains the fleet's dedicated image endpoint; this -# makes frank-medium multimodal too, which is why the seam flips from -# "text only" to "vision loads with the model". +# Vision enabled: llama-server loads mmproj-Qwen3.8-27B-Uncensored-f16.gguf +# (931 MiB, lands on ROCm0 -- the fit charges it there) via the explicit +# --mmproj flag. This keeps frank-medium multimodal across the model swap. NO_MMPROJ=0 diff --git a/tests/frank_medium_unit.bats b/tests/frank_medium_unit.bats index 188db47..a41939a 100755 --- a/tests/frank_medium_unit.bats +++ b/tests/frank_medium_unit.bats @@ -20,8 +20,8 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" # Measured constants (rocm-smi / HF blob sizes), used by the fit arithmetic. CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it -MODEL_BYTES=31457990112 # UD-Q8_K_XL target -DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter +MODEL_BYTES=29047084416 # orcarouter Q8_0 target +DRAFT_BYTES=1143006816 # DFlash2 Q4_K_M drafter @test "frank-medium's unit, serve script, env and its build/fetch scripts exist" { [ -f "$UNIT" ] @@ -48,17 +48,19 @@ DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter grep -q '^SPLIT_MODE=layer$' "$ENV_FILE" grep -q '^DEVICES=ROCm0,ROCm1$' "$ENV_FILE" # The split is a layer PAIR: the shares add up to the model's 65 blocks (one - # -ts unit per layer) and ROCm1 -- the card that also carries the drafter and - # the output layers -- takes the FEWER of the two. WHICH pair balances the - # cards is the seam test's job, which recomputes it from this file; pinning a - # literal here would only mean that re-measuring the balance breaks a test - # about parallelism. + # -ts unit per layer). The operator asked for the tensors BALANCED across + # the two cards (2026-09-19), which for 65 layers is 32:33 -- the most even + # split that exists, with ROCm1 (the last-layers card, which also carries + # the drafter and the output layers) holding the extra one. WHICH pair + # balances the cards is the fit test's job, which recomputes it from this + # file; pinning a literal here would only mean that re-measuring the + # balance breaks a test about parallelism. local split_hi split_lo split_hi="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\1/p' "$ENV_FILE")" split_lo="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\2/p' "$ENV_FILE")" [ -n "$split_hi" ] && [ -n "$split_lo" ] [ $(( split_hi + split_lo )) -eq 65 ] - [ "$split_lo" -lt "$split_hi" ] + [ $(( split_lo - split_hi )) -le 1 ] grep -q -- '--split-mode "$SPLIT_MODE"' "$SERVE" grep -q -- '--device "$DEVICES"' "$SERVE" grep -q -- '--tensor-split "$TENSOR_SPLIT"' "$SERVE" @@ -72,24 +74,26 @@ DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter grep -q '^export HIP_VISIBLE_DEVICES$' "$SERVE" } -@test "the drafter is the model's own MTP head, at the measured depth" { - # MTP's head lives in the target file (blk..nextn.*): no sidecar, no - # second copy of the weights, and it is the only drafter that fits the - # 393216 pool. n_max 3 is the measured MTP winner -- n=7 accepts 33 % of - # 147, n=3 accepts 58 % of 72 (the env's table; identical greedy text at - # every depth, so this is pure speed). - grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE" +@test "the drafter is the DFlash2 sidecar at the measured depth" { + # The DFlash2 sidecar is the deployed drafter (operator call 2026-09-19). + # n_max 7 is the measured DFlash2 winner -- 43.5 tok/s at 41 % acceptance + # against 32.4 for the in-file MTP head at n=3 (the env's table; identical + # greedy text at every depth, so this is pure speed). The target file + # also carries its own MTP head (nextn_predict_layers=1), so draft-mtp is + # one SPEC_TYPE change away; the drafter file stays on disk for that. + grep -qE '^SPEC_TYPE=draft-dflash$' "$ENV_FILE" grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE" local n_max n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")" - [ -n "$n_max" ] && [ "$n_max" -eq 3 ] - # The DFlash2 sidecar stays on disk and in the fetch script -- one - # SPEC_TYPE change brings the faster chain back -- and ngram-mod's window - # is retained with it, so neither may be dropped from the env. - grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q8_0\.gguf$' "$ENV_FILE" + [ -n "$n_max" ] && [ "$n_max" -eq 7 ] + # The Q4_K_M rung is the operator's requested drafter quant -- the + # drafter's job is throughput, not output quality, and the target + # verifies every drafted token. ngram-mod's window is retained in the env + # for the chain, so it may not be dropped. + grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q4_K_M\.gguf$' "$ENV_FILE" grep -q '^SPEC_NGRAM_N_MATCH=24$' "$ENV_FILE" # ...and neither is loaded: ngram-mod's flags only go out when the type - # contains ngram, so a pure MTP configuration is not handed them. + # contains ngram, so a pure dflash configuration is not handed them. python3 -c " lines = open('$SERVE').read().splitlines() guard = next(i for i, l in enumerate(lines) if '*ngram*' in l) @@ -130,11 +134,14 @@ print('draft-model wiring ok') grep -q 'frank-medium.env' "$SERVE" } -@test "the payload is fetched resumably and verified by size" { - grep -q 'UD-Q8_K_XL\.gguf' "$FETCH" - grep -q 'Qwen3\.8-27B-DFlash2-Q8_0\.gguf' "$FETCH" - grep -q 'curl -fL -C -' "$FETCH" - grep -q 'verifying sizes' "$FETCH" +@test "the payload is fetched resumably and verified by size and sha256" { + grep -q 'Qwen3\.8-27B-Uncensored-Q8_0\.gguf' "$FETCH" + grep -q 'Qwen3\.8-27B-DFlash2-Q4_K_M\.gguf' "$FETCH" + grep -q -- '-fL -C -' "$FETCH" + grep -q 'verifying sizes and sha256' "$FETCH" + # The target repo is gated: the script must fail loud without a token + # rather than write a 401 body to disk and report success. + grep -q 'no HF token is available' "$FETCH" } @test "both cards stay balanced and keep the operator's 1.5 GiB floor" { @@ -158,9 +165,9 @@ print('draft-model wiring ok') # measured 6.44 / 6.90 GiB free. The fit reproduces card 1 to 0.00 GiB # and card 0 to 0.86 GiB at that point; it is a GATE, not a meter -- the # deployed point is re-measured on the host after every split/ctx/batch - # move (the current one is 35:30 / 393216 / MTP, moved from 34:31 on - # 2026-09-17 to lean work off ROCm1's Gen4-x1 chipset path -- see the - # TENSOR_SPLIT header in frank-medium.env). + # move (the current one is 32:33 / 327680 / DFlash2 Q4_K_M, the balanced + # split the operator asked for on 2026-09-19 -- see the TENSOR_SPLIT + # header in frank-medium.env). # # card 0 = its share of the weights and of the KV (qwen35 spends KV on # its 16 full-attention layers only: 2 (K,V) x 4 heads x 256 dim @@ -197,22 +204,22 @@ print('draft-model wiring ok') # allocation. local kv_pos=65536 draft_kv_pos=20480 mtp_kv_pos=4096 floor slack local card_hi card_lo diff - local mmproj=931145888 # mmproj-BF16.gguf, autoloaded onto the primary device + local mmproj=931145984 # mmproj-Qwen3.8-27B-Uncensored-f16.gguf, on the primary device # 1.5 GiB was the floor until the 2x196k shape: the 34:31 / 393216 / # MTP point measured 28.91 / 30.78 GiB, leaving 2.95 / 1.08 GiB, so the - # floor is the measured 1.08 GiB rounded down. The margin for one big - # allocation is thinner than it used to be. The layer-toward-ROCm0 - # lever has since been spent (35:30, 2026-09-17): if a large allocation - # fails at load now, the next lever is a ctx cut, not another layer. + # floor is the measured 1.08 GiB rounded down. The 32:33 / 327680 / + # DFlash2 point is roomier than that (fit: 6.97 / 4.89 GiB free), so the + # floor stays at the measured 1.0 GiB -- if a large allocation fails at + # load, the next lever is a ctx cut, not a floor change. floor=$(( 1024 * 1024 * 1024 )) # 1.0 GiB per card - # The deployed point (35:30 at ctx 393216 with MTP) has a fit gap of - # 0.58 GiB -- closer to balanced than its 34:31 predecessor (1.06 GiB). - # The slack sits at 1.483 GiB: it accepts the deployed split and still - # rejects the next step toward ROCm0, 36:29, which fails BOTH gates - # (gap 2.22 GiB, card 0 free 0.45 GiB under the floor). (The 36:29 / - # 327680 / DFlash2 configuration's slack was 3225 MiB -- its 3.15 GiB - # gap was by construction, and is no longer the deployed shape.) - slack=$(( 1483 * 1024 * 1024 )) + # The deployed point (32:33 at ctx 327680 with DFlash2) has a fit gap of + # 2082 MiB -- the balanced split the operator asked for, not the + # tightest reachable one. The slack sits at 2150 MiB: it accepts the + # deployed split and rejects a two-layer re-skew (34:31 / 31:34 both + # move the fit gap well past it). The x16-lean 35:30 that served the + # predecessor payload (gap 0.58 GiB) no longer applies: the operator + # chose symmetry over the x1-card bottleneck. + slack=$(( 2150 * 1024 * 1024 )) card_hi=$(( MODEL_BYTES * split_hi / (split_hi + split_lo) \ + kv_pos * ctx * split_hi / (split_hi + split_lo) \ + 381898377 \ @@ -243,8 +250,9 @@ print('draft-model wiring ok') diff=$(( card_hi - card_lo )); [ "$diff" -ge 0 ] || diff=$(( -diff )) [ "$diff" -le "$slack" ] # ...and the window may not wander far past what the checkpoint was - # trained for (262144; the GGUF carries no rope scaling, so the top 131k - # of the 393216 pool extrapolates on the base rope). + # trained for (262144; the GGUF carries no rope scaling, so positions + # past that extrapolate on the base rope). The deployed 327680 pool is + # two 163840 slots -- entirely trained territory. [ "$ctx" -le 393216 ] } @@ -280,16 +288,16 @@ print('draft-model wiring ok') grep -q -- '--cache-prompt' "$SERVE" } -@test "vision is enabled: the BF16 projector loads with the model" { +@test "vision is enabled: the f16 projector loads with the model" { grep -q '^NO_MMPROJ=0$' "$ENV_FILE" - grep -q '^MM_PROJ_FILE=mmproj-BF16.gguf$' "$ENV_FILE" + grep -q '^MM_PROJ_FILE=mmproj-Qwen3\.8-27B-Uncensored-f16\.gguf$' "$ENV_FILE" # Explicit flag, not autoload: this build starts text-only without it, # and refuses to start without the projector file, so a missing file is # a deploy error, not a silent text-only server. grep -q -- '--mmproj "$MM_PROJ"' "$SERVE" grep -q 'vision is enabled but the projector is missing' "$SERVE" # And the download script fetches and byte-verifies it. - grep -q '^MM_PROJ_SIZE=931145888$' "$FETCH" + grep -q '^MM_PROJ_SIZE=931145984$' "$FETCH" grep -q -- 'check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE"' "$FETCH" }