dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-medium: orcarouter Q8_0 + DFlash2 Q4_K_M, 2x160k, balanced 32:33

Browse Source 
Operator call 2026-09-19: switch the target to
orcarouter/Qwen3.8-27B-Uncensored-GGUF Q8_0 (gated repo; the fetch script now
requires an HF token and verifies sha256, not just size), take the drafter to
the Q4_K_M rung of z-lab/Qwen3.8-27B-DFlash2-GGUF, cut the pool to two
163840-token conversations (327680, entirely inside the trained 262144
window), and balance the tensors across the cards (32:33) in place of the
35:30 x16-lean.  The in-file MTP head stays one SPEC_TYPE change away.
This commit is contained in:
Frank committed 2026-09-19 02:10:38 -07:00
1 parent 374041cd7a
commit d374ab963f
4 files changed
+210 -236
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/frank_medium_unit.bats
+55 -47
View File
@@ -20,8 +20,8 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
# Measured constants (rocm-smi / HF blob sizes), used by the fit arithmetic.
CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it
MODEL_BYTES=31457990112 # UD-Q8_K_XL target
DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter
MODEL_BYTES=29047084416 # orcarouter Q8_0 target
DRAFT_BYTES=1143006816 # DFlash2 Q4_K_M drafter
@test "frank-medium's unit, serve script, env and its build/fetch scripts exist" {
[ -f "$UNIT" ]
@@ -48,17 +48,19 @@ DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter
grep -q '^SPLIT_MODE=layer$' "$ENV_FILE"
grep -q '^DEVICES=ROCm0,ROCm1$' "$ENV_FILE"
# The split is a layer PAIR: the shares add up to the model's 65 blocks (one
# -ts unit per layer) and ROCm1 -- the card that also carries the drafter and
# the output layers -- takes the FEWER of the two. WHICH pair balances the
# cards is the seam test's job, which recomputes it from this file; pinning a
# literal here would only mean that re-measuring the balance breaks a test
# about parallelism.
# -ts unit per layer). The operator asked for the tensors BALANCED across
# the two cards (2026-09-19), which for 65 layers is 32:33 -- the most even
# split that exists, with ROCm1 (the last-layers card, which also carries
# the drafter and the output layers) holding the extra one. WHICH pair
# balances the cards is the fit test's job, which recomputes it from this
# file; pinning a literal here would only mean that re-measuring the
# balance breaks a test about parallelism.
local split_hi split_lo
split_hi="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\1/p' "$ENV_FILE")"
split_lo="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\2/p' "$ENV_FILE")"
[ -n "$split_hi" ] && [ -n "$split_lo" ]
[ $(( split_hi + split_lo )) -eq 65 ]
[ "$split_lo" -lt "$split_hi" ]
[ $(( split_lo - split_hi )) -le 1 ]
grep -q -- '--split-mode "$SPLIT_MODE"' "$SERVE"
grep -q -- '--device "$DEVICES"' "$SERVE"
grep -q -- '--tensor-split "$TENSOR_SPLIT"' "$SERVE"
@@ -72,24 +74,26 @@ DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter
grep -q '^export HIP_VISIBLE_DEVICES$' "$SERVE"
}
@test "the drafter is the model's own MTP head, at the measured depth" {
# MTP's head lives in the target file (blk.<n>.nextn.*): no sidecar, no
# second copy of the weights, and it is the only drafter that fits the
# 393216 pool. n_max 3 is the measured MTP winner -- n=7 accepts 33 % of
# 147, n=3 accepts 58 % of 72 (the env's table; identical greedy text at
# every depth, so this is pure speed).
grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE"
@test "the drafter is the DFlash2 sidecar at the measured depth" {
# The DFlash2 sidecar is the deployed drafter (operator call 2026-09-19).
# n_max 7 is the measured DFlash2 winner -- 43.5 tok/s at 41 % acceptance
# against 32.4 for the in-file MTP head at n=3 (the env's table; identical
# greedy text at every depth, so this is pure speed). The target file
# also carries its own MTP head (nextn_predict_layers=1), so draft-mtp is
# one SPEC_TYPE change away; the drafter file stays on disk for that.
grep -qE '^SPEC_TYPE=draft-dflash$' "$ENV_FILE"
grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE"
local n_max
n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")"
[ -n "$n_max" ] && [ "$n_max" -eq 3 ]
# The DFlash2 sidecar stays on disk and in the fetch script -- one
# SPEC_TYPE change brings the faster chain back -- and ngram-mod's window
# is retained with it, so neither may be dropped from the env.
grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q8_0\.gguf$' "$ENV_FILE"
[ -n "$n_max" ] && [ "$n_max" -eq 7 ]
# The Q4_K_M rung is the operator's requested drafter quant -- the
# drafter's job is throughput, not output quality, and the target
# verifies every drafted token. ngram-mod's window is retained in the env
# for the chain, so it may not be dropped.
grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q4_K_M\.gguf$' "$ENV_FILE"
grep -q '^SPEC_NGRAM_N_MATCH=24$' "$ENV_FILE"
# ...and neither is loaded: ngram-mod's flags only go out when the type
# contains ngram, so a pure MTP configuration is not handed them.
# contains ngram, so a pure dflash configuration is not handed them.
python3 -c "
lines = open('$SERVE').read().splitlines()
guard = next(i for i, l in enumerate(lines) if '*ngram*' in l)
@@ -130,11 +134,14 @@ print('draft-model wiring ok')
grep -q 'frank-medium.env' "$SERVE"
}
@test "the payload is fetched resumably and verified by size" {
grep -q 'UD-Q8_K_XL\.gguf' "$FETCH"
grep -q 'Qwen3\.8-27B-DFlash2-Q8_0\.gguf' "$FETCH"
grep -q 'curl -fL -C -' "$FETCH"
grep -q 'verifying sizes' "$FETCH"
@test "the payload is fetched resumably and verified by size and sha256" {
grep -q 'Qwen3\.8-27B-Uncensored-Q8_0\.gguf' "$FETCH"
grep -q 'Qwen3\.8-27B-DFlash2-Q4_K_M\.gguf' "$FETCH"
grep -q -- '-fL -C -' "$FETCH"
grep -q 'verifying sizes and sha256' "$FETCH"
# The target repo is gated: the script must fail loud without a token
# rather than write a 401 body to disk and report success.
grep -q 'no HF token is available' "$FETCH"
}
@test "both cards stay balanced and keep the operator's 1.5 GiB floor" {
@@ -158,9 +165,9 @@ print('draft-model wiring ok')
# measured 6.44 / 6.90 GiB free. The fit reproduces card 1 to 0.00 GiB
# and card 0 to 0.86 GiB at that point; it is a GATE, not a meter -- the
# deployed point is re-measured on the host after every split/ctx/batch
# move (the current one is 35:30 / 393216 / MTP, moved from 34:31 on
# 2026-09-17 to lean work off ROCm1's Gen4-x1 chipset path -- see the
# TENSOR_SPLIT header in frank-medium.env).
# move (the current one is 32:33 / 327680 / DFlash2 Q4_K_M, the balanced
# split the operator asked for on 2026-09-19 -- see the TENSOR_SPLIT
# header in frank-medium.env).
#
# card 0 = its share of the weights and of the KV (qwen35 spends KV on
# its 16 full-attention layers only: 2 (K,V) x 4 heads x 256 dim
@@ -197,22 +204,22 @@ print('draft-model wiring ok')
# allocation.
local kv_pos=65536 draft_kv_pos=20480 mtp_kv_pos=4096 floor slack
local card_hi card_lo diff
local mmproj=931145888 # mmproj-BF16.gguf, autoloaded onto the primary device
local mmproj=931145984 # mmproj-Qwen3.8-27B-Uncensored-f16.gguf, on the primary device
# 1.5 GiB was the floor until the 2x196k shape: the 34:31 / 393216 /
# MTP point measured 28.91 / 30.78 GiB, leaving 2.95 / 1.08 GiB, so the
# floor is the measured 1.08 GiB rounded down. The margin for one big
# allocation is thinner than it used to be. The layer-toward-ROCm0
# lever has since been spent (35:30, 2026-09-17): if a large allocation
# fails at load now, the next lever is a ctx cut, not another layer.
# floor is the measured 1.08 GiB rounded down. The 32:33 / 327680 /
# DFlash2 point is roomier than that (fit: 6.97 / 4.89 GiB free), so the
# floor stays at the measured 1.0 GiB -- if a large allocation fails at
# load, the next lever is a ctx cut, not a floor change.
floor=$(( 1024 * 1024 * 1024 )) # 1.0 GiB per card
# The deployed point (35:30 at ctx 393216 with MTP) has a fit gap of
# 0.58 GiB -- closer to balanced than its 34:31 predecessor (1.06 GiB).
# The slack sits at 1.483 GiB: it accepts the deployed split and still
# rejects the next step toward ROCm0, 36:29, which fails BOTH gates
# (gap 2.22 GiB, card 0 free 0.45 GiB under the floor). (The 36:29 /
# 327680 / DFlash2 configuration's slack was 3225 MiB -- its 3.15 GiB
# gap was by construction, and is no longer the deployed shape.)
slack=$(( 1483 * 1024 * 1024 ))
# The deployed point (32:33 at ctx 327680 with DFlash2) has a fit gap of
# 2082 MiB -- the balanced split the operator asked for, not the
# tightest reachable one. The slack sits at 2150 MiB: it accepts the
# deployed split and rejects a two-layer re-skew (34:31 / 31:34 both
# move the fit gap well past it). The x16-lean 35:30 that served the
# predecessor payload (gap 0.58 GiB) no longer applies: the operator
# chose symmetry over the x1-card bottleneck.
slack=$(( 2150 * 1024 * 1024 ))
card_hi=$(( MODEL_BYTES * split_hi / (split_hi + split_lo) \
+ kv_pos * ctx * split_hi / (split_hi + split_lo) \
+ 381898377 \
@@ -243,8 +250,9 @@ print('draft-model wiring ok')
diff=$(( card_hi - card_lo )); [ "$diff" -ge 0 ] || diff=$(( -diff ))
[ "$diff" -le "$slack" ]
# ...and the window may not wander far past what the checkpoint was
# trained for (262144; the GGUF carries no rope scaling, so the top 131k
# of the 393216 pool extrapolates on the base rope).
# trained for (262144; the GGUF carries no rope scaling, so positions
# past that extrapolate on the base rope). The deployed 327680 pool is
# two 163840 slots -- entirely trained territory.
[ "$ctx" -le 393216 ]
}
@@ -280,16 +288,16 @@ print('draft-model wiring ok')
grep -q -- '--cache-prompt' "$SERVE"
}
@test "vision is enabled: the BF16 projector loads with the model" {
@test "vision is enabled: the f16 projector loads with the model" {
grep -q '^NO_MMPROJ=0$' "$ENV_FILE"
grep -q '^MM_PROJ_FILE=mmproj-BF16.gguf$' "$ENV_FILE"
grep -q '^MM_PROJ_FILE=mmproj-Qwen3\.8-27B-Uncensored-f16\.gguf$' "$ENV_FILE"
# Explicit flag, not autoload: this build starts text-only without it,
# and refuses to start without the projector file, so a missing file is
# a deploy error, not a silent text-only server.
grep -q -- '--mmproj "$MM_PROJ"' "$SERVE"
grep -q 'vision is enabled but the projector is missing' "$SERVE"
# And the download script fetches and byte-verifies it.
grep -q '^MM_PROJ_SIZE=931145888$' "$FETCH"
grep -q '^MM_PROJ_SIZE=931145984$' "$FETCH"
grep -q -- 'check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE"' "$FETCH"
}