diff --git a/dankserver/frank-primary/frank-primary-entrypoint.sh b/dankserver/frank-primary/frank-primary-entrypoint.sh index c1a0d70..22cfdd6 100755 --- a/dankserver/frank-primary/frank-primary-entrypoint.sh +++ b/dankserver/frank-primary/frank-primary-entrypoint.sh @@ -8,22 +8,37 @@ # C++ server reads LUCE_* environment variables straight from the process # environment, so no env plumbing is needed here. # -# LAUNCH SHAPE (server/docs/DS4.md, server/docs/RECOMMENDED_SETUPS.md): -# --ds4-fused-decode requests a monolithic GPU load; on a 377 GiB model -# that allocation fails and the backend falls back -# to hybrid expert placement + layered decode (dense -# work on the R9700, routed experts computed on CPU -# from the mmapped page cache). That fallback IS the -# intended serving mode on this host. -# --ds4-prefill sparse the batched prefill; use for prompts beyond a few -# hundred tokens (exact is token-wise decode cost). -# --ds4-expert-top-k 6 the model default. Do NOT reduce on adaptive -# (qtype-105/106) artifacts: measured degenerate -# output at top-4 (DS4.md, 2026-08-05). -# --chunk 8192 the qualified Strix Halo profile's chunk. -# --max-ctx 131072 the qualified Strix Halo window; DS4's MLA KV is -# compressed, so the pool is small. -# --cache-type-k/v q4_0 the qualified launch's KV dtype. +# SOURCE BRANCH. The `deepseek41` architecture exists only on the +# `feat/ds41-engine` branch of Luce-Org/lucebox: main carries `deepseek4` +# alone and rejects this checkpoint with "unsupported model architecture +# 'deepseek41'" (server/src/common/backend_plan.cpp). The image is built from +# that branch — see serverAppConfigs-docs/LLM/frank-primary-lucebox-ds41.md. +# +# LAUNCH SHAPE (server/docs/DS41.md — the V4.1 page, not the V4 DS4.md one): +# --ds4-prefill dense the V4.1 batched prefill. The default `exact` walks +# the prompt token by token, and the V4-only modes are +# REFUSED for V4.1: `sparse`, --ds4-fused-decode, +# --ds4-fused-verify-f16-kv, layer split and --mmproj +# all fail validate_model_features at load. +# --chunk 4096 the chunk the V4.1 dense-prefill numbers were taken +# at; the runtime caps it to what the device holds. +# --max-ctx 131072 the V4.1 default window. The MLA compressed caches +# are F16 by design and stay small; --cache-type-k/v +# are ignored by the DeepSeek4 backend, so they are +# not passed. +# --prefix-cache-slots 0 --prefill-cache-slots 0 +# the DS41.md bring-up recipe (the server default is +# 32 prefix slots). Lift them once capacity under load +# has been measured, not before. +# +# EXPERT PLACEMENT. 377 GiB of weights against 64 GiB of VRAM: only the dense +# weights, the compressed caches and a hot expert stack fit on hip:0, and the +# remaining routed experts are streamed out of the mmapped page cache/NVMe by +# the hybrid expert tier. LUCE_EXPERT_STREAM_DEVICE=1 moves the streamed- +# expert slot cache onto the second R9700 — it defaults to the target device, +# where it would compete with the hot stack. Both the owner stack and the +# slot cache then auto-size to their device's free memory minus the 1.5 GiB +# discrete headroom. # # No DSpark draft is published for DeepSeek-V4.1-Flash, so this is plain # autoregressive decode (no --draft, no LUCE_DS4_SPEC). @@ -32,6 +47,12 @@ set -euo pipefail MODEL="${FRANK_PRIMARY_MODEL:-/models/DeepSeek-V4.1-Flash/DeepSeek-V4.1-Flash-ROCMFP23.gguf}" BIN="${FRANK_PRIMARY_BIN:-/opt/lucebox-hub/server/build/luce_server}" +# The second R9700 holds the streamed-expert slot cache and computes them. +export LUCE_EXPERT_STREAM_DEVICE="${LUCE_EXPERT_STREAM_DEVICE:-1}" +# rocBLAS/BLAS dispatch and queue depth, as the fleet's other ROCm servers. +export ROCBLAS_USE_HIPBLASLT="${ROCBLAS_USE_HIPBLASLT:-0}" +export GPU_MAX_HW_QUEUES="${GPU_MAX_HW_QUEUES:-8}" + if [[ ! -f "$MODEL" ]]; then echo "frank-primary: model missing at $MODEL" >&2 exit 1 @@ -47,9 +68,7 @@ exec "$BIN" "$MODEL" \ --model-name frank-primary \ --target-device hip:0 \ --max-ctx 131072 \ - --chunk 8192 \ - --cache-type-k q4_0 \ - --cache-type-v q4_0 \ - --ds4-fused-decode \ - --ds4-prefill sparse \ - --ds4-expert-top-k 6 + --chunk 4096 \ + --ds4-prefill dense \ + --prefix-cache-slots 0 \ + --prefill-cache-slots 0 diff --git a/dankserver/frank-primary/quadlet/frank-primary.container b/dankserver/frank-primary/quadlet/frank-primary.container index 3040496..700a639 100644 --- a/dankserver/frank-primary/quadlet/frank-primary.container +++ b/dankserver/frank-primary/quadlet/frank-primary.container @@ -7,13 +7,14 @@ # runtime (github.com/Luce-Org/lucebox, local image localhost/lucebox-rocm: # gfx1201 built from Dockerfile.rocm with LUCE_HIP_ARCHES=gfx1201 and # ROCM_VERSION=7.14.1-full to match the host driver's ROCm 7.x line) -# serving DeepSeek-V4.1-Flash-ROCMFP23 (377.2 GiB GGUF, 43-layer MoE, -# 256 routed experts + 1 shared per layer, top-6 routing) on a single -# Radeon AI PRO R9700 (gfx1201). The model does not fit VRAM: dense work -# and the MLA KV cache sit on the R9700, the routed experts are computed -# on CPU from the mmapped page cache (246 GiB RAM holds the working set; -# the rest streams from NVMe). Lazy mmap page-in is the engine default — -# no flag. +# serving DeepSeek-V4.1-Flash-ROCMFP23 (377.2 GiB GGUF, 40-layer MoE, +# 384 routed experts + 1 shared per layer, top-6 routing, Engram tables at +# layers 1 and 14) on the two Radeon AI PRO R9700s (gfx1201). The model +# cannot be resident: dense weights, the compressed MLA caches and a hot +# expert stack sit on hip:0, the streamed-expert slot cache on hip:1, and +# every other routed expert is pulled out of the mmapped page cache (246 GiB +# RAM holds most of the 179 GiB expert set; the rest reads from NVMe). Lazy +# mmap page-in is the engine default — no flag. # # The image's entrypoint only forwards a fixed Qwen/DFlash flag set and # cannot express the DS4 flags, so the unit bypasses it (--entrypoint) and @@ -25,11 +26,14 @@ # Open WebUI metadata) — the fleet contract is unchanged from # frank-medium. # -# DEVICES. /dev/kfd + /dev/dri through the amdgpu/KFD stack. HIP_VISIBLE_ -# DEVICES=0 pins the first R9700 (the Raphael iGPU enumerates as index 2 -# and stays out). The second R9700 is not used: the in-process -# heterogeneous expert-parallel path (LUCE_DS4_MOE_TP_INPROC) is qualified -# for gfx1151+gfx1201 pairs, not gfx1201+gfx1201. +# DEVICES. /dev/kfd + /dev/dri through the amdgpu/KFD stack. Both R9700s are +# visible (indices 0 and 1; the Raphael iGPU enumerates as index 2 and stays +# out). They are NOT driven as a symmetric pair: the in-process expert- +# parallel path (LUCE_DS4_MOE_TP_INPROC) is qualified for gfx1151+gfx1201 +# pairs, not gfx1201+gfx1201. Instead hip:0 owns the dense weights and the +# hot expert stack while LUCE_EXPERT_STREAM_DEVICE=1 (entrypoint) gives hip:1 +# the streamed-expert slot cache, so each card carries one tier and neither +# waits on the other. # # MOUNTS. /models (ro — the GGUF is mmapped, nothing writes), and the # entrypoint + healthcheck scripts straight out of this checkout @@ -58,9 +62,10 @@ AddDevice=/dev/kfd AddDevice=/dev/dri GroupAdd=keep-groups PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add SYS_PTRACE --network=host --entrypoint /usr/local/bin/frank-primary-entrypoint.sh -# First R9700 only (index pin excludes the Raphael iGPU at index 2). -Environment=HIP_VISIBLE_DEVICES=0 -Environment=ROCR_VISIBLE_DEVICES=0 +# Both R9700s: hip:0 target, hip:1 the streamed-expert slot cache. Index 2, +# the Raphael iGPU, stays out. +Environment=HIP_VISIBLE_DEVICES=0,1 +Environment=ROCR_VISIBLE_DEVICES=0,1 # The 377 GiB GGUF, mmapped; pages pull from disk/page cache on demand. Volume=/home/dank/models:/models:ro # Entrypoint + healthcheck from this checkout, mounted read-only.