diff --git a/.AGENTTRASH/frank-default-large.env.ciru-activefpx-2026-08-21 b/.AGENTTRASH/frank-default-large.env.ciru-activefpx-2026-08-21 new file mode 100644 index 0000000..074e0e4 --- /dev/null +++ b/.AGENTTRASH/frank-default-large.env.ciru-activefpx-2026-08-21 @@ -0,0 +1,74 @@ +# Tunables for frank-default-large (CIRU llama.cpp, dankclaw, Strix Halo +# gfx1151). +# +# Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag +# qwen3.8-activefpx-promptforge-v2.3 serving +# jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge (v2). ActiveFPX compact +# GGUF with PromptForge companions for CIRU DualView prompt acceleration and +# native depth-4 MTP. Alias frank-default-large, port 52625. +# +# This runtime replaced the retired vLLM service that previously occupied +# this slot. See docs/ciru-activefpx-promptforge.md. +# +# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm +# HIP runtime reads those bare env vars (see the 2026-08-13 TEMP incident). +# llama.cpp reads --host/--port/--temp as CLI flags, never as env vars. + +# CIRU llama-server binary (ROCmFPX v2.3 build, see docs/ciru runtime page). +# Built against the private TheRock ROCm 7.15 SDK kept in the retired +# ling3-ciru venv. ROCm 7.15 is required for the advertised Strix Halo +# performance; the system ROCm 7.2.4 selects slower/fallback kernels. +LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server + +# Model files. The three-file payload (v2 GGUF + FFN + GDN) enables CIRU +# DualView prompt acceleration; omit the two .pfs files to run the GGUF alone +# and save ~19.7 GiB of memory at the cost of prompt throughput. +MODEL_DIR=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX +MODEL_FILE=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v2.gguf +FFN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs +GDN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs + +# ROCm 7.15 SDK paths (inside the retired ling3-ciru venv). The serve script +# exports ROCM_PATH and LD_LIBRARY_PATH from these so the 7.15 libraries +# precede any system ROCm 7.2.4. +ROCM_DEVEL=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel +ROCM_LIB=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries +ROCM_CORE=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core + +ALIAS=frank-default-large +BIND_ADDR=0.0.0.0 +LISTEN_PORT=52625 + +# Serving: full native 262144 context (n_ctx_train), 1 slot (user request +# 2026-08-19: parallel slots degraded performance). -c is the TOTAL +# context; with PARALLEL=1 it equals the per-slot context. Concurrent +# requests queue on the single slot; the server never preempts a running +# generation. f16 KV, context checkpoints, prompt cache, native MTP +# depth-4. The Strix Halo is unified memory; rocm-smi reports the 64 GiB +# budget. KV at 262144 x 64 KiB/cell is ~16 GiB. +CTX_SIZE=262144 +PARALLEL=1 +BATCH_SIZE=4096 +UBATCH_SIZE=4096 +CACHE_TYPE_K=f16 +CACHE_TYPE_V=f16 +CTX_CHECKPOINTS=16 +CACHE_RAM=8192 + +# Native MTP draft (depth-4, greedy, ROCm-safe CPU argmax fast path). +SPEC_DRAFT_N_MAX=4 +SPEC_DRAFT_N_MIN=0 +SPEC_DRAFT_P_MIN=0.0 +SPEC_DRAFT_P_SPLIT=0.10 + +# Sampling (model card defaults). +TEMPERATURE=0.7 +TOP_P=0.95 + +# Thinking enabled (user request, 2026-08-18). deepseek format moves the +# reasoning into message.reasoning_content (OpenAI-compatible); +# none would leave the thought tags unparsed in content. +REASONING=on +REASONING_FORMAT=deepseek + +VERBOSITY=4 diff --git a/.AGENTTRASH/serve-frank-large.sh.ciru-activefpx-2026-08-21 b/.AGENTTRASH/serve-frank-large.sh.ciru-activefpx-2026-08-21 new file mode 100755 index 0000000..540d449 --- /dev/null +++ b/.AGENTTRASH/serve-frank-large.sh.ciru-activefpx-2026-08-21 @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Serve jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge as frank-default-large +# via the CIRU ROCmFPX llama.cpp runtime on Dankclaw (Strix Halo gfx1151). +# +# Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag +# qwen3.8-activefpx-promptforge-v2.3 (commit 103af8b41) against the private +# TheRock ROCm 7.15 SDK (kept in the retired ling3-ciru venv). The model is +# a GGUF with optional PromptForge companion views (FFN/GDN .pfs) that enable +# CIRU DualView prompt acceleration. Native depth-4 MTP speculative decoding. +# +# The model card requires ROCm 7.15 (not the system ROCm 7.2.4) for the +# advertised Strix Halo performance. LD_LIBRARY_PATH therefore points at the +# 7.15 SDK libraries first. An older system ROCm must not precede them. +# +# Tunables come from frank-default-large.env in this directory unless +# FRANK_LARGE_ENV_FILE overrides that path. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}" +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +# --- defaults (used when the env file does not set them) ------------------- +: "${LLAMA_BIN:=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server}" +: "${MODEL_DIR:=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX}" +: "${MODEL_FILE:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v2.gguf}" +: "${FFN_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs}" +: "${GDN_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs}" +: "${ROCM_DEVEL:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel}" +: "${ROCM_LIB:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries}" +: "${ROCM_CORE:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core}" +: "${ALIAS:=frank-default-large}" +: "${BIND_ADDR:=0.0.0.0}" +: "${LISTEN_PORT:=52625}" +: "${CTX_SIZE:=524288}" +: "${PARALLEL:=2}" +: "${BATCH_SIZE:=4096}" +: "${UBATCH_SIZE:=4096}" +: "${CACHE_TYPE_K:=f16}" +: "${CACHE_TYPE_V:=f16}" +: "${CTX_CHECKPOINTS:=16}" +: "${CACHE_RAM:=8192}" +: "${SPEC_DRAFT_N_MAX:=4}" +: "${SPEC_DRAFT_N_MIN:=0}" +: "${SPEC_DRAFT_P_MIN:=0.0}" +: "${SPEC_DRAFT_P_SPLIT:=0.10}" +: "${TEMPERATURE:=0.7}" +: "${TOP_P:=0.95}" +: "${REASONING:=on}" +: "${REASONING_FORMAT:=deepseek}" +: "${VERBOSITY:=4}" + +MODEL="$MODEL_DIR/$MODEL_FILE" +export PROMPTFORGE_SIDECAR="$MODEL_DIR/$FFN_PFS" +export PROMPTFORGE_GDN_SIDECAR="$MODEL_DIR/$GDN_PFS" +export PROMPTFORGE_MODE="m2048_fused_tail1476" +export GGML_CUDA_GRAPH_OPT=0 +export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 + +if [[ ! -f "$MODEL" ]]; then + echo "missing required file: $MODEL" >&2 + exit 1 +fi +if [[ ! -f "$PROMPTFORGE_SIDECAR" ]]; then + echo "missing PromptForge FFN companion: $PROMPTFORGE_SIDECAR" >&2 + exit 1 +fi +if [[ ! -f "$PROMPTFORGE_GDN_SIDECAR" ]]; then + echo "missing PromptForge GDN companion: $PROMPTFORGE_GDN_SIDECAR" >&2 + exit 1 +fi +if [[ ! -x "$LLAMA_BIN" ]]; then + echo "missing llama-server binary: $LLAMA_BIN" >&2 + exit 1 +fi + +# ROCm 7.15 SDK libraries must precede any system ROCm in LD_LIBRARY_PATH. +export ROCM_PATH="$ROCM_DEVEL" +export LD_LIBRARY_PATH="${ROCM_LIB}/lib:${ROCM_CORE}/lib:${ROCM_DEVEL}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + +exec "$LLAMA_BIN" \ + --model "$MODEL" \ + --alias "$ALIAS" \ + --host "$BIND_ADDR" \ + --port "$LISTEN_PORT" \ + -dev ROCm0 \ + -ngl 999 \ + -c "$CTX_SIZE" \ + -b "$BATCH_SIZE" \ + -ub "$UBATCH_SIZE" \ + -fa on \ + -ctk "$CACHE_TYPE_K" \ + -ctv "$CACHE_TYPE_V" \ + -np "$PARALLEL" \ + -ctxcp "$CTX_CHECKPOINTS" \ + --cache-ram "$CACHE_RAM" \ + --cache-prompt \ + --metrics \ + --no-mmap \ + --spec-type draft-mtp \ + --spec-draft-device ROCm0 \ + --spec-draft-ngl 999 \ + --spec-draft-type-k f16 \ + --spec-draft-type-v f16 \ + --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \ + --spec-draft-n-min "$SPEC_DRAFT_N_MIN" \ + --spec-draft-p-min "$SPEC_DRAFT_P_MIN" \ + --spec-draft-p-split "$SPEC_DRAFT_P_SPLIT" \ + --spec-draft-backend-sampling \ + --temp "$TEMPERATURE" \ + --top-p "$TOP_P" \ + --reasoning "$REASONING" \ + --reasoning-format "$REASONING_FORMAT" \ + --reasoning-budget -1 \ + -lv "$VERBOSITY" diff --git a/AGENTS.md b/AGENTS.md index bbac1d8..73b1653 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,23 +3,21 @@ ## Overview This project manages deployment scripts for the LLM inference fleet on two -local machines. Topology (2026-08-18): +local machines. Topology (2026-08-21): | Target | Host | Runtime | Service | Model | Port | |--------|------|---------|---------|-------|------| | Dankserver | dank@dankserver | llama.cpp (upstream b10121) | frank-default-small | HauhauCS Qwen3.6-35B-A3B-Uncensored-Aggressive Q6_K_P + ngram-mod, 36 MoE experts on CPU | 7770 | -| Dankclaw | dank@dankclaw | CIRU ROCmFPX llama.cpp (ROCm 7.15 SDK in the ling3-ciru venv) | frank-default-large | jcbtc Qwen3.8-27B-CIRU-ActiveFPX-PromptForge v2, native MTP, PromptForge DualView, 262K context / 1 slot | 52625 | +| Dankclaw | dank@dankclaw | llama.cpp PR #27342 DFlash2 (Vulkan RADV) | frank-default-large | unsloth Qwen3.8-27B-UD-Q8_K_XL + z-lab Qwen3.8-27B-DFlash2-Q4_K_M drafter, 262K context / 1 slot | 52625 | The 2026-08-17 redo retired the pipeline-parallel split and the FastFlowLM -NPU endpoint. The 2026-08-18 swap retired the vLLM service on Dankclaw; the -CIRU ROCmFPX llama.cpp runtime now runs on the Strix Halo iGPU (gfx1151). -A second 2026-08-18 swap renamed the services and restored the Qwen3.6 35B -A3B Q6_K_P model on Dankserver: Dankserver is now `frank-default-small` -(Qwen3.6-35B-A3B Q6_K_P + ngram-mod, MoE experts on CPU) and Dankclaw is -now `frank-default-large` (CIRU Qwen3.8-27B, 262K context / 1 slot). The -retired Qwen3.8-27B IQ4_XS FastMTP files live in `.AGENTTRASH/`. See -docs/ciru-activefpx-promptforge.md. The old scripts live in `old/scripts/` -for reference; they describe the retired PP topology. +NPU endpoint. The 2026-08-18 swap retired the vLLM service and the CIRU +ROCmFPX runtime on Dankclaw. On 2026-08-21 `frank-default-large` moved to +upstream llama.cpp PR #27342 with DFlash2 speculative decoding on the +Vulkan backend (`Vulkan0`, RADV gfx1151). The retired CIRU AI and IQ4_XS +files live in `.AGENTTRASH/`. See docs/dflash2-large-vulkan.md. The old +scripts live in `old/scripts/` for reference; they describe the retired PP +topology. **Sibling project:** [../serverAppConfigs](../serverAppConfigs) manages the Podman app stack on the same hosts (OpenWebUI, Gitea, observability). Its @@ -65,7 +63,7 @@ Use `scripts/deploy.sh` to deploy committed changes to a target. It pushes `main ``` FRANK_SUDO_PW=... ./scripts/deploy.sh dankserver # llama.cpp FastMTP, port 7770 -FRANK_SUDO_PW=... ./scripts/deploy.sh dankclaw # CIRU llama.cpp, port 52625 +FRANK_SUDO_PW=... ./scripts/deploy.sh dankclaw # DFlash2 llama.cpp, port 52625 ./scripts/deploy.sh dankserver --dry-run # preview without changes ``` @@ -127,23 +125,19 @@ back to a slow path and prefill collapses ~12x (measured 2026-08-17: 95 t/s vs 1300+ t/s at an 8K prompt on the 3090). See the HauhauCS HF repo README for the base recipe. -### CIRU ROCmFPX llama.cpp (Dankclaw) +### DFlash2 llama.cpp (Dankclaw, Vulkan) -The CIRU llama-server serves `jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge` -from `/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server`. -The runtime is the ciru-ai/ROCmFPX tag `qwen3.8-activefpx-promptforge-v2.3` -(commit `103af8b41`) built against the private TheRock ROCm 7.15 SDK kept -in the retired ling3-ciru venv -(`.venv/lib/python3.12/site-packages/_rocm_sdk_*`). The model card requires -ROCm 7.15 for the advertised Strix Halo performance; the system ROCm 7.2.4 -selects slower kernels. The serve script exports the 7.15 SDK libraries -first in `LD_LIBRARY_PATH`. +The upstream llama.cpp PR #27342 llama-server serves +`unsloth/Qwen3.8-27B-UD-Q8_K_XL.gguf` with a z-lab DFlash2 drafter +(`Qwen3.8-27B-DFlash2-Q4_K_M.gguf`) on the Vulkan backend. The binary lives +at `/home/dank/llama.cpp-dflash2/build-dflash2/bin/llama-server`. DFlash2 +needs `TOP_K` + `ARGMAX`, which HIP does not implement, so the stack runs on +`Vulkan0` (RADV, gfx1151). See docs/dflash2-large-vulkan.md. -The model files (v2 GGUF + FFN.pfs + GDN.pfs, ~34.9 GiB) live in -`/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX/`. The full build runbook is -in docs/ciru-activefpx-promptforge.md. The retired vLLM venvs -(`/home/dank/vllm-venv`, `/home/dank/vllm-vk-venv`) were moved to -`/home/dank/.AGENTTRASH/` on 2026-08-18. +The target and drafter live in `/home/dank/models/`. The retired CIRU +runtime (ROCmFPX v2.3, ROCm 7.15 SDK in the ling3-ciru venv, FFN/GDN +companions) is archived in `.AGENTTRASH/` and documented in +circu-activefpx-promptforge.md. **Env var naming rule still applies:** the HIP runtime reads `TEMP`, `HOST`, and `GPU_MAX_HEAP_SIZE`. The serve script passes `--host`/`--port`/`--temp` @@ -192,7 +186,7 @@ graph TD - `sudo` on both boxes needs a tty. Remote sudo steps fail non-interactively. Use `echo '' | sudo -S `. - Never run `pkill -f` with a pattern that appears in your own command line. It kills your ssh session. Use `pkill -x `. -- Model load is slow: Dankserver loads the 30.6 GB Q6_K_P through RAM (`--no-mmap`, MoE experts on CPU) in a few minutes cold; Dankclaw CIRU loads the 34.9 GiB ActiveFPX payload in ~5-10 minutes. Health returns 503 while loading. Do not kill the process. `deploy.sh` waits up to 20 minutes on both targets (see `HEALTH_WAIT`). +- Model load is slow: Dankserver loads the 30.6 GB Q6_K_P through RAM (`--no-mmap`, MoE experts on CPU) in a few minutes cold; Dankclaw DFlash2 loads the 31.4 GB UD-Q8_K_XL + 2.0 GB drafter in ~1 minute with a warm OS page cache and a few minutes cold. Health returns 503 while loading. Do not kill the process. `deploy.sh` waits up to 20 minutes on both targets (see `HEALTH_WAIT`). - A client abort and a server release look different in the journal. A client abort logs `W srv stop: cancel task, id_task = N`. A natural stop (EOS or limit) logs only `slot release` with `truncated = 0`. The server never preempts a running generation; when all slots are busy, new tasks defer into a queue. See the 2026-08-13 incident in [docs/deployment.md](docs/deployment.md). - Retired hardware history: the Dankserver 35B cuBLAS crash and the DFlash premature-EOS bug are historical; see [docs/dankserver-vram-growth.md](docs/dankserver-vram-growth.md) and [docs/qwen3.8-spec-matrix.md](docs/qwen3.8-spec-matrix.md). The FastFlowLM NPU endpoint is retired; see [docs/FastFlowLM NPU Inference.md](docs/FastFlowLM%20NPU%20Inference.md). diff --git a/docs/deployment.md b/docs/deployment.md index d06182b..b886f52 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -9,7 +9,7 @@ Two servers run LLM inference services. Each server pulls its scripts from this | Target | Host | Runtime | Service | Port | |--------|------|---------|---------|------| | Dankserver | dank@dankserver | upstream llama.cpp (Qwen3.6-35B-A3B Q6_K_P + ngram-mod, MoE on CPU) | frank-default-small | 7770 | -| Dankclaw | dank@dankclaw | CIRU ROCmFPX llama.cpp (jcbtc Qwen3.8-27B-CIRU-ActiveFPX-PromptForge v2, native MTP + PromptForge DualView, 262K ctx / 1 slot) | frank-default-large | 52625 | +| Dankclaw | dank@dankclaw | upstream llama.cpp PR #27342 DFlash2 (unsloth Qwen3.8-27B-UD-Q8_K_XL + z-lab DFlash2 Q4_K_M drafter, Vulkan, 262K ctx / 1 slot) | frank-default-large | 52625 | ## Workflow @@ -115,19 +115,19 @@ launches through `scripts/serve-frank-small.sh`, which reads tunables from ### Dankclaw — frank-default-large -The active service on Dankclaw is the CIRU ROCmFPX llama.cpp runtime serving -`jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge` (v2 GGUF + FFN/GDN -companions) under the alias `frank-default-large` on port 52625. It runs -`/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server` (ROCm -7.15 SDK in the retired ling3-ciru venv) with native depth-4 MTP -(`--spec-type draft-mtp`), f16 KV for both K and V, full 262144 context -(1 slot; concurrent requests queue, the server never preempts a running -generation), batch 4096 / ubatch 4096, 16 -context checkpoints, 8 GiB +The active service on Dankclaw is upstream llama.cpp PR #27342 (DFlash2 +support) built with GGML_VULKAN=ON, serving +`/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL.gguf` (unsloth Dynamic v3.0, +31.4 GB) under the alias `frank-default-large` on port 52625. It runs +`/home/dank/llama.cpp-dflash2/build-dflash2/bin/llama-server` with a +z-lab DFlash2 drafter (`/home/dank/models/Qwen3.8-27B-DFlash2-Q4_K_M.gguf`, +block size 8) via `--spec-type draft-dflash`, on the Vulkan (RADV) backend +(`Vulkan0`, gfx1151, 113942 MiB). f16 KV for both K and V, full 262144 +context (1 slot), batch 4096 / ubatch 4096, 16 context checkpoints, 8 GiB prompt cache, and reasoning in deepseek format. The unit file is `scripts/frank-default-large.service`. It launches through `scripts/serve-frank-large.sh`, which reads tunables from -`scripts/frank-default-large.env`. See [[ciru-activefpx-promptforge]]. +`scripts/frank-default-large.env`. See [[dflash2-large-vulkan]]. ## Validation diff --git a/docs/dflash2-large-vulkan.md b/docs/dflash2-large-vulkan.md new file mode 100644 index 0000000..5b42bc7 --- /dev/null +++ b/docs/dflash2-large-vulkan.md @@ -0,0 +1,170 @@ +--- +type: reference +title: DFlash2 on Dankclaw (frank-default-large) +description: unsloth Qwen3.8-27B-UD-Q8_K_XL served by upstream llama.cpp PR #27342 with a z-lab DFlash2 drafter on the Vulkan backend. Install runbook and the frank-default-large service. +tags: + - llama.cpp + - vulkan + - dankclaw + - gfx1151 + - dflash + - dflash2 + - speculative-decoding +timestamp: 2026-08-21 +--- + +# DFlash2 on Dankclaw (frank-default-large) + +`frank-default-large` (port 52625) serves +`unsloth/Qwen3.8-27B-UD-Q8_K_XL` with a DFlash2 drafter from z-lab. The +runtime is upstream llama.cpp PR #27342 built with `GGML_VULKAN=ON`. +This deployment replaced the CIRU HIP runtime on 2026-08-21. + +## Why Vulkan + +DFlash2 selects its draft candidates with a lightweight path selector +built on the `TOP_K` and `ARGMAX` graph ops. The HIP backend does not +implement `TOP_K`. That is why the retired CIRU setup routed the MTP +draft around it with `LLAMA_MTP_CPU_ARGMAX_FASTPATH`. + +The PR reference fork (Nathanw1014/strix-halo-llamacpp) validated DFlash2 +on this exact gfx1151 through Vulkan (RADV). Its `test-backend-ops` passed +the new graph ops: + +| Op | Cases | +|----|-------| +| TOP_K | 445 | +| CONCAT | 193 | +| PAD | 28 | +| REPEAT | 18 | +| FILL | 4 | + +The whole stack (target + drafter) runs on `Vulkan0`. + +## What Runs + +| Artifact | Repo / File | Size | +|----------|-------------|------| +| Target | `unsloth/Qwen3.8-27B-GGUF` `Qwen3.8-27B-UD-Q8_K_XL.gguf` | 31.4 GB | +| Drafter | `z-lab/Qwen3.8-27B-DFlash2-GGUF` `Qwen3.8-27B-DFlash2-Q4_K_M.gguf` | 2.0 GB | +| Runtime | upstream llama.cpp PR #27342, GGML_VULKAN=ON | - | + +The target was already on disk. The drafter was added for this deploy. +Both live in `/home/dank/models/`. + +## Drafter Hyperparameters + +Read from the drafter GGUF metadata: + +| Parameter | Value | +|-----------|-------| +| `dflash.block_count` | 5 | +| `dflash.block_size` | 8 | +| `dflash.conv_kernel_size` | 2 | +| `dflash.conv_group_size` | 16 | +| `dflash.selector_rank` | 256 | +| `dflash.selector_top_k` | 16 | +| `dflash.context_length` | 262144 | +| `dflash.embedding_length` | 5120 | +| architecture | `dflash` | + +The target and drafter share the Qwen3.8 architecture +(`qwen35`, dense 27B, vocab 248320), so the drafter inherits the target +tokenizer and LM head. + +## Build Runbook (2026-08-21) + +Performed on Dankclaw. + +1. In `/home/dank/llama.cpp`, register the `ggml` remote and fetch the PR + head: + ``` + git -C /home/dank/llama.cpp remote add ggml https://github.com/ggml-org/llama.cpp.git + git -C /home/dank/llama.cpp fetch ggml pull/27342/head:pr-27342 + ``` +2. Add a detached worktree that does not disturb the running build: + ``` + git -C /home/dank/llama.cpp worktree add --detach /home/dank/llama.cpp-dflash2 pr-27342 + ``` +3. Configure with Vulkan only (glslang and `glslc` must be installed): + ``` + cmake -B /home/dank/llama.cpp-dflash2/build-dflash2 -G Ninja \ + -DGGML_VULKAN=ON -DGGML_HIP=OFF -DGGML_CUDA=OFF -DGGML_METAL=OFF \ + -DGGML_NATIVE=ON -DCMAKE_BUILD_TYPE=Release \ + -DLLAMA_BUILD_SERVER=ON -DLLAMA_BUILD_WEBUI=OFF + ``` +4. Build the server: + ``` + cmake --build /home/dank/llama.cpp-dflash2/build-dflash2 --target llama-server -j16 + ``` +5. Verify the Vulkan device: + ``` + /home/dank/llama.cpp-dflash2/build-dflash2/bin/llama-server --list-devices + ``` + Output: `Vulkan0: AMD Radeon Graphics (RADV STRIX_HALO)`. + +```mermaid +graph TD + A[Fetch PR #27342 head] --> B[Worktree on DFlash2 head] + B --> C[Configure Vulkan build] + C --> D[Build llama-server] + D --> E[Verify Vulkan0 device] + E --> F[Add DFlash2 drafter] + F --> G[Swap frank-default-large] + G --> H[Health check + completion] +``` + +## Service + +- Unit: `frank-default-large.service`, port 52625 +- Script: `scripts/serve-frank-large.sh`, env `scripts/frank-default-large.env` +- Binary: `/home/dank/llama.cpp-dflash2/build-dflash2/bin/llama-server` + +### Launch profile + +- 262144 context (native max), 1 slot, f16 KV (~16 GiB), 16 context + checkpoints, 8 GiB prompt cache +- DFlash2 draft, block size 8, device Vulkan0, drafter fully offloaded +- `--spec-type draft-dflash --spec-draft-n-max 8` + (clamped to block size 8 - 1 = 7 at runtime) +- `--temp 0.7 --top-p 0.95`, reasoning on (`--reasoning-format deepseek`) +- `--no-mmap`, flash attention `-fa on`, `--metrics` + +### Required environment + +No PromptForge variables are needed. The `TEMP`, `HOST`, and +`GPU_MAX_HEAP_SIZE` naming rule still applies for portability, even though +this build uses Vulkan, not HIP. + +## Validation + +The smoke test on 2026-08-21 used a scratch port (52626) and confirmed: + +- Both models load and the server reports healthy. +- A chat completion returns reasoning and content. +- The DFlash2 drafter engaged: + `draft-dflash: #gen drafts = 11, #acc drafts = 10, #mean acc len = 5.27`. +- The per-position acceptance curve decays along the block + `(0.909, 0.818, 0.727, 0.545, 0.545, 0.364, 0.364)`, as expected. + +The published mean acceptance for the Q4_K_M drafter is ~5.39, so 5.27 +is close. + +## Operational Notes + +- Health: `curl http://127.0.0.1:52625/health`. Returns 503 while loading. +- Model load takes about a minute with a warm OS page cache (31 GB target) + and longer on a cold cache. Do not kill the process during warmup. +- `deploy.sh dankclaw` waits 20 minutes for health. +- The Vulkan device reports 113942 MiB. rocm-smi reports the 64 GiB + unified-memory budget. Trust the runtime for the addressable SVM pool. +- The MTP head of the target GGUF is ignored (`--spec-type draft-dflash` + replaces it). The load log warns about unused MTP tensors; this is normal. + +## See Also + +- [[DFlash DSpark Speculative Decoding]] — DFlash/DSpark design and history +- [[ciru-activefpx-promptforge]] — the retired CIRU runtime (archived) +- [[deployment]] — the deployment and update process +- [[qwen3.8-strix-optimizations]] — llama.cpp Optimizations on this iGPU +- [[index]] — knowledge base index diff --git a/docs/index.md b/docs/index.md index 8df3bf3..4cd4a71 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,8 +1,8 @@ # vLLM Scripts — Knowledge Base Documentation for the LLM inference fleet on two local machines -(2026-08-18): upstream llama.cpp on Dankserver (frank-default-small) and -CIRU ROCmFPX llama.cpp on Dankclaw (frank-default-large). +(2026-08-21): upstream llama.cpp on Dankserver (frank-default-small) and +upstream llama.cpp PR #27342 DFlash2 on Dankclaw (frank-default-large). ## Deployment Topology @@ -19,10 +19,11 @@ ngram-mod, 36 MoE experts on CPU, 700K ctx] subgraph Dankclaw[AMD Strix Halo - Ubuntu 26.04] D[gfx1151 iGPU 64 GiB unified memory] - E[CIRU ROCmFPX llama.cpp + E[upstream llama.cpp PR #27342 DFlash2, Vulkan frank-default-large, port 52625] - F[jcbtc Qwen3.8-27B-CIRU-ActiveFPX-PromptForge -262K ctx / 1 slot, native MTP, PromptForge DualView] + F[unsloth Qwen3.8-27B-UD-Q8_K_XL + +z-lab Qwen3.8-27B-DFlash2-Q4_K_M drafter +262K ctx / 1 slot, DFlash2 spec] D --> E --> F end @@ -41,16 +42,21 @@ frank-default-large, port 52625] - [[deployment]] — deploy and update process for both servers - [[frank-redo-2026-08-17]] — the 2026-08-17 topology change -- [[ciru-activefpx-promptforge]] — the CIRU runtime on Dankclaw - (frank-default-large) +- [[dflash2-large-vulkan]] — the DFlash2 runtime on Dankclaw + (frank-default-large, Vulkan) - [[Dankserver]] — the upstream llama.cpp box (frank-default-small) -- [[Dankclaw]] — the CIRU ROCmFPX box (frank-default-large) +- [[Dankclaw]] — the DFlash2 Vulkan box (frank-default-large) -## CIRU Reference (Dankclaw) +## DFlash2 Reference (Dankclaw) -- [[ciru-activefpx-promptforge]] — model, ROCm 7.15 toolchain, build, +- [[dflash2-large-vulkan]] — DFlash2 model, Vulkan build, build runbook, and the frank-default-large service +## CIRU Reference (retired) + +- [[ciru-activefpx-promptforge]] — the retired CIRU ROCmFPX runtime + (archived in .AGENTTRASH, 2026-08-21) + ## vLLM Reference (historical) - [[VLLM Overview]] — installation, CLI, API, architecture, quantization, parallelism diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 074e0e4..cb4eabc 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -1,51 +1,36 @@ -# Tunables for frank-default-large (CIRU llama.cpp, dankclaw, Strix Halo -# gfx1151). +# Tunables for frank-default-large (upstream llama.cpp PR #27342 DFlash2, +# Vulkan backend, dankclaw, Strix Halo gfx1151). # -# Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag -# qwen3.8-activefpx-promptforge-v2.3 serving -# jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge (v2). ActiveFPX compact -# GGUF with PromptForge companions for CIRU DualView prompt acceleration and -# native depth-4 MTP. Alias frank-default-large, port 52625. +# Runtime: llama.cpp PR #27342 built with GGML_VULKAN=ON serving +# unsloth/Qwen3.8-27B-UD-Q8_K_XL (target) with a z-lab DFlash2 drafter +# (Qwen3.8-27B-DFlash2-Q4_K_M). Target at 31.4 GB, DFlash2 drafter at 2.0 GB. +# Alias frank-default-large, port 52625. # -# This runtime replaced the retired vLLM service that previously occupied -# this slot. See docs/ciru-activefpx-promptforge.md. +# DFlash2 runs on the Vulkan (RADV) backend. Its candidate selector needs +# TOP_K + ARGMAX, which the HIP backend does not implement. This replaced +# the CIRU HIP runtime on 2026-08-21. See docs/dflash2-large-vulkan.md. # -# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm -# HIP runtime reads those bare env vars (see the 2026-08-13 TEMP incident). -# llama.cpp reads --host/--port/--temp as CLI flags, never as env vars. +# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The +# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP +# incident). This deployment uses Vulkan, not HIP, but the rule stays for +# portability. -# CIRU llama-server binary (ROCmFPX v2.3 build, see docs/ciru runtime page). -# Built against the private TheRock ROCm 7.15 SDK kept in the retired -# ling3-ciru venv. ROCm 7.15 is required for the advertised Strix Halo -# performance; the system ROCm 7.2.4 selects slower/fallback kernels. -LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server +# Upstream llama.cpp DFlash2 binary (PR #27342, Vulkan build). +LLAMA_BIN=/home/dank/llama.cpp-dflash2/build-dflash2/bin/llama-server -# Model files. The three-file payload (v2 GGUF + FFN + GDN) enables CIRU -# DualView prompt acceleration; omit the two .pfs files to run the GGUF alone -# and save ~19.7 GiB of memory at the cost of prompt throughput. -MODEL_DIR=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX -MODEL_FILE=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v2.gguf -FFN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs -GDN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs - -# ROCm 7.15 SDK paths (inside the retired ling3-ciru venv). The serve script -# exports ROCM_PATH and LD_LIBRARY_PATH from these so the 7.15 libraries -# precede any system ROCm 7.2.4. -ROCM_DEVEL=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel -ROCM_LIB=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries -ROCM_CORE=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core +# Model files. The target (UD-Q8_K_XL) was already on disk; the DFlash2 +# drafter was added for this deployment. Both live in /home/dank/models. +MODEL_DIR=/home/dank/models +MODEL_FILE=Qwen3.8-27B-UD-Q8_K_XL.gguf +DRAFT_FILE=Qwen3.8-27B-DFlash2-Q4_K_M.gguf ALIAS=frank-default-large BIND_ADDR=0.0.0.0 LISTEN_PORT=52625 -# Serving: full native 262144 context (n_ctx_train), 1 slot (user request -# 2026-08-19: parallel slots degraded performance). -c is the TOTAL -# context; with PARALLEL=1 it equals the per-slot context. Concurrent -# requests queue on the single slot; the server never preempts a running -# generation. f16 KV, context checkpoints, prompt cache, native MTP -# depth-4. The Strix Halo is unified memory; rocm-smi reports the 64 GiB -# budget. KV at 262144 x 64 KiB/cell is ~16 GiB. +# Serving: full native 262144 context, 1 slot. The Vulkan device reports +# 113942 MiB; KV at 262144 x 64 KiB/cell is ~16 GiB. rocm-smi reports the +# 64 GiB budget; trust the runtime for the addressable SVM pool. CTX_SIZE=262144 PARALLEL=1 BATCH_SIZE=4096 @@ -55,20 +40,21 @@ CACHE_TYPE_V=f16 CTX_CHECKPOINTS=16 CACHE_RAM=8192 -# Native MTP draft (depth-4, greedy, ROCm-safe CPU argmax fast path). -SPEC_DRAFT_N_MAX=4 +# DFlash2 speculative decoding. The drafter's trained block size is 8; +# SPEC_DRAFT_N_MAX clamps to it. TOP_K selector (top 16 candidates). +SPEC_TYPE=draft-dflash +SPEC_DRAFT_N_MAX=8 SPEC_DRAFT_N_MIN=0 SPEC_DRAFT_P_MIN=0.0 SPEC_DRAFT_P_SPLIT=0.10 -# Sampling (model card defaults). +# Sampling (model card defaults, thinking-general). TEMPERATURE=0.7 TOP_P=0.95 -# Thinking enabled (user request, 2026-08-18). deepseek format moves the -# reasoning into message.reasoning_content (OpenAI-compatible); -# none would leave the thought tags unparsed in content. +# Thinking enabled. deepseek format moves the thinking reasoning into +# message.reasoning_content (OpenAI-compatible). REASONING=on REASONING_FORMAT=deepseek -VERBOSITY=4 +VERBOSITY=4 \ No newline at end of file diff --git a/scripts/frank-default-large.service b/scripts/frank-default-large.service index 70cdd45..0cd5561 100644 --- a/scripts/frank-default-large.service +++ b/scripts/frank-default-large.service @@ -1,5 +1,5 @@ [Unit] -Description=CIRU llama.cpp Qwen3.8-27B ActiveFPX PromptForge MTP (dankclaw, frank-default-large) +Description=llama.cpp DFlash2 Qwen3.8-27B UD-Q8_K_XL + Q4_K_M drafter (dankclaw, frank-default-large) After=network-online.target Wants=network-online.target diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 540d449..bb1217b 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -1,16 +1,15 @@ #!/usr/bin/env bash -# Serve jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge as frank-default-large -# via the CIRU ROCmFPX llama.cpp runtime on Dankclaw (Strix Halo gfx1151). +# Serve unsloth/Qwen3.8-27B-UD-Q8_K_XL as frank-default-large with a +# z-lab DFlash2 drafter (Qwen3.8-27B-DFlash2-Q4_K_M) on the Vulkan backend. # -# Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag -# qwen3.8-activefpx-promptforge-v2.3 (commit 103af8b41) against the private -# TheRock ROCm 7.15 SDK (kept in the retired ling3-ciru venv). The model is -# a GGUF with optional PromptForge companion views (FFN/GDN .pfs) that enable -# CIRU DualView prompt acceleration. Native depth-4 MTP speculative decoding. +# Runtime: upstream llama.cpp PR #27342 (DFlash2 support) built with +# GGML_VULKAN=ON for the Strix Halo gfx1151 (RADV). DFlash2 selects its +# candidates through TOP_K + ARGMAX ops that the HIP backend does not +# implement, so the whole stack runs on Vulkan0. # -# The model card requires ROCm 7.15 (not the system ROCm 7.2.4) for the -# advertised Strix Halo performance. LD_LIBRARY_PATH therefore points at the -# 7.15 SDK libraries first. An older system ROCm must not precede them. +# The target and the drafter share the Qwen3.8 arch. DFlash is lossless +# under strict rejection sampling. --spec-draft-n-max clamps to the drafter's +# trained block size (8). # # Tunables come from frank-default-large.env in this directory unless # FRANK_LARGE_ENV_FILE overrides that path. @@ -26,26 +25,23 @@ if [[ -f "$ENV_FILE" ]]; then fi # --- defaults (used when the env file does not set them) ------------------- -: "${LLAMA_BIN:=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server}" -: "${MODEL_DIR:=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX}" -: "${MODEL_FILE:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v2.gguf}" -: "${FFN_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs}" -: "${GDN_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs}" -: "${ROCM_DEVEL:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel}" -: "${ROCM_LIB:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries}" -: "${ROCM_CORE:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core}" +: "${LLAMA_BIN:=/home/dank/llama.cpp-dflash2/build-dflash2/bin/llama-server}" +: "${MODEL_DIR:=/home/dank/models}" +: "${MODEL_FILE:=Qwen3.8-27B-UD-Q8_K_XL.gguf}" +: "${DRAFT_FILE:=Qwen3.8-27B-DFlash2-Q4_K_M.gguf}" : "${ALIAS:=frank-default-large}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=52625}" -: "${CTX_SIZE:=524288}" -: "${PARALLEL:=2}" +: "${CTX_SIZE:=262144}" +: "${PARALLEL:=1}" : "${BATCH_SIZE:=4096}" : "${UBATCH_SIZE:=4096}" : "${CACHE_TYPE_K:=f16}" : "${CACHE_TYPE_V:=f16}" : "${CTX_CHECKPOINTS:=16}" : "${CACHE_RAM:=8192}" -: "${SPEC_DRAFT_N_MAX:=4}" +: "${SPEC_TYPE:=draft-dflash}" +: "${SPEC_DRAFT_N_MAX:=8}" : "${SPEC_DRAFT_N_MIN:=0}" : "${SPEC_DRAFT_P_MIN:=0.0}" : "${SPEC_DRAFT_P_SPLIT:=0.10}" @@ -56,22 +52,14 @@ fi : "${VERBOSITY:=4}" MODEL="$MODEL_DIR/$MODEL_FILE" -export PROMPTFORGE_SIDECAR="$MODEL_DIR/$FFN_PFS" -export PROMPTFORGE_GDN_SIDECAR="$MODEL_DIR/$GDN_PFS" -export PROMPTFORGE_MODE="m2048_fused_tail1476" -export GGML_CUDA_GRAPH_OPT=0 -export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 +DRAFT="$MODEL_DIR/$DRAFT_FILE" if [[ ! -f "$MODEL" ]]; then echo "missing required file: $MODEL" >&2 exit 1 fi -if [[ ! -f "$PROMPTFORGE_SIDECAR" ]]; then - echo "missing PromptForge FFN companion: $PROMPTFORGE_SIDECAR" >&2 - exit 1 -fi -if [[ ! -f "$PROMPTFORGE_GDN_SIDECAR" ]]; then - echo "missing PromptForge GDN companion: $PROMPTFORGE_GDN_SIDECAR" >&2 +if [[ ! -f "$DRAFT" ]]; then + echo "missing DFlash2 drafter: $DRAFT" >&2 exit 1 fi if [[ ! -x "$LLAMA_BIN" ]]; then @@ -79,16 +67,12 @@ if [[ ! -x "$LLAMA_BIN" ]]; then exit 1 fi -# ROCm 7.15 SDK libraries must precede any system ROCm in LD_LIBRARY_PATH. -export ROCM_PATH="$ROCM_DEVEL" -export LD_LIBRARY_PATH="${ROCM_LIB}/lib:${ROCM_CORE}/lib:${ROCM_DEVEL}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - exec "$LLAMA_BIN" \ --model "$MODEL" \ --alias "$ALIAS" \ --host "$BIND_ADDR" \ --port "$LISTEN_PORT" \ - -dev ROCm0 \ + -dev Vulkan0 \ -ngl 999 \ -c "$CTX_SIZE" \ -b "$BATCH_SIZE" \ @@ -102,8 +86,9 @@ exec "$LLAMA_BIN" \ --cache-prompt \ --metrics \ --no-mmap \ - --spec-type draft-mtp \ - --spec-draft-device ROCm0 \ + --spec-type "$SPEC_TYPE" \ + --spec-draft-model "$DRAFT" \ + --spec-draft-device Vulkan0 \ --spec-draft-ngl 999 \ --spec-draft-type-k f16 \ --spec-draft-type-v f16 \ @@ -111,10 +96,9 @@ exec "$LLAMA_BIN" \ --spec-draft-n-min "$SPEC_DRAFT_N_MIN" \ --spec-draft-p-min "$SPEC_DRAFT_P_MIN" \ --spec-draft-p-split "$SPEC_DRAFT_P_SPLIT" \ - --spec-draft-backend-sampling \ --temp "$TEMPERATURE" \ --top-p "$TOP_P" \ --reasoning "$REASONING" \ --reasoning-format "$REASONING_FORMAT" \ --reasoning-budget -1 \ - -lv "$VERBOSITY" + -lv "$VERBOSITY" \ No newline at end of file