From d6ea71b9a4477725a409957d365fc638fcd1d20d Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 25 Aug 2026 16:52:26 -0700 Subject: [PATCH] Configure DeepSeek reasoning and large-model ubatch sweep --- docs/LLM/deployment.md | 2 +- docs/LLM/index.md | 1 + docs/LLM/ubatch-benchmark-large.md | 59 ++++++++++++++++++++++++++++++ docs/index.md | 1 + scripts/bench-ubatch-rocmi4.sh | 30 ++++++++------- scripts/frank-default-large.env | 11 +++--- tests/frank_large_config.bats | 15 ++++++++ 7 files changed, 99 insertions(+), 20 deletions(-) create mode 100644 docs/LLM/ubatch-benchmark-large.md diff --git a/docs/LLM/deployment.md b/docs/LLM/deployment.md index 28eecc6..64f618e 100644 --- a/docs/LLM/deployment.md +++ b/docs/LLM/deployment.md @@ -138,7 +138,7 @@ from `peculiar-ragdoll/Qwen-Sharp-Chat-Templates`. See [[dealign-ornith-crack]]. ### Dankclaw — frank-default-large The active service on Dankclaw uses the ROCmFPX llama.cpp runtime to serve -[kingjones777/Qwen3.8-Uncensored-ROCmFP4-STRIX-MTP-GGUF](https://huggingface.co/kingjones777/Qwen3.8-Uncensored-ROCmFP4-STRIX-MTP-GGUF) +[kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF](https://huggingface.co/kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF) under the alias `frank-default-large` on port 7770. The main model is `Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf`; the standalone MTP draft is `mtp-Qwen3.8-27B-Q4_0.gguf`, and the Q8_0 vision projector is loaded alongside diff --git a/docs/LLM/index.md b/docs/LLM/index.md index 697e3d2..47a9e7e 100644 --- a/docs/LLM/index.md +++ b/docs/LLM/index.md @@ -50,6 +50,7 @@ MTP draft + Turbo4 V cache, 262K ctx / 1 slot] - [[frank-redo-2026-08-17]] — the 2026-08-17 topology change - [kingjones Qwen3.8 Uncensored ROCmFP4 STRIX MTP](https://huggingface.co/kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF) — active frank-default-large model on Dankclaw; Q6 AGENT main model with standalone MTP - [[kingjones-strix-mtp]] — the active Q6 AGENT runbook: serving profile, standalone MTP, reasoning + preserve_thinking, 114 GiB TTM/GTT +- [[ubatch-benchmark-large]] — active-model ubatch sweep plan and results - [[kingjones-q8agent-rocmfpx]] — the retired Q8_0-AGENT investigation and `set_rows` failure - [[eemin-rocmi4-mtp]] — the former caf-eemin Qwen3.8-27B embedded-MTP model on Dankclaw (retired) diff --git a/docs/LLM/ubatch-benchmark-large.md b/docs/LLM/ubatch-benchmark-large.md new file mode 100644 index 0000000..b8ad1e3 --- /dev/null +++ b/docs/LLM/ubatch-benchmark-large.md @@ -0,0 +1,59 @@ +--- +type: benchmark plan +title: frank-default-large ubatch sweep +description: Standardized ubatch sweep for the active kingjones Qwen3.8 ROCmFPX deployment. +tags: + - llama.cpp + - rocm + - benchmark + - ubatch +timestamp: 2026-09-01 +--- + +# frank-default-large ubatch sweep + +This benchmark sweeps the active `frank-default-large` model on Dankclaw using +scratch ports and an 8K-context workload. The production service is stopped +before the sweep and restarted by the benchmark's exit handler. + +## Sweep + +The default points are `256, 512, 768, 1024, 1280, 1536, 2048, 3072, 4096, +5120, 6144`. This implements the requested 256-step range through 1536 and +1024-step range from 2048 through 6144. Each point runs three 256-token decode +requests by default. + +Run on Dankclaw from the repository checkout: + +```bash +STOPSVC=1 ./scripts/bench-ubatch-rocmi4.sh +``` + +The script uses the active Q6 AGENT model, standalone MTP draft, tracked Sharp +template, and `--reasoning-format deepseek`. It reports server timing lines and +per-run generation throughput. Run only when the production service can be +interrupted; the GPU is shared with live workloads. + +## Results + +Record results below after the sweep, including failures or OOM points. + +| ubatch | prefill | generation | notes | +| ---: | ---: | ---: | --- | +| 256 | — | — | pending | +| 512 | — | — | pending | +| 768 | — | — | pending | +| 1024 | — | — | pending | +| 1280 | — | — | pending | +| 1536 | — | — | pending | +| 2048 | — | — | pending | +| 3072 | — | — | pending | +| 4096 | — | — | pending | +| 5120 | — | — | pending | +| 6144 | — | — | pending | + +## See also + +- [LLM fleet index](index.md) +- [Deployment](deployment.md) +- [Active kingjones runbook](kingjones-strix-mtp.md) diff --git a/docs/index.md b/docs/index.md index 7a96627..8da7500 100644 --- a/docs/index.md +++ b/docs/index.md @@ -116,6 +116,7 @@ The LLM inference fleet on the two hosts. Full index: ### Benchmarks and Sizing - [[frank-default-large-benchmark]] — backend/spec benchmark behind the old head +- [[ubatch-benchmark-large]] — active-model ubatch sweep plan and results - [[frank-default-small-benchmark]] — the small-model benchmark suite - [[Running Qwen3.6 35B A3B FP8 on Dankserver]] — Qwen3.6-35B-A3B hardware notes - [[Running Qwen3.6 27B FP8 on Dankclaw]] — Qwen3.6-27B sizing on the iGPU diff --git a/scripts/bench-ubatch-rocmi4.sh b/scripts/bench-ubatch-rocmi4.sh index b2a8866..c97cf9e 100755 --- a/scripts/bench-ubatch-rocmi4.sh +++ b/scripts/bench-ubatch-rocmi4.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # bench-ubatch-rocmi4.sh — sweep ubatch size at 8k context on the -# frank-default-large ROCmI4 embedded-MTP stack (dankclaw, gfx1151). +# frank-default-large ROCmFPX standalone-MTP stack (dankclaw, gfx1151). # # Boots one llama-server instance per ubatch on a scratch port with a fixed # -b and -c 8192, runs a standardized ~75%-of-ctx prose workload, and reads @@ -26,21 +26,23 @@ if [[ -f "$ENV_FILE" ]]; then fi # --- sweep parameters ------------------------------------------------------- -: "${UBATCH_MIN:=512}" -: "${UBATCH_MAX:=4096}" -: "${UBATCH_STEP:=512}" +: "${UBATCH_MIN:=256}" +: "${UBATCH_MAX:=6144}" +: "${UBATCH_STEP:=256}" +: "${UBATCH_VALUES:=256 512 768 1024 1280 1536 2048 3072 4096 5120 6144}" : "${CTX:=8192}" # 8k contexts : "${BENCH_BATCH:=$CTX}" # fixed batch = ctx, so the full 8k prompt fits : "${OUT_TOKENS:=256}" : "${RUNS:=3}" # decode runs per ubatch : "${PORT_BASE:=7894}" # scratch ports 7894.. -: "${STOPSVC:=0}" # stop/restart the live unit for a clean GPU +: "${STOPSVC:=1}" # stop/restart the live unit for a clean GPU # --- resolved paths (fall back to serve-script defaults) -------------------- -: "${LLAMA_BIN:=/home/dank/rocmi4-qwen38/ROCmFPX/build-rocmi4-w4a4/bin/llama-server}" -: "${MODEL_DIR:=/home/dank/models/eemin-Qwen3.8-27B-ROCMI4-MTP}" -: "${MODEL_FILE:=Qwen3.8-27B-Q4_0_ROCMI4.gguf}" -: "${MMPROJ_FILE:=mmproj-BF16.gguf}" +: "${LLAMA_BIN:=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server}" +: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-27B-STRIX-MTP}" +: "${MODEL_FILE:=Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf}" +: "${MTP_DRAFT_FILE:=$MODEL_DIR/mtp-Qwen3.8-27B-Q4_0.gguf}" +: "${MMPROJ_FILE:=mmproj-Qwen3.8-27B-Q8_0.gguf}" : "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}" : "${ALIAS:=frank-default-large}" : "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}" @@ -114,7 +116,7 @@ run_one() { --model "$MODEL" \ "${MMPROJ_ARGS[@]}" \ --chat-template-file "$CHAT_TEMPLATE_FILE" \ - --chat-template-kwargs '{"enable_thinking":true,"preserve_thinking":true}' \ + --chat-template-kwargs '{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}' \ --alias bench-ubatch \ --host 127.0.0.1 --port "$port" \ -dev ROCm0 -ngl 999 -np 1 \ @@ -122,14 +124,14 @@ run_one() { -c "$CTX" -b "$BENCH_BATCH" -ub "$ubatch" \ -fa on -ctk f16 -ctv f16 -ctxcp 16 \ --cache-ram 8192 --cache-prompt --metrics --no-mmap \ - --spec-type draft-mtp --spec-mtp-strict-qwen \ - --spec-draft-device ROCm0 --spec-draft-ngl all \ + --spec-type draft-mtp --model-draft "$MTP_DRAFT_FILE" \ + --spec-draft-device ROCm0 --spec-draft-ngl 999 \ --spec-draft-type-k f16 --spec-draft-type-v f16 \ --spec-draft-n-max "$MTP_N_MAX" --spec-draft-n-min 0 \ --spec-draft-p-min "$MTP_P_MIN" \ --spec-draft-backend-sampling \ --temp 0.7 --top-p 0.95 \ - --reasoning on --reasoning-format deepseek --reasoning-budget -1 \ + --reasoning "$REASONING" --reasoning-format "$REASONING_FORMAT" --reasoning-budget -1 \ -lv 4 >>"$log" 2>&1 & local pid=$! wait_health "$pid" "$port" || { tail -30 "$log"; return 2; } @@ -207,7 +209,7 @@ main() { stop_live_service local idx=0 ubatch port printf "%-10s %-14s %-10s\n" "UBATCH" "PREFILL" "GEN_TPS" - for ubatch in $(seq "$UBATCH_MIN" "$UBATCH_STEP" "$UBATCH_MAX"); do + for ubatch in $UBATCH_VALUES; do port=$((PORT_BASE + idx)) run_one "$ubatch" "$port" "$OUT_TOKENS" "$RUNS" idx=$((idx + 1)) diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 9feaac4..fd1e940 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -22,8 +22,9 @@ MTP_DRAFT_FILE=$MODEL_DIR/mtp-Qwen3.8-27B-Q4_0.gguf MMPROJ_FILE=mmproj-Qwen3.8-27B-Q8_0.gguf # Qwen-Sharp chat template (tracked in the repo, staged on the host). The Sharp -# template drives thinking on by default (medium effort) and produces the -# peg-native + deepseek chat/reasoning shape the Pi agent expects. +# template drives thinking on by default (medium effort); the server emits +# reasoning in DeepSeek format so clients receive reasoning_content separately +# instead of literal tags in content. CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}' @@ -63,10 +64,10 @@ MIN_P=0.0 PRESENCE_PENALTY=1.5 # Reasoning on (Sharp template drives it; preserve_thinking + preserve_reasoning -# keep the reasoning trail in the conversation). reasoning-format none lets the -# template project the reasoning content into reasoning_content. +# keep the reasoning trail in the conversation). DeepSeek format projects the +# reasoning into the API's reasoning_content field instead of content tags. REASONING=on -REASONING_FORMAT=none +REASONING_FORMAT=deepseek # Strix Halo unified memory + native gfx1151 detection for this ROCmFPX build. HSA_OVERRIDE_GFX_VERSION=11.5.1 diff --git a/tests/frank_large_config.bats b/tests/frank_large_config.bats index fbd1c7d..ac4808d 100644 --- a/tests/frank_large_config.bats +++ b/tests/frank_large_config.bats @@ -39,6 +39,11 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" grep -q ': "${CTX_SIZE:=262144}"' "$SERVE_FILE" } +@test "frank-default-large uses DeepSeek reasoning output" { + grep -q '^REASONING_FORMAT=deepseek$' "$ENV_FILE" + grep -q -- '--reasoning-format "$REASONING_FORMAT"' "$SERVE_FILE" +} + @test "frank-default-large has reasoning and preserved reasoning/thinking enabled" { grep -q '^REASONING=on$' "$ENV_FILE" grep -q 'enable_thinking":true' "$ENV_FILE" @@ -58,6 +63,16 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" grep -q -- '-ub "$UBATCH_SIZE"' "$SERVE_FILE" } +@test "large ubatch bench defaults to the active model and requested sweep" { + local bench="$REPO_ROOT/scripts/bench-ubatch-rocmi4.sh" + grep -q ': "${UBATCH_MIN:=256}"' "$bench" + grep -q ': "${UBATCH_MAX:=6144}"' "$bench" + grep -q 'UBATCH_VALUES.*256 512 768 1024 1280 1536 2048 3072 4096 5120 6144' "$bench" + grep -q ': "${STOPSVC:=1}"' "$bench" + grep -q 'MODEL_FILE' "$bench" + grep -q 'REASONING_FORMAT' "$bench" +} + @test "frank-default-large uses the tracked Qwen-Sharp template on :7770" { grep -q '^CHAT_TEMPLATE_FILE=' "$ENV_FILE" grep -q -- '--jinja' "$SERVE_FILE"