From 132e1478cf2017dd7c8260cdef3c5e559b161318 Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 21 Aug 2026 20:00:31 -0700 Subject: [PATCH] frank-default-large: upgrade to CIRU ActiveFPX PromptForge v3 (Output-K8 runtime, full-W8 views, BF16 vision) --- scripts/frank-default-large.env | 34 ++++++++++++--------- scripts/serve-frank-large.sh | 52 ++++++++++++++++++++------------- 2 files changed, 51 insertions(+), 35 deletions(-) diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 074e0e4..ded4d07 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -2,10 +2,11 @@ # gfx1151). # # Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag -# qwen3.8-activefpx-promptforge-v2.3 serving -# jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge (v2). ActiveFPX compact -# GGUF with PromptForge companions for CIRU DualView prompt acceleration and -# native depth-4 MTP. Alias frank-default-large, port 52625. +# qwen3.8-activefpx-promptforge-v2.3 plus the v3 output-K8 runtime patch +# serving jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge (v3). v3 tensor +# allocation (FFN-equalized, late-Q6 writers, full-Q8 output) with the +# full-W8 PromptForge companions, the Output-K8 draft proxy, and the BF16 +# vision projector. Alias frank-default-large, port 52625. # # This runtime replaced the retired vLLM service that previously occupied # this slot. See docs/ciru-activefpx-promptforge.md. @@ -14,19 +15,22 @@ # HIP runtime reads those bare env vars (see the 2026-08-13 TEMP incident). # llama.cpp reads --host/--port/--temp as CLI flags, never as env vars. -# CIRU llama-server binary (ROCmFPX v2.3 build, see docs/ciru runtime page). -# Built against the private TheRock ROCm 7.15 SDK kept in the retired -# ling3-ciru venv. ROCm 7.15 is required for the advertised Strix Halo -# performance; the system ROCm 7.2.4 selects slower/fallback kernels. +# CIRU llama-server binary (ROCmFPX v2.3 + v3 output-K8 patch build, see +# docs/ciru runtime page). Built against the private TheRock ROCm 7.15 SDK +# kept in the retired ling3-ciru venv. ROCm 7.15 is required for the +# advertised Strix Halo performance; the system ROCm 7.2.4 selects +# slower/fallback kernels. LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server -# Model files. The three-file payload (v2 GGUF + FFN + GDN) enables CIRU -# DualView prompt acceleration; omit the two .pfs files to run the GGUF alone -# and save ~19.7 GiB of memory at the cost of prompt throughput. -MODEL_DIR=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX -MODEL_FILE=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v2.gguf +# Model files (v3). The five-file payload (GGUF + FFN/GDN .pfs + Output-K8 +# proxy + BF16 vision projector) is the full v3 configuration. The v3 files +# live in a separate directory so the v2 artifacts stay for rollback. +MODEL_DIR=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX-v3 +MODEL_FILE=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v3-Q8-FFNEQ-LateQ6.gguf FFN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs GDN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs +OUTPUT_K8_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-Output-K8.pfs +MMPROJ_FILE=Qwen3.8-27B-mmproj-BF16.mmproj # ROCm 7.15 SDK paths (inside the retired ling3-ciru venv). The serve script # exports ROCM_PATH and LD_LIBRARY_PATH from these so the 7.15 libraries @@ -55,7 +59,9 @@ CACHE_TYPE_V=f16 CTX_CHECKPOINTS=16 CACHE_RAM=8192 -# Native MTP draft (depth-4, greedy, ROCm-safe CPU argmax fast path). +# Native MTP draft (depth-4, p_min 0, backend sampling). The qualified v3 +# profile drives the draft via the Output-K8 proxy; the CPU argmax fast +# path and IU4/ngram lanes are disabled in the serve script. SPEC_DRAFT_N_MAX=4 SPEC_DRAFT_N_MIN=0 SPEC_DRAFT_P_MIN=0.0 diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 540d449..ba09b77 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -1,12 +1,15 @@ #!/usr/bin/env bash -# Serve jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge as frank-default-large -# via the CIRU ROCmFPX llama.cpp runtime on Dankclaw (Strix Halo gfx1151). +# Serve jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge (v3) as +# frank-default-large via the CIRU ROCmFPX llama.cpp runtime on Dankclaw +# (Strix Halo gfx1151). # # Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag -# qwen3.8-activefpx-promptforge-v2.3 (commit 103af8b41) against the private -# TheRock ROCm 7.15 SDK (kept in the retired ling3-ciru venv). The model is -# a GGUF with optional PromptForge companion views (FFN/GDN .pfs) that enable -# CIRU DualView prompt acceleration. Native depth-4 MTP speculative decoding. +# qwen3.8-activefpx-promptforge-v2.3 (commit 103af8b41) plus the v3 output-K8 +# runtime patch (runtime/qwen38-v3-output-k8-runtime.patch, commit af0eac6 on +# Dankclaw) against the private TheRock ROCm 7.15 SDK (kept in the retired +# ling3-ciru venv). The model is the v3 GGUF with the full-W8 PromptForge +# companion views (FFN/GDN .pfs), the Output-K8 draft proxy, and the BF16 +# vision projector. Native depth-4 MTP speculative decoding. # # The model card requires ROCm 7.15 (not the system ROCm 7.2.4) for the # advertised Strix Halo performance. LD_LIBRARY_PATH therefore points at the @@ -27,10 +30,12 @@ fi # --- defaults (used when the env file does not set them) ------------------- : "${LLAMA_BIN:=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server}" -: "${MODEL_DIR:=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX}" -: "${MODEL_FILE:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v2.gguf}" +: "${MODEL_DIR:=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX-v3}" +: "${MODEL_FILE:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v3-Q8-FFNEQ-LateQ6.gguf}" : "${FFN_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs}" : "${GDN_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs}" +: "${OUTPUT_K8_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-Output-K8.pfs}" +: "${MMPROJ_FILE:=Qwen3.8-27B-mmproj-BF16.mmproj}" : "${ROCM_DEVEL:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel}" : "${ROCM_LIB:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries}" : "${ROCM_CORE:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core}" @@ -58,22 +63,26 @@ fi MODEL="$MODEL_DIR/$MODEL_FILE" export PROMPTFORGE_SIDECAR="$MODEL_DIR/$FFN_PFS" export PROMPTFORGE_GDN_SIDECAR="$MODEL_DIR/$GDN_PFS" +# v3 qualified runtime: Output-K8 draft proxy + small-row W8 prompt route. export PROMPTFORGE_MODE="m2048_fused_tail1476" export GGML_CUDA_GRAPH_OPT=0 -export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 +export PROMPTFORGE_ENABLE_SMALLM_W8=1 +export PROMPTFORGE_MTP_OUTPUT_K8=1 +export PROMPTFORGE_MTP_OUTPUT_K8_VALIDATE=0 +export PROMPTFORGE_MTP_OUTPUT_K8_PROXY="$MODEL_DIR/$OUTPUT_K8_PFS" +# The qualified v3 profile disables the CPU argmax draft fast path; the +# Output-K8 proxy + backend sampling replace it. Also drop any IU4/ngram +# companions from the non-IU4 development lane. +unset LLAMA_MTP_CPU_ARGMAX_FASTPATH LLAMA_MTP_BACKEND_GREEDY || true +unset PROMPTFORGE_IU4_SIDECAR PROMPTFORGE_GDN_IU4_SIDECAR || true +unset PROMPTFORGE_ENABLE_NGRAM_MOD PROMPTFORGE_ENABLE_NGRAM_M65_IU4 || true -if [[ ! -f "$MODEL" ]]; then - echo "missing required file: $MODEL" >&2 - exit 1 -fi -if [[ ! -f "$PROMPTFORGE_SIDECAR" ]]; then - echo "missing PromptForge FFN companion: $PROMPTFORGE_SIDECAR" >&2 - exit 1 -fi -if [[ ! -f "$PROMPTFORGE_GDN_SIDECAR" ]]; then - echo "missing PromptForge GDN companion: $PROMPTFORGE_GDN_SIDECAR" >&2 - exit 1 -fi +for f in "$MODEL" "$PROMPTFORGE_SIDECAR" "$PROMPTFORGE_GDN_SIDECAR" "$PROMPTFORGE_MTP_OUTPUT_K8_PROXY" "$MODEL_DIR/$MMPROJ_FILE"; do + if [[ ! -f "$f" ]]; then + echo "missing required file: $f" >&2 + exit 1 + fi +done if [[ ! -x "$LLAMA_BIN" ]]; then echo "missing llama-server binary: $LLAMA_BIN" >&2 exit 1 @@ -85,6 +94,7 @@ export LD_LIBRARY_PATH="${ROCM_LIB}/lib:${ROCM_CORE}/lib:${ROCM_DEVEL}/lib${LD_L exec "$LLAMA_BIN" \ --model "$MODEL" \ + --mmproj "$MODEL_DIR/$MMPROJ_FILE" \ --alias "$ALIAS" \ --host "$BIND_ADDR" \ --port "$LISTEN_PORT" \