diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 7980a17..9feaac4 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -9,8 +9,11 @@ # HIP runtime reads those bare env vars (see the 2026-08-13 TEMP incident). # llama.cpp reads --host/--port/--temp as CLI flags, never as env vars. -# ROCmFPX build with TurboQuant KV-cache support. +# PromptForge ROCmFPX build and its historical ROCm 7.15 SDK environment. LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server +ROCM_DEVEL=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel +ROCM_LIB=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries +ROCM_CORE=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core # Model files from https://huggingface.co/kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF. MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-27B-STRIX-MTP @@ -37,13 +40,13 @@ LISTEN_PORT=7770 CTX_SIZE=262144 PARALLEL=1 # Batch/ubatch follow the existing qualified Dankclaw profile. -BATCH_SIZE=512 -UBATCH_SIZE=512 +BATCH_SIZE=4096 +UBATCH_SIZE=4096 NUM_THREADS=16 THREAD_BATCH=32 CACHE_TYPE_K=f16 -CACHE_TYPE_V=turbo4 -CTX_CHECKPOINTS=32 +CACHE_TYPE_V=f16 +CTX_CHECKPOINTS=16 CACHE_RAM=8192 # Standalone MTP draft head, loaded from MTP_DRAFT_FILE by the serve script. diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 2eeb4b5..28fa305 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Serve the kingjones Qwen3.8 Uncensored Q6 AGENT model as frank-default-large. -# The deployment uses a standalone MTP draft head and Turbo4 V-side KV cache. +# The deployment uses a standalone MTP draft head and f16 KV cache. set -euo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" @@ -22,13 +22,13 @@ fi : "${LISTEN_PORT:=7770}" : "${CTX_SIZE:=262144}" : "${PARALLEL:=1}" -: "${BATCH_SIZE:=2048}" -: "${UBATCH_SIZE:=512}" +: "${BATCH_SIZE:=4096}" +: "${UBATCH_SIZE:=4096}" : "${NUM_THREADS:=16}" : "${THREAD_BATCH:=32}" : "${CACHE_TYPE_K:=f16}" -: "${CACHE_TYPE_V:=turbo4}" -: "${CTX_CHECKPOINTS:=32}" +: "${CACHE_TYPE_V:=f16}" +: "${CTX_CHECKPOINTS:=16}" : "${CACHE_RAM:=8192}" : "${MTP_N_MAX:=4}" : "${MTP_N_MIN:=0}" @@ -55,14 +55,18 @@ if [[ ! -x "$LLAMA_BIN" ]]; then exit 1 fi SERVER_DIR="$(cd -- "$(dirname -- "$LLAMA_BIN")" && pwd)" -ROCM_PATH="${ROCM_PATH:-/opt/rocm}" -LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib:$ROCM_PATH/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +ROCM_PATH="${ROCM_DEVEL:-${ROCM_PATH:-/opt/rocm}}" +if [[ -n "${ROCM_LIB:-}" && -n "${ROCM_CORE:-}" ]]; then + LIBRARY_PATH="$ROCM_LIB/lib:$ROCM_CORE/lib:$ROCM_DEVEL/lib/llvm/lib:$SERVER_DIR:${LD_LIBRARY_PATH:-}" +else + LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib:$ROCM_PATH/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +fi exec /usr/bin/env \ ROCM_PATH="$ROCM_PATH" \ HIP_VISIBLE_DEVICES=0 \ - HSA_OVERRIDE_GFX_VERSION="$HSA_OVERRIDE_GFX_VERSION" \ LD_LIBRARY_PATH="$LIBRARY_PATH" \ + HSA_OVERRIDE_GFX_VERSION="$HSA_OVERRIDE_GFX_VERSION" \ GGML_CUDA_GRAPH_OPT=0 \ LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0 \ LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 \ diff --git a/tests/frank_large_config.bats b/tests/frank_large_config.bats index fe6a3ba..fbd1c7d 100644 --- a/tests/frank_large_config.bats +++ b/tests/frank_large_config.bats @@ -46,15 +46,15 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" grep -q 'preserve_reasoning":true' "$ENV_FILE" } -@test "frank-default-large uses Turbo4 only for the V cache" { +@test "frank-default-large uses f16 for both KV cache sides" { grep -q '^CACHE_TYPE_K=f16$' "$ENV_FILE" - grep -q '^CACHE_TYPE_V=turbo4$' "$ENV_FILE" + grep -q '^CACHE_TYPE_V=f16$' "$ENV_FILE" grep -qF -- '-ctk "$CACHE_TYPE_K" -ctv "$CACHE_TYPE_V"' "$SERVE_FILE" } @test "frank-default-large exposes a swappable ubatch" { - grep -q '^UBATCH_SIZE=512$' "$ENV_FILE" - grep -q ': "${UBATCH_SIZE:=512}"' "$SERVE_FILE" + grep -q '^UBATCH_SIZE=4096$' "$ENV_FILE" + grep -q ': "${UBATCH_SIZE:=4096}"' "$SERVE_FILE" grep -q -- '-ub "$UBATCH_SIZE"' "$SERVE_FILE" }