dank/serverAppConfigs
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

Restore historical Q6 large model runtime

Browse Source
This commit is contained in:
Frank
2026-08-25 14:23:47 -07:00
parent ddfc64013f
commit 6ee195dd8f
3 changed files with 24 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/frank-default-large.env
+8 -5
View File
@@ -9,8 +9,11 @@
# HIP runtime reads those bare env vars (see the 2026-08-13 TEMP incident).
# llama.cpp reads --host/--port/--temp as CLI flags, never as env vars.
# ROCmFPX build with TurboQuant KV-cache support.
# PromptForge ROCmFPX build and its historical ROCm 7.15 SDK environment.
LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server
ROCM_DEVEL=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel
ROCM_LIB=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries
ROCM_CORE=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core
# Model files from https://huggingface.co/kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF.
MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-27B-STRIX-MTP
@@ -37,13 +40,13 @@ LISTEN_PORT=7770
CTX_SIZE=262144
PARALLEL=1
# Batch/ubatch follow the existing qualified Dankclaw profile.
BATCH_SIZE=512
UBATCH_SIZE=512
BATCH_SIZE=4096
UBATCH_SIZE=4096
NUM_THREADS=16
THREAD_BATCH=32
CACHE_TYPE_K=f16
CACHE_TYPE_V=turbo4
CTX_CHECKPOINTS=32
CACHE_TYPE_V=f16
CTX_CHECKPOINTS=16
CACHE_RAM=8192
# Standalone MTP draft head, loaded from MTP_DRAFT_FILE by the serve script.
scripts/serve-frank-large.sh
+12 -8
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
# Serve the kingjones Qwen3.8 Uncensored Q6 AGENT model as frank-default-large.
# The deployment uses a standalone MTP draft head and Turbo4 V-side KV cache.
# The deployment uses a standalone MTP draft head and f16 KV cache.
set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
@@ -22,13 +22,13 @@ fi
: "${LISTEN_PORT:=7770}"
: "${CTX_SIZE:=262144}"
: "${PARALLEL:=1}"
: "${BATCH_SIZE:=2048}"
: "${UBATCH_SIZE:=512}"
: "${BATCH_SIZE:=4096}"
: "${UBATCH_SIZE:=4096}"
: "${NUM_THREADS:=16}"
: "${THREAD_BATCH:=32}"
: "${CACHE_TYPE_K:=f16}"
: "${CACHE_TYPE_V:=turbo4}"
: "${CTX_CHECKPOINTS:=32}"
: "${CACHE_TYPE_V:=f16}"
: "${CTX_CHECKPOINTS:=16}"
: "${CACHE_RAM:=8192}"
: "${MTP_N_MAX:=4}"
: "${MTP_N_MIN:=0}"
@@ -55,14 +55,18 @@ if [[ ! -x "$LLAMA_BIN" ]]; then
exit 1
fi
SERVER_DIR="$(cd -- "$(dirname -- "$LLAMA_BIN")" && pwd)"
ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib:$ROCM_PATH/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
ROCM_PATH="${ROCM_DEVEL:-${ROCM_PATH:-/opt/rocm}}"
if [[ -n "${ROCM_LIB:-}" && -n "${ROCM_CORE:-}" ]]; then
LIBRARY_PATH="$ROCM_LIB/lib:$ROCM_CORE/lib:$ROCM_DEVEL/lib/llvm/lib:$SERVER_DIR:${LD_LIBRARY_PATH:-}"
else
LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib:$ROCM_PATH/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
fi
exec /usr/bin/env \
ROCM_PATH="$ROCM_PATH" \
HIP_VISIBLE_DEVICES=0 \
HSA_OVERRIDE_GFX_VERSION="$HSA_OVERRIDE_GFX_VERSION" \
LD_LIBRARY_PATH="$LIBRARY_PATH" \
HSA_OVERRIDE_GFX_VERSION="$HSA_OVERRIDE_GFX_VERSION" \
GGML_CUDA_GRAPH_OPT=0 \
LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0 \
LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 \