Restore historical Q6 large model runtime

This commit is contained in:
Frank committed 2026-08-25 14:23:47 -07:00
1 parent ddfc64013f
commit 6ee195dd8f
3 files changed
+24 -17

No files matched your search

+8 -5
View File
@@ -9,8 +9,11 @@
# HIP runtime reads those bare env vars (see the 2026-08-13 TEMP incident).
# llama.cpp reads --host/--port/--temp as CLI flags, never as env vars.
# ROCmFPX build with TurboQuant KV-cache support.
# PromptForge ROCmFPX build and its historical ROCm 7.15 SDK environment.
LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server
ROCM_DEVEL=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel
ROCM_LIB=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries
ROCM_CORE=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core
# Model files from https://huggingface.co/kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF.
MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-27B-STRIX-MTP
@@ -37,13 +40,13 @@ LISTEN_PORT=7770
CTX_SIZE=262144
PARALLEL=1
# Batch/ubatch follow the existing qualified Dankclaw profile.
BATCH_SIZE=512
UBATCH_SIZE=512
BATCH_SIZE=4096
UBATCH_SIZE=4096
NUM_THREADS=16
THREAD_BATCH=32
CACHE_TYPE_K=f16
CACHE_TYPE_V=turbo4
CTX_CHECKPOINTS=32
CACHE_TYPE_V=f16
CTX_CHECKPOINTS=16
CACHE_RAM=8192
# Standalone MTP draft head, loaded from MTP_DRAFT_FILE by the serve script.