frank-small: Q5_K_M GGUF on llama.cpp, two full native windows
One quant step down from Q6_K on the same CUDA build and the same shape: 2 x 262,144 with K f16 + V q8_0. The 0.86 GiB freed from the weights is headroom, not a new window — the native 262,144 cap is already reached, and f16 V would still cost 3.96 GiB more than the pool can carry.
This commit is contained in:
1 parent
4bb10cd20e
commit
d82c5c03b7
5 files changed
+36
-26
No files matched your search
@@ -1186,7 +1186,7 @@ deploy_llm_large() {
|
||||
#
|
||||
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
|
||||
# not shared: frank-large's engine runs on the gfx1151 iGPU.
|
||||
LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}"
|
||||
LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}"
|
||||
LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
|
||||
deploy_llm_small() {
|
||||
@@ -1202,7 +1202,7 @@ deploy_llm_small() {
|
||||
fi
|
||||
if [[ ! -f "$LLM_SMALL_MODEL_FILE" ]]; then
|
||||
echo "ERROR: payload GGUF absent at $LLM_SMALL_MODEL_FILE" >&2
|
||||
echo "frank-small: fetch bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF Q6_K first;" >&2
|
||||
echo "frank-small: fetch bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF Q5_K_M first;" >&2
|
||||
echo "frank-small: refusing to start a unit that cannot load." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
Reference in new issue
Block a user