dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: disable --fit in frank-default-large serve (full offload fits; fit aborted on explicit n-gpu-layers)

Browse Source
This commit is contained in:
Frank committed 2026-08-11 16:28:03 -07:00
1 parent 25e90bbbf7
commit f5a959bfa2
1 file changed
+3 -2
Download Patch File Download Diff File Expand all files Collapse all files
scripts/serve-llama-frank-large.sh
+3 -2
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Serve HauhauCS Qwen3.6-27B-Uncensored-HauhauCS-Aggressive:Q8_K_P # Serve HauhauCS Qwen3.6-27B-Uncensored-HauhauCS-Aggressive:Q8_K_P
# with DFlash draft + ngram-mod speculative decoding on dankclaw # with DFlash draft speculative decoding (draft-dflash) on dankclaw
# (AMD Strix Halo gfx1151, ROCm or Vulkan backend). # (AMD Strix Halo gfx1151, ROCm backend).
# Runtime: llama.cpp llama-server. # Runtime: llama.cpp llama-server.
# #
# Tunables come from frank-default-large.env in this directory unless the # Tunables come from frank-default-large.env in this directory unless the
@@ -61,6 +61,7 @@ exec "$LLAMA_SERVER" \
--spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH" \ --spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH" \
--device "$DEVICE" \ --device "$DEVICE" \
--gpu-layers "$N_GPU_LAYERS" \ --gpu-layers "$N_GPU_LAYERS" \
--fit off \
--cache-type-k "$CACHE_TYPE_K" \ --cache-type-k "$CACHE_TYPE_K" \
--cache-type-v "$CACHE_TYPE_V" \ --cache-type-v "$CACHE_TYPE_V" \
--ctx-size "$CTX_SIZE" \ --ctx-size "$CTX_SIZE" \