feat: frank-default-large → llama.cpp Qwen3.6-27B Q8_K_P + DFlash draft + ngram-mod (bench pending)

2026-08-11 13:47:37 -07:00
parent a74c1c5bcb
commit aa2f4bfe39
5 changed files with 268 additions and 4 deletions
@@ -0,0 +1,31 @@
+# Tunables for frank-default-large (HauhauCS Qwen3.6-27B Q8_K_P, DFlash+ngram-mod, dankclaw)
+# Runtime: llama.cpp llama-server; AMD Strix Halo (gfx1151) 96GB UMA; Ryzen AI MAX+ 395.
+MODEL_DIR=/home/dank/models
+ALIAS=frank-default-large
+HOST=0.0.0.0
+PORT=7770
+# 6 parallel slots x 65536 ctx each = 393216 total context pool.
+CTX_SIZE=393216
+PARALLEL=6
+# Spec: DFlash draft model layered with ngram-mod. --spec-type accepts a
+# comma-separated list; llama.cpp chains the implementations (verified in
+# common/speculative.cpp). --spec-draft-n-max must stay below the DFlash
+# block size (BF16 sidecar = 8 token block), so 8 is the ceiling.
+SPEC_TYPE=draft-dflash,ngram-mod
+SPEC_DRAFT_N_MAX=8
+SPEC_NGRAM_MOD_N_MIN=48
+SPEC_NGRAM_MOD_N_MAX=64
+SPEC_NGRAM_MOD_N_MATCH=24
+# Backend device for offload, from --list-devices. Pick the faster of:
+#   ROCm0    (HIP / ROCm backend)
+#   Vulkan0  (RADV Vulkan backend)
+DEVICE=ROCm0
+# Dense model (Qwen3.6-27B, not MoE): no MoE CPU offload. Full offload.
+N_GPU_LAYERS=all
+# KV on GPU, quantized (K q8_0, V f16); mirrors the small service profile.
+CACHE_TYPE_K=q8_0
+CACHE_TYPE_V=f16
+BATCH_SIZE=2048
+UBATCH_SIZE=512
+THREADS=8
+THREADS_HTTP=8