Frank
b8de25d70b
Switch frank-default-small on Dankserver from upstream llama.cpp (b10121) to ikawrakow/ik_llama.cpp (b166e269, built with the staged pip cu13 toolkit). The model (Ornith-1.5-35B-A3B CRACK Q6_K) is unchanged and the old binary is kept for rollback. Runtime changes: - KV cache quantized to q8_KV with --k-cache-hadamard/--v-cache-hadamard - --split-mode graph (single-GPU no-op, retained per operator pref) - migrate ngram speculation to canonical --spec-type ngram-mod:n_min=48,n_max=64,ngram_size_n=24 (legacy --spec-ngram-mod-* are rejected in ik main) - N_GPU_LAYERS 999 (ik -ngl rejects the literal "all"); experts stay pinned to CPU via --override-tensor Build: GGML_CUDA_FA_ALL_QUANTS=ON + GGML_IQK_FA_ALL_QUANTS=ON mandatory for the quantized KV cache; recipe + reasoning baked into docs/LLM/ikllama-ornith.md. Tests and both indices updated.
70 lines
2.8 KiB
Bash
70 lines
2.8 KiB
Bash
#!/usr/bin/env bats
|
|
# Seam: frank-default-small model, context, tensor placement, and template.
|
|
# This test covers only the Q6_K deployment contract.
|
|
|
|
load helpers
|
|
|
|
ENV_FILE="$REPO_ROOT/scripts/frank-default-small.env"
|
|
SERVE_FILE="$REPO_ROOT/scripts/serve-frank-small.sh"
|
|
UNIT_FILE="$REPO_ROOT/scripts/frank-default-small.service"
|
|
TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
|
|
|
|
@test "frank-default-small selects the Ornith Q6_K model" {
|
|
grep -q '^MODEL_FILE=Ornith-1\.5-35B-A3B-CRACK-Q6_K\.gguf$' "$ENV_FILE"
|
|
grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q6_K\.gguf' "$SERVE_FILE"
|
|
}
|
|
|
|
@test "frank-default-small uses 163840 context per slot across three slots" {
|
|
grep -q '^CTX_SIZE=491520$' "$ENV_FILE"
|
|
grep -q '163840 ctx each' "$ENV_FILE"
|
|
grep -q '^PARALLEL=3$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "frank-default-small keeps full-attention and final blocks on CUDA" {
|
|
source "$ENV_FILE"
|
|
[[ "$OVERRIDE_EXPERT" == 'blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps' ]]
|
|
[[ "$OVERRIDE_EXPERT" != *'|3|'* ]]
|
|
[[ "$OVERRIDE_EXPERT" != *'|7|'* ]]
|
|
[[ "$OVERRIDE_EXPERT" != *'|11|'* ]]
|
|
[[ "$OVERRIDE_EXPERT" != *'|15|'* ]]
|
|
[[ "$OVERRIDE_EXPERT" != *'|19|'* ]]
|
|
[[ "$OVERRIDE_EXPERT" != *'|23|'* ]]
|
|
[[ "$OVERRIDE_EXPERT" != *'|27|'* ]]
|
|
[[ "$OVERRIDE_EXPERT" != *'|31|'* ]]
|
|
[[ "$OVERRIDE_EXPERT" != *'|35|'* ]]
|
|
[[ "$OVERRIDE_EXPERT" != *'|36|'* ]]
|
|
[[ "$OVERRIDE_EXPERT" != *'|40|'* ]]
|
|
[[ "$OVERRIDE_BUFT" == CPU ]]
|
|
# ik_llama --gpu-layers/-ngl requires an integer ("all" is rejected); 999
|
|
# loads as many layers as possible while OVERRIDE_EXPERT pins exps to CPU.
|
|
[[ "$N_GPU_LAYERS" == 999 ]]
|
|
}
|
|
|
|
@test "frank-default-small uses ngram-mod without MTP" {
|
|
# ik_llama main uses the canonical --spec-type ngram-mod:<k=v> form; the
|
|
# legacy --spec-ngram-mod-n-* flags are rejected.
|
|
grep -q '^SPEC_TYPE=ngram-mod:n_min=48,n_max=64,ngram_size_n=24$' "$ENV_FILE"
|
|
! grep -q '^SPEC_NGRAM_MOD_N_MIN=' "$ENV_FILE"
|
|
! grep -q '^SPEC_NGRAM_MOD_N_MAX=' "$ENV_FILE"
|
|
! grep -q '^SPEC_NGRAM_MOD_N_MATCH=' "$ENV_FILE"
|
|
! grep -q -- 'SPEC_DRAFT_' "$ENV_FILE"
|
|
! grep -q -- '--spec-type.*draft-mtp' "$SERVE_FILE"
|
|
grep -q -- '--spec-type' "$SERVE_FILE"
|
|
! grep -q -- '--spec-ngram-mod-' "$SERVE_FILE"
|
|
}
|
|
|
|
@test "frank-default-small uses ubatch 1024" {
|
|
grep -q '^BATCH_SIZE=4096$' "$ENV_FILE"
|
|
grep -q '^UBATCH_SIZE=1024$' "$ENV_FILE"
|
|
grep -q ': "${UBATCH_SIZE:=1024}"' "$SERVE_FILE"
|
|
}
|
|
|
|
@test "frank-default-small uses the tracked Qwen-Sharp template" {
|
|
grep -q '^CHAT_TEMPLATE_FILE=' "$ENV_FILE"
|
|
grep -q 'peculiar-ragdoll/Qwen-Sharp-Chat-Templates' "$ENV_FILE"
|
|
grep -q 'chat-template\.jinja' "$SERVE_FILE"
|
|
[ -s "$TEMPLATE_FILE" ]
|
|
grep -q 'qwen3\.8-froggeric-v22\.4\.0' "$TEMPLATE_FILE"
|
|
grep -q 'Qwen-Sharp' "$UNIT_FILE"
|
|
}
|