dankclaw: retire DSpark, use llama.cpp --spec-default (ngram-mod) preset; PARALLEL=2 trial

This commit is contained in:
Frank committed 2026-08-15 15:15:06 -07:00
1 parent 020e874a5c
commit 1dbc70cb32
3 files changed
+40 -40

No files matched your search

+26 -18
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
# Serve unsloth Qwen3.8-27B UD-Q8_K_XL on dankclaw with a DSpark
# sidecar draft model (draft-dspark). ROCm backend.
# Serve unsloth Qwen3.8-27B UD-Q8_K_XL on dankclaw with llama.cpp's
# --spec-default preset (ngram-mod). ROCm backend.
# Runtime: llama.cpp llama-server (>= b10411).
#
# Tunables come from frank-default-large.env in this directory unless the
@@ -25,7 +25,7 @@ fi
: "${PORT:=7770}"
: "${CTX_SIZE:=262144}" # Qwen3.8 native context (env may raise for parallel≥2)
: "${PARALLEL:=2}"
: "${SPEC_TYPE:=draft-mtp}"
: "${SPEC_DEFAULT:=0}" # 1: use --spec-default (ngram-mod preset) instead of --spec-type
: "${SPEC_DRAFT_N_MAX:=10}"
: "${SPEC_NGRAM_MOD_N_MIN:=48}"
: "${SPEC_NGRAM_MOD_N_MAX:=64}"
@@ -52,29 +52,37 @@ if [[ ! -f "$MODEL" ]]; then
exit 1
fi
# Draft-model flag applies only when a separate draft file is provided.
# Qwen3.8-UD bundles MTP in-model; leave DRAFT_FILE empty for bundled MTP,
# or set it to a DSpark/DFlash sidecar GGUF for a dedicated draft.
# Spec flags. --spec-default is a preset that enables ngram-mod
# (n_match=24, n_min=48, n_max=64) and takes no draft model. The explicit
# --spec-type / draft-model path is used when SPEC_DEFAULT is 0.
SPEC_ARGS=()
if [[ -n "$DRAFT_FILE" ]]; then
if [[ ! -f "$DRAFT" ]]; then
echo "missing required file: $DRAFT" >&2
if [[ "${SPEC_DEFAULT}" == "1" ]]; then
SPEC_ARGS+=(--spec-default)
else
if [[ -z "${SPEC_TYPE:-}" ]]; then
echo "SPEC_TYPE must be set when SPEC_DEFAULT=0" >&2
exit 1
fi
SPEC_ARGS+=(--spec-draft-model "$DRAFT")
fi
if [[ -n "${SPEC_DRAFT_P_MIN:-}" ]]; then
SPEC_ARGS+=(--spec-draft-p-min "$SPEC_DRAFT_P_MIN")
SPEC_ARGS+=(--spec-type "$SPEC_TYPE")
if [[ -n "$DRAFT_FILE" ]]; then
if [[ ! -f "$DRAFT" ]]; then
echo "missing required file: $DRAFT" >&2
exit 1
fi
SPEC_ARGS+=(--spec-draft-model "$DRAFT")
fi
if [[ -n "${SPEC_DRAFT_P_MIN:-}" ]]; then
SPEC_ARGS+=(--spec-draft-p-min "$SPEC_DRAFT_P_MIN")
fi
SPEC_ARGS+=(--spec-draft-n-max "$SPEC_DRAFT_N_MAX")
SPEC_ARGS+=(--spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN")
SPEC_ARGS+=(--spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX")
SPEC_ARGS+=(--spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH")
fi
exec "$LLAMA_SERVER" \
--model "$MODEL" \
"${SPEC_ARGS[@]}" \
--spec-type "$SPEC_TYPE" \
--spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
--spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN" \
--spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX" \
--spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH" \
--device "$DEVICE" \
--gpu-layers "$N_GPU_LAYERS" \
--fit off \