dankclaw: retire DSpark, use llama.cpp --spec-default (ngram-mod) preset; PARALLEL=2 trial
This commit is contained in:
1 parent
020e874a5c
commit
1dbc70cb32
3 files changed
+40
-40
No files matched your search
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve unsloth Qwen3.8-27B UD-Q8_K_XL on dankclaw with a DSpark
|
||||
# sidecar draft model (draft-dspark). ROCm backend.
|
||||
# Serve unsloth Qwen3.8-27B UD-Q8_K_XL on dankclaw with llama.cpp's
|
||||
# --spec-default preset (ngram-mod). ROCm backend.
|
||||
# Runtime: llama.cpp llama-server (>= b10411).
|
||||
#
|
||||
# Tunables come from frank-default-large.env in this directory unless the
|
||||
@@ -25,7 +25,7 @@ fi
|
||||
: "${PORT:=7770}"
|
||||
: "${CTX_SIZE:=262144}" # Qwen3.8 native context (env may raise for parallel≥2)
|
||||
: "${PARALLEL:=2}"
|
||||
: "${SPEC_TYPE:=draft-mtp}"
|
||||
: "${SPEC_DEFAULT:=0}" # 1: use --spec-default (ngram-mod preset) instead of --spec-type
|
||||
: "${SPEC_DRAFT_N_MAX:=10}"
|
||||
: "${SPEC_NGRAM_MOD_N_MIN:=48}"
|
||||
: "${SPEC_NGRAM_MOD_N_MAX:=64}"
|
||||
@@ -52,29 +52,37 @@ if [[ ! -f "$MODEL" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Draft-model flag applies only when a separate draft file is provided.
|
||||
# Qwen3.8-UD bundles MTP in-model; leave DRAFT_FILE empty for bundled MTP,
|
||||
# or set it to a DSpark/DFlash sidecar GGUF for a dedicated draft.
|
||||
# Spec flags. --spec-default is a preset that enables ngram-mod
|
||||
# (n_match=24, n_min=48, n_max=64) and takes no draft model. The explicit
|
||||
# --spec-type / draft-model path is used when SPEC_DEFAULT is 0.
|
||||
SPEC_ARGS=()
|
||||
if [[ -n "$DRAFT_FILE" ]]; then
|
||||
if [[ ! -f "$DRAFT" ]]; then
|
||||
echo "missing required file: $DRAFT" >&2
|
||||
if [[ "${SPEC_DEFAULT}" == "1" ]]; then
|
||||
SPEC_ARGS+=(--spec-default)
|
||||
else
|
||||
if [[ -z "${SPEC_TYPE:-}" ]]; then
|
||||
echo "SPEC_TYPE must be set when SPEC_DEFAULT=0" >&2
|
||||
exit 1
|
||||
fi
|
||||
SPEC_ARGS+=(--spec-draft-model "$DRAFT")
|
||||
fi
|
||||
if [[ -n "${SPEC_DRAFT_P_MIN:-}" ]]; then
|
||||
SPEC_ARGS+=(--spec-draft-p-min "$SPEC_DRAFT_P_MIN")
|
||||
SPEC_ARGS+=(--spec-type "$SPEC_TYPE")
|
||||
if [[ -n "$DRAFT_FILE" ]]; then
|
||||
if [[ ! -f "$DRAFT" ]]; then
|
||||
echo "missing required file: $DRAFT" >&2
|
||||
exit 1
|
||||
fi
|
||||
SPEC_ARGS+=(--spec-draft-model "$DRAFT")
|
||||
fi
|
||||
if [[ -n "${SPEC_DRAFT_P_MIN:-}" ]]; then
|
||||
SPEC_ARGS+=(--spec-draft-p-min "$SPEC_DRAFT_P_MIN")
|
||||
fi
|
||||
SPEC_ARGS+=(--spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
||||
SPEC_ARGS+=(--spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN")
|
||||
SPEC_ARGS+=(--spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX")
|
||||
SPEC_ARGS+=(--spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH")
|
||||
fi
|
||||
|
||||
exec "$LLAMA_SERVER" \
|
||||
--model "$MODEL" \
|
||||
"${SPEC_ARGS[@]}" \
|
||||
--spec-type "$SPEC_TYPE" \
|
||||
--spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
|
||||
--spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN" \
|
||||
--spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX" \
|
||||
--spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH" \
|
||||
--device "$DEVICE" \
|
||||
--gpu-layers "$N_GPU_LAYERS" \
|
||||
--fit off \
|
||||
|
||||
Reference in new issue
Block a user