diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env new file mode 100644 index 0000000..444aa79 --- /dev/null +++ b/scripts/frank-default-small.env @@ -0,0 +1,18 @@ +# Tunables for frank-default-small (HauhauCS Q6_K_P + DFlash draft, dankserver) +# Runtime: llama-server b10108; RTX 3090 24GB; Ryzen 7 7700X; 62GB RAM. +MODEL_DIR=/home/dank/models +ALIAS=frank-default-small +HOST=0.0.0.0 +PORT=7770 +CTX_SIZE=262144 +PARALLEL=3 +SPEC_TYPE=draft-dflash +SPEC_N_MAX=7 +CACHE_TYPE_K=q8_0 +CACHE_TYPE_V=f16 +# All 40 layers keep their MoE expert weights on CPU (targeted expert offload). +N_CPU_MOE=40 +# Every non-expert tensor (attention, norms, embedding, LM head) goes to GPU. +N_GPU_LAYERS=all +THREADS=8 +THREADS_HTTP=8 \ No newline at end of file diff --git a/scripts/frank-default-small.service b/scripts/frank-default-small.service new file mode 100644 index 0000000..770ebb4 --- /dev/null +++ b/scripts/frank-default-small.service @@ -0,0 +1,14 @@ +[Unit] +Description=llama.cpp DFlash inference (frank-default-small) +After=network.target + +[Service] +Type=simple +User=dank +Group=dank +ExecStart=/home/dank/vLLM/scripts/serve-llama-frank-small.sh +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/scripts/serve-llama-frank-small.sh b/scripts/serve-llama-frank-small.sh new file mode 100755 index 0000000..5fe8969 --- /dev/null +++ b/scripts/serve-llama-frank-small.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Serve HauhauCS Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q6_K_P +# with a DFlash draft model on dankserver (RTX 3090, MoE experts on CPU). +# Runtime: upstream llama.cpp llama-server (b10108+). +# +# Tunables come from frank-default-small.env in this directory unless the +# FRANK_SMALL_ENV_FILE environment variable overrides that path. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}" +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +# --- defaults (used when the env file does not set them) ----------------- +: "${MODEL_DIR:=/home/dank/models}" +: "${MODEL_FILE:=Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q6_K_P.gguf}" +: "${DRAFT_FILE:=Qwen3.6-35B-A3B-DFlash-Q8_0.gguf}" +: "${MMPROJ_FILE:=mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf}" +: "${ALIAS:=frank-default-small}" +: "${HOST:=0.0.0.0}" +: "${PORT:=7770}" +: "${CTX_SIZE:=262144}" +: "${PARALLEL:=3}" +: "${SPEC_TYPE:=draft-dflash}" +: "${SPEC_N_MAX:=7}" +: "${CACHE_TYPE_K:=q8_0}" +: "${CACHE_TYPE_V:=f16}" +: "${N_CPU_MOE:=40}" # all 40 layers: MoE expert weights stay on CPU +: "${N_GPU_LAYERS:=all}" # every non-expert tensor goes to GPU +: "${THREADS:=8}" # 7700X physical cores +: "${THREADS_HTTP:=8}" + +LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp/build/bin/llama-server}" +MODEL="$MODEL_DIR/$MODEL_FILE" +DRAFT="$MODEL_DIR/$DRAFT_FILE" +MMPROJ="$MODEL_DIR/$MMPROJ_FILE" + +for f in "$MODEL" "$DRAFT" "$MMPROJ"; do + if [[ ! -f "$f" ]]; then + echo "missing required file: $f" >&2 + exit 1 + fi +done + +exec "$LLAMA_SERVER" \ + --model "$MODEL" \ + --model-draft "$DRAFT" \ + --mmproj "$MMPROJ" \ + --jinja \ + --spec-type "$SPEC_TYPE" \ + --spec-draft-n-max "$SPEC_N_MAX" \ + --cache-type-k "$CACHE_TYPE_K" \ + --cache-type-v "$CACHE_TYPE_V" \ + --ctx-size "$CTX_SIZE" \ + --parallel "$PARALLEL" \ + --n-cpu-moe "$N_CPU_MOE" \ + --gpu-layers "$N_GPU_LAYERS" \ + --threads "$THREADS" \ + --threads-http "$THREADS_HTTP" \ + --alias "$ALIAS" \ + --host "$HOST" \ + --port "$PORT" \ No newline at end of file