Frank
baf33d016f
The parallel session's uncommitted swap, committed at Frank's direction (commit whatever's in the working tree): - llama.cpp master on the RTX 3090 now serves ukisai's Swift-1.5-Qwen3.8-27B GSQ-RCO IQ3_S WITH the in-file MTP head, ONE 131,072-token slot at Frank's 128k cap, q8_0 K + q8_0 V, vision through the base-model projector; MiMo-V2.6 Q6_K (2x200704 f16 KV) retired to rollback target. - env/serve/download + seam twins updated together (deployer payload gates rode the previous commit's deploy-remote.sh). - docs: new swift15 runbook; MiMo Q6K runbook retired; AGENTS.md topology and both vault indexes follow (root index ACTIVE/RETIRED flip, test 135 seam green). - comfyui: two qwen-image t2i workflow drops under dankclaw/comfyui/ workflows/ (12 KiB).
116 lines
5.0 KiB
Bash
Executable File
116 lines
5.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF (IQ3_S **-mtp**) as
|
|
# frank-small on Dankclaw's RTX 3090 via llama.cpp (master build, CUDA
|
|
# sm_86).
|
|
#
|
|
# 2026-10-05 evening (Frank): a 27B on the 24 GiB card — ONE 131,072-token
|
|
# conversation (Frank's cap), KV at FP8 class (q8_0 both halves), the MTP
|
|
# head that lives IN the -mtp file drafting 3, vision on. The q8_0 KV
|
|
# contract supersedes the same-day f16/f16 rule, which was written for the
|
|
# Ornith 9B fit; the per-tensor flags stay explicit either way.
|
|
#
|
|
# Architecture: qwen35 hybrid — 64 real layers, 16 full-attention + 48
|
|
# Gated DeltaNet (full_attention_interval 4), native 262144 window, plus
|
|
# the blk.64 MTP head (nextn_predict_layers=1). KV lives only in the 16
|
|
# attention layers (34 KiB/token at q8_0); the DeltaNet state is per-slot
|
|
# fixed size. Vision comes from the projector file (the GGUF carries no
|
|
# vision tensors; unsloth's mmproj built from the shared Qwen3.8-27B base —
|
|
# proven byte-identical to Swift's own tower, see download-frank-small.sh),
|
|
# so --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the
|
|
# documented text-only opt-out.
|
|
#
|
|
# Sampling: llama.cpp takes it per request (the fleet harness sends its
|
|
# recipe from models.yml); greedy is the harness default.
|
|
#
|
|
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) -------------------
|
|
: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
|
: "${MODEL_FILE:=/home/dank/models/Swift15-IQ3S/Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
|
|
: "${ALIAS:=frank-small}"
|
|
: "${BIND_ADDR:=0.0.0.0}"
|
|
: "${LISTEN_PORT:=7771}"
|
|
# ctx-size is the POOL llama.cpp divides across slots: 131072 / 1 gives
|
|
# 131,072 per conversation — Frank's 128k cap. A second 131,072 slot does
|
|
# not fit the 2-3 GiB floor on the 24 GiB card with the 27B payload
|
|
# (frank-small.env carries the arithmetic); "as many 128k contexts as fit"
|
|
# = 1.
|
|
: "${CTX_SIZE:=131072}"
|
|
: "${PARALLEL:=1}"
|
|
# Compute-buffer geometry. Ubatch is a prefill knob (decode never moves),
|
|
# and Frank's standing law for a resize is LOWER UBATCH FIRST; the shipped
|
|
# pair is 4096/1024. If boot lands well above the 2-3 GiB floor, ubatch
|
|
# 2048 is the first A/B for prefill (a miss fails loud at graph capture).
|
|
# The logical batch is VRAM- and throughput-neutral at a fixed ubatch.
|
|
: "${BATCH_SIZE:=4096}"
|
|
: "${UBATCH_SIZE:=1024}"
|
|
# KV at FP8 class = q8_0 on llama.cpp (1.0625 B/elem => 8.5 effective bits;
|
|
# there is no true fp8 KV dtype). Explicit either way so the contract is
|
|
# visible in the argv.
|
|
: "${CACHE_TYPE_K:=q8_0}"
|
|
: "${CACHE_TYPE_V:=q8_0}"
|
|
# Speculative decoding: draft-mtp loads the MTP head that is IN the -mtp
|
|
# GGUF; the flags move as a set.
|
|
: "${SPEC_TYPE:=draft-mtp}"
|
|
: "${SPEC_DRAFT_N_MAX:=3}"
|
|
: "${NGL:=99}"
|
|
# Vision projector. No colon on purpose: an explicitly empty value means
|
|
# "text-only, on purpose" and must NOT be replaced by the default path.
|
|
: "${MMPROJ_FILE=/home/dank/models/Swift15-IQ3S/mmproj-Qwen3.8-27B-F16.gguf}"
|
|
|
|
# Fail loudly rather than half-loading: a missing file is a download that
|
|
# has not finished, and Restart=always would turn it into a crash loop that
|
|
# reads like a runtime fault.
|
|
for path in "$LLAMA_BIN" "$MODEL_FILE"; do
|
|
[[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
|
|
done
|
|
EXTRA_ARGS=()
|
|
if [[ -n "$MMPROJ_FILE" ]]; then
|
|
[[ -e "$MMPROJ_FILE" ]] || { echo "frank-small: missing $MMPROJ_FILE" >&2; exit 1; }
|
|
EXTRA_ARGS+=(--mmproj "$MMPROJ_FILE")
|
|
fi
|
|
|
|
# The MTP head ships IN the -mtp GGUF (blk.64.nextn.*,
|
|
# nextn_predict_layers=1): draft-mtp loads it against the main model's own
|
|
# weights — no --model-draft and no second KV pool. Speculative flags move
|
|
# as a set: a no-spec baseline must strip BOTH of these, not just one.
|
|
EXTRA_ARGS+=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
|
|
|
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
|
|
# open after boot; make sure it exists before the server needs it.
|
|
nvidia-modprobe -u -c=0 2>/dev/null || true
|
|
|
|
# Flash attention stays pinned on: it is REQUIRED for quantized V (q8_0),
|
|
# and the build carries GGML_CUDA_FA_ALL_QUANTS=ON (since 2026-09-17), so
|
|
# the q8_0 KV contract in frank-small.env is a real path — without this
|
|
# flag or that build option the engine silently falls back and the
|
|
# contract becomes a lie.
|
|
ARGS=(
|
|
--model "$MODEL_FILE"
|
|
--alias "$ALIAS"
|
|
--host "$BIND_ADDR"
|
|
--port "$LISTEN_PORT"
|
|
--ctx-size "$CTX_SIZE"
|
|
--parallel "$PARALLEL"
|
|
--batch-size "$BATCH_SIZE"
|
|
--ubatch-size "$UBATCH_SIZE"
|
|
--n-gpu-layers "$NGL"
|
|
--flash-attn on
|
|
--cache-type-k "$CACHE_TYPE_K"
|
|
--cache-type-v "$CACHE_TYPE_V"
|
|
--jinja
|
|
--reasoning auto
|
|
--reasoning-format deepseek
|
|
)
|
|
exec "$LLAMA_BIN" "${ARGS[@]}" ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}
|