diff --git a/.AGENTTRASH/frank-default-large.env.dflash2-vulkan-2026-08-21 b/.AGENTTRASH/frank-default-large.env.dflash2-vulkan-2026-08-21 deleted file mode 100644 index cb4eabc..0000000 --- a/.AGENTTRASH/frank-default-large.env.dflash2-vulkan-2026-08-21 +++ /dev/null @@ -1,60 +0,0 @@ -# Tunables for frank-default-large (upstream llama.cpp PR #27342 DFlash2, -# Vulkan backend, dankclaw, Strix Halo gfx1151). -# -# Runtime: llama.cpp PR #27342 built with GGML_VULKAN=ON serving -# unsloth/Qwen3.8-27B-UD-Q8_K_XL (target) with a z-lab DFlash2 drafter -# (Qwen3.8-27B-DFlash2-Q4_K_M). Target at 31.4 GB, DFlash2 drafter at 2.0 GB. -# Alias frank-default-large, port 52625. -# -# DFlash2 runs on the Vulkan (RADV) backend. Its candidate selector needs -# TOP_K + ARGMAX, which the HIP backend does not implement. This replaced -# the CIRU HIP runtime on 2026-08-21. See docs/dflash2-large-vulkan.md. -# -# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The -# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP -# incident). This deployment uses Vulkan, not HIP, but the rule stays for -# portability. - -# Upstream llama.cpp DFlash2 binary (PR #27342, Vulkan build). -LLAMA_BIN=/home/dank/llama.cpp-dflash2/build-dflash2/bin/llama-server - -# Model files. The target (UD-Q8_K_XL) was already on disk; the DFlash2 -# drafter was added for this deployment. Both live in /home/dank/models. -MODEL_DIR=/home/dank/models -MODEL_FILE=Qwen3.8-27B-UD-Q8_K_XL.gguf -DRAFT_FILE=Qwen3.8-27B-DFlash2-Q4_K_M.gguf - -ALIAS=frank-default-large -BIND_ADDR=0.0.0.0 -LISTEN_PORT=52625 - -# Serving: full native 262144 context, 1 slot. The Vulkan device reports -# 113942 MiB; KV at 262144 x 64 KiB/cell is ~16 GiB. rocm-smi reports the -# 64 GiB budget; trust the runtime for the addressable SVM pool. -CTX_SIZE=262144 -PARALLEL=1 -BATCH_SIZE=4096 -UBATCH_SIZE=4096 -CACHE_TYPE_K=f16 -CACHE_TYPE_V=f16 -CTX_CHECKPOINTS=16 -CACHE_RAM=8192 - -# DFlash2 speculative decoding. The drafter's trained block size is 8; -# SPEC_DRAFT_N_MAX clamps to it. TOP_K selector (top 16 candidates). -SPEC_TYPE=draft-dflash -SPEC_DRAFT_N_MAX=8 -SPEC_DRAFT_N_MIN=0 -SPEC_DRAFT_P_MIN=0.0 -SPEC_DRAFT_P_SPLIT=0.10 - -# Sampling (model card defaults, thinking-general). -TEMPERATURE=0.7 -TOP_P=0.95 - -# Thinking enabled. deepseek format moves the thinking reasoning into -# message.reasoning_content (OpenAI-compatible). -REASONING=on -REASONING_FORMAT=deepseek - -VERBOSITY=4 \ No newline at end of file diff --git a/.AGENTTRASH/frank-default-small.env.llama.cpp b/.AGENTTRASH/frank-default-small.env.llama.cpp new file mode 100644 index 0000000..d11f10c --- /dev/null +++ b/.AGENTTRASH/frank-default-small.env.llama.cpp @@ -0,0 +1,71 @@ +# Tunables for frank-default-small (Tiel-Coder-35B-A3B UD-Q6_K_XL, +# qwen35moe hybrid MoE, Dankserver, RTX 3090). +# +# Runtime: upstream llama.cpp (latest master, CUDA build with +# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86) on RTX 3090 +# 24 GB; Ryzen 7 7700X; 62 GB RAM. Replaced the ik_llama.cpp Ornith +# CRACK Q8_0 deployment on 2026-09-01. +# +# Model: peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF, UD-Q6_K_XL quant +# (30 GB). Architecture: qwen35moe — 40 layers, full_attention_interval=4 +# (10 attention + 30 SSM/Mamba), 256 experts / 8 used, 2 KV heads × 256 +# dim, embedding 2048, rope dimension_count 64, rope freq_base 10 000 000. +# Text-only (no mmproj). Built-in chat template in the GGUF. +# +# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM. +# Non-expert weights (~1.6 GB) fit entirely on the 24 GB card. +# +# KV cache: Q8_0 for both K and V. With CTX_SIZE=200700 (66 900 per slot +# × 3 parallel) and 10 attention layers, the KV cache is ~2 GB — the +# requested 2 GB VRAM buffer. +# +# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The +# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP +# incident). This box is NVIDIA/CUDA, but the rule keeps the env portable. + +# Model file (UD-Q6_K_XL, 30 GB, text-only MoE hybrid). +MODEL_DIR=/home/dank/models/tiel-coder-35b +MODEL_FILE=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf + +# Upstream llama.cpp llama-server (CUDA build, latest master). +# Built with -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86 +# using the staged CUDA 13.2 toolkit at /home/dank/cuda13-stage. +LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server + +ALIAS=frank-default-small +BIND_ADDR=0.0.0.0 +LISTEN_PORT=7770 + +# Total context budget across all parallel slots. 200700 / 3 = 66 900 +# per slot. The model's context_length is 262 144, so 66 900 is well +# within range. +CTX_SIZE=200700 +PARALLEL=3 + +# Expert offload: all MoE expert weights pinned to CPU RAM. +CPU_MOE=1 + +# All non-expert layers on GPU (auto-fit will determine the exact count; +# with --cpu-moe the non-expert footprint is ~1.6 GB, leaving ample room). +N_GPU_LAYERS=999 + +# KV cache: Q8_0 for both K and V. ~2 GB total for 200700 ctx × 3 slots +# × 10 attention layers × (2 KV heads × 256 dim × 1 byte) × 2 (K+V). +CACHE_TYPE_K=q8_0 +CACHE_TYPE_V=q8_0 + +VERBOSITY=2 + +# Prefill batch sizing. +BATCH_SIZE=4096 +UBATCH_SIZE=1024 + +# Sampling. +TEMPERATURE=1.0 +TOP_K=20 +TOP_P=0.95 +MIN_P=0.0 + +# Compute threads lowered so other processes keep a core. +THREADS=7 +THREADS_HTTP=8 \ No newline at end of file diff --git a/.AGENTTRASH/frank-default-small.service.llama.cpp b/.AGENTTRASH/frank-default-small.service.llama.cpp new file mode 100644 index 0000000..0805e2c --- /dev/null +++ b/.AGENTTRASH/frank-default-small.service.llama.cpp @@ -0,0 +1,16 @@ +[Unit] +Description=llama.cpp Tiel-Coder-35B-A3B Q6_K_XL qwen35moe hybrid MoE (dankserver, frank-default-small) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=dank +Group=dank +Environment="FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-default-small.env" +ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/.AGENTTRASH/frank_large_config.bats b/.AGENTTRASH/frank_large_config.bats deleted file mode 100644 index 59df127..0000000 --- a/.AGENTTRASH/frank_large_config.bats +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash -# Seam: frank-default-large model, ROCmFPX runtime, MTP, KV cache, template. -# -# load helpers - -load helpers - -ENV_FILE="$REPO_ROOT/scripts/frank-default-large.env" -SERVE_FILE="$REPO_ROOT/scripts/serve-frank-large.sh" -UNIT_FILE="$REPO_ROOT/scripts/frank-default-large.service" -TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" - -@test "frank-default-large serves the kingjones AGENT model" { - grep -q '^MODEL_FILE=Qwen3\.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT\.gguf$' "$ENV_FILE" - grep -q '^MTP_DRAFT_FILE=' "$ENV_FILE" - grep -q '^MMPROJ_FILE=mmproj-Qwen3\.8-27B-Q8_0\.gguf$' "$ENV_FILE" - ! grep -q -- '--kairic-edge' "$SERVE_FILE" -} - -@test "frank-default-large uses the ROCmFPX runtime" { - grep -q 'ROCmFPX/.*/bin/llama-server' "$ENV_FILE" - grep -q 'build-promptforge/bin/llama-server' "$ENV_FILE" -} - -@test "frank-default-large runs standalone MTP" { - grep -q -- '--spec-type draft-mtp' "$SERVE_FILE" - grep -qF -- '--model-draft "$MTP_DRAFT_FILE"' "$SERVE_FILE" - grep -qF -- '--spec-draft-n-max "$MTP_N_MAX"' "$SERVE_FILE" - grep -q '^MTP_N_MAX=4$' "$ENV_FILE" -} - -@test "frank-default-large has no obsolete PromptForge sidecar wiring" { - ! grep -qE 'KAIRIC_|PROMPTFORGE_' "$SERVE_FILE" - ! grep -qE 'KAIRIC_|PROMPTFORGE_' "$ENV_FILE" -} - -@test "frank-default-large fits the model and KV cache in unified memory" { - grep -q '^CTX_SIZE=262144$' "$ENV_FILE" - grep -q ': "${CTX_SIZE:=262144}"' "$SERVE_FILE" -} - -@test "frank-default-large uses DeepSeek reasoning output" { - grep -q '^REASONING_FORMAT=deepseek$' "$ENV_FILE" - grep -q -- '--reasoning-format "$REASONING_FORMAT"' "$SERVE_FILE" -} - -@test "frank-default-large has reasoning and preserved reasoning/thinking enabled" { - grep -q '^REASONING=on$' "$ENV_FILE" - grep -q 'enable_thinking":true' "$ENV_FILE" - grep -q 'preserve_thinking":true' "$ENV_FILE" - grep -q 'preserve_reasoning":true' "$ENV_FILE" -} - -@test "frank-default-large uses f16 for both KV cache sides" { - grep -q '^CACHE_TYPE_K=f16$' "$ENV_FILE" - grep -q '^CACHE_TYPE_V=f16$' "$ENV_FILE" - grep -qF -- '--cache-type-k "$CACHE_TYPE_K"' "$SERVE_FILE" - grep -qF -- '--cache-type-v "$CACHE_TYPE_V"' "$SERVE_FILE" -} - -@test "frank-default-large exposes a swappable ubatch" { - grep -q '^UBATCH_SIZE=4096$' "$ENV_FILE" - grep -q ': "${UBATCH_SIZE:=4096}"' "$SERVE_FILE" - grep -q -- '--ubatch-size "$UBATCH_SIZE"' "$SERVE_FILE" -} - -@test "llama-bench ubatch harness uses the active model" { - local bench="$REPO_ROOT/scripts/bench-ubatch.sh" - [ -x "$bench" ] - grep -q 'llama-bench' "$bench" - grep -q 'Qwen3\.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT\.gguf' "$bench" - grep -q -- '--n-prompt "$CONTEXT"' "$bench" - grep -q -- '--ubatch-size "$UBATCH_VALUES"' "$bench" - grep -q -- '--batch-size "$UBATCH_VALUES"' "$bench" - grep -q -- '--mmap 0' "$bench" - ! grep -qE '(^|[[:space:]])llama-server|(^|[[:space:]])curl|print_timing' "$bench" -} - -@test "frank-default-large uses the tracked Qwen-Sharp template on :7770" { - grep -q '^CHAT_TEMPLATE_FILE=' "$ENV_FILE" - grep -q -- '--jinja' "$SERVE_FILE" - [ -s "$TEMPLATE_FILE" ] - grep -q '^ALIAS=frank-default-large$' "$ENV_FILE" - grep -q '^LISTEN_PORT=7770$' "$ENV_FILE" - grep -q 'frank-default-large' "$UNIT_FILE" -} - -@test "frank-default-large env avoids the HIP TEMP/HOST collision" { - # The 2026-08-13 incident: TEMP/HOST/GPU_MAX_HEAP_SIZE are HIP runtime vars. - ! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE" - grep -q '^TEMPERATURE=' "$ENV_FILE" -} diff --git a/.AGENTTRASH/frank_small_config.bats b/.AGENTTRASH/frank_small_config.bats deleted file mode 100644 index 70be16f..0000000 --- a/.AGENTTRASH/frank_small_config.bats +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bats -# Seam: frank-default-small model, context, tensor placement, and template. -# This test covers only the Q6_K deployment contract. - -load helpers - -ENV_FILE="$REPO_ROOT/scripts/frank-default-small.env" -SERVE_FILE="$REPO_ROOT/scripts/serve-frank-small.sh" -UNIT_FILE="$REPO_ROOT/scripts/frank-default-small.service" -TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" - -@test "frank-default-small selects the Ornith Q8_0 model" { - grep -q '^MODEL_FILE=Ornith-1\.5-35B-A3B-CRACK-Q8_0\.gguf$' "$ENV_FILE" - grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q8_0\.gguf' "$SERVE_FILE" -} - -@test "frank-default-small uses three 163840-slot context (491520 pool)" { - grep -q '^CTX_SIZE=491520$' "$ENV_FILE" - grep -q '^PARALLEL=3$' "$ENV_FILE" - grep -q '163840 per slot' "$ENV_FILE" -} - -@test "frank-default-small uses default f16 K and q6_0 V KV cache" { - grep -q '^CACHE_TYPE_K=f16$' "$ENV_FILE" - grep -q '^CACHE_TYPE_V=q6_0$' "$ENV_FILE" - # K at default f16 has no Hadamard; V at q6_0 keeps Hadamard. - grep -q '^K_CACHE_HADAMARD=0$' "$ENV_FILE" - grep -q '^V_CACHE_HADAMARD=1$' "$ENV_FILE" -} - -@test "frank-default-small keeps full-attention and final blocks on CUDA" { - source "$ENV_FILE" - [[ "$OVERRIDE_EXPERT" == 'blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|27|28|29|30|31|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps' ]] - # output-adjacent final blocks stay on CUDA - [[ "$OVERRIDE_EXPERT" != *'|35|'* ]] - [[ "$OVERRIDE_EXPERT" != *'|36|'* ]] - [[ "$OVERRIDE_EXPERT" != *'|37|'* ]] - [[ "$OVERRIDE_EXPERT" != *'|38|'* ]] - [[ "$OVERRIDE_EXPERT" != *'|39|'* ]] - [[ "$OVERRIDE_EXPERT" != *'|40|'* ]] - # full-attention blocks (3,7,11,15,19,23) are untouched (no exps) - [[ "$OVERRIDE_EXPERT" != *'|3|'* ]] - [[ "$OVERRIDE_EXPERT" != *'|7|'* ]] - [[ "$OVERRIDE_EXPERT" != *'|11|'* ]] - [[ "$OVERRIDE_EXPERT" != *'|15|'* ]] - [[ "$OVERRIDE_EXPERT" != *'|19|'* ]] - [[ "$OVERRIDE_EXPERT" != *'|23|'* ]] - [[ "$OVERRIDE_BUFT" == CPU ]] - # ik_llama --gpu-layers/-ngl requires an integer ("all" is rejected); 999 - # loads as many layers as possible while OVERRIDE_EXPERT pins exps to CPU. - [[ "$N_GPU_LAYERS" == 999 ]] -} - -@test "frank-default-small disables speculative decoding (3 parallel slots)" { - # ik_llama limits speculative decoding to -np 1; with PARALLEL=3 a - # configured spec stage hard-exits, so SPEC_TYPE=none. - grep -q '^SPEC_TYPE=none$' "$ENV_FILE" - grep -q '^PARALLEL=3$' "$ENV_FILE" - ! grep -q '^SPEC_NGRAM_MOD_N_MIN=' "$ENV_FILE" - ! grep -q '^SPEC_NGRAM_MOD_N_MAX=' "$ENV_FILE" - ! grep -q '^SPEC_NGRAM_MOD_N_MATCH=' "$ENV_FILE" - ! grep -q -- 'SPEC_DRAFT_' "$ENV_FILE" - ! grep -q -- '--spec-type.*draft-mtp' "$SERVE_FILE" - grep -q -- '--spec-type' "$SERVE_FILE" - ! grep -q -- 'ngram' "$SERVE_FILE" -} - -@test "frank-default-small uses ubatch 1024" { - grep -q '^BATCH_SIZE=4096$' "$ENV_FILE" - grep -q '^UBATCH_SIZE=1024$' "$ENV_FILE" - grep -q ': "${UBATCH_SIZE:=1024}"' "$SERVE_FILE" -} - -@test "frank-default-small uses the tracked Qwen-Sharp template" { - grep -q '^CHAT_TEMPLATE_FILE=' "$ENV_FILE" - grep -q 'peculiar-ragdoll/Qwen-Sharp-Chat-Templates' "$ENV_FILE" - grep -q 'chat-template\.jinja' "$SERVE_FILE" - [ -s "$TEMPLATE_FILE" ] - grep -q 'qwen3\.8-froggeric-v22\.4\.0' "$TEMPLATE_FILE" - grep -q 'Qwen-Sharp' "$UNIT_FILE" -} - -@test "frank-default-small renders the custom template with --jinja" { - # Without --jinja, ik_llama only supports built-in non-Jinja handlers and - # silently drops the custom Qwen-Sharp template formatting. - grep -q '^USE_JINJA=1$' "$ENV_FILE" - grep -q -- '--jinja' "$SERVE_FILE" -} diff --git a/.AGENTTRASH/serve-frank-large.sh.dflash2-vulkan-2026-08-21 b/.AGENTTRASH/serve-frank-large.sh.dflash2-vulkan-2026-08-21 deleted file mode 100755 index bb1217b..0000000 --- a/.AGENTTRASH/serve-frank-large.sh.dflash2-vulkan-2026-08-21 +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash -# Serve unsloth/Qwen3.8-27B-UD-Q8_K_XL as frank-default-large with a -# z-lab DFlash2 drafter (Qwen3.8-27B-DFlash2-Q4_K_M) on the Vulkan backend. -# -# Runtime: upstream llama.cpp PR #27342 (DFlash2 support) built with -# GGML_VULKAN=ON for the Strix Halo gfx1151 (RADV). DFlash2 selects its -# candidates through TOP_K + ARGMAX ops that the HIP backend does not -# implement, so the whole stack runs on Vulkan0. -# -# The target and the drafter share the Qwen3.8 arch. DFlash is lossless -# under strict rejection sampling. --spec-draft-n-max clamps to the drafter's -# trained block size (8). -# -# Tunables come from frank-default-large.env in this directory unless -# FRANK_LARGE_ENV_FILE overrides that path. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}" -if [[ -f "$ENV_FILE" ]]; then - set -a - # shellcheck disable=SC1090 - source "$ENV_FILE" - set +a -fi - -# --- defaults (used when the env file does not set them) ------------------- -: "${LLAMA_BIN:=/home/dank/llama.cpp-dflash2/build-dflash2/bin/llama-server}" -: "${MODEL_DIR:=/home/dank/models}" -: "${MODEL_FILE:=Qwen3.8-27B-UD-Q8_K_XL.gguf}" -: "${DRAFT_FILE:=Qwen3.8-27B-DFlash2-Q4_K_M.gguf}" -: "${ALIAS:=frank-default-large}" -: "${BIND_ADDR:=0.0.0.0}" -: "${LISTEN_PORT:=52625}" -: "${CTX_SIZE:=262144}" -: "${PARALLEL:=1}" -: "${BATCH_SIZE:=4096}" -: "${UBATCH_SIZE:=4096}" -: "${CACHE_TYPE_K:=f16}" -: "${CACHE_TYPE_V:=f16}" -: "${CTX_CHECKPOINTS:=16}" -: "${CACHE_RAM:=8192}" -: "${SPEC_TYPE:=draft-dflash}" -: "${SPEC_DRAFT_N_MAX:=8}" -: "${SPEC_DRAFT_N_MIN:=0}" -: "${SPEC_DRAFT_P_MIN:=0.0}" -: "${SPEC_DRAFT_P_SPLIT:=0.10}" -: "${TEMPERATURE:=0.7}" -: "${TOP_P:=0.95}" -: "${REASONING:=on}" -: "${REASONING_FORMAT:=deepseek}" -: "${VERBOSITY:=4}" - -MODEL="$MODEL_DIR/$MODEL_FILE" -DRAFT="$MODEL_DIR/$DRAFT_FILE" - -if [[ ! -f "$MODEL" ]]; then - echo "missing required file: $MODEL" >&2 - exit 1 -fi -if [[ ! -f "$DRAFT" ]]; then - echo "missing DFlash2 drafter: $DRAFT" >&2 - exit 1 -fi -if [[ ! -x "$LLAMA_BIN" ]]; then - echo "missing llama-server binary: $LLAMA_BIN" >&2 - exit 1 -fi - -exec "$LLAMA_BIN" \ - --model "$MODEL" \ - --alias "$ALIAS" \ - --host "$BIND_ADDR" \ - --port "$LISTEN_PORT" \ - -dev Vulkan0 \ - -ngl 999 \ - -c "$CTX_SIZE" \ - -b "$BATCH_SIZE" \ - -ub "$UBATCH_SIZE" \ - -fa on \ - -ctk "$CACHE_TYPE_K" \ - -ctv "$CACHE_TYPE_V" \ - -np "$PARALLEL" \ - -ctxcp "$CTX_CHECKPOINTS" \ - --cache-ram "$CACHE_RAM" \ - --cache-prompt \ - --metrics \ - --no-mmap \ - --spec-type "$SPEC_TYPE" \ - --spec-draft-model "$DRAFT" \ - --spec-draft-device Vulkan0 \ - --spec-draft-ngl 999 \ - --spec-draft-type-k f16 \ - --spec-draft-type-v f16 \ - --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \ - --spec-draft-n-min "$SPEC_DRAFT_N_MIN" \ - --spec-draft-p-min "$SPEC_DRAFT_P_MIN" \ - --spec-draft-p-split "$SPEC_DRAFT_P_SPLIT" \ - --temp "$TEMPERATURE" \ - --top-p "$TOP_P" \ - --reasoning "$REASONING" \ - --reasoning-format "$REASONING_FORMAT" \ - --reasoning-budget -1 \ - -lv "$VERBOSITY" \ No newline at end of file diff --git a/.AGENTTRASH/serve-frank-small.sh.llama.cpp b/.AGENTTRASH/serve-frank-small.sh.llama.cpp new file mode 100755 index 0000000..5b3b6e8 --- /dev/null +++ b/.AGENTTRASH/serve-frank-small.sh.llama.cpp @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Serve peculiar-ragdoll/Tiel-Coder-35B-A3B (UD-Q6_K_XL) as +# frank-default-small via stock llama.cpp on Dankserver (RTX 3090). +# +# Runtime: upstream llama.cpp (latest master, CUDA build with +# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86). +# The model is a 35B-total / 8-active MoE hybrid (qwen35moe arch) with +# SSM/Mamba layers (full_attention_interval=4: 10 attention + 30 SSM +# layers out of 40). 256 experts, 8 used per token. +# +# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM. +# Non-expert weights (attention, SSM, shared expert, embeddings, output) +# are ~1.6 GB and fit entirely on the 24 GB RTX 3090. +# +# KV cache: Q8_0 for both K and V. With CTX_SIZE=200700 (66 900 per +# slot × 3 parallel) and 10 attention layers (2 KV heads × 256 dim), +# the KV cache is ~2 GB — the requested 2 GB VRAM buffer. +# +# Model is loaded with mmap (default). The 30 GB Q6_K_XL file is +# memory-mapped; only accessed pages enter RAM. +# +# Tunables come from frank-default-small.env unless FRANK_SMALL_ENV_FILE +# overrides that path. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}" +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +# --- defaults (used when the env file does not set them) ----------------- +: "${MODEL_DIR:=/home/dank/models/tiel-coder-35b}" +: "${MODEL_FILE:=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf}" +: "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}" +: "${ALIAS:=frank-default-small}" +: "${BIND_ADDR:=0.0.0.0}" +: "${LISTEN_PORT:=7770}" +: "${CTX_SIZE:=200700}" +: "${PARALLEL:=3}" +: "${CACHE_TYPE_K:=q8_0}" +: "${CACHE_TYPE_V:=q8_0}" +: "${N_GPU_LAYERS:=999}" +: "${BATCH_SIZE:=4096}" +: "${UBATCH_SIZE:=1024}" +: "${VERBOSITY:=2}" +: "${CPU_MOE:=1}" +: "${TEMPERATURE:=1.0}" +: "${TOP_K:=20}" +: "${TOP_P:=0.95}" +: "${MIN_P:=0.0}" +: "${THREADS:=7}" +: "${THREADS_HTTP:=8}" + +MODEL="$MODEL_DIR/$MODEL_FILE" + +if [[ ! -f "$MODEL" ]]; then + echo "missing required file: $MODEL" >&2 + exit 1 +fi +if [[ ! -x "$LLAMA_SERVER" ]]; then + echo "missing llama-server binary: $LLAMA_SERVER" >&2 + exit 1 +fi + +exec "$LLAMA_SERVER" \ + --model "$MODEL" \ + $([ "$CPU_MOE" = "1" ] && printf -- "--cpu-moe") \ + --cache-type-k "$CACHE_TYPE_K" \ + --cache-type-v "$CACHE_TYPE_V" \ + --ctx-size "$CTX_SIZE" \ + --parallel "$PARALLEL" \ + --gpu-layers "$N_GPU_LAYERS" \ + --batch-size "$BATCH_SIZE" \ + --ubatch-size "$UBATCH_SIZE" \ + --temp "$TEMPERATURE" \ + --top-k "$TOP_K" \ + --top-p "$TOP_P" \ + --min-p "$MIN_P" \ + --threads "$THREADS" \ + --threads-http "$THREADS_HTTP" \ + --alias "$ALIAS" \ + --host "$BIND_ADDR" \ + --port "$LISTEN_PORT" \ No newline at end of file diff --git a/docs/LLM/freetoken-benchmark-sweep.md b/docs/LLM/freetoken-benchmark-sweep.md new file mode 100644 index 0000000..8676da4 --- /dev/null +++ b/docs/LLM/freetoken-benchmark-sweep.md @@ -0,0 +1,94 @@ +--- +type: research +title: FreeToken Benchmark Sweep — Dankserver RTX 3090 +description: KV pool vs decode speed sweep across bs1/bs3/bs4/bs6 on Dankserver with Qwen3.6-35B-A3B-FP8 via FreeToken v0.1.2. +tags: + - freetoken + - benchmark + - dankserver + - frank-default-small + - qwen3.6 +timestamp: 2026-09-02 +--- + +# FreeToken Benchmark Sweep — Dankserver RTX 3090 + +Full sweep of KV pool size vs decode speed at batch sizes 1, 3, 4, and 6. +Measured 2026-09-02 with FreeToken v0.1.2 serving `Qwen/Qwen3.6-35B-A3B-FP8` +on Dankserver (RTX 3090 24 GB, 60 GB RAM, driver 595.84). + +## Methodology + +- Each point: restart at low KV (8k), live-rebuild upward via + `ft ctl cache --kv N --moe M` (auto-computed MoE slot count). +- Warmup: 2–3 iterations at 64 tokens (fills LRU expert cache). +- Measured: 2–3 iterations at 128 tokens, `temperature=0`, `ignore_eos=true`. +- Prompt: "Count from 1 to 100." (short — isolates decode bandwidth). +- Per-token KV cost: ~20 KiB (10 full-attention layers × 2 KV heads × + 256 dim × bf16). Per-slot MoE cost: ~3 MiB. +- GPU budget: ~17.6 GiB after weights + fixed caches. + +## bs1 vs bs3 vs bs4 vs bs6 + +| KV pool | Residency | bs1 | bs3 agg | bs3/req | bs4 agg | bs4/req | bs6 agg | bs6/req | +|---------|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| 8,192 | ~89% | 56.7 | 108.8 | 36.3 | 150.6 | 37.7 | 132.1 | 22.0 | +| 65,536 | ~83% | ~52 | 106.1 | 35.4 | 148.0 | 37.0 | 131.8 | 22.0 | +| 131,072 | ~76% | ~48 | 99.9 | 33.3 | 144.4 | 36.1 | 131.8 | 22.0 | +| 262,144 | ~62% | ~48 | 85.3 | 28.4 | 131.3 | 32.8 | 130.5 | 21.8 | +| 400,000 | ~47% | ~42 | 73.3 | 24.4 | 116.5 | 29.1 | 161.1 | 26.9 | +| 524,288 | ~30% | 34.8 | 59.8 | 19.9 | 102.2 | 25.5 | 150.6 | 25.1 | +| 655,360 | ~19% | 30.4 | 42.8 | 14.3 | 86.9 | 21.7 | 131.5 | 21.9 | +| 700,000 | ~15% | 28.3 | 38.5 | 12.8 | 83.0 | 20.8 | 121.1 | 20.2 | + +## Key Findings + +### No cliff — gentle slope + +The LRU expert cache absorbs residency reductions gracefully. Even at +5% residency (544 slots), bs1 decode is 25.2 tok/s — 44% of baseline. + +### Free zone up to ~262k + +Decode speed is flat from 8k through 262k at bs1 and bs4. Giving up +expert residency (89% → 62%) costs nothing because the LRU working set +fits in fewer slots. + +### bs4 is the sweet spot + +bs4 dominates at every high-residency operating point. Per-request +speed at bs4 (37.7 tok/s at 8k) is *faster* than bs3 (36.3) because 4 +concurrent requests better fill the PCIe expert-streaming pipeline. + +### bs6 crossover at ~400k + +At high residency, bs6 aggregate (132) < bs4 aggregate (150) — the +bottleneck is GPU compute on attention layers, not PCIe. At low +residency (400k+), the bottleneck flips to PCIe expert streaming and +bs6 wins (161 vs 116 at 400k) because more concurrency fills the +pipeline. + +### bs3 is strictly worse than bs4 + +Fewer concurrent users leave PCIe idle gaps. At every point past 8k, +bs3 per-request is slower than bs4 per-request. Don't run bs3 — set +`--max-running-requests 4` regardless of actual user count. + +### 800k doesn't fit + +4 × 200k = 800k exceeds the GPU budget even at minimum MoE (256 slots). +The absolute ceiling is ~750k tokens. With radix prefix sharing, +effective capacity exceeds naive `pool ÷ concurrency`. + +## Comparison: old llama.cpp service + +The old Tiel-Coder llama.cpp service ran 200,700 total context across +3 parallel slots (66,900/slot) at ~99 tok/s aggregate at concurrency 4. +FreeToken at 131k KV (bs4) does 144 tok/s aggregate — 45% faster — with +the caveat that it serves base Qwen3.6-35B-A3B weights, not the +Tiel-Coder finetune. + +## See Also + +- [[freetoken-frank-default-small]] — production runbook +- [[dealign-ornith-crack]] — the prior llama.cpp deployment (retired) \ No newline at end of file diff --git a/docs/LLM/freetoken-frank-default-small.md b/docs/LLM/freetoken-frank-default-small.md new file mode 100644 index 0000000..ef6cfc8 --- /dev/null +++ b/docs/LLM/freetoken-frank-default-small.md @@ -0,0 +1,127 @@ +--- +type: configuration +title: FreeToken frank-default-small Runbook +description: FreeToken v0.1.2 serving Qwen3.6-35B-A3B-FP8 as frank-default-small on Dankserver (RTX 3090). +tags: + - freetoken + - dankserver + - frank-default-small + - qwen3.6 + - moe + - fp8 +timestamp: 2026-09-02 +--- + +# FreeToken frank-default-small Runbook + +The `frank-default-small` service on Dankserver is served by +**FreeToken v0.1.2** — an edge-native MoE serving engine that keeps +expert weights in host RAM with an LRU GPU cache. It replaced the +llama.cpp Tiel-Coder-35B-A3B (UD-Q6_K_XL) deployment on 2026-09-02. + +## Model + +| Field | Value | +|-------|-------| +| HuggingFace | `Qwen/Qwen3.6-35B-A3B-FP8` | +| Architecture | `qwen3_5_moe` (Qwen3_5MoeForConditionalGeneration) | +| Quant | modelopt block-fp8 (128×128 `weight_block_size`) | +| Size on disk | 35 GB | +| Path | `/home/dank/models/qwen36-35b-a3b-fp8` | +| Layers | 40 (10 full-attention + 30 GDN/linear-attention) | +| Experts | 256 per layer, 8 used per token | +| KV heads | 2 × 256 dim | +| Context | 262,144 (model max) | + +**Important:** This serves base Qwen3.6-35B-A3B weights, not the +Tiel-Coder finetune that was previously served via llama.cpp. + +## Runtime + +| Field | Value | +|-------|-------| +| Engine | FreeToken v0.1.2 (`ft` CLI) | +| Venv | `/home/dank/freetoken/.venv` (uv-managed CPython 3.12) | +| Dependencies | torch 2.11.0+cu130, flashinfer 0.6.17, apache-tvm-ffi | +| CUDA toolchain | pip-installed nvidia-cuda-nvcc/crt/nvvm/runtime 13.2.86 | +| API | OpenAI-compatible (`/v1/chat/completions`, `/v1/models`) | + +## Pool Configuration + +The GPU budget (~17.6 GiB after weights + fixed caches) is split +MoE-first between expert residency and KV pages: + +| Pool | Size | VRAM | Notes | +|------|------|------|-------| +| MoE cache | 2,048 slots (20%) | 6.0 GiB | LRU expert cache, out of 10,240 total | +| KV cache | 453,005 tokens | 8.6 GiB | ~20 KiB/token (10 attn layers only) | +| Mamba state | 24 slots | 1.4 GiB | Per-sequence GDN state (caps concurrency) | + +### Live Resize + +KV and MoE can be re-partitioned without restart: + +```bash +ft ctl --base-url http://127.0.0.1:7770 cache --kv N --moe M --wait 300 +``` + +- **Growing KV** (shrinking MoE): works live. +- **Shrinking KV** (growing MoE): requires restart (old + new pools + cannot coexist in VRAM during the swap → OOM). +- The fit check refuses overcommit: if the plan doesn't fit, the old + cache is retained and serving continues. + +### Benchmark Results (2026-09-02) + +See [[freetoken-benchmark-sweep]] for the full bs1/bs3/bs4/bs6 sweep +across KV pool sizes 8k–700k. + +At the production config (20% residency, bs4): + +| Metric | Value | +|--------|-------| +| bs1 decode | ~30 tok/s | +| bs4 aggregate | ~87 tok/s | +| bs4 per-request | ~22 tok/s | +| Prefill (4k) | ~1,130 tok/s | + +## Quantization Compatibility + +FreeToken v0.1.2 supports these safetensor quants for `qwen3_5_moe`: + +| Format | Detection | Works | +|--------|-----------|-------| +| BF16 (no quant config) | default | ✅ (experts too large for 60 GB RAM) | +| modelopt block-fp8 (128×128) | `quant_method: "fp8"` + `weight_block_size` | ✅ (current) | +| NVFP4 (modelopt) | `quant_algo: "NVFP4"` | ✅ (best for VRAM) | +| NVFP4 (compressed-tensors) | 4-bit float, group_size=16, strategy=tensor_group | ✅ | +| Per-tensor FP8 (modelopt) | scalar `weight_scale` | ✅ | +| Per-channel FP8 (compressed-tensors) | `weight_scale: [O, 1]` | ❌ loader bug | +| Quark W8A8 INT8 | per-channel scales | ❌ loader bug | +| GGUF | only `gemma4` arch | ❌ for qwen3_5_moe | + +## Files + +| File | Purpose | +|------|---------| +| `scripts/serve-frank-small.sh` | Serve script (launches `ft serve`) | +| `scripts/frank-default-small.env` | Tunables (model path, pool sizes) | +| `scripts/frank-default-small.service` | Systemd unit | + +## Limitations + +- **Model identity**: serves base Qwen3.6-35B-A3B, not Tiel-Coder finetune. +- **JIT compilation**: first startup compiles flashinfer/tvm-ffi kernels + (~5–15 min cold). Cached afterward in `~/.cache/flashinfer` and + `~/.cache/tvm-ffi`. +- **CUDA toolchain**: driver-only install (no system CUDA toolkit). The + venv provides pip-installed nvcc 13.2.86 for JIT compilation. All + nvidia-cu13 packages must be at the same version. +- **Env var collisions**: never name tunables `TEMP`, `HOST`, or + `GPU_MAX_HEAP_SIZE` (HIP runtime reads these). + +## Citations + +- FreeToken paper: arXiv 2608.16157 +- FreeToken PyPI: `freetoken[accel]` v0.1.2 +- HuggingFace model: `Qwen/Qwen3.6-35B-A3B-FP8` \ No newline at end of file diff --git a/docs/LLM/index.md b/docs/LLM/index.md index 24a86af..9aafb97 100644 --- a/docs/LLM/index.md +++ b/docs/LLM/index.md @@ -1,8 +1,8 @@ # LLM Fleet — Knowledge Base Documentation for the LLM inference fleet on two local machines -(2026-08-26): ik_llama.cpp on Dankserver (frank-default-small, -Ornith-1.5-35B-A3B CRACK) and ROCmFPX llama.cpp on Dankclaw +(2026-09-02): FreeToken v0.1.2 on Dankserver (frank-default-small, +Qwen3.6-35B-A3B-FP8) and ROCmFPX llama.cpp on Dankclaw (frank-default-large, kingjones Qwen3.8 Uncensored Q6 AGENT). Both services run the Qwen-Sharp chat template. @@ -16,11 +16,11 @@ services run the Qwen-Sharp chat template. ```mermaid graph LR subgraph Dankserver[NVIDIA Dankserver - Ubuntu 26.04] - A[RTX 3090 24GB + 62GB RAM] - B2[ik_llama.cpp build b166e269 + A[RTX 3090 24GB + 60GB RAM] + B2[FreeToken v0.1.2 frank-default-small, port 7770] - C[Ornith-1.5-35B-A3B-CRACK Q6_K -no speculation, q8_0 + Hadamard cache, 491520 ctx] + C[Qwen3.6-35B-A3B-FP8 +block-fp8, 20% MoE residency, 453k KV / 4 streams] A --> B2 --> C end @@ -46,6 +46,11 @@ MTP draft + Turbo4 V cache, 262K ctx / 1 slot] ## Active Deployment +- [[freetoken-frank-default-small]] — the active frank-default-small + runbook: FreeToken v0.1.2 serving Qwen3.6-35B-A3B-FP8 on Dankserver + (2026-09-02) +- [[freetoken-benchmark-sweep]] — KV pool vs decode speed sweep + (bs1/bs3/bs4/bs6, 8k–700k KV) on Dankserver RTX 3090 - [[deployment]] — deploy and update process for both servers - [[frank-redo-2026-08-17]] — the 2026-08-17 topology change - [kingjones Qwen3.8 Uncensored ROCmFP4 STRIX MTP](https://huggingface.co/kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF) — active frank-default-large model on Dankclaw; Q6 AGENT main model with standalone MTP @@ -54,9 +59,9 @@ MTP draft + Turbo4 V cache, 262K ctx / 1 slot] - [[kingjones-q8agent-rocmfpx]] — the retired Q8_0-AGENT investigation and `set_rows` failure - [[eemin-rocmi4-mtp]] — the former caf-eemin Qwen3.8-27B embedded-MTP model on Dankclaw (retired) -- [[dealign-ornith-crack]] — the dealign Ornith-1.5-35B-A3B CRACK model on - Dankserver (frank-default-small, active 2026-08-24) -- [[ikllama-ornith-dankserver]] — the ik_llama.cpp runtime for +- [[dealign-ornith-crack]] — the retired Ornith-1.5-35B-A3B CRACK model on + Dankserver (frank-default-small, retired 2026-09-02) +- [[ikllama-ornith-dankserver]] — the retired ik_llama.cpp runtime for frank-default-small (2026-08-26): q8_0 + Hadamard KV cache, split-mode none (single-GPU), mandatory FA_ALL_QUANTS build flags - [[kingjones-strix-mtp]] — the active kingjones Qwen3.8-27B ROCmFP4 STRIX MTP diff --git a/docs/index.md b/docs/index.md index e62de6b..f019947 100644 --- a/docs/index.md +++ b/docs/index.md @@ -105,13 +105,15 @@ The LLM inference fleet on the two hosts. Full index: - [[deployment]] — unified deploy and update process for both servers - [[frank-redo-2026-08-17]] — the 2026-08-17 topology change +- [[freetoken-frank-default-small]] — the active frank-default-small runbook: FreeToken v0.1.2 serving Qwen3.6-35B-A3B-FP8 on Dankserver (2026-09-02) +- [[freetoken-benchmark-sweep]] — KV pool vs decode speed sweep (bs1/bs3/bs4/bs6, 8k–700k KV) on Dankserver RTX 3090 - [[kingjones-strix-mtp]] — the active kingjones Qwen3.8-27B ROCmFP4 STRIX MTP model on Dankclaw (restored 2026-09-01) - [[kingjones-q8agent-rocmfpx]] — the retired Q8_0-AGENT investigation and `set_rows` failure - [[eemin-rocmi4-mtp]] — the former caf-eemin Qwen3.8-27B ROCmI4 embedded-MTP model on Dankclaw - [[ciru-activefpx-promptforge]] — the RETIRED v3 CIRU PromptForge runtime on Dankclaw (superseded 2026-08-24) - [[kingjones-strix-mtp]] — the kingjones Qwen3.8-27B ROCmFP4 STRIX MTP model on Dankclaw (active) -- [[dealign-ornith-crack]] — the dealign Ornith-1.5-35B-A3B CRACK model on Dankserver (active) -- [[ikllama-ornith-dankserver]] — the ik_llama.cpp runtime for frank-default-small (q8_0 + Hadamard KV cache, split-mode none; 2026-08-26) +- [[dealign-ornith-crack]] — the retired Ornith-1.5-35B-A3B CRACK model on Dankserver (retired 2026-09-02) +- [[ikllama-ornith-dankserver]] — the retired ik_llama.cpp runtime for frank-default-small (q8_0 + Hadamard KV cache, split-mode none; 2026-08-26) - [[kairic-edge]] — the retired Kairic Edge IU4 experiment (2026-08-22) - [[dflash2-large-vulkan]] — the retired DFlash2 Vulkan deployment (2026-08-21) - [[Dankserver System Specs]] — the RTX 3090 box hardware diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index d11f10c..46b96de 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -1,71 +1,51 @@ -# Tunables for frank-default-small (Tiel-Coder-35B-A3B UD-Q6_K_XL, -# qwen35moe hybrid MoE, Dankserver, RTX 3090). +# Tunables for frank-default-small (Qwen3.6-35B-A3B-FP8, +# qwen3_5_moe hybrid MoE, Dankserver, RTX 3090). # -# Runtime: upstream llama.cpp (latest master, CUDA build with -# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86) on RTX 3090 -# 24 GB; Ryzen 7 7700X; 62 GB RAM. Replaced the ik_llama.cpp Ornith -# CRACK Q8_0 deployment on 2026-09-01. +# Runtime: FreeToken v0.1.2 (edge-native MoE serving engine). +# Installed via uv into /home/dank/freetoken/.venv (CPython 3.12). +# FreeToken keeps expert weights in host RAM with an LRU GPU cache; +# KV is paged with radix prefix sharing. Only the 10 full-attention +# layers page KV (~20 KiB/token); the 30 GDN layers use constant-size +# per-sequence state (mamba slot pool). # -# Model: peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF, UD-Q6_K_XL quant -# (30 GB). Architecture: qwen35moe — 40 layers, full_attention_interval=4 -# (10 attention + 30 SSM/Mamba), 256 experts / 8 used, 2 KV heads × 256 -# dim, embedding 2048, rope dimension_count 64, rope freq_base 10 000 000. -# Text-only (no mmproj). Built-in chat template in the GGUF. +# Model: Qwen/Qwen3.6-35B-A3B-FP8 (HuggingFace safetensors, modelopt +# block-fp8 quant, 128x128 weight_block_size, 35 GB on disk). +# Architecture: qwen3_5_moe — 40 layers (10 attention + 30 GDN), +# 256 experts / 8 used, 2 KV heads × 256 dim, full_attention_interval=4. # -# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM. -# Non-expert weights (~1.6 GB) fit entirely on the 24 GB card. +# Replaced the llama.cpp Tiel-Coder-35B-A3B (UD-Q6_K_XL) deployment on +# 2026-09-02. The old serve script and env are in .AGENTTRASH/. # -# KV cache: Q8_0 for both K and V. With CTX_SIZE=200700 (66 900 per slot -# × 3 parallel) and 10 attention layers, the KV cache is ~2 GB — the -# requested 2 GB VRAM buffer. +# Pool sizing: GPU budget (~17.6 GiB after weights + fixed caches) is +# split MoE-first. MOE_CACHE_SIZE pins resident expert slots; NUM_TOKENS +# pins the KV pool. At 2048 slots / 453005 tokens the split is ~20% +# residency (2048/10240) and ~8.6 GiB KV. Live resize via +# `ft ctl cache --kv N --moe M` can grow KV by shrinking MoE without +# restart (the reverse requires a restart). # -# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The -# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP -# incident). This box is NVIDIA/CUDA, but the rule keeps the env portable. +# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. +# The ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP +# incident). This box is NVIDIA/CUDA, but the rule keeps the env portable. -# Model file (UD-Q6_K_XL, 30 GB, text-only MoE hybrid). -MODEL_DIR=/home/dank/models/tiel-coder-35b -MODEL_FILE=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf +# Model directory (safetensors checkpoint, 35 GB). +MODEL_DIR=/home/dank/models/qwen36-35b-a3b-fp8 -# Upstream llama.cpp llama-server (CUDA build, latest master). -# Built with -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86 -# using the staged CUDA 13.2 toolkit at /home/dank/cuda13-stage. -LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server +# FreeToken venv (uv-managed CPython 3.12, ft CLI, flashinfer, torch+cu130). +FT_VENV=/home/dank/freetoken/.venv +CUDA_HOME_DIR=/home/dank/freetoken/.venv/lib/python3.12/site-packages/nvidia/cu13 ALIAS=frank-default-small BIND_ADDR=0.0.0.0 LISTEN_PORT=7770 -# Total context budget across all parallel slots. 200700 / 3 = 66 900 -# per slot. The model's context_length is 262 144, so 66 900 is well -# within range. -CTX_SIZE=200700 -PARALLEL=3 +# Expert residency: 2048 of 10240 total expert instances (20%). +# Each slot ~3 MiB → ~6.0 GiB VRAM for the MoE cache. +MOE_CACHE_SIZE=2048 -# Expert offload: all MoE expert weights pinned to CPU RAM. -CPU_MOE=1 +# KV pool: 453005 tokens (~8.6 GiB at ~20 KiB/token). +# Covers 4 concurrent sessions at ~113k unique context each +# (more with radix prefix sharing for common prefixes). +NUM_TOKENS=453005 -# All non-expert layers on GPU (auto-fit will determine the exact count; -# with --cpu-moe the non-expert footprint is ~1.6 GB, leaving ample room). -N_GPU_LAYERS=999 - -# KV cache: Q8_0 for both K and V. ~2 GB total for 200700 ctx × 3 slots -# × 10 attention layers × (2 KV heads × 256 dim × 1 byte) × 2 (K+V). -CACHE_TYPE_K=q8_0 -CACHE_TYPE_V=q8_0 - -VERBOSITY=2 - -# Prefill batch sizing. -BATCH_SIZE=4096 -UBATCH_SIZE=1024 - -# Sampling. -TEMPERATURE=1.0 -TOP_K=20 -TOP_P=0.95 -MIN_P=0.0 - -# Compute threads lowered so other processes keep a core. -THREADS=7 -THREADS_HTTP=8 \ No newline at end of file +# Max concurrent decoding requests (also caps mamba slot allocation). +MAX_RUNNING_REQUESTS=4 \ No newline at end of file diff --git a/scripts/frank-default-small.service b/scripts/frank-default-small.service index 0805e2c..0a06079 100644 --- a/scripts/frank-default-small.service +++ b/scripts/frank-default-small.service @@ -1,5 +1,5 @@ [Unit] -Description=llama.cpp Tiel-Coder-35B-A3B Q6_K_XL qwen35moe hybrid MoE (dankserver, frank-default-small) +Description=FreeToken Qwen3.6-35B-A3B-FP8 qwen3_5_moe hybrid MoE (dankserver, frank-default-small) After=network-online.target Wants=network-online.target @@ -13,4 +13,4 @@ Restart=on-failure RestartSec=10 [Install] -WantedBy=multi-user.target +WantedBy=multi-user.target \ No newline at end of file diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 5b3b6e8..745066c 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -1,23 +1,25 @@ #!/usr/bin/env bash -# Serve peculiar-ragdoll/Tiel-Coder-35B-A3B (UD-Q6_K_XL) as -# frank-default-small via stock llama.cpp on Dankserver (RTX 3090). +# Serve Qwen/Qwen3.6-35B-A3B-FP8 as frank-default-small via FreeToken +# on Dankserver (RTX 3090, 60 GB RAM). # -# Runtime: upstream llama.cpp (latest master, CUDA build with -# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86). -# The model is a 35B-total / 8-active MoE hybrid (qwen35moe arch) with -# SSM/Mamba layers (full_attention_interval=4: 10 attention + 30 SSM -# layers out of 40). 256 experts, 8 used per token. +# Runtime: FreeToken v0.1.2 (edge-native MoE serving engine). +# - Experts live in host RAM; an LRU cache of expert slots is resident +# on the GPU. Misses are streamed over PCIe. +# - Paged KV cache (radix prefix sharing). Only the 10 full-attention +# layers page KV (~20 KiB/token); the 30 GDN/linear-attention layers +# hold constant-size per-sequence state (mamba slots). +# - OpenAI-compatible API (/v1/chat/completions, /v1/models). # -# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM. -# Non-expert weights (attention, SSM, shared expert, embeddings, output) -# are ~1.6 GB and fit entirely on the 24 GB RTX 3090. +# Model: Qwen/Qwen3.6-35B-A3B-FP8 (safetensors, modelopt block-fp8, +# 128x128 weight_block_size). Architecture: qwen3_5_moe — 40 layers, +# full_attention_interval=4 (10 attention + 30 GDN), 256 experts / 8 used, +# 2 KV heads × 256 dim. 35 GB on disk. # -# KV cache: Q8_0 for both K and V. With CTX_SIZE=200700 (66 900 per -# slot × 3 parallel) and 10 attention layers (2 KV heads × 256 dim), -# the KV cache is ~2 GB — the requested 2 GB VRAM buffer. -# -# Model is loaded with mmap (default). The 30 GB Q6_K_XL file is -# memory-mapped; only accessed pages enter RAM. +# Pool sizing: the GPU budget is split MoE-first between expert residency +# and KV pages. --moe-cache-size pins the number of resident expert +# slots; --num-tokens pins the KV pool capacity. Live resize is possible +# via `ft ctl cache --kv N --moe M` (grows KV by shrinking MoE; shrinking +# KV requires a restart due to the swap-during-rebuild memory cost). # # Tunables come from frank-default-small.env unless FRANK_SMALL_ENV_FILE # overrides that path. @@ -33,55 +35,39 @@ if [[ -f "$ENV_FILE" ]]; then fi # --- defaults (used when the env file does not set them) ----------------- -: "${MODEL_DIR:=/home/dank/models/tiel-coder-35b}" -: "${MODEL_FILE:=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf}" -: "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}" +: "${MODEL_DIR:=/home/dank/models/qwen36-35b-a3b-fp8}" : "${ALIAS:=frank-default-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7770}" -: "${CTX_SIZE:=200700}" -: "${PARALLEL:=3}" -: "${CACHE_TYPE_K:=q8_0}" -: "${CACHE_TYPE_V:=q8_0}" -: "${N_GPU_LAYERS:=999}" -: "${BATCH_SIZE:=4096}" -: "${UBATCH_SIZE:=1024}" -: "${VERBOSITY:=2}" -: "${CPU_MOE:=1}" -: "${TEMPERATURE:=1.0}" -: "${TOP_K:=20}" -: "${TOP_P:=0.95}" -: "${MIN_P:=0.0}" -: "${THREADS:=7}" -: "${THREADS_HTTP:=8}" +: "${FT_VENV:=/home/dank/freetoken/.venv}" +: "${CUDA_HOME_DIR:=${FT_VENV}/lib/python3.12/site-packages/nvidia/cu13}" -MODEL="$MODEL_DIR/$MODEL_FILE" +# Pool sizing (expert residency vs KV cache tradeoff). +: "${MOE_CACHE_SIZE:=2048}" +: "${NUM_TOKENS:=453005}" +: "${MAX_RUNNING_REQUESTS:=4}" -if [[ ! -f "$MODEL" ]]; then - echo "missing required file: $MODEL" >&2 +# Paths +FT_BIN="${FT_VENV}/bin/ft" +MODEL="${MODEL_DIR}" + +if [[ ! -x "$FT_BIN" ]]; then + echo "missing FreeToken CLI: $FT_BIN" >&2 exit 1 fi -if [[ ! -x "$LLAMA_SERVER" ]]; then - echo "missing llama-server binary: $LLAMA_SERVER" >&2 +if [[ ! -d "$MODEL" ]]; then + echo "missing model directory: $MODEL" >&2 exit 1 fi -exec "$LLAMA_SERVER" \ +export PATH="${FT_VENV}/bin:${PATH}" +export CUDA_HOME="${CUDA_HOME_DIR}" + +exec "$FT_BIN" serve \ --model "$MODEL" \ - $([ "$CPU_MOE" = "1" ] && printf -- "--cpu-moe") \ - --cache-type-k "$CACHE_TYPE_K" \ - --cache-type-v "$CACHE_TYPE_V" \ - --ctx-size "$CTX_SIZE" \ - --parallel "$PARALLEL" \ - --gpu-layers "$N_GPU_LAYERS" \ - --batch-size "$BATCH_SIZE" \ - --ubatch-size "$UBATCH_SIZE" \ - --temp "$TEMPERATURE" \ - --top-k "$TOP_K" \ - --top-p "$TOP_P" \ - --min-p "$MIN_P" \ - --threads "$THREADS" \ - --threads-http "$THREADS_HTTP" \ - --alias "$ALIAS" \ + --served-model-name "$ALIAS" \ --host "$BIND_ADDR" \ - --port "$LISTEN_PORT" \ No newline at end of file + --port "$LISTEN_PORT" \ + --moe-cache-size "$MOE_CACHE_SIZE" \ + --num-tokens "$NUM_TOKENS" \ + --max-running-requests "$MAX_RUNNING_REQUESTS" \ No newline at end of file