diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 0f9b968..e09d392 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -1,59 +1,18 @@ -# Tunables for frank-default-small (HauhauCS Q6_K_P, ngram-mod spec, dankserver) -# Runtime: llama-server b10108; RTX 3090 24GB; Ryzen 7 7700X; 62GB RAM. -MODEL_DIR=/home/dank/models -ALIAS=frank-default-small -HOST=0.0.0.0 -PORT=7770 -# 4 parallel slots x 175104 ctx each = 700416 total context pool. -# (2026-08-14: refitted from 800000 when the KV cache went dual-f16; the -# f16/f16 pool costs 15640 MiB at 800768 cells and did not fit the 3090's -# 24 GiB with the image-warmup graph. 700416 cells leaves ~2.3 GiB free.) -CTX_SIZE=700000 -PARALLEL=4 -# Spec: ngram-mod. n_min=48, n_max=64, n_match=24 (defaults). 4 expert -# layers back on GPU; 36 on CPU. Batch 8192 / ubatch 3072. -SPEC_TYPE=ngram-mod -NGRAM_MOD_N_MIN=48 -NGRAM_MOD_N_MAX=64 -NGRAM_MOD_N_MATCH=24 -# Expert offload: 36 of 40 layers keep MoE weights on CPU; remaining -# 4 layers have their expert weights resident on GPU for faster routing. -# Chosen for prefill-max stability at ubatch 3072: frees ~0.75 GiB peak -# VRAM vs ncmoe 35 (grid: peak 8606 vs 9360 MiB) for -2% prefill, -2% -# decode. Fixes the cuBLAS OOM crash class during big prefills. -N_CPU_MOE=36 -N_GPU_LAYERS=all -# KV cache on GPU, both K and V at f16 (2026-08-14 diagnostic). -# F16 K removes the flash-attention F16-conversion workspace (the MMA_F16 -# prefill kernel needs no copy when K is already f16), trading static pool -# size for the context-proportional workspace growth that caused the -# cublasSgemm_v2 crashes. Run with VERBOSITY=4 so the load log shows the -# real allocation sizes; refit CTX_SIZE if it does not fit. -CACHE_TYPE_K=f16 -CACHE_TYPE_V=f16 -# Diagnostic verbosity for this run; logs the KV/graph allocation sizes. -VERBOSITY=4 -# Store SWA checkpoints in RAM to hold peak GPU memory down for the -# larger batch/ubatch (leaves headroom for ctx-checkpoints operation). -CTX_CHECKPOINTS=4 -# Larger batches / ubatch for big prefill workloads. -# ubatch 3072 (benchmark-chosen): prefill is the priority; this is the top -# prefill tier (~1458 t/s) with ~1.2 GiB headroom. Decode is independent. -BATCH_SIZE=8192 -UBATCH_SIZE=3072 -# Load the model into RAM rather than lazy-mapping it. The MoE experts live on -# CPU, so eager RAM load avoids page faults during decode (uses ~31GB RAM). -NO_MMAP=1 -# Compute threads lowered to 7 so other processes on the box keep a core. -# Sampling settings — per HauhauCS Qwen3.6-35B-A3B model card. -# Thinking-general: temp=1.0, top_p=0.95, top_k=20, min_p=0. -# Clients may override per-request; these are the server-side defaults. -# WARNING: do not name this variable TEMP. The ROCm HIP runtime reads the -# TEMP environment variable (JIT kernel-cache dir); TEMP=1.0 made the HIP -# runtime SEGV (NULL unref) during model load on Aug 13 2026. -TEMPERATURE=1.0 -TOP_K=20 -TOP_P=0.95 -MIN_P=0.0 -THREADS=7 -THREADS_HTTP=8 \ No newline at end of file +# Tunables for frank-default-small (FastFlowLM NPU endpoint, dankclaw) +# Runtime: FLM 0.9.45 on the XDNA2 NPU (50 TOPS). Model choice, measured +# results, and the ~15 GiB NPU budget are in docs/FastFlowLM NPU Inference.md. +# The NPU allocates from the unified 30 GiB pool shared with +# frank-default-large (the PP head). Keep the resident footprint small +# enough that the head keeps its headroom. +# +# 2026-08-16: gemma4-it:e4b chosen as the default after the 4-model bakeoff +# (12.2 t/s decode, 582 t/s 8k prefill, 4/4 vision, 32k ctx, small +# footprint). Swap FLM_MODEL for qwen3.5:9b, gpt-oss:20b, or +# qwen3.6-moe:35b-a3b to change the served model. gpt-oss:20b is capped at +# 8k context and has no vision; 35b-a3b needs ~29 GiB resident (do not run +# alongside the PP head at full load). + +FLM_MODEL=gemma4-it:e4b +FLM_HOST=0.0.0.0 +FLM_PORT=52625 +FLM_PMODE=performance diff --git a/scripts/frank-default-small.service b/scripts/frank-default-small.service index 770ebb4..cf8b12e 100644 --- a/scripts/frank-default-small.service +++ b/scripts/frank-default-small.service @@ -1,14 +1,17 @@ [Unit] -Description=llama.cpp DFlash inference (frank-default-small) -After=network.target +Description=FastFlowLM NPU inference (frank-default-small) on Dankclaw +After=network-online.target +Wants=network-online.target [Service] Type=simple User=dank Group=dank -ExecStart=/home/dank/vLLM/scripts/serve-llama-frank-small.sh +Environment="FLM_SMALL_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-small.env" +ExecStart=/home/dank/vllmScripts/scripts/serve-flm-small.sh +LimitMEMLOCK=infinity Restart=on-failure RestartSec=10 [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target diff --git a/scripts/serve-flm-small.sh b/scripts/serve-flm-small.sh new file mode 100644 index 0000000..5464a9e --- /dev/null +++ b/scripts/serve-flm-small.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Serve the FastFlowLM (FLM) NPU endpoint on Dankclaw — frank-default-small. +# +# Runs flm serve with the model tag in frank-default-small.env. The NPU +# (XDNA2, 50 TOPS) is a single-lane device: generations serialize in FIFO +# order, so this is a background/triage endpoint, not an interactive +# multi-user chat backend. The NPU shares the 30 GiB unified pool with the +# frank-default-large PP head; see docs/FastFlowLM NPU Inference.md for the +# memory budget and the measured model matrix. +# +# NOTE: FLM requires unlimited memlock. The unit sets LimitMEMLOCK=infinity. +# Running this script by hand from a login shell needs the pam_limits rule +# in /etc/security/limits.d/99-flm-memlock.conf. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FLM_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}" +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +# --- defaults (used when the env file does not set them) ------------------- +: "${FLM_BIN:=/usr/bin/flm}" +: "${FLM_MODEL:=gemma4-it:e4b}" +: "${FLM_HOST:=0.0.0.0}" +: "${FLM_PORT:=52625}" +: "${FLM_PMODE:=performance}" + +if [[ ! -x "$FLM_BIN" ]]; then + echo "missing flm binary: $FLM_BIN" >&2 + exit 1 +fi + +exec "$FLM_BIN" serve "$FLM_MODEL" \ + --host "$FLM_HOST" \ + -p "$FLM_PORT" \ + --pmode "$FLM_PMODE"