promote NPU endpoint to frank-default-small: FLM serve unit + wrapper + env (gemma4-it:e4b default), LimitMEMLOCK=infinity (2026-08-16)

2026-08-16 14:19:37 -07:00
parent 48142f5d8e
commit f6737f234d
3 changed files with 65 additions and 63 deletions
@@ -1,59 +1,18 @@
-# Tunables for frank-default-small (HauhauCS Q6_K_P, ngram-mod spec, dankserver)
-# Runtime: llama-server b10108; RTX 3090 24GB; Ryzen 7 7700X; 62GB RAM.
-MODEL_DIR=/home/dank/models
-ALIAS=frank-default-small
-HOST=0.0.0.0
-PORT=7770
-# 4 parallel slots x 175104 ctx each = 700416 total context pool.
-# (2026-08-14: refitted from 800000 when the KV cache went dual-f16; the
-#  f16/f16 pool costs 15640 MiB at 800768 cells and did not fit the 3090's
-#  24 GiB with the image-warmup graph. 700416 cells leaves ~2.3 GiB free.)
-CTX_SIZE=700000
-PARALLEL=4
-# Spec: ngram-mod. n_min=48, n_max=64, n_match=24 (defaults). 4 expert
-# layers back on GPU; 36 on CPU. Batch 8192 / ubatch 3072.
-SPEC_TYPE=ngram-mod
-NGRAM_MOD_N_MIN=48
-NGRAM_MOD_N_MAX=64
-NGRAM_MOD_N_MATCH=24
-# Expert offload: 36 of 40 layers keep MoE weights on CPU; remaining
-# 4 layers have their expert weights resident on GPU for faster routing.
-# Chosen for prefill-max stability at ubatch 3072: frees ~0.75 GiB peak
-# VRAM vs ncmoe 35 (grid: peak 8606 vs 9360 MiB) for -2% prefill, -2%
-# decode. Fixes the cuBLAS OOM crash class during big prefills.
-N_CPU_MOE=36
-N_GPU_LAYERS=all
-# KV cache on GPU, both K and V at f16 (2026-08-14 diagnostic).
-# F16 K removes the flash-attention F16-conversion workspace (the MMA_F16
-# prefill kernel needs no copy when K is already f16), trading static pool
-# size for the context-proportional workspace growth that caused the
-# cublasSgemm_v2 crashes. Run with VERBOSITY=4 so the load log shows the
-# real allocation sizes; refit CTX_SIZE if it does not fit.
-CACHE_TYPE_K=f16
-CACHE_TYPE_V=f16
-# Diagnostic verbosity for this run; logs the KV/graph allocation sizes.
-VERBOSITY=4
-# Store SWA checkpoints in RAM to hold peak GPU memory down for the
-# larger batch/ubatch (leaves headroom for ctx-checkpoints operation).
-CTX_CHECKPOINTS=4
-# Larger batches / ubatch for big prefill workloads.
-# ubatch 3072 (benchmark-chosen): prefill is the priority; this is the top
-# prefill tier (~1458 t/s) with ~1.2 GiB headroom. Decode is independent.
-BATCH_SIZE=8192
-UBATCH_SIZE=3072
-# Load the model into RAM rather than lazy-mapping it. The MoE experts live on
-# CPU, so eager RAM load avoids page faults during decode (uses ~31GB RAM).
-NO_MMAP=1
-# Compute threads lowered to 7 so other processes on the box keep a core.
-# Sampling settings — per HauhauCS Qwen3.6-35B-A3B model card.
-# Thinking-general: temp=1.0, top_p=0.95, top_k=20, min_p=0.
-# Clients may override per-request; these are the server-side defaults.
-# WARNING: do not name this variable TEMP. The ROCm HIP runtime reads the
-# TEMP environment variable (JIT kernel-cache dir); TEMP=1.0 made the HIP
-# runtime SEGV (NULL unref) during model load on Aug 13 2026.
-TEMPERATURE=1.0
-TOP_K=20
-TOP_P=0.95
-MIN_P=0.0
-THREADS=7
-THREADS_HTTP=8
+# Tunables for frank-default-small (FastFlowLM NPU endpoint, dankclaw)
+# Runtime: FLM 0.9.45 on the XDNA2 NPU (50 TOPS). Model choice, measured
+# results, and the ~15 GiB NPU budget are in docs/FastFlowLM NPU Inference.md.
+# The NPU allocates from the unified 30 GiB pool shared with
+# frank-default-large (the PP head). Keep the resident footprint small
+# enough that the head keeps its headroom.
+#
+# 2026-08-16: gemma4-it:e4b chosen as the default after the 4-model bakeoff
+# (12.2 t/s decode, 582 t/s 8k prefill, 4/4 vision, 32k ctx, small
+# footprint). Swap FLM_MODEL for qwen3.5:9b, gpt-oss:20b, or
+# qwen3.6-moe:35b-a3b to change the served model. gpt-oss:20b is capped at
+# 8k context and has no vision; 35b-a3b needs ~29 GiB resident (do not run
+# alongside the PP head at full load).
+
+FLM_MODEL=gemma4-it:e4b
+FLM_HOST=0.0.0.0
+FLM_PORT=52625
+FLM_PMODE=performance
@@ -1,14 +1,17 @@
 [Unit]
-Description=llama.cpp DFlash inference (frank-default-small)
-After=network.target
+Description=FastFlowLM NPU inference (frank-default-small) on Dankclaw
+After=network-online.target
+Wants=network-online.target

 [Service]
 Type=simple
 User=dank
 Group=dank
-ExecStart=/home/dank/vLLM/scripts/serve-llama-frank-small.sh
+Environment="FLM_SMALL_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-small.env"
+ExecStart=/home/dank/vllmScripts/scripts/serve-flm-small.sh
+LimitMEMLOCK=infinity
 Restart=on-failure
 RestartSec=10

 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Serve the FastFlowLM (FLM) NPU endpoint on Dankclaw — frank-default-small.
+#
+# Runs flm serve with the model tag in frank-default-small.env. The NPU
+# (XDNA2, 50 TOPS) is a single-lane device: generations serialize in FIFO
+# order, so this is a background/triage endpoint, not an interactive
+# multi-user chat backend. The NPU shares the 30 GiB unified pool with the
+# frank-default-large PP head; see docs/FastFlowLM NPU Inference.md for the
+# memory budget and the measured model matrix.
+#
+# NOTE: FLM requires unlimited memlock. The unit sets LimitMEMLOCK=infinity.
+# Running this script by hand from a login shell needs the pam_limits rule
+# in /etc/security/limits.d/99-flm-memlock.conf.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FLM_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -------------------
+: "${FLM_BIN:=/usr/bin/flm}"
+: "${FLM_MODEL:=gemma4-it:e4b}"
+: "${FLM_HOST:=0.0.0.0}"
+: "${FLM_PORT:=52625}"
+: "${FLM_PMODE:=performance}"
+
+if [[ ! -x "$FLM_BIN" ]]; then
+    echo "missing flm binary: $FLM_BIN" >&2
+    exit 1
+fi
+
+exec "$FLM_BIN" serve "$FLM_MODEL" \
+    --host "$FLM_HOST" \
+    -p "$FLM_PORT" \
+    --pmode "$FLM_PMODE"