From df0c66981542fd0e79e81e4eb15c4aea50cc3070 Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 3 Sep 2026 17:37:41 -0700 Subject: [PATCH] feat(npu): boot nuextract3:4b as frank-npu's resident model Boot tag flips from qwen3.8-distilled-heretic:9b to nuextract3:4b (verified live: per-request routing already serves every registered tag; the boot tag only decides which model is resident at start, so the first request after a swap pays the load cost). FLM_CTX_LEN=-1 drops the global --ctx-len override so each pack loads with its own trained context (nuextract3 32768, gemma4 131072). --- scripts/frank-npu.env | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/scripts/frank-npu.env b/scripts/frank-npu.env index 244d903..8f531b5 100644 --- a/scripts/frank-npu.env +++ b/scripts/frank-npu.env @@ -1,13 +1,13 @@ -# Tunables for frank-npu (FastFlowLM NPU sight+sound endpoint, Dankclaw) -# Runtime: FLM 1.0.1 (/usr/bin/flm) on the XDNA2 NPU (50 TOPS). Model: -# gemma4-it:e4b = Gemma4-E4B-IT-NPU2 (8B total / ~4.5B effective params, -# Q4_1; vision + audio + reasoning; ~9.1 GB footprint). It is the only NPU2 -# model family with BOTH an image and an audio tower — the uncensored -# candidate qwythos:9b is text-only and fails the audio requirement, so this -# endpoint trades "uncensored" for "sight + sound". The NPU draws from the -# unified RAM pool shared with frank-large (iGPU); ~31 GiB was -# available there at idle, so a 9.1 GB pack fits with headroom. -FLM_MODEL=qwen3.8-distilled-heretic:9b +# Tunables for frank-npu (FastFlowLM NPU endpoint, Dankclaw) +# Runtime: FLM 1.0.1 (/usr/bin/flm) on the XDNA2 NPU (50 TOPS). Boot tag is +# nuextract3:4b (NuExtract3-4B-NPU2, Qwen3.5-4B skeleton, structured-extraction +# model); serve routes every request by the exact FLM registry tag in the +# request's model field, so any registered tag (gemma4-it:e4b, qwythos:9b, ...) +# is reachable on this port regardless of the boot tag. The NPU is a +# single-lane device: generations serialize FIFO, and the first request after a +# model swap pays the ~6-11 s load cost. The NPU draws from the unified RAM +# pool shared with frank-large (iGPU). +FLM_MODEL=nuextract3:4b FLM_HOST=0.0.0.0 FLM_PORT=7771 # turbo = max NPU power mode (xrt-smi configure --pmode turbo at serve start). @@ -15,9 +15,9 @@ FLM_PORT=7771 # neighbours; drop to `performance` if iGPU power-budget contention ever shows. FLM_PMODE=turbo # FLM_ASR=1 additionally loads Whisper-V3-Turbo-NPU2 (+0.62 GB) and exposes -# /v1/audio/transcriptions. gemma4-it:e4b answers audio natively through its -# own audio tower in /v1/chat/completions, so ASR stays off by default. +# /v1/audio/transcriptions. NuExtract3 answers text-only; audio stays off. FLM_ASR=0 -# 131072 = verified max for gemma4-it:e4b on this box (native 128K context; -# the registry's default_context_length 32768 is only the fallback default). -FLM_CTX_LEN=131072 +# -1 = no --ctx-len flag: every pack loads with its own trained context +# (nuextract3:4b -> 32768, gemma4-it:e4b -> 131072). A global override would +# over-allocate KV for the small packs. +FLM_CTX_LEN=-1