24 lines
1.4 KiB
Bash
24 lines
1.4 KiB
Bash
# Tunables for frank-default-sightsound (FastFlowLM NPU sight+sound endpoint, Dankclaw)
|
|
# Runtime: FLM 1.0.1 (/usr/bin/flm) on the XDNA2 NPU (50 TOPS). Model:
|
|
# gemma4-it:e4b = Gemma4-E4B-IT-NPU2 (8B total / ~4.5B effective params,
|
|
# Q4_1; vision + audio + reasoning; ~9.1 GB footprint). It is the only NPU2
|
|
# model family with BOTH an image and an audio tower — the uncensored
|
|
# candidate qwythos:9b is text-only and fails the audio requirement, so this
|
|
# endpoint trades "uncensored" for "sight + sound". The NPU draws from the
|
|
# unified RAM pool shared with frank-default-large (iGPU); ~31 GiB was
|
|
# available there at idle, so a 9.1 GB pack fits with headroom.
|
|
FLM_MODEL=qwen3.8-distilled-heretic:9b
|
|
FLM_HOST=0.0.0.0
|
|
FLM_PORT=7771
|
|
# turbo = max NPU power mode (xrt-smi configure --pmode turbo at serve start).
|
|
# No other tenant uses the NPU on this host, so the top mode costs nothing to
|
|
# neighbours; drop to `performance` if iGPU power-budget contention ever shows.
|
|
FLM_PMODE=turbo
|
|
# FLM_ASR=1 additionally loads Whisper-V3-Turbo-NPU2 (+0.62 GB) and exposes
|
|
# /v1/audio/transcriptions. gemma4-it:e4b answers audio natively through its
|
|
# own audio tower in /v1/chat/completions, so ASR stays off by default.
|
|
FLM_ASR=0
|
|
# 131072 = verified max for gemma4-it:e4b on this box (native 128K context;
|
|
# the registry's default_context_length 32768 is only the fallback default).
|
|
FLM_CTX_LEN=131072
|