28 lines
1.4 KiB
Bash
28 lines
1.4 KiB
Bash
# Tunables for frank-default-small (FastFlowLM NPU endpoint, dankclaw)
|
|
# Runtime: FLM 0.9.45 on the XDNA2 NPU (50 TOPS). Model choice, measured
|
|
# results, and the ~15 GiB NPU budget are in docs/FastFlowLM NPU Inference.md.
|
|
# The NPU allocates from the unified 30 GiB pool shared with
|
|
# frank-default-large (the PP head). Keep the resident footprint small
|
|
# enough that the head keeps its headroom.
|
|
#
|
|
# 2026-08-16: gemma4-it:e4b chosen as the default after the 4-model bakeoff
|
|
# (12.2 t/s decode, 582 t/s 8k prefill, 4/4 vision, 32k ctx, small
|
|
# footprint). Swap FLM_MODEL for qwen3.5:9b, gpt-oss:20b, or
|
|
# qwen3.6-moe:35b-a3b to change the served model. gpt-oss:20b is capped at
|
|
# 8k context and has no vision; 35b-a3b needs ~29 GiB resident (do not run
|
|
# alongside the PP head at full load).
|
|
|
|
FLM_MODEL=gemma4-it:e4b
|
|
FLM_HOST=0.0.0.0
|
|
FLM_PORT=52625
|
|
# Power mode: turbo. Measured decode is bandwidth-bound (turbo ~= performance
|
|
# for this model), but the user requested turbo; it may help prefill.
|
|
FLM_PMODE=turbo
|
|
# Load the ASR model (Whisper-V3-Turbo-NPU2) alongside Gemma4. Exposes
|
|
# /v1/audio/transcriptions. Adds ~1 GiB to the resident footprint.
|
|
FLM_ASR=1
|
|
# Maximum context length. Gemma4 natively supports 128K; the NPU budget
|
|
# (~15 GiB minus the 9.1 GiB footprint) fits 131072. Verified 2026-08-16:
|
|
# loads, KV cache allocates to ~115k, decode ~12 t/s, 13 GiB headroom.
|
|
FLM_CTX_LEN=131072
|