28 lines
1.4 KiB
Bash
28 lines
1.4 KiB
Bash
# Tunables for frank-default-small (FastFlowLM NPU endpoint, dankclaw)
|
|
# Runtime: FLM 0.9.45 on the XDNA2 NPU (50 TOPS). Model choice, measured
|
|
# results, and the ~15 GiB NPU budget are in docs/FastFlowLM NPU Inference.md.
|
|
# The NPU allocates from the unified 30 GiB pool shared with
|
|
# frank-default-large (the PP head). Keep the resident footprint small
|
|
# enough that the head keeps its headroom.
|
|
#
|
|
# 2026-08-16: qwen3.6-moe:35b-a3b chosen after the 4-model bakeoff and the
|
|
# user's model swap (was gemma4-it:e4b). Swap FLM_MODEL back to gemma4-it:e4b
|
|
# to restore the lighter footprint. gpt-oss:20b is capped at 8k context and
|
|
# has no vision.
|
|
|
|
FLM_MODEL=qwen3.6-moe:35b-a3b
|
|
FLM_HOST=0.0.0.0
|
|
FLM_PORT=52625
|
|
# Power mode: turbo. Measured decode ~13.1 t/s on the 35B-A3B at 262k ctx.
|
|
FLM_PMODE=turbo
|
|
# Load the ASR model (Whisper-V3-Turbo-NPU2) alongside the LLM. Exposes
|
|
# /v1/audio/transcriptions. Adds ~1 GiB to the resident footprint.
|
|
FLM_ASR=1
|
|
# Maximum context length. Qwen3.6-35B-A3B native max is 262144 (extensible
|
|
# to ~1M via RoPE/YaRN, which FLM does not serve). Verified 2026-08-16:
|
|
# loads at -c 262144 with turbo+asr; KV occupancy 0.0687% at ~180 tokens;
|
|
# decode 13.1 t/s; PP head unaffected on 7770. The box runs at ~70 MiB free
|
|
# RAM while this loads (footprint 24.3 GiB + Whisper) — the user plans a
|
|
# BIOS RAM/VRAM reallocation for headroom.
|
|
FLM_CTX_LEN=262144
|