34 lines
1.7 KiB
Bash
34 lines
1.7 KiB
Bash
# Tunables for frank-default-small (FastFlowLM NPU endpoint, dankclaw)
|
|
# Runtime: FLM 1.0.1 on the XDNA2 NPU (50 TOPS). Model choice, measured
|
|
# results, and the NPU memory budget are in docs/FastFlowLM NPU Inference.md.
|
|
# The NPU allocates from the unified pool shared with frank-default-large
|
|
# (the PP head). Keep the resident footprint small enough that the head
|
|
# keeps its headroom.
|
|
#
|
|
# 2026-08-17: qwythos:9b (Huihui-Qwythos-9B-Claude-Mythos-5-1M-abliterated,
|
|
# Qwen3.5-9B hybrid linear-attention base, Q4NX, 7.76 GB) chosen after the
|
|
# user's model swap (was qwen3.6-moe:35b-a3b, which stays installed for
|
|
# rollback). FLM upgraded 0.9.45 -> 1.0.1: the pack declares flm_version
|
|
# 1.0.0 and 0.9.45 refused to serve it.
|
|
|
|
FLM_MODEL=qwythos:9b
|
|
FLM_HOST=0.0.0.0
|
|
FLM_PORT=52625
|
|
# Power mode: turbo. Smoke decode ~5-7 t/s on the 9B dense at 262k ctx
|
|
# (not benchmarked; see docs/FastFlowLM NPU Inference.md).
|
|
FLM_PMODE=turbo
|
|
# Load the ASR model (Whisper-V3-Turbo-NPU2) alongside the LLM. Exposes
|
|
# /v1/audio/transcriptions. Adds ~1 GiB to the resident footprint.
|
|
FLM_ASR=1
|
|
# Maximum context length. Qwythos-9B native max is 262144 (Qwen3.5-9B
|
|
# hybrid: 8 of 32 layers are full attention, so the KV cache stays small).
|
|
# Verified 2026-08-17: loads at -c 262144 with turbo+asr on FLM 1.0.1;
|
|
# KV occupancy 0.007% at 19 tokens.
|
|
FLM_CTX_LEN=262144
|
|
# Custom-model resolution: the qwythos:9b tag and its xclbin symlinks live
|
|
# in the user config, not the system registry. Without these, flm treats
|
|
# the tag as unknown and silently serves llama3.2:1b (see the unknown-tag
|
|
# fallback in docs/FastFlowLM NPU Inference.md).
|
|
FLM_CONFIG_PATH=/home/dank/.config/flm/model_list.json
|
|
FLM_XCLBIN_PATH=/home/dank/.config/flm
|