diff --git a/scripts/frank-default-npu.env b/scripts/frank-default-npu.env new file mode 100644 index 0000000..d1c33fe --- /dev/null +++ b/scripts/frank-default-npu.env @@ -0,0 +1,17 @@ +# Tunables for frank-default-npu (FastFlowLM NPU endpoint, Dankclaw) +# Runtime: FLM 1.0.1 on the XDNA2 NPU (50 TOPS). Model: Gemma4-E4B-IT +# (8B, Q4_1, vision + audio + reasoning). The NPU allocates from the +# unified pool shared with frank-default-large (the CIRU llama.cpp +# service on the iGPU). See docs/FastFlowLM NPU Inference.md. +# +# 2026-08-20: gemma4-it:e4b deployed. Footprint ~9.1 GB. 64k context. + +FLM_MODEL=gemma4-it:e4b +FLM_HOST=0.0.0.0 +FLM_PORT=52626 +FLM_PMODE=performance +FLM_ASR=0 +# 64K context (native max for gemma4-it is 32768, but NPU supports up to 256k). +# The model's max_prefill_len is 4096, so prompts longer than that get +# truncated at prefill time. +FLM_CTX_LEN=65536 diff --git a/scripts/frank-default-npu.service b/scripts/frank-default-npu.service new file mode 100644 index 0000000..1dadeb6 --- /dev/null +++ b/scripts/frank-default-npu.service @@ -0,0 +1,17 @@ +[Unit] +Description=FastFlowLM NPU inference (frank-default-npu) on Dankclaw +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=dank +Group=dank +Environment="FLM_NPU_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-npu.env" +ExecStart=/home/dank/vllmScripts/scripts/serve-frank-default-npu.sh +LimitMEMLOCK=infinity +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/serve-frank-default-npu.sh b/scripts/serve-frank-default-npu.sh new file mode 100755 index 0000000..3322c18 --- /dev/null +++ b/scripts/serve-frank-default-npu.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Serve the FastFlowLM (FLM) NPU endpoint — frank-default-npu. +# +# Runs flm serve with the model tag in frank-default-npu.env. The NPU +# (XDNA2, 50 TOPS) is a single-lane device: generations serialize in FIFO +# order, so this is a background/triage endpoint. The NPU shares the ~30 GiB +# unified pool with frank-default-large (the CIRU llama.cpp service on the +# iGPU); see docs/FastFlowLM NPU Inference.md for the memory budget. +# +# NOTE: FLM requires unlimited memlock. The unit sets LimitMEMLOCK=infinity. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FLM_NPU_ENV_FILE:-$SCRIPT_DIR/frank-default-npu.env}" +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +# --- defaults (used when the env file does not set them) ------------------- +: "${FLM_BIN:=/usr/bin/flm}" +: "${FLM_MODEL:=gemma4-it:e4b}" +: "${FLM_HOST:=0.0.0.0}" +: "${FLM_PORT:=52626}" +: "${FLM_PMODE:=performance}" +: "${FLM_ASR:=0}" +: "${FLM_CTX_LEN:=-1}" + +if [[ ! -x "$FLM_BIN" ]]; then + echo "missing flm binary: $FLM_BIN" >&2 + exit 1 +fi + +# --- optional flags ---------------------------------------------------------- +OPT_ARGS=() +if [[ "${FLM_ASR}" != "0" ]]; then + OPT_ARGS+=(-a "${FLM_ASR}") +fi +if [[ "${FLM_CTX_LEN}" != "-1" ]]; then + OPT_ARGS+=(-c "${FLM_CTX_LEN}") +fi + +exec "$FLM_BIN" serve "$FLM_MODEL" \ + --host "$FLM_HOST" \ + -p "$FLM_PORT" \ + --pmode "$FLM_PMODE" \ + "${OPT_ARGS[@]}"