frank-default-npu: deploy gemma4-it:e4b on NPU, port 52626, 64k context

2026-08-20 15:12:58 -07:00
parent b38e384f6e
commit ed9e1cabfd
3 changed files with 83 additions and 0 deletions
@@ -0,0 +1,17 @@
+# Tunables for frank-default-npu (FastFlowLM NPU endpoint, Dankclaw)
+# Runtime: FLM 1.0.1 on the XDNA2 NPU (50 TOPS). Model: Gemma4-E4B-IT
+# (8B, Q4_1, vision + audio + reasoning). The NPU allocates from the
+# unified pool shared with frank-default-large (the CIRU llama.cpp
+# service on the iGPU). See docs/FastFlowLM NPU Inference.md.
+#
+# 2026-08-20: gemma4-it:e4b deployed. Footprint ~9.1 GB. 64k context.
+
+FLM_MODEL=gemma4-it:e4b
+FLM_HOST=0.0.0.0
+FLM_PORT=52626
+FLM_PMODE=performance
+FLM_ASR=0
+# 64K context (native max for gemma4-it is 32768, but NPU supports up to 256k).
+# The model's max_prefill_len is 4096, so prompts longer than that get
+# truncated at prefill time.
+FLM_CTX_LEN=65536
@@ -0,0 +1,17 @@
+[Unit]
+Description=FastFlowLM NPU inference (frank-default-npu) on Dankclaw
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=dank
+Group=dank
+Environment="FLM_NPU_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-npu.env"
+ExecStart=/home/dank/vllmScripts/scripts/serve-frank-default-npu.sh
+LimitMEMLOCK=infinity
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Serve the FastFlowLM (FLM) NPU endpoint — frank-default-npu.
+#
+# Runs flm serve with the model tag in frank-default-npu.env. The NPU
+# (XDNA2, 50 TOPS) is a single-lane device: generations serialize in FIFO
+# order, so this is a background/triage endpoint. The NPU shares the ~30 GiB
+# unified pool with frank-default-large (the CIRU llama.cpp service on the
+# iGPU); see docs/FastFlowLM NPU Inference.md for the memory budget.
+#
+# NOTE: FLM requires unlimited memlock. The unit sets LimitMEMLOCK=infinity.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FLM_NPU_ENV_FILE:-$SCRIPT_DIR/frank-default-npu.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -------------------
+: "${FLM_BIN:=/usr/bin/flm}"
+: "${FLM_MODEL:=gemma4-it:e4b}"
+: "${FLM_HOST:=0.0.0.0}"
+: "${FLM_PORT:=52626}"
+: "${FLM_PMODE:=performance}"
+: "${FLM_ASR:=0}"
+: "${FLM_CTX_LEN:=-1}"
+
+if [[ ! -x "$FLM_BIN" ]]; then
+    echo "missing flm binary: $FLM_BIN" >&2
+    exit 1
+fi
+
+# --- optional flags ----------------------------------------------------------
+OPT_ARGS=()
+if [[ "${FLM_ASR}" != "0" ]]; then
+    OPT_ARGS+=(-a "${FLM_ASR}")
+fi
+if [[ "${FLM_CTX_LEN}" != "-1" ]]; then
+    OPT_ARGS+=(-c "${FLM_CTX_LEN}")
+fi
+
+exec "$FLM_BIN" serve "$FLM_MODEL" \
+    --host "$FLM_HOST" \
+    -p "$FLM_PORT" \
+    --pmode "$FLM_PMODE" \
+    "${OPT_ARGS[@]}"