diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py index 2962f86..cbf9e0b 100755 --- a/scripts/deploy-plan.py +++ b/scripts/deploy-plan.py @@ -56,6 +56,7 @@ SERVICE_ORDER = ( "readarr", "frank-default-large", "frank-default-npu", + "frank-default-sightsound", "frank-default-small", ) @@ -84,6 +85,7 @@ DANKCLAW_SERVICES = { "homepage", "frank-default-large", "frank-default-npu", + "frank-default-sightsound", } DANKSERVER_SERVICES = { @@ -304,6 +306,8 @@ def services_for_path(path: str, target: str) -> set[str]: _add(services, "frank-default-large") elif name in {"frank-default-npu.env", "frank-default-npu.service", "serve-frank-default-npu.sh"}: _add(services, "frank-default-npu") + elif name in {"frank-default-sightsound.env", "frank-default-sightsound.service", "serve-frank-default-sightsound.sh"}: + _add(services, "frank-default-sightsound") elif name in {"frank-default-small.env", "frank-default-small.service", "serve-frank-small.sh"}: _add(services, "frank-default-small") elif name == "chat-template.jinja": diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index fa5422d..d18cddc 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -610,6 +610,18 @@ deploy_llm_npu() { echo "Relinked frank-default-npu; it was not restarted." } +# frank-default-sightsound is a live serving endpoint (FLM on the XDNA2 NPU), +# not a relink-only unit: it is enabled at boot and restarted on deploy. The +# health gate polls /v1/version, FLM's cheapest always-on route (there is no +# /health route in the FLM server). +deploy_llm_sightsound() { + relink_unit frank-default-sightsound + run_privileged systemctl daemon-reload + run_privileged systemctl enable frank-default-sightsound + run_privileged systemctl restart frank-default-sightsound + wait_for_health frank-default-sightsound http://127.0.0.1:52626/v1/version 300 +} + wait_for_health() { local service="$1" url="$2" limit="$3" local i @@ -646,6 +658,7 @@ deploy_service() { frank-default-large) deploy_llm_large ;; frank-default-small) deploy_llm_small ;; frank-default-npu) deploy_llm_npu ;; + frank-default-sightsound) deploy_llm_sightsound ;; *) deploy_compose_service "$service" ;; esac } diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 52d991b..dc95ec9 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -121,6 +121,7 @@ valid_service() { dankclaw:node-exporter|dankclaw:authentik-postgres|dankclaw:authentik-server|\ dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:anubis|\ dankclaw:frank-default-large|dankclaw:frank-default-npu|\ + dankclaw:frank-default-sightsound|\ dankserver:mariadb|dankserver:redis|dankserver:romm|dankserver:copyparty|\ dankserver:jellyfin|dankserver:calibre-web|dankserver:prowlarr|\ dankserver:qbittorrent|dankserver:radarr|dankserver:sonarr|\ diff --git a/scripts/frank-default-sightsound.env b/scripts/frank-default-sightsound.env new file mode 100644 index 0000000..e9d9049 --- /dev/null +++ b/scripts/frank-default-sightsound.env @@ -0,0 +1,23 @@ +# Tunables for frank-default-sightsound (FastFlowLM NPU sight+sound endpoint, Dankclaw) +# Runtime: FLM 1.0.1 (/usr/bin/flm) on the XDNA2 NPU (50 TOPS). Model: +# gemma4-it:e4b = Gemma4-E4B-IT-NPU2 (8B total / ~4.5B effective params, +# Q4_1; vision + audio + reasoning; ~9.1 GB footprint). It is the only NPU2 +# model family with BOTH an image and an audio tower — the uncensored +# candidate qwythos:9b is text-only and fails the audio requirement, so this +# endpoint trades "uncensored" for "sight + sound". The NPU draws from the +# unified RAM pool shared with frank-default-large (iGPU); ~31 GiB was +# available there at idle, so a 9.1 GB pack fits with headroom. +FLM_MODEL=gemma4-it:e4b +FLM_HOST=0.0.0.0 +FLM_PORT=52626 +# turbo = max NPU power mode (xrt-smi configure --pmode turbo at serve start). +# No other tenant uses the NPU on this host, so the top mode costs nothing to +# neighbours; drop to `performance` if iGPU power-budget contention ever shows. +FLM_PMODE=turbo +# FLM_ASR=1 additionally loads Whisper-V3-Turbo-NPU2 (+0.62 GB) and exposes +# /v1/audio/transcriptions. gemma4-it:e4b answers audio natively through its +# own audio tower in /v1/chat/completions, so ASR stays off by default. +FLM_ASR=0 +# 131072 = verified max for gemma4-it:e4b on this box (native 128K context; +# the registry's default_context_length 32768 is only the fallback default). +FLM_CTX_LEN=131072 diff --git a/scripts/frank-default-sightsound.service b/scripts/frank-default-sightsound.service new file mode 100644 index 0000000..d185faa --- /dev/null +++ b/scripts/frank-default-sightsound.service @@ -0,0 +1,17 @@ +[Unit] +Description=FastFlowLM NPU sight+sound inference (frank-default-sightsound) on Dankclaw +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=dank +Group=dank +Environment="FLM_SIGHTSOUND_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-default-sightsound.env" +ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-default-sightsound.sh +LimitMEMLOCK=infinity +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/serve-frank-default-sightsound.sh b/scripts/serve-frank-default-sightsound.sh new file mode 100755 index 0000000..3720e8d --- /dev/null +++ b/scripts/serve-frank-default-sightsound.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Serve the FastFlowLM (FLM) sight+sound endpoint — frank-default-sightsound. +# +# Runs `flm serve ` on the XDNA2 NPU (50 TOPS, Dankclaw). The NPU is a +# single-lane device: generations serialize in FIFO order, so this is a +# background/triage endpoint, not an interactive multi-user chat. The NPU +# draws from the unified RAM pool shared with frank-default-large (the iGPU +# llama.cpp service); see serverAppConfigs-docs/LLM/frank-default-sightsound.md +# for the memory budget and the model-selection tradeoff. +# +# NOTE: FLM requires unlimited memlock. The unit sets LimitMEMLOCK=infinity. +# NOTE: an unknown model tag does NOT 404 — FLM logs an error and silently +# serves llama3.2:1b instead. Client model IDs must be the exact FLM registry +# tag (e.g. gemma4-it:e4b), never a service alias. Custom tags only resolve +# when FLM_CONFIG_PATH points at the user registry that carries them and their +# kernels are symlinked into /home/dank/.config/flm/xclbins/. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FLM_SIGHTSOUND_ENV_FILE:-$SCRIPT_DIR/frank-default-sightsound.env}" +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +# --- defaults (used when the env file does not set them) ------------------- +: "${FLM_BIN:=/usr/bin/flm}" +: "${FLM_MODEL:=gemma4-it:e4b}" +: "${FLM_HOST:=0.0.0.0}" +: "${FLM_PORT:=52626}" +: "${FLM_PMODE:=turbo}" +: "${FLM_ASR:=0}" +: "${FLM_CTX_LEN:=131072}" + +# The user registry is a superset of the system registry (it carries the +# custom tags too), and its model_path resolves relative to the registry file, +# so exports keep a tag swap in the env file from needing script edits. +export FLM_CONFIG_PATH="${FLM_CONFIG_PATH:-/home/dank/.config/flm/model_list.json}" +export FLM_XCLBIN_PATH="${FLM_XCLBIN_PATH:-/home/dank/.config/flm/xclbins}" + +if [[ ! -x "$FLM_BIN" ]]; then + echo "missing flm binary: $FLM_BIN" >&2 + exit 1 +fi + +# --- optional flags ---------------------------------------------------------- +OPT_ARGS=() +if [[ "${FLM_ASR}" != "0" ]]; then + OPT_ARGS+=(--asr "${FLM_ASR}") +fi +if [[ "${FLM_CTX_LEN}" != "-1" ]]; then + OPT_ARGS+=(--ctx-len "${FLM_CTX_LEN}") +fi + +exec "$FLM_BIN" serve "$FLM_MODEL" \ + --host "$FLM_HOST" \ + -p "$FLM_PORT" \ + --pmode "$FLM_PMODE" \ + "${OPT_ARGS[@]}"