Frank
9ef034e5e6
Env file is the source of truth (unit ExecStart runs the serve script which sources it); serve-script default and the deployer health-gate URL flipped in lockstep so the post-restart health gate polls the new port. Docs, fleet table, and both index bullets updated to match.
63 lines
2.5 KiB
Bash
Executable File
63 lines
2.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve the FastFlowLM (FLM) sight+sound endpoint — frank-default-sightsound.
|
|
#
|
|
# Runs `flm serve <tag>` on the XDNA2 NPU (50 TOPS, Dankclaw). The NPU is a
|
|
# single-lane device: generations serialize in FIFO order, so this is a
|
|
# background/triage endpoint, not an interactive multi-user chat. The NPU
|
|
# draws from the unified RAM pool shared with frank-default-large (the iGPU
|
|
# llama.cpp service); see serverAppConfigs-docs/LLM/frank-default-sightsound.md
|
|
# for the memory budget and the model-selection tradeoff.
|
|
#
|
|
# NOTE: FLM requires unlimited memlock. The unit sets LimitMEMLOCK=infinity.
|
|
# NOTE: an unknown model tag does NOT 404 — /v1/chat/completions answers HTTP 200 with a
|
|
# flat {"error":"Failed to load <tag> model!"} body and no choices; /v1/completions returns a
|
|
# nested OpenAI-shaped server_error (code 500) also under HTTP 200. Client model IDs must be the exact FLM registry
|
|
# tag (e.g. gemma4-it:e4b), never a service alias. Custom tags only resolve
|
|
# when FLM_CONFIG_PATH points at the user registry that carries them and their
|
|
# kernels are symlinked into /home/dank/.config/flm/xclbins/<PackName>.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FLM_SIGHTSOUND_ENV_FILE:-$SCRIPT_DIR/frank-default-sightsound.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) -------------------
|
|
: "${FLM_BIN:=/usr/bin/flm}"
|
|
: "${FLM_MODEL:=gemma4-it:e4b}"
|
|
: "${FLM_HOST:=0.0.0.0}"
|
|
: "${FLM_PORT:=7771}"
|
|
: "${FLM_PMODE:=turbo}"
|
|
: "${FLM_ASR:=0}"
|
|
: "${FLM_CTX_LEN:=131072}"
|
|
|
|
# The user registry is a superset of the system registry (it carries the
|
|
# custom tags too), and its model_path resolves relative to the registry file,
|
|
# so exports keep a tag swap in the env file from needing script edits.
|
|
export FLM_CONFIG_PATH="${FLM_CONFIG_PATH:-/home/dank/.config/flm/model_list.json}"
|
|
export FLM_XCLBIN_PATH="${FLM_XCLBIN_PATH:-/home/dank/.config/flm/xclbins}"
|
|
|
|
if [[ ! -x "$FLM_BIN" ]]; then
|
|
echo "missing flm binary: $FLM_BIN" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# --- optional flags ----------------------------------------------------------
|
|
OPT_ARGS=()
|
|
if [[ "${FLM_ASR}" != "0" ]]; then
|
|
OPT_ARGS+=(--asr "${FLM_ASR}")
|
|
fi
|
|
if [[ "${FLM_CTX_LEN}" != "-1" ]]; then
|
|
OPT_ARGS+=(--ctx-len "${FLM_CTX_LEN}")
|
|
fi
|
|
|
|
exec "$FLM_BIN" serve "$FLM_MODEL" \
|
|
--host "$FLM_HOST" \
|
|
-p "$FLM_PORT" \
|
|
--pmode "$FLM_PMODE" \
|
|
"${OPT_ARGS[@]}"
|