feat(llm): swap frank-large to halogen-flash-server, retire frank-npu
frank-large moves from the kingjones30/ROCmFPX llama.cpp build to peonist-ai/halogen-flash-server 0.5.8 serving Qwen3.8-Flash-Next (w4b .hgn 115.55 GiB + quality overlay). Upstream measures this engine at roughly 4x the prefill throughput of the llama.cpp family on the same gfx1151 silicon, and prefill dominates any prompt with real context. Shape: a rootless Quadlet container (dankclaw/frank-large/quadlet/), engine on container loopback :8730 (never published), OpenAI front-end :8731 published as host 7770, HALOGEN_MODEL_ID=frank-large so no client renamed anything. Context 262144, KV pool 262144 positions (~7.2 GiB, conservative for a shared 121 GiB host), 4 slots, host reserve 24, vision tower on, weights pinned, all other flags at vendor defaults. Weights are fetched by HALOGEN_DOWNLOAD on first start into a writable /home/dank/models/halogen. The llama.cpp era (unit, env, serve script) is staged in .AGENTTRASH/ rather than deleted; the STRIX payload and the ROCmFPX build stay on disk as the rollback target. deploy_llm_large becomes a user-quadlet handler: it retires the old ROOT unit (moving its checkout symlink into the home trash, never rm) and gates the first start on the download rather than reporting a failure for a fetch that is working exactly as designed. frank-npu is retired in the same change. The kernel cmdline moves from iommu=pt to amd_iommu=off for the 13-16% of prefill upstream measured, and that takes the XDNA2 NPU with it: without PASID/SVA the driver cannot open /dev/accel/accel0 (err=-19). Its five dependent routes are repointed (Hermes vision -> frank-large, Hermes title generation -> frank-small, harness vision -> frank-large, tiny -> frank-small, task -> frank-large:medium) and its provider blocks are removed from both the harness config and Hermes's live config. Audio input leaves the fleet with no replacement; image input survives on halogen's vision tower. Docs: new halogen-flash-server runbook, new IOMMU decision record, and frank-npu rewritten as a retirement record. AGENTS.md topology, runtime sections, benchmarking policy (frank-large is a user unit now) and the harness model-config section are corrected.
This commit is contained in:
Frank
@@ -59,7 +59,6 @@ SERVICE_ORDER = (
|
||||
"lidarr",
|
||||
"readarr",
|
||||
"frank-large",
|
||||
"frank-npu",
|
||||
"frank-small",
|
||||
)
|
||||
|
||||
@@ -89,7 +88,6 @@ DANKCLAW_SERVICES = {
|
||||
"caddy",
|
||||
"homepage",
|
||||
"frank-large",
|
||||
"frank-npu",
|
||||
}
|
||||
|
||||
DANKSERVER_SERVICES = {
|
||||
@@ -297,6 +295,11 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
if len(parts) >= 2 and parts[1] == "open-webui":
|
||||
_add(services, "open-webui")
|
||||
return services
|
||||
if len(parts) >= 2 and parts[1] == "frank-large":
|
||||
# The service is a rootless Quadlet container since 2026-09-11;
|
||||
# its unit and fleet settings live in one .container file.
|
||||
_add(services, "frank-large")
|
||||
return services
|
||||
if len(parts) >= 2 and parts[1] == "gitea":
|
||||
if len(parts) >= 4 and parts[2] == "quadlet":
|
||||
name = parts[3]
|
||||
@@ -346,15 +349,12 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
|
||||
if parts[0] == "scripts":
|
||||
name = p.name
|
||||
if name in {"frank-large.env", "frank-large.service", "serve-frank-large.sh"}:
|
||||
_add(services, "frank-large")
|
||||
elif name in {"frank-npu.env", "frank-npu.service", "serve-frank-npu.sh"}:
|
||||
_add(services, "frank-npu")
|
||||
elif name in {"frank-small.env", "frank-small.service", "serve-frank-small.sh"}:
|
||||
if name in {"frank-small.env", "frank-small.service", "serve-frank-small.sh"}:
|
||||
_add(services, "frank-small")
|
||||
elif name == "chat-template.jinja":
|
||||
# Only the selected target's LLM consumes this shared template.
|
||||
_add(services, "frank-large" if target == "dankclaw" else "frank-small")
|
||||
# Only frank-small consumes this template now: frank-large runs
|
||||
# halogen-flash-server, which owns its own chat handling.
|
||||
_add(services, "frank-small")
|
||||
elif name in {"homepage-status.py", "homepage-status.service", "homepage-status.timer"}:
|
||||
_add(services, "homepage")
|
||||
# Deployment/test/benchmark scripts and other repo tooling do not
|
||||
|
||||
+76
-30
@@ -90,7 +90,7 @@ service_is_allowed() {
|
||||
dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
|
||||
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|\
|
||||
dankclaw:frank-large|dankclaw:frank-npu|\
|
||||
dankclaw:frank-large|\
|
||||
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
|
||||
@@ -183,7 +183,6 @@ add_pending_services() {
|
||||
[[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue
|
||||
if [[ "$TARGET" == dankclaw && "$service" == frank-small ]]; then continue; fi
|
||||
if [[ "$TARGET" == dankserver && "$service" == frank-large ]]; then continue; fi
|
||||
if [[ "$TARGET" == dankserver && "$service" == frank-npu ]]; then continue; fi
|
||||
if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \
|
||||
--repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then
|
||||
echo "WARN: unable to compare state for $service; retaining it as pending" >&2
|
||||
@@ -826,21 +825,85 @@ deploy_anubis() {
|
||||
wait_container_healthy anubis 60 || return 1
|
||||
}
|
||||
|
||||
# Rename cutover (2026-09-03): frank-default-large -> frank-large,
|
||||
# frank-default-small -> frank-small, frank-default-sightsound -> frank-npu.
|
||||
# The legacy units are stopped and disabled before the renamed unit takes the
|
||||
# port so no legacy process is left holding it; stop/disable of an already
|
||||
# unloaded legacy unit fail harmlessly and are tolerated.
|
||||
deploy_llm_large() {
|
||||
run_privileged systemctl stop frank-default-large || true
|
||||
run_privileged systemctl disable frank-default-large || true
|
||||
relink_unit frank-large
|
||||
# frank-large runs as a rootless Quadlet unit since 2026-09-11 (halogen-flash-
|
||||
# server replaced the llama.cpp/kingjones runtime). Unlike the sibling LLM
|
||||
# services it is therefore NOT a root unit: no relink_unit, no `systemctl
|
||||
# enable` (this host's quadlet generator produces units that cannot be
|
||||
# enabled — boot persistence comes from [Install] WantedBy=default.target in
|
||||
# the .container file plus linger).
|
||||
#
|
||||
# The llama.cpp-era ROOT unit is retired first. Its unit file is a symlink
|
||||
# into this checkout (the old relink_unit did `ln -sfn scripts/frank-large.
|
||||
# service /etc/systemd/system/`), so leaving it behind after the repo file
|
||||
# moves to the trash would leave systemd holding a dangling unit that can
|
||||
# still claim port 7770. Stop it, drop its enable symlinks, move the unit file
|
||||
# into the home trash (never rm), then reload the root manager.
|
||||
#
|
||||
# FIRST START: the container fetches ~127 GB of weights before it can serve,
|
||||
# so /health is legitimately down for the whole fetch. wait_for_health would
|
||||
# burn its budget and report a failure that is not one. The gate below
|
||||
# recognises the bootstrap (checkpoint absent) and waits for the checkpoint
|
||||
# instead, falling through to the normal health gate once it lands.
|
||||
LLM_LARGE_CHECKPOINT="${LLM_LARGE_CHECKPOINT:-/home/dank/models/halogen/qwen38-flash-next-w4b.hgn}"
|
||||
|
||||
retire_root_llm_large() {
|
||||
local trash="$HOME/.AGENTTRASH/20260911-halogen-swap/host-units"
|
||||
run_privileged systemctl stop frank-default-large 2>/dev/null || true
|
||||
run_privileged systemctl disable frank-default-large 2>/dev/null || true
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
# On this host `disable` has been observed to delete the unit file itself
|
||||
# (2026-08-26 incident, see AGENTS.md); tolerate either behaviour and keep
|
||||
# whatever remains by moving it to the trash.
|
||||
run_privileged systemctl disable frank-large.service 2>/dev/null || true
|
||||
if [[ -e /etc/systemd/system/frank-large.service || -L /etc/systemd/system/frank-large.service ]]; then
|
||||
mkdir -p "$trash"
|
||||
run_privileged mv -- /etc/systemd/system/frank-large.service \
|
||||
"$trash/frank-large.service"
|
||||
echo "retired root unit frank-large.service -> $trash"
|
||||
fi
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-large
|
||||
run_privileged systemctl restart frank-large
|
||||
}
|
||||
|
||||
deploy_llm_large() {
|
||||
retire_root_llm_large
|
||||
# Pull once, then run the localhost/-aliased tag: this host's quadlet path
|
||||
# cannot resolve registry-prefixed refs from local storage at create time
|
||||
# (same workaround as anubis/hermes).
|
||||
local image="ghcr.io/peonist-ai/halogen-flash-server:0.5.8"
|
||||
local alias="localhost/peonist-ai/halogen-flash-server:0.5.8"
|
||||
podman pull "$image"
|
||||
podman tag "$image" "$alias"
|
||||
# The weights volume: read-write, because the container downloads into it.
|
||||
mkdir -p /home/dank/models/halogen
|
||||
install_quadlet frank-large dankclaw/frank-large/quadlet frank-large
|
||||
systemctl --user restart frank-large.service
|
||||
wait_for_llm_large_ready
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
# wait_for_llm_large_ready — bootstrap-aware gate. If the checkpoint is absent
|
||||
# the container is downloading it; wait for the file (resumable, and the unit
|
||||
# restarts it on failure) rather than gating on a /health that cannot answer
|
||||
# yet. Reports success either way, so the first deploy does not look like a
|
||||
# failure while the engine does exactly what it was told to.
|
||||
wait_for_llm_large_ready() {
|
||||
[[ -f "$LLM_LARGE_CHECKPOINT" ]] && return 0
|
||||
echo "frank-large: checkpoint absent at $LLM_LARGE_CHECKPOINT"
|
||||
echo "frank-large: first start — the container is downloading ~127 GB from"
|
||||
echo "frank-large: peonist-ai/halogen-qwen3.8-flash-next (resumes if interrupted)."
|
||||
local i
|
||||
for ((i = 1; i <= 1200; i++)); do
|
||||
if [[ -f "$LLM_LARGE_CHECKPOINT" ]]; then
|
||||
echo "frank-large: weights landed after approximately $((i * 2))s; gating on /health"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "WARN: frank-large download still in progress after 40m — NOT a failure."
|
||||
echo "WARN: the unit keeps fetching; re-run this deploy (or re-check /health) later." >&2
|
||||
return 0
|
||||
}
|
||||
|
||||
deploy_llm_small() {
|
||||
run_privileged systemctl stop frank-default-small || true
|
||||
run_privileged systemctl disable frank-default-small || true
|
||||
@@ -851,22 +914,6 @@ deploy_llm_small() {
|
||||
wait_for_health frank-small http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
# frank-npu is a live serving endpoint (FLM on the XDNA2 NPU), not a
|
||||
# relink-only unit: it is enabled at boot and restarted on deploy. The health
|
||||
# gate polls /v1/version, FLM's cheapest always-on route (there is no /health
|
||||
# route in the FLM server).
|
||||
deploy_llm_npu() {
|
||||
run_privileged systemctl stop frank-default-sightsound || true
|
||||
run_privileged systemctl disable frank-default-sightsound || true
|
||||
run_privileged systemctl stop frank-default-npu || true
|
||||
run_privileged systemctl disable frank-default-npu || true
|
||||
relink_unit frank-npu
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-npu
|
||||
run_privileged systemctl restart frank-npu
|
||||
wait_for_health frank-npu http://127.0.0.1:7771/v1/version 300
|
||||
}
|
||||
|
||||
wait_for_health() {
|
||||
local service="$1" url="$2" limit="$3"
|
||||
local i
|
||||
@@ -1052,7 +1099,6 @@ deploy_service() {
|
||||
seerr) deploy_seerr ;;
|
||||
frank-large) deploy_llm_large ;;
|
||||
frank-small) deploy_llm_small ;;
|
||||
frank-npu) deploy_llm_npu ;;
|
||||
hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
|
||||
romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
|
||||
jellyfin) deploy_jellyfin ;;
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
# Tunables for frank-large (kingjones777/Qwen3.8-Flash-Next-ROCmFP4-
|
||||
# STRIX-GGUF on the kingjones30/ROCmFPX llama.cpp runtime, Dankclaw,
|
||||
# Strix Halo gfx1151, unified memory).
|
||||
#
|
||||
# Runtime: kingjones30/ROCmFPX (fork of charlie12345/ROCmFPX with the
|
||||
# qwen4exp-on-rocmfpx-d3ca537 patch applied — the model card's pinned
|
||||
# runtime: ROCmFP4 tensor types AND the qwen4exp architecture in one
|
||||
# tree; stock llama.cpp and charlie's fork alone cannot load this GGUF).
|
||||
# Built per the card recipe:
|
||||
# cmake -B build -DGGML_HIP=ON -DGPU_TARGETS=gfx1151 -DGGML_NATIVE=ON -DCMAKE_BUILD_TYPE=Release
|
||||
# at /home/dank/GitPackages/kingjones-ROCmFPX/build.
|
||||
# Replaced the agentionai ROCmFP4-FAST imatrix deployment (charlie's
|
||||
# build-qwen38 runtime) on 2026-08-30.
|
||||
#
|
||||
# Model: Qwen3.8-Flash-Next base (qwen4exp MoE: 512 experts / 10 used,
|
||||
# 48 layers = 12 attention + 36 SSM, QSA indexer, hyper-connections, PLE
|
||||
# at layer 1), Q4_0_ROCMFP4_STRIX (file_type 105), 113.47 GiB single
|
||||
# file. STRIX is the top quality tier: PLE table at Q8_0 (50.66 GiB),
|
||||
# token embeddings + lm head at Q6_K. The PLE table NEVER enters GPU
|
||||
# memory — it streams from the file through the OS page cache. Two hard
|
||||
# card constraints:
|
||||
# 1. mmap MUST stay enabled (default) — --no-mmap forces the 50.66 GiB
|
||||
# table into anonymous memory and the cgroup killer takes the
|
||||
# process with nothing in the server log (only dmesg).
|
||||
# 2. Do NOT pin the PLE tensor to CPU with -ot — measured gen fell
|
||||
# ~23 -> 13.4 tok/s; the kernel streams it better.
|
||||
# NOTE: the card's own build_attn_qsa still calls ggml_fill on the
|
||||
# F16-cast kq mask (the path that aborted our charlie build with
|
||||
# --flash-attn on before scripts/patches/qwen4exp-qsa-f32-mask.patch).
|
||||
# If a clean kingjones-tree build aborts graph build with
|
||||
# GGML_ASSERT(a-type == GGML_TYPE_F32), port that fix (F32 base mask +
|
||||
# cast at the attention call) into this tree and rebuild.
|
||||
#
|
||||
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
|
||||
# incident). llama.cpp reads --host/--port/--temp as CLI flags, never as
|
||||
# env vars.
|
||||
|
||||
# kingjones30/ROCmFPX runtime binary (card recipe build).
|
||||
LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server
|
||||
|
||||
# Model package from
|
||||
# https://huggingface.co/kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX-GGUF
|
||||
# Layout: single unsharded GGUF, text-only (the repo ships no mmproj;
|
||||
# the vision tower below is borrowed from the agentionai FAST package —
|
||||
# same base model, f16 — kept only if it loads clean, drop
|
||||
# MMPROJ_PATH if it does not).
|
||||
MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX
|
||||
MODEL_FILE=Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf
|
||||
MMPROJ_PATH=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf
|
||||
|
||||
# Qwen-Sharp chat template (tracked in the repo, staged on the host). The
|
||||
# Sharp template drives thinking on by default (medium effort); the server
|
||||
# emits reasoning in DeepSeek format so clients receive reasoning_content
|
||||
# separately instead of literal thinking tags in content.
|
||||
CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
|
||||
CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}'
|
||||
|
||||
# Alias + network.
|
||||
ALIAS=frank-large
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7770
|
||||
|
||||
# Serving: 401408-token context, TWO slots (parallel 2), batch 4096 /
|
||||
# ubatch 512, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB
|
||||
# prompt cache. --fit off: auto-fit segfaults on this model's
|
||||
# hyper-connection/indexer config; -ngl 999 puts every layer on the
|
||||
# SVM-backed GPU pool. ubatch 512 matches the card's re-measurement
|
||||
# (-ub 1024/2048/4096 showed no effect; -t 32 vs 16 is noise).
|
||||
#
|
||||
# CTX_SIZE=401408 / PARALLEL=2: carried over from the FAST deployment
|
||||
# where it loaded clean (~74 s, health 200). The card measures the full
|
||||
# 262144 window at ~71.7 GiB GTT at load on the same 128 GB board; QSA
|
||||
# caps KV growth so the extra window is cheap. KV footprint scales
|
||||
# with the 12 attention layers only. Verify load on first STRIX boot.
|
||||
CTX_SIZE=401408
|
||||
PARALLEL=2
|
||||
|
||||
BATCH_SIZE=4096
|
||||
UBATCH_SIZE=512
|
||||
NUM_THREADS=16
|
||||
THREAD_BATCH=16
|
||||
CACHE_TYPE_K=f16
|
||||
CACHE_TYPE_V=f16
|
||||
CTX_CHECKPOINTS=16
|
||||
CACHE_RAM=8192
|
||||
SLOT_SAVE_PATH=/home/dank/serverAppConfigs/scripts/slot-state
|
||||
SPLIT_MODE=none
|
||||
FIT=off
|
||||
NGL=999
|
||||
FLASH_ATTN=on
|
||||
|
||||
# Sampling (card instruct table, non-thinking).
|
||||
TEMPERATURE=0.7
|
||||
TOP_K=20
|
||||
TOP_P=0.8
|
||||
MIN_P=0.0
|
||||
PRESENCE_PENALTY=1.5
|
||||
|
||||
# Reasoning on (Sharp template drives it; preserve_thinking + preserve_reasoning
|
||||
# keep the reasoning trail in the conversation). DeepSeek format projects the
|
||||
# reasoning into the API's reasoning_content field instead of literal thinking
|
||||
# tags.
|
||||
REASONING=on
|
||||
REASONING_FORMAT=deepseek
|
||||
|
||||
# Strix Halo unified memory + native gfx1151.
|
||||
HSA_OVERRIDE_GFX_VERSION=11.5.1
|
||||
|
||||
VERBOSITY=4
|
||||
|
||||
|
||||
# Speculative decoding: disabled. The card's Usage block ships no spec
|
||||
# flags. The Flash-Next MTP heads do not load on this loader (requires
|
||||
# per-layer blk.N.hc_attn_norm; every head carries only blk.48 —
|
||||
# verified 3x on build-qwen38). DRAFT_MODEL_* point at the retired
|
||||
# agentionai FAST head and are unused while SPEC_TYPE=none.
|
||||
DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST
|
||||
DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf
|
||||
SPEC_TYPE=none
|
||||
SPEC_DRAFT_N_MAX=3
|
||||
SPEC_DRAFT_N_MIN=0
|
||||
SPEC_DRAFT_P_MIN=0.75
|
||||
SPEC_DRAFT_NGL=999
|
||||
SPEC_DRAFT_DEVICE=ROCm0
|
||||
SPEC_DRAFT_TYPE_K=f16
|
||||
SPEC_DRAFT_TYPE_V=f16
|
||||
@@ -1,16 +0,0 @@
|
||||
[Unit]
|
||||
Description=ROCmFPX (kingjones30 fork) llama.cpp Qwen3.8-Flash-Next ROCmFP4 STRIX 177B MoE (dankclaw, frank-large)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,23 +0,0 @@
|
||||
# Tunables for frank-npu (FastFlowLM NPU endpoint, Dankclaw)
|
||||
# Runtime: FLM 1.0.1 (/usr/bin/flm) on the XDNA2 NPU (50 TOPS). Boot tag is
|
||||
# nuextract3:4b (NuExtract3-4B-NPU2, Qwen3.5-4B skeleton, structured-extraction
|
||||
# model); serve routes every request by the exact FLM registry tag in the
|
||||
# request's model field, so any registered tag (gemma4-it:e4b, qwythos:9b, ...)
|
||||
# is reachable on this port regardless of the boot tag. The NPU is a
|
||||
# single-lane device: generations serialize FIFO, and the first request after a
|
||||
# model swap pays the ~6-11 s load cost. The NPU draws from the unified RAM
|
||||
# pool shared with frank-large (iGPU).
|
||||
FLM_MODEL=nuextract3:4b
|
||||
FLM_HOST=0.0.0.0
|
||||
FLM_PORT=7771
|
||||
# turbo = max NPU power mode (xrt-smi configure --pmode turbo at serve start).
|
||||
# No other tenant uses the NPU on this host, so the top mode costs nothing to
|
||||
# neighbours; drop to `performance` if iGPU power-budget contention ever shows.
|
||||
FLM_PMODE=turbo
|
||||
# FLM_ASR=1 additionally loads Whisper-V3-Turbo-NPU2 (+0.62 GB) and exposes
|
||||
# /v1/audio/transcriptions. NuExtract3 answers text-only; audio stays off.
|
||||
FLM_ASR=0
|
||||
# -1 = no --ctx-len flag: every pack loads with its own trained context
|
||||
# (nuextract3:4b -> 32768, gemma4-it:e4b -> 131072). A global override would
|
||||
# over-allocate KV for the small packs.
|
||||
FLM_CTX_LEN=-1
|
||||
@@ -1,17 +0,0 @@
|
||||
[Unit]
|
||||
Description=FastFlowLM NPU sight+sound inference (frank-npu) on Dankclaw
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
Environment="FRANK_NPU_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-npu.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-npu.sh
|
||||
LimitMEMLOCK=infinity
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,180 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX-GGUF as
|
||||
# frank-large via the kingjones30/ROCmFPX llama.cpp runtime on
|
||||
# Dankclaw (Strix Halo gfx1151, unified memory).
|
||||
#
|
||||
# Runtime: kingjones30/ROCmFPX (fork of charlie12345/ROCmFPX with the
|
||||
# model card's qwen4exp-on-rocmfpx patch) built at
|
||||
# /home/dank/GitPackages/kingjones-ROCmFPX/build (HIP, gfx1151,
|
||||
# ROCm 7.2.4, card recipe). The ROCmFP4 quant types plus the qwen4exp
|
||||
# architecture must be in one tree; stock llama.cpp cannot load this
|
||||
# GGUF.
|
||||
#
|
||||
# Model: Qwen3.8-Flash-Next base (qwen4exp MoE: 512 experts / 10 used,
|
||||
# 48 layers = 12 attention + 36 SSM, QSA indexer, hyper-connections, PLE
|
||||
# at layer 1) quantized Q4_0_ROCMFP4_STRIX, 113.47 GiB single file. The
|
||||
# 50.66 GiB PLE n-gram table stays OUT of GPU memory and streams via
|
||||
# mmap from the OS page cache — never add --no-mmap or -ot for it.
|
||||
#
|
||||
# Profile: card sampling table (temp 0.7, top_p 0.80, top_k 20, min_p 0.0,
|
||||
# presence 1.5 — non-thinking instruct) mixed with the validated serving
|
||||
# profile: ctx 401408 x 2 slots, b 4096 / ub 512, t 16 / tb 16, f16 KV,
|
||||
# fa on (if the graph build aborts on the QSA F16 mask, see the note in
|
||||
# the env file: port scripts/patches/qwen4exp-qsa-f32-mask.patch into
|
||||
# this tree and rebuild), cont-batching, prompt cache + 8 GiB cache-ram
|
||||
# + idle-slot cache, 16 ctx checkpoints, --fit off (auto-fit segfaults
|
||||
# on qwen4exp hyper-connection config). Speculative decoding off: the
|
||||
# Flash-Next MTP heads do not load on this loader; the card ships no
|
||||
# spec flags.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${LLAMA_BIN:=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}"
|
||||
: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX}"
|
||||
: "${MODEL_FILE:=Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf}"
|
||||
: "${MMPROJ_PATH:=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf}"
|
||||
: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
|
||||
: "${CHAT_TEMPLATE_KWARGS:={}}"
|
||||
: "${ALIAS:=frank-large}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7770}"
|
||||
: "${CTX_SIZE:=401408}"
|
||||
: "${PARALLEL:=2}"
|
||||
: "${BATCH_SIZE:=4096}"
|
||||
: "${UBATCH_SIZE:=512}"
|
||||
: "${NUM_THREADS:=16}"
|
||||
: "${THREAD_BATCH:=16}"
|
||||
: "${CACHE_TYPE_K:=f16}"
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
: "${CTX_CHECKPOINTS:=16}"
|
||||
: "${CACHE_RAM:=8192}"
|
||||
: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}"
|
||||
: "${SPLIT_MODE:=none}"
|
||||
: "${FIT:=off}"
|
||||
: "${NGL:=999}"
|
||||
: "${FLASH_ATTN:=on}"
|
||||
: "${TEMPERATURE:=0.7}"
|
||||
: "${TOP_K:=20}"
|
||||
: "${TOP_P:=0.8}"
|
||||
: "${MIN_P:=0.0}"
|
||||
: "${PRESENCE_PENALTY:=1.5}"
|
||||
: "${REASONING:=on}"
|
||||
: "${REASONING_FORMAT:=deepseek}"
|
||||
: "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}"
|
||||
: "${SPEC_TYPE:=none}"
|
||||
: "${DRAFT_MODEL_DIR:=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST}"
|
||||
: "${DRAFT_MODEL_FILE:=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf}"
|
||||
: "${SPEC_DRAFT_N_MAX:=3}"
|
||||
: "${SPEC_DRAFT_N_MIN:=0}"
|
||||
: "${SPEC_DRAFT_P_MIN:=0.75}"
|
||||
: "${SPEC_DRAFT_NGL:=999}"
|
||||
: "${SPEC_DRAFT_DEVICE:=ROCm0}"
|
||||
: "${SPEC_DRAFT_TYPE_K:=f16}"
|
||||
: "${SPEC_DRAFT_TYPE_V:=f16}"
|
||||
|
||||
MODEL="$MODEL_DIR/$MODEL_FILE"
|
||||
for f in "$LLAMA_BIN" "$MODEL" "$CHAT_TEMPLATE_FILE"; do
|
||||
if [[ ! -r "$f" ]]; then
|
||||
echo "missing or unreadable required artifact: $f" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if [[ ! -x "$LLAMA_BIN" ]]; then
|
||||
echo "missing executable runtime: $LLAMA_BIN" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- speculative decoding (MTP draft head) ---------------------------------
|
||||
# The MTP draft head is a separate GGUF sidecar loaded via --spec-draft-model
|
||||
# (alias --model-draft). ngram-only specs do not need a draft file.
|
||||
# Set SPEC_TYPE=none in the env to disable speculative decoding entirely.
|
||||
SPEC_ARGS=()
|
||||
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
|
||||
SPEC_ARGS=(
|
||||
--spec-type "$SPEC_TYPE"
|
||||
--spec-draft-n-max "$SPEC_DRAFT_N_MAX"
|
||||
--spec-draft-n-min "$SPEC_DRAFT_N_MIN"
|
||||
--spec-draft-p-min "$SPEC_DRAFT_P_MIN"
|
||||
)
|
||||
if [[ "$SPEC_TYPE" == *draft-mtp* ]]; then
|
||||
DRAFT_MODEL="$DRAFT_MODEL_DIR/$DRAFT_MODEL_FILE"
|
||||
if [[ ! -r "$DRAFT_MODEL" ]]; then
|
||||
echo "missing or unreadable draft model: $DRAFT_MODEL" >&2
|
||||
exit 1
|
||||
fi
|
||||
SPEC_ARGS+=(
|
||||
--spec-draft-model "$DRAFT_MODEL"
|
||||
--spec-draft-ngl "$SPEC_DRAFT_NGL"
|
||||
--spec-draft-device "$SPEC_DRAFT_DEVICE"
|
||||
--spec-draft-type-k "$SPEC_DRAFT_TYPE_K"
|
||||
--spec-draft-type-v "$SPEC_DRAFT_TYPE_V"
|
||||
)
|
||||
fi
|
||||
if [[ -n "${SPEC_DRAFT_P_SPLIT:-}" ]]; then
|
||||
SPEC_ARGS+=(--spec-draft-p-split "$SPEC_DRAFT_P_SPLIT")
|
||||
fi
|
||||
fi
|
||||
|
||||
# The STRIX repo ships no mmproj; this is the agentionai FAST package's
|
||||
# f16 vision tower for the same base model. Passed only when readable.
|
||||
MMPROJ_ARGS=()
|
||||
if [[ -n "${MMPROJ_PATH:-}" && -r "${MMPROJ_PATH:-}" ]]; then
|
||||
MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
|
||||
fi
|
||||
|
||||
SERVER_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
mkdir -p "$SLOT_SAVE_PATH"
|
||||
ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
|
||||
LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
||||
|
||||
exec /usr/bin/env \
|
||||
ROCM_PATH="$ROCM_PATH" \
|
||||
HIP_VISIBLE_DEVICES=0 \
|
||||
LD_LIBRARY_PATH="$LIBRARY_PATH" \
|
||||
HSA_OVERRIDE_GFX_VERSION="$HSA_OVERRIDE_GFX_VERSION" \
|
||||
"$LLAMA_BIN" \
|
||||
--model "$MODEL" \
|
||||
--alias "$ALIAS" \
|
||||
--chat-template-file "$CHAT_TEMPLATE_FILE" \
|
||||
--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS" \
|
||||
--host "$BIND_ADDR" \
|
||||
--port "$LISTEN_PORT" \
|
||||
--jinja \
|
||||
--device ROCm0 \
|
||||
--gpu-layers "$NGL" \
|
||||
--split-mode "$SPLIT_MODE" \
|
||||
--parallel "$PARALLEL" \
|
||||
--ctx-size "$CTX_SIZE" \
|
||||
--batch-size "$BATCH_SIZE" \
|
||||
--ubatch-size "$UBATCH_SIZE" \
|
||||
--fit "$FIT" \
|
||||
--flash-attn "$FLASH_ATTN" \
|
||||
--cache-type-k "$CACHE_TYPE_K" \
|
||||
--cache-type-v "$CACHE_TYPE_V" \
|
||||
--threads "$NUM_THREADS" \
|
||||
--threads-batch "$THREAD_BATCH" \
|
||||
--ctx-checkpoints "$CTX_CHECKPOINTS" \
|
||||
--cache-ram "$CACHE_RAM" \
|
||||
--slot-save-path "$SLOT_SAVE_PATH" \
|
||||
--cache-prompt \
|
||||
--cache-idle-slots \
|
||||
--metrics \
|
||||
--temp "$TEMPERATURE" \
|
||||
--top-p "$TOP_P" \
|
||||
--top-k "$TOP_K" \
|
||||
--min-p "$MIN_P" \
|
||||
--presence-penalty "$PRESENCE_PENALTY" \
|
||||
--reasoning "$REASONING" \
|
||||
--reasoning-format "$REASONING_FORMAT" \
|
||||
--reasoning-budget -1 \
|
||||
"${MMPROJ_ARGS[@]}" \
|
||||
"${SPEC_ARGS[@]}"
|
||||
@@ -1,62 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve the FastFlowLM (FLM) sight+sound endpoint — frank-npu.
|
||||
#
|
||||
# Runs `flm serve <tag>` on the XDNA2 NPU (50 TOPS, Dankclaw). The NPU is a
|
||||
# single-lane device: generations serialize in FIFO order, so this is a
|
||||
# background/triage endpoint, not an interactive multi-user chat. The NPU
|
||||
# draws from the unified RAM pool shared with frank-large (the iGPU
|
||||
# llama.cpp service); see serverAppConfigs-docs/LLM/frank-npu.md
|
||||
# for the memory budget and the model-selection tradeoff.
|
||||
#
|
||||
# NOTE: FLM requires unlimited memlock. The unit sets LimitMEMLOCK=infinity.
|
||||
# NOTE: an unknown model tag does NOT 404 — /v1/chat/completions answers HTTP 200 with a
|
||||
# flat {"error":"Failed to load <tag> model!"} body and no choices; /v1/completions returns a
|
||||
# nested OpenAI-shaped server_error (code 500) also under HTTP 200. Client model IDs must be the exact FLM registry
|
||||
# tag (e.g. gemma4-it:e4b), never a service alias. Custom tags only resolve
|
||||
# when FLM_CONFIG_PATH points at the user registry that carries them and their
|
||||
# kernels are symlinked into /home/dank/.config/flm/xclbins/<PackName>.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_NPU_ENV_FILE:-$SCRIPT_DIR/frank-npu.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${FLM_BIN:=/usr/bin/flm}"
|
||||
: "${FLM_MODEL:=gemma4-it:e4b}"
|
||||
: "${FLM_HOST:=0.0.0.0}"
|
||||
: "${FLM_PORT:=7771}"
|
||||
: "${FLM_PMODE:=turbo}"
|
||||
: "${FLM_ASR:=0}"
|
||||
: "${FLM_CTX_LEN:=131072}"
|
||||
|
||||
# The user registry is a superset of the system registry (it carries the
|
||||
# custom tags too), and its model_path resolves relative to the registry file,
|
||||
# so exports keep a tag swap in the env file from needing script edits.
|
||||
export FLM_CONFIG_PATH="${FLM_CONFIG_PATH:-/home/dank/.config/flm/model_list.json}"
|
||||
export FLM_XCLBIN_PATH="${FLM_XCLBIN_PATH:-/home/dank/.config/flm/xclbins}"
|
||||
|
||||
if [[ ! -x "$FLM_BIN" ]]; then
|
||||
echo "missing flm binary: $FLM_BIN" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- optional flags ----------------------------------------------------------
|
||||
OPT_ARGS=()
|
||||
if [[ "${FLM_ASR}" != "0" ]]; then
|
||||
OPT_ARGS+=(--asr "${FLM_ASR}")
|
||||
fi
|
||||
if [[ "${FLM_CTX_LEN}" != "-1" ]]; then
|
||||
OPT_ARGS+=(--ctx-len "${FLM_CTX_LEN}")
|
||||
fi
|
||||
|
||||
exec "$FLM_BIN" serve "$FLM_MODEL" \
|
||||
--host "$FLM_HOST" \
|
||||
-p "$FLM_PORT" \
|
||||
--pmode "$FLM_PMODE" \
|
||||
"${OPT_ARGS[@]}"
|
||||
@@ -134,6 +134,14 @@ for path in "${CHANGED[@]}"; do
|
||||
add_test tests/homepage_caddy.bats
|
||||
add_test tests/deployment_scoping.bats
|
||||
;;
|
||||
# frank-large moved to a rootless quadlet (halogen-flash-server) on
|
||||
# 2026-09-11; its unit carries the fleet's GPU and podman posture, so
|
||||
# it gets its own seam rather than the generic fallback below.
|
||||
dankclaw/frank-large/*)
|
||||
add_test tests/frank_large_quadlet.bats
|
||||
add_test tests/deployment_scoping.bats
|
||||
add_test tests/docs_completeness.bats
|
||||
;;
|
||||
# Quadlets live under dankclaw/<svc>/quadlet/ (reorganized out of the
|
||||
# top-level dankclaw/*.container globs above). Map the whole subtree so
|
||||
# changes to the pihole, open-webui, searxng, and gitea quadlets hit
|
||||
@@ -160,11 +168,6 @@ for path in "${CHANGED[@]}"; do
|
||||
scripts/mount-entropy.sh)
|
||||
add_test tests/mount_entropy.bats
|
||||
;;
|
||||
scripts/frank-large.env|scripts/serve-frank-large.sh)
|
||||
add_test tests/llm_units.bats
|
||||
add_test tests/llm_secrets.bats
|
||||
add_test tests/scripts_syntax.bats
|
||||
;;
|
||||
scripts/frank-small.env|scripts/serve-frank-small.sh)
|
||||
add_test tests/llm_units.bats
|
||||
add_test tests/llm_secrets.bats
|
||||
|
||||
Reference in New Issue
Block a user