feat(llm): swap frank-large to halogen-flash-server, retire frank-npu
frank-large moves from the kingjones30/ROCmFPX llama.cpp build to peonist-ai/halogen-flash-server 0.5.8 serving Qwen3.8-Flash-Next (w4b .hgn 115.55 GiB + quality overlay). Upstream measures this engine at roughly 4x the prefill throughput of the llama.cpp family on the same gfx1151 silicon, and prefill dominates any prompt with real context. Shape: a rootless Quadlet container (dankclaw/frank-large/quadlet/), engine on container loopback :8730 (never published), OpenAI front-end :8731 published as host 7770, HALOGEN_MODEL_ID=frank-large so no client renamed anything. Context 262144, KV pool 262144 positions (~7.2 GiB, conservative for a shared 121 GiB host), 4 slots, host reserve 24, vision tower on, weights pinned, all other flags at vendor defaults. Weights are fetched by HALOGEN_DOWNLOAD on first start into a writable /home/dank/models/halogen. The llama.cpp era (unit, env, serve script) is staged in .AGENTTRASH/ rather than deleted; the STRIX payload and the ROCmFPX build stay on disk as the rollback target. deploy_llm_large becomes a user-quadlet handler: it retires the old ROOT unit (moving its checkout symlink into the home trash, never rm) and gates the first start on the download rather than reporting a failure for a fetch that is working exactly as designed. frank-npu is retired in the same change. The kernel cmdline moves from iommu=pt to amd_iommu=off for the 13-16% of prefill upstream measured, and that takes the XDNA2 NPU with it: without PASID/SVA the driver cannot open /dev/accel/accel0 (err=-19). Its five dependent routes are repointed (Hermes vision -> frank-large, Hermes title generation -> frank-small, harness vision -> frank-large, tiny -> frank-small, task -> frank-large:medium) and its provider blocks are removed from both the harness config and Hermes's live config. Audio input leaves the fleet with no replacement; image input survives on halogen's vision tower. Docs: new halogen-flash-server runbook, new IOMMU decision record, and frank-npu rewritten as a retirement record. AGENTS.md topology, runtime sections, benchmarking policy (frank-large is a user unit now) and the harness model-config section are corrected.
This commit is contained in:
Frank
+76
-30
@@ -90,7 +90,7 @@ service_is_allowed() {
|
||||
dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
|
||||
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|\
|
||||
dankclaw:frank-large|dankclaw:frank-npu|\
|
||||
dankclaw:frank-large|\
|
||||
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
|
||||
@@ -183,7 +183,6 @@ add_pending_services() {
|
||||
[[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue
|
||||
if [[ "$TARGET" == dankclaw && "$service" == frank-small ]]; then continue; fi
|
||||
if [[ "$TARGET" == dankserver && "$service" == frank-large ]]; then continue; fi
|
||||
if [[ "$TARGET" == dankserver && "$service" == frank-npu ]]; then continue; fi
|
||||
if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \
|
||||
--repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then
|
||||
echo "WARN: unable to compare state for $service; retaining it as pending" >&2
|
||||
@@ -826,21 +825,85 @@ deploy_anubis() {
|
||||
wait_container_healthy anubis 60 || return 1
|
||||
}
|
||||
|
||||
# Rename cutover (2026-09-03): frank-default-large -> frank-large,
|
||||
# frank-default-small -> frank-small, frank-default-sightsound -> frank-npu.
|
||||
# The legacy units are stopped and disabled before the renamed unit takes the
|
||||
# port so no legacy process is left holding it; stop/disable of an already
|
||||
# unloaded legacy unit fail harmlessly and are tolerated.
|
||||
deploy_llm_large() {
|
||||
run_privileged systemctl stop frank-default-large || true
|
||||
run_privileged systemctl disable frank-default-large || true
|
||||
relink_unit frank-large
|
||||
# frank-large runs as a rootless Quadlet unit since 2026-09-11 (halogen-flash-
|
||||
# server replaced the llama.cpp/kingjones runtime). Unlike the sibling LLM
|
||||
# services it is therefore NOT a root unit: no relink_unit, no `systemctl
|
||||
# enable` (this host's quadlet generator produces units that cannot be
|
||||
# enabled — boot persistence comes from [Install] WantedBy=default.target in
|
||||
# the .container file plus linger).
|
||||
#
|
||||
# The llama.cpp-era ROOT unit is retired first. Its unit file is a symlink
|
||||
# into this checkout (the old relink_unit did `ln -sfn scripts/frank-large.
|
||||
# service /etc/systemd/system/`), so leaving it behind after the repo file
|
||||
# moves to the trash would leave systemd holding a dangling unit that can
|
||||
# still claim port 7770. Stop it, drop its enable symlinks, move the unit file
|
||||
# into the home trash (never rm), then reload the root manager.
|
||||
#
|
||||
# FIRST START: the container fetches ~127 GB of weights before it can serve,
|
||||
# so /health is legitimately down for the whole fetch. wait_for_health would
|
||||
# burn its budget and report a failure that is not one. The gate below
|
||||
# recognises the bootstrap (checkpoint absent) and waits for the checkpoint
|
||||
# instead, falling through to the normal health gate once it lands.
|
||||
LLM_LARGE_CHECKPOINT="${LLM_LARGE_CHECKPOINT:-/home/dank/models/halogen/qwen38-flash-next-w4b.hgn}"
|
||||
|
||||
retire_root_llm_large() {
|
||||
local trash="$HOME/.AGENTTRASH/20260911-halogen-swap/host-units"
|
||||
run_privileged systemctl stop frank-default-large 2>/dev/null || true
|
||||
run_privileged systemctl disable frank-default-large 2>/dev/null || true
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
# On this host `disable` has been observed to delete the unit file itself
|
||||
# (2026-08-26 incident, see AGENTS.md); tolerate either behaviour and keep
|
||||
# whatever remains by moving it to the trash.
|
||||
run_privileged systemctl disable frank-large.service 2>/dev/null || true
|
||||
if [[ -e /etc/systemd/system/frank-large.service || -L /etc/systemd/system/frank-large.service ]]; then
|
||||
mkdir -p "$trash"
|
||||
run_privileged mv -- /etc/systemd/system/frank-large.service \
|
||||
"$trash/frank-large.service"
|
||||
echo "retired root unit frank-large.service -> $trash"
|
||||
fi
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-large
|
||||
run_privileged systemctl restart frank-large
|
||||
}
|
||||
|
||||
deploy_llm_large() {
|
||||
retire_root_llm_large
|
||||
# Pull once, then run the localhost/-aliased tag: this host's quadlet path
|
||||
# cannot resolve registry-prefixed refs from local storage at create time
|
||||
# (same workaround as anubis/hermes).
|
||||
local image="ghcr.io/peonist-ai/halogen-flash-server:0.5.8"
|
||||
local alias="localhost/peonist-ai/halogen-flash-server:0.5.8"
|
||||
podman pull "$image"
|
||||
podman tag "$image" "$alias"
|
||||
# The weights volume: read-write, because the container downloads into it.
|
||||
mkdir -p /home/dank/models/halogen
|
||||
install_quadlet frank-large dankclaw/frank-large/quadlet frank-large
|
||||
systemctl --user restart frank-large.service
|
||||
wait_for_llm_large_ready
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
# wait_for_llm_large_ready — bootstrap-aware gate. If the checkpoint is absent
|
||||
# the container is downloading it; wait for the file (resumable, and the unit
|
||||
# restarts it on failure) rather than gating on a /health that cannot answer
|
||||
# yet. Reports success either way, so the first deploy does not look like a
|
||||
# failure while the engine does exactly what it was told to.
|
||||
wait_for_llm_large_ready() {
|
||||
[[ -f "$LLM_LARGE_CHECKPOINT" ]] && return 0
|
||||
echo "frank-large: checkpoint absent at $LLM_LARGE_CHECKPOINT"
|
||||
echo "frank-large: first start — the container is downloading ~127 GB from"
|
||||
echo "frank-large: peonist-ai/halogen-qwen3.8-flash-next (resumes if interrupted)."
|
||||
local i
|
||||
for ((i = 1; i <= 1200; i++)); do
|
||||
if [[ -f "$LLM_LARGE_CHECKPOINT" ]]; then
|
||||
echo "frank-large: weights landed after approximately $((i * 2))s; gating on /health"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "WARN: frank-large download still in progress after 40m — NOT a failure."
|
||||
echo "WARN: the unit keeps fetching; re-run this deploy (or re-check /health) later." >&2
|
||||
return 0
|
||||
}
|
||||
|
||||
deploy_llm_small() {
|
||||
run_privileged systemctl stop frank-default-small || true
|
||||
run_privileged systemctl disable frank-default-small || true
|
||||
@@ -851,22 +914,6 @@ deploy_llm_small() {
|
||||
wait_for_health frank-small http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
# frank-npu is a live serving endpoint (FLM on the XDNA2 NPU), not a
|
||||
# relink-only unit: it is enabled at boot and restarted on deploy. The health
|
||||
# gate polls /v1/version, FLM's cheapest always-on route (there is no /health
|
||||
# route in the FLM server).
|
||||
deploy_llm_npu() {
|
||||
run_privileged systemctl stop frank-default-sightsound || true
|
||||
run_privileged systemctl disable frank-default-sightsound || true
|
||||
run_privileged systemctl stop frank-default-npu || true
|
||||
run_privileged systemctl disable frank-default-npu || true
|
||||
relink_unit frank-npu
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-npu
|
||||
run_privileged systemctl restart frank-npu
|
||||
wait_for_health frank-npu http://127.0.0.1:7771/v1/version 300
|
||||
}
|
||||
|
||||
wait_for_health() {
|
||||
local service="$1" url="$2" limit="$3"
|
||||
local i
|
||||
@@ -1052,7 +1099,6 @@ deploy_service() {
|
||||
seerr) deploy_seerr ;;
|
||||
frank-large) deploy_llm_large ;;
|
||||
frank-small) deploy_llm_small ;;
|
||||
frank-npu) deploy_llm_npu ;;
|
||||
hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
|
||||
romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
|
||||
jellyfin) deploy_jellyfin ;;
|
||||
|
||||
Reference in New Issue
Block a user