frank-large: CIRU v4.4.1 Orca cutover — root unit, one stream, full context
Operator call 2026-09-19: switch the iGPU endpoint from halogen-flash-server to jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca on the CIRU v4.4.1 runtime with the QSA window hotfix (host build against the private ROCm 10 SDK), one stream of the full 262144 trained window. The service leaves the quadlet shape for a root unit (the same shape as frank-small and frank-medium: a private-SDK host binary has no image to install and no generated unit to dodge); the halogen quadlet is archived, the deploy handler retires it on the host, gates on 7770, and its stray-run sweep matches the model path so it cannot take frank-small down.
This commit is contained in:
Frank
@@ -314,11 +314,8 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
if len(parts) >= 2 and parts[1] == "open-webui":
|
||||
_add(services, "open-webui")
|
||||
return services
|
||||
if len(parts) >= 2 and parts[1] == "frank-large":
|
||||
# The service is a rootless Quadlet container since 2026-09-11;
|
||||
# its unit and fleet settings live in one .container file.
|
||||
_add(services, "frank-large")
|
||||
return services
|
||||
# frank-large left the quadlet shape on 2026-09-19 (CIRU v4.4.1 host
|
||||
# build, root unit): its files live in scripts/ and map there.
|
||||
if len(parts) >= 2 and parts[1] == "observability":
|
||||
# The five observability containers became Quadlet units on
|
||||
# 2026-09-11. A unit change selects exactly its own service;
|
||||
@@ -414,6 +411,12 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
# runtime and payload are host-side one-offs: they change no
|
||||
# deployed state, so they select nothing.
|
||||
_add(services, "frank-medium")
|
||||
if name in {"frank-large.env", "frank-large.service", "serve-frank-large.sh"}:
|
||||
# CIRU v4.4.1 frank-large (2026-09-19) keeps its unit, env and
|
||||
# serve files here the same way: a root unit on a host-built
|
||||
# runtime. The download script fetches the payload, which is
|
||||
# host-side state, so it selects nothing.
|
||||
_add(services, "frank-large")
|
||||
# Deployment/test/benchmark scripts and other repo tooling do not
|
||||
# restart a runtime service.
|
||||
return services
|
||||
|
||||
+56
-82
@@ -1028,41 +1028,32 @@ deploy_anubis() {
|
||||
wait_container_healthy anubis 60 || return 1
|
||||
}
|
||||
|
||||
# frank-large runs as a rootless Quadlet unit again since 2026-09-16: the
|
||||
# halogen-flash-server runtime is back (it replaced ik_llama.cpp, which had
|
||||
# replaced it), and the RTX 3090 it shared during that interlude now belongs
|
||||
# to frank-small.
|
||||
# frank-large is the CIRU v4.4.1 runtime (QSA window hotfix) serving
|
||||
# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca on the Strix Halo iGPU, as a ROOT
|
||||
# unit since the 2026-09-19 cutover: the runtime is a host build against
|
||||
# the private ROCm 10 SDK, so there is no container image to install and no
|
||||
# generated unit to dodge. relink_unit points
|
||||
# /etc/systemd/system/frank-large.service at the repo file and boot
|
||||
# persistence is that unit's [Install] section.
|
||||
#
|
||||
# TWO MANAGERS, ONE UNIT NAME. The ik incarnation ran as a ROOT unit whose
|
||||
# file is a symlink into this checkout; the halogen unit is a USER (quadlet)
|
||||
# unit. Both must never exist at once — they would race for port 7770 — so the
|
||||
# root unit is retired first: stop, disable, move the /etc symlink into the
|
||||
# home trash (never `rm`), reload the root manager. This host's `systemctl
|
||||
# disable` has been observed to delete the unit file itself (2026-08-26
|
||||
# incident), so the move tolerates a file that is already gone.
|
||||
#
|
||||
# FIRST START fetches ~127 GB of weights, and /health cannot answer while it
|
||||
# does. wait_for_llm_large_ready recognises that (checkpoint absent) and waits
|
||||
# for the file instead of burning the health budget on a failure that is not
|
||||
# one.
|
||||
LLM_LARGE_CHECKPOINT="${LLM_LARGE_CHECKPOINT:-/home/dank/models/halogen/qwen38-flash-next-w4b.hgn}"
|
||||
# It replaces the halogen-flash-server quadlet, which served
|
||||
# peonist-ai/halogen-qwen3.8-flash-next on this same iGPU since 2026-09-16.
|
||||
# The quadlet is retired HERE (the unit file moves into the home trash,
|
||||
# never `rm`), and the port gate is the same as the 2026-09-16 cutover:
|
||||
# nothing starts until 7770 is released.
|
||||
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf}"
|
||||
LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/mtp/Qwen3.8-Flash-CIRU-STRIX-Orca-MTP-Q8_0.gguf}"
|
||||
LLM_LARGE_PLE="${LLM_LARGE_PLE:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/ple/ple.payload.bin}"
|
||||
|
||||
retire_root_llm_large() {
|
||||
local trash="$HOME/.AGENTTRASH/20260916-halogen-restore/host-units"
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
run_privileged systemctl disable frank-large.service 2>/dev/null || true
|
||||
if [[ -e /etc/systemd/system/frank-large.service || -L /etc/systemd/system/frank-large.service ]]; then
|
||||
retire_halogen_large_quadlet() {
|
||||
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
||||
systemctl --user stop frank-large.service 2>/dev/null || true
|
||||
if [[ -e "$quadlet_unit_dir/frank-large.container" ]]; then
|
||||
mkdir -p "$trash"
|
||||
run_privileged mv -- /etc/systemd/system/frank-large.service \
|
||||
"$trash/frank-large.service"
|
||||
echo "retired root unit frank-large.service -> $trash"
|
||||
else
|
||||
# This host's `systemctl disable` is allowed to delete the unit file
|
||||
# outright (2026-08-26 incident), so there is often nothing left to
|
||||
# move. Say so: previously no message meant no evidence either way.
|
||||
echo "root unit frank-large.service was already removed by disable"
|
||||
mv -- "$quadlet_unit_dir/frank-large.container" "$trash/frank-large.container"
|
||||
echo "retired quadlet frank-large.container -> $trash"
|
||||
fi
|
||||
run_privileged systemctl daemon-reload
|
||||
systemctl --user daemon-reload
|
||||
}
|
||||
|
||||
# Wait until a TCP port has no socket on it at all -- listening, accepted, or
|
||||
@@ -1085,58 +1076,41 @@ wait_port_free() {
|
||||
}
|
||||
|
||||
deploy_llm_large() {
|
||||
retire_root_llm_large
|
||||
# The predecessor can also be THIS quadlet -- a redeploy of a running
|
||||
# frank-large -- so stop the user unit before waiting on the port. On a
|
||||
# first cutover it is not running and this is a no-op.
|
||||
systemctl --user stop frank-large.service 2>/dev/null || true
|
||||
# Pull once, then run the localhost/-aliased tag: this host's quadlet path
|
||||
# cannot resolve registry-prefixed refs from local storage at create time
|
||||
# (same workaround as anubis/hermes), and Pull=never keeps boots off the
|
||||
# registry.
|
||||
local image="ghcr.io/peonist-ai/halogen-flash-server:0.5.8"
|
||||
local alias="localhost/peonist-ai/halogen-flash-server:0.5.8"
|
||||
podman pull "$image"
|
||||
podman tag "$image" "$alias"
|
||||
# The weights volume must exist, and the container needs it read-write:
|
||||
# that is where a first start downloads into.
|
||||
mkdir -p /home/dank/models/halogen
|
||||
# Nothing may start until the predecessor has released the port. On
|
||||
# 2026-09-16 this sequence failed exactly here: the ik server logged
|
||||
# "Deactivated successfully", pasta tried to bind 7770 a second later and
|
||||
# reported "Address already in use", and the unit exited 126. Restart=
|
||||
# on-failure recovered it minutes later, but a deploy that starts a doomed
|
||||
# container has not deployed anything -- so gate on the port, not on hope.
|
||||
wait_port_free 7770 || return 1
|
||||
install_quadlet frank-large dankclaw/frank-large/quadlet frank-large
|
||||
# No enable: generated units cannot be enabled on this host (v1 quadlet
|
||||
# generator); boot persistence comes from [Install] in the unit file.
|
||||
systemctl --user restart frank-large.service
|
||||
wait_for_llm_large_ready
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
# Bootstrap-aware gate: if the checkpoint is absent the container is fetching
|
||||
# it; wait for the file (the fetch resumes, and the unit restarts it on
|
||||
# failure) rather than gating on a /health that cannot answer yet. Reports
|
||||
# success either way, so a first deploy does not look like a failure while the
|
||||
# engine does exactly what it was told to.
|
||||
wait_for_llm_large_ready() {
|
||||
[[ -f "$LLM_LARGE_CHECKPOINT" ]] && return 0
|
||||
echo "frank-large: checkpoint absent at $LLM_LARGE_CHECKPOINT"
|
||||
echo "frank-large: first start — the container is downloading ~127 GB from"
|
||||
echo "frank-large: peonist-ai/halogen-qwen3.8-flash-next (resumes if interrupted)."
|
||||
local i
|
||||
for ((i = 1; i <= 1200; i++)); do
|
||||
if [[ -f "$LLM_LARGE_CHECKPOINT" ]]; then
|
||||
echo "frank-large: weights landed after approximately $((i * 2))s; gating on /health"
|
||||
return 0
|
||||
# Refuse rather than restart-loop. A unit whose runtime or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the deploy log to say why.
|
||||
local missing=0 f
|
||||
for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT" "$LLM_LARGE_PLE"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: payload absent at $f" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large-orca.sh" >&2
|
||||
missing=1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "WARN: frank-large download still in progress after 40m — NOT a failure." >&2
|
||||
echo "WARN: the unit keeps fetching; re-run this deploy (or re-check /health) later." >&2
|
||||
return 0
|
||||
[[ "$missing" -eq 0 ]] || return 1
|
||||
|
||||
retire_halogen_large_quadlet
|
||||
# On a re-deploy the predecessor is THIS unit: stop it before waiting on
|
||||
# the port, or the wait deadlocks on the server it just stopped serving.
|
||||
# Then sweep a stray manual run of the SAME model -- that is how a bench
|
||||
# harness leaves the port busy. Match the model path, not the binary name:
|
||||
# frank-small on this host is also a llama-server and must not be killed.
|
||||
# The pattern never appears in the deploying shell's own command line, so
|
||||
# -f is safe here (it is not in `bash deploy-remote.sh`).
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
pkill -f 'Qwen3.8-Flash-CIRU-STRIX-Orca' 2>/dev/null || true
|
||||
sleep 2
|
||||
# Nothing may start until the predecessor has released the port. On
|
||||
# 2026-09-16 the cutover failed exactly here: the old server logged
|
||||
# "Deactivated successfully", the next engine tried to bind 7770 a second
|
||||
# later and reported "Address already in use". Gate on the port, not on
|
||||
# hope.
|
||||
wait_port_free 7770 || return 1
|
||||
relink_unit frank-large
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-large.service
|
||||
run_privileged systemctl restart frank-large.service
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
# deploy_llm_small — frank-small is mainline llama.cpp on the RTX 3090, as a
|
||||
@@ -1146,7 +1120,7 @@ wait_for_llm_large_ready() {
|
||||
# is that unit's [Install] section.
|
||||
#
|
||||
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
|
||||
# not shared: frank-large's halogen engine runs on the gfx1151 iGPU.
|
||||
# not shared: frank-large's engine runs on the gfx1151 iGPU.
|
||||
LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/Gemma4-12B-QAT/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf}"
|
||||
|
||||
deploy_llm_small() {
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
# frank-large — CIRU v4.4.1 (QSA window hotfix) serving
|
||||
# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca on the Strix Halo iGPU (gfx1151).
|
||||
#
|
||||
# These are the launcher tunables read by the runtime's
|
||||
# scripts/ciru/run-orca-server.sh (see the serve script). The 36-variable
|
||||
# kernel bundle and the PLE pager defaults live in the runtime's own
|
||||
# profiles/strix-halo-production.env, which the launcher sources AFTER this
|
||||
# file; every profile line is a ${VAR:-default}, so a value set here wins.
|
||||
#
|
||||
# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP runtime
|
||||
# reads those bare (the 2026-08-13 TEMP incident). HOST is the one exception
|
||||
# because the launcher itself consumes it for --host; its value doubles as a
|
||||
# JIT-cache directory name, which is why 0.0.0.0 is safe.
|
||||
|
||||
# The assembled payload (scripts/download-frank-large-orca.sh): the launcher
|
||||
# refuses to start without the main GGUF, the MTP head and the PLE sidecar.
|
||||
MODEL_DIR=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca
|
||||
|
||||
# Fleet endpoint: the same port and alias the halogen predecessor served, so
|
||||
# no client, probe or models.yml entry changes address.
|
||||
HOST=0.0.0.0
|
||||
PORT=7770
|
||||
|
||||
# One stream, full context (operator call, 2026-09-19). MTP requires exactly
|
||||
# one slot in this runtime, so the drafter's constraint and the operator's
|
||||
# ask are the same number. 262144 is the model's trained window; the GGUF
|
||||
# carries no rope scaling.
|
||||
CONTEXT_SIZE=262144
|
||||
PARALLEL_SLOTS=1
|
||||
|
||||
# MTP head: on. Depth 4 is the Orca variant's own default (mtp_depth_default
|
||||
# in the launcher); the sidecar is the Q8_0 MTP file from the model repo.
|
||||
ENABLE_MTP=1
|
||||
|
||||
# Text-only for now: the F16 projector is on disk at
|
||||
# vision/mmproj-Qwen3.8-Flash-Orca-F16.mmproj; flip to 1 to enable it. The
|
||||
# fleet's image path is frank-small either way, so models.yml stays
|
||||
# input: [text] until this changes.
|
||||
ENABLE_VISION=0
|
||||
|
||||
# PLE pager: the 52.4 GiB payload is mmap'd and paged by the engine's own
|
||||
# workers; this is the hot window kept resident. The vendor's production
|
||||
# profile default is 4096.
|
||||
PLE_CACHE_MIB=4096
|
||||
@@ -0,0 +1,21 @@
|
||||
[Unit]
|
||||
Description=CIRU v4.4.1 Orca Qwen3.8-Flash (dankclaw, frank-large, Strix Halo iGPU gfx1151)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
# The engine pins staging buffers and the PLE pager's hot window; keep the
|
||||
# memlock budget unbounded so an RLIMIT smaller than the pinned set cannot
|
||||
# fail the load. Parity with the halogen quadlet's Ulimit=memlock=-1:-1 and
|
||||
# the other LLM units on this fleet.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+80
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca as frank-large on Dankclaw's
|
||||
# Strix Halo iGPU (gfx1151), using the CIRU v4.4.1 runtime with the QSA
|
||||
# window hotfix.
|
||||
#
|
||||
# Runtime: the v4.4.1 source with the QSA window hotfix pre-applied
|
||||
# (ciru-runtime-v4.4.1-qsa-hotfix-source.tar.gz, shipped in the model repo
|
||||
# under runtime/v4.4.1/qsa-window-hotfix/), built ON the host against the
|
||||
# private ROCm 10.0.0 SDK that scripts/ciru/setup-linux-amd.sh installs into
|
||||
# the runtime checkout's own venv. The launcher below re-verifies on every
|
||||
# start that the built libraries carry the v4.4.1 corrections (it greps the
|
||||
# common and HIP libraries for the fix markers) before exec.
|
||||
#
|
||||
# The launcher owns the flag list: -sm none (one iGPU, no split), -ngl all,
|
||||
# f16 KV, flash attention, cont-batching, the PLE sidecar, the MTP drafter
|
||||
# (depth 4 for the Orca variant) and the production kernel-bundle profile.
|
||||
# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides it;
|
||||
# anything not set there falls through to the launcher's own defaults.
|
||||
#
|
||||
# It replaces the halogen-flash-server quadlet, which served
|
||||
# peonist-ai/halogen-qwen3.8-flash-next on this same iGPU since 2026-09-16.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) ------------------
|
||||
: "${RUNTIME_DIR:=/home/dank/GitPackages/ciru-runtime-v4.4.1-qsa-hotfix}"
|
||||
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca}"
|
||||
: "${HOST:=0.0.0.0}"
|
||||
: "${PORT:=7770}"
|
||||
: "${CONTEXT_SIZE:=262144}"
|
||||
: "${PARALLEL_SLOTS:=1}"
|
||||
|
||||
LAUNCHER="$RUNTIME_DIR/scripts/ciru/run-orca-server.sh"
|
||||
SERVER_BIN="$RUNTIME_DIR/build-gfx1151-sdk/bin/llama-server"
|
||||
|
||||
if [[ ! -x "$LAUNCHER" ]]; then
|
||||
echo "missing runtime launcher: $LAUNCHER" >&2
|
||||
echo "fetch the hotfix source tarball from the model repo" >&2
|
||||
echo "(runtime/v4.4.1/qsa-window-hotfix/) and build it:" >&2
|
||||
echo " bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -x "$SERVER_BIN" ]]; then
|
||||
echo "missing built runtime: $SERVER_BIN" >&2
|
||||
echo "build it: bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -r "$MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" ]]; then
|
||||
echo "missing target model: $MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" >&2
|
||||
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The launcher checks the MTP head and the PLE sidecar itself, but a missing
|
||||
# payload should be a deploy error with a pointer, not a runtime exit 2.
|
||||
for required in \
|
||||
"$MODEL_DIR/mtp/Qwen3.8-Flash-CIRU-STRIX-Orca-MTP-Q8_0.gguf" \
|
||||
"$MODEL_DIR/ple/ple.payload.bin" \
|
||||
"$MODEL_DIR/ple/ple.manifest.json" \
|
||||
"$MODEL_DIR/ple/ple.scale.bf16"; do
|
||||
if [[ ! -r "$required" ]]; then
|
||||
echo "missing payload: $required" >&2
|
||||
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# The Qwen3.8 family template keeps prior thinking blocks across turns when
|
||||
# preserve_thinking is set; the launcher passes --jinja but no template
|
||||
# kwargs, so name them here. A request-level chat_template_kwargs still
|
||||
# wins over this.
|
||||
exec "$LAUNCHER" --chat-template-kwargs '{"preserve_thinking":true}'
|
||||
Reference in New Issue
Block a user