From b6ce3afb1fa78894dca708b19896f66061b32de8 Mon Sep 17 00:00:00 2001 From: Frank Date: Sat, 19 Sep 2026 02:59:01 -0700 Subject: [PATCH] =?UTF-8?q?frank-large:=20CIRU=20v4.4.1=20Orca=20cutover?= =?UTF-8?q?=20=E2=80=94=20root=20unit,=20one=20stream,=20full=20context?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator call 2026-09-19: switch the iGPU endpoint from halogen-flash-server to jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca on the CIRU v4.4.1 runtime with the QSA window hotfix (host build against the private ROCm 10 SDK), one stream of the full 262144 trained window. The service leaves the quadlet shape for a root unit (the same shape as frank-small and frank-medium: a private-SDK host binary has no image to install and no generated unit to dodge); the halogen quadlet is archived, the deploy handler retires it on the host, gates on 7770, and its stray-run sweep matches the model path so it cannot take frank-small down. --- .../frank-large/quadlet/frank-large.container | 219 ------------------ scripts/deploy-plan.py | 13 +- scripts/deploy-remote.sh | 138 +++++------ scripts/frank-large.env | 44 ++++ scripts/frank-large.service | 21 ++ scripts/serve-frank-large.sh | 80 +++++++ tests/deployment_scoping.bats | 2 +- tests/frank_large_unit.bats | 194 ++++++++-------- tests/frank_small_unit.bats | 12 +- 9 files changed, 317 insertions(+), 406 deletions(-) delete mode 100644 dankclaw/frank-large/quadlet/frank-large.container create mode 100644 scripts/frank-large.env create mode 100644 scripts/frank-large.service create mode 100755 scripts/serve-frank-large.sh diff --git a/dankclaw/frank-large/quadlet/frank-large.container b/dankclaw/frank-large/quadlet/frank-large.container deleted file mode 100644 index 16b6906..0000000 --- a/dankclaw/frank-large/quadlet/frank-large.container +++ /dev/null @@ -1,219 +0,0 @@ -# Quadlet unit: frank-large (rootless, boot-persistent). -# -# Install path on dankclaw: ~/.config/containers/systemd/frank-large.container -# Unit name: frank-large.service (container name: frank-large) -# -# The large inference service. The halogen-flash-server runtime (since -# 2026-09-11, restored 2026-09-16 after a two-day ik_llama.cpp hybrid -# interlude) serving peonist-ai/halogen-qwen3.8-flash-next (Qwen3.8-Flash-Next, -# w4b .hgn checkpoint + quality overlay sidecar). It replaced the llama.cpp / -# kingjones30-ROCmFPX STRIX deployment because the engine is ~4x faster on -# prefill (the dominant cost of any real prompt) on this exact SKU. The RTX -# 3090 it briefly shared with frank-large is now frank-small's card. -# -# SHAPE. The image runs both halves in one container: the engine on container -# loopback (:8730, UNAUTHENTICATED — never publish it) and the OpenAI -# front-end on :8731, published here as host 7770 so the fleet's clients, -# services.json probe and docs keep the port they have always used. Same for -# HALOGEN_MODEL_ID=frank-large: it is the id at /v1/models, in /health and in -# every response, so no client had to be renamed. Halogen never rejects a -# request for naming a different model, it just serves it. -# -# DEVICES. gfx1151 through the amdgpu/KFD stack: /dev/kfd plus the DRM render -# node. Both are 0666 on this host, so the rootless user namespace can open -# them (id dank is also in render/video). seccomp=unconfined and --ipc=host are -# the image's documented requirements; memlock must be unlimited. -# -# DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP -# runtime reads bare TEMP for its JIT cache path (2026-08-13 incident). -# -# MEMORY. Weights are pinned (HALOGEN_FLASH_PIN_TRUNK default 1) — that is the -# configuration the published decode numbers were measured with, and `free`, -# MemAvailable and every monitor built on them will overstate free memory by -# the size of the model (~68 GiB) because the kernel counts the locked weights -# as reclaimable page cache. Halogen prints the corrected figure at startup. -# The pinned trunk means this container holds most of the host; a co-tenant -# that then takes the remainder can stall both for minutes at full CPU with no -# disk activity and no output. That is documented behaviour, not a crash, and -# the escape hatch is HALOGEN_FLASH_PIN_TRUNK=0 (at several times the decode -# cost). Sizing is chosen for a SHARED 121 GiB host, and HALOGEN_KV_POOL_FIT -# stays on as the safety net: if the machine refuses the configured pool the -# engine lowers it itself and prints what it settled on. Read that ledger on -# every restart — it is the ground truth for sizing, not this comment. -# -# POOL RETUNED 2026-09-16: 400,384 positions -- exactly two 200,192-token -# conversations -- beside a per-conversation ceiling of 200,192 and two slots. -# It replaces the 401,408 the 2026-09-13 trim left behind, which held one full -# native-262k conversation beside a ~139k one, or three at 131k. -# -# MEASURED AT 401,408 (2026-09-13 deploy ledger): 68.0 weights + 11.0 KV + -# 21.0 working = 100.0 GiB held, 9.1 GiB / 1396 contiguous 2 MiB blocks left, -# engine need printed ~31.5 against the modelled ~33.4, and NO stall-warning -# NOTE. Fit-check budget here: 121.2 - 67.7 - 20 = 33.5. That boot is the -# baseline the 09-16 retune is measured against: the vision tower (1.97 GiB) -# and roughly 4.2 GiB of arena come back to the host, which is what pays for -# the co-tenant below. -# -# CO-TENANT since 2026-09-16: this host also serves frank-small on the RTX -# 3090 (llama.cpp, ~12.2 GiB of weights that load through the page cache and -# then live in VRAM, plus a CUDA context). It is the first real co-tenant this -# service has had, which is why the arena was halved and the reserve raised. -# -# BOOT NOTE: on the first request after a (re)start the engine can go silent -# for ~45-60 s ("has not answered PING") while it first-touches the freshly -# reserved pool; /health reads engine_unresponsive during it and recovers by -# itself. The quadlet health check's HealthStartPeriod=300s covers it; do not -# restart the unit over it. -# -# WEIGHTS. HALOGEN_DOWNLOAD fetches the whole weights repo on FIRST START ONLY -# (the entrypoint checks whether the checkpoint is absent) into the bind mount -# below, which must therefore be read-WRITE. huggingface_hub resumes partial -# files natively, and the quadlet runs the container with --replace --rm, so a -# restart mid-fetch continues rather than starting over. Roughly 127 GB: the -# 115.55 GiB checkpoint, the 2.31 GiB quality overlay, the 0.84 GiB vision -# tower and the tokenizer. The repo is public and ungated, so no token is -# needed; later starts open no outbound connections at all. -# -# VISION. Deliberately NOT set since 2026-09-16. The tower is a 1.97 GiB -# charge against the same device budget as the pool and the arena, and the -# fleet's image path moved to frank-small (llama.cpp on the RTX 3090, loaded -# with --mmproj mmproj-Qwen3.8-27B-BF16). This server refuses image content -# parts, and models.yml declares frank-large text-only to match. -# -# CLIENT CONTRACT (see serverAppConfigs-docs/LLM/halogen-flash-server.md). -# There are no server-side sampler defaults: a request that sends no -# temperature decodes GREEDY. The per-request budget defaults to 8,192 tokens -# and REASONING COUNTS AGAINST IT, so a budget that runs out mid-thought -# returns finish_reason "length" with empty content and the partial reasoning -# in reasoning_content. The cap is 65,536; above it a request is a 400, never a -# silent truncation. -# -# IMAGE. The unit runs the localhost/-aliased tag: this host's podman quadlet -# path cannot resolve registry-prefixed refs from local storage at create -# time. deploy-remote.sh pulls the ghcr.io ref and re-tags it before the -# restart. Pull=never keeps boots off the registry. -[Unit] -Description=frank-large — halogen-flash-server, Qwen3.8-Flash-Next (w4b) on gfx1151 -After=network-online.target - -[Container] -ContainerName=frank-large -Image=localhost/peonist-ai/halogen-flash-server:0.5.8 -# Published port stays 7770: Hermes, open-webui, the harness and the homepage -# probe all point here. 8730 (the engine) is deliberately not published. -PublishPort=7770:8731 -AddDevice=/dev/kfd -AddDevice=/dev/dri -# Read-write: the first start downloads into it. Contains only weights. -Volume=/home/dank/models/halogen:/models -# Fetch the weights on first start only (absent checkpoint gates it). -Environment=HALOGEN_DOWNLOAD=peonist-ai/halogen-qwen3.8-flash-next -# Fleet identity: the id at /v1/models, /health and every response body. -Environment=HALOGEN_MODEL_ID=frank-large -# Per-conversation ceiling: exactly half the pool, so two conversations of the -# full window are always resident. 200,192 rather than a round 200,000 is -# pool/2. The model's native window is 262,144; a conversation past this -# ceiling is refused, never silently truncated. Clients declare 200192 (see -# the frank-large contextWindow in models.yml). -Environment=HALOGEN_CTX=200192 -# The KV pool: 400,384 positions = 2 x 200,192 (the 2026-09-16 retune, down -# from 401,408). ~28 KiB per position including block scratch, so ~11.0 GiB -# held -- the 401,408 ledger entry measured exactly 11.0. The two slots still -# OVERSUBSCRIBE the pool for any real request, because admission reserves -# prompt + max_tokens: two full-window conversations each holding an output -# budget are pool-bound, and the second waits in arrival order for the first -# to release its region. Read the `kv pool:` lines on every boot; -# HALOGEN_KV_POOL_FIT lowers the pool itself if the budget refuses the -# configured one, and a green health check does not mean the pool you asked -# for is the pool you got. -Environment=HALOGEN_KV_POOL_POSITIONS=400384 -# Concurrency cap: two, one per half of the pool. Each slot is 111.0 MiB of -# O(1) state, so this hands ~111 MiB back against the three set on 2026-09-12 -# for the fleet size of the time. -Environment=HALOGEN_KV_SLOTS=2 -# Sizing is budget arithmetic, all of it. ONE budget covers the KV pool, the -# per-call prefill arena and the vision tower together: -# -# budget = 121.2 host - 67.7 resident weights - reserve -# need = KV (~28 KiB/position) + arena + vision -# arena = 3.80 GiB + 8.40 GiB x (HALOGEN_MAX_TOK / 16384) -# -# The arena term is linear in max_tok, and it reproduces five live restarts -# AND upstream's three published "on the device" figures to within 0.1 GiB — -# note those published figures are pool+arena totals, not pool alone, which is -# the misreading that first made a doubling look affordable here. Corners this -# box offered at a reserve of 24, at the old 262,144-position pool: -# -# 16384 -> need 28.6 vs 29.5 FITS, 0.9 spare (doubling, 8-9% prefill cost) -# 32768 -> need 37.0 vs 29.5 POOL CLAMPED to 262144, deploy still green -# 65536 -> need 46.5 vs 29.5 OOM, will not boot; unit still reads ACTIVE -# -# LOWERED 32768 -> 24576 on 2026-09-16 for the frank-small co-tenant on this -# same host. The arena's variable term goes 8.40 x 2 -> 8.40 x 1.5, handing -# ~4.2 GiB back to the host at roughly a 4-5 % prefill cost -- the same lever -# at 16,384 costs 8-9 %. The vendor's own notes name 24,576 as the largest -# arena that allocated with headroom to spare, so this stays inside the range -# it has tested. -# -# The arena is a PER-CALL PREFILL CHUNK, not a prompt ceiling: the -# 200,192-token conversation is prefilled in arena-sized pieces, so a smaller -# arena means more pieces and a slower prefill, not a smaller conversation. -# -# THE POOL, ARENA AND RESERVE ARE A MATCHED SET, NOT independent tunables -- -# the fit check's printed ledger on each boot is the arbiter, not these -# comments. -Environment=HALOGEN_MAX_TOK=24576 -# RAM the fit check refuses to count on, watched over by the engine's own -# n-gram lookup table (47.7 GiB, read through the page cache and never held) -# and, since 2026-09-16, by frank-small on the same host. -# -# RAISED 15 -> 20 with the 2026-09-16 retune. This knob feeds the fit-check -# budget and is NOT an allocation: raising it cannot hand memory back to the -# host, because the actual footprint is weights + KV + arena + vision, none of -# which it touches — it can only take the pool away. What it does is keep the -# fit check honest at the smaller arena. With the arena at 24,576 and no -# vision tower, the engine's printed need falls to roughly 25 GiB against a -# budget of 121.2 - 67.7 - 20 = 33.5, leaving ~8 GiB of margin, so the pool is -# not clamped. -# -# The 2026-09-13 profile for comparison: 103.5 GiB held and "7.1 GiB left for -# everything else" (310 contiguous 2 MiB blocks) at the doubled 524,288 pool, -# which is why the pool came down; at 401,408 the same ledger read 9.1 GiB / -# 1396 blocks. The 09-16 retune gives back the vision tower (1.97) and ~4.2 of -# arena on top of that for the co-tenant, so the expected figure at this boot -# is ~15 GiB left. Read the ledger; do not trust this arithmetic. -# -# To free host memory, change the POOL or the ARENA, and the reserve then -# becomes free to rise: HALOGEN_KV_POOL_POSITIONS=262144 hands back 3.6 GiB, -# HALOGEN_MAX_TOK=16384 hands back another 4.2. Both are one-line edits the -# fit check confirms on the next boot. -Environment=HALOGEN_HOST_RESERVE_GIB=20 -# Verbose engine logging: the vendor's first-line evidence for load, -# allocation and admission questions. Shipping it on means an incident report -# already has the full startup account instead of needing a restart-into-debug. -# HALOGEN_DMALLOC_LOG is deliberately NOT set — it logs every allocation -# over 64 MB, which is steady-state noise here; raise it only for a live -# OOM hunt. -Environment=HALOGEN_VERBOSE=1 -# Required by the image: the engine maps the checkpoint and registers it with -# the GPU in place, which needs host IPC and an unconfined seccomp profile. -PodmanArgs=--ipc=host --security-opt seccomp=unconfined -Ulimit=memlock=-1:-1 -# The image ships its own HEALTHCHECK; this pins it. /health asks the ENGINE -# for a PONG before answering, so 200 means "ready to serve", not "process -# alive". During the first-start download it stays unhealthy by design. -HealthCmd=halogen-healthcheck api -HealthInterval=30s -HealthTimeout=10s -HealthRetries=3 -HealthStartPeriod=300s -Pull=never - -[Service] -# The first start reads ~68 GiB off disk and pins it before serving. -TimeoutStartSec=900 -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=default.target diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py index 2fa5aea..ad2ae9d 100755 --- a/scripts/deploy-plan.py +++ b/scripts/deploy-plan.py @@ -314,11 +314,8 @@ def services_for_path(path: str, target: str) -> set[str]: if len(parts) >= 2 and parts[1] == "open-webui": _add(services, "open-webui") return services - if len(parts) >= 2 and parts[1] == "frank-large": - # The service is a rootless Quadlet container since 2026-09-11; - # its unit and fleet settings live in one .container file. - _add(services, "frank-large") - return services + # frank-large left the quadlet shape on 2026-09-19 (CIRU v4.4.1 host + # build, root unit): its files live in scripts/ and map there. if len(parts) >= 2 and parts[1] == "observability": # The five observability containers became Quadlet units on # 2026-09-11. A unit change selects exactly its own service; @@ -414,6 +411,12 @@ def services_for_path(path: str, target: str) -> set[str]: # runtime and payload are host-side one-offs: they change no # deployed state, so they select nothing. _add(services, "frank-medium") + if name in {"frank-large.env", "frank-large.service", "serve-frank-large.sh"}: + # CIRU v4.4.1 frank-large (2026-09-19) keeps its unit, env and + # serve files here the same way: a root unit on a host-built + # runtime. The download script fetches the payload, which is + # host-side state, so it selects nothing. + _add(services, "frank-large") # Deployment/test/benchmark scripts and other repo tooling do not # restart a runtime service. return services diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 15e1cf1..6444a85 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1028,41 +1028,32 @@ deploy_anubis() { wait_container_healthy anubis 60 || return 1 } -# frank-large runs as a rootless Quadlet unit again since 2026-09-16: the -# halogen-flash-server runtime is back (it replaced ik_llama.cpp, which had -# replaced it), and the RTX 3090 it shared during that interlude now belongs -# to frank-small. +# frank-large is the CIRU v4.4.1 runtime (QSA window hotfix) serving +# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca on the Strix Halo iGPU, as a ROOT +# unit since the 2026-09-19 cutover: the runtime is a host build against +# the private ROCm 10 SDK, so there is no container image to install and no +# generated unit to dodge. relink_unit points +# /etc/systemd/system/frank-large.service at the repo file and boot +# persistence is that unit's [Install] section. # -# TWO MANAGERS, ONE UNIT NAME. The ik incarnation ran as a ROOT unit whose -# file is a symlink into this checkout; the halogen unit is a USER (quadlet) -# unit. Both must never exist at once — they would race for port 7770 — so the -# root unit is retired first: stop, disable, move the /etc symlink into the -# home trash (never `rm`), reload the root manager. This host's `systemctl -# disable` has been observed to delete the unit file itself (2026-08-26 -# incident), so the move tolerates a file that is already gone. -# -# FIRST START fetches ~127 GB of weights, and /health cannot answer while it -# does. wait_for_llm_large_ready recognises that (checkpoint absent) and waits -# for the file instead of burning the health budget on a failure that is not -# one. -LLM_LARGE_CHECKPOINT="${LLM_LARGE_CHECKPOINT:-/home/dank/models/halogen/qwen38-flash-next-w4b.hgn}" +# It replaces the halogen-flash-server quadlet, which served +# peonist-ai/halogen-qwen3.8-flash-next on this same iGPU since 2026-09-16. +# The quadlet is retired HERE (the unit file moves into the home trash, +# never `rm`), and the port gate is the same as the 2026-09-16 cutover: +# nothing starts until 7770 is released. +LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf}" +LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/mtp/Qwen3.8-Flash-CIRU-STRIX-Orca-MTP-Q8_0.gguf}" +LLM_LARGE_PLE="${LLM_LARGE_PLE:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/ple/ple.payload.bin}" -retire_root_llm_large() { - local trash="$HOME/.AGENTTRASH/20260916-halogen-restore/host-units" - run_privileged systemctl stop frank-large.service 2>/dev/null || true - run_privileged systemctl disable frank-large.service 2>/dev/null || true - if [[ -e /etc/systemd/system/frank-large.service || -L /etc/systemd/system/frank-large.service ]]; then +retire_halogen_large_quadlet() { + local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" + systemctl --user stop frank-large.service 2>/dev/null || true + if [[ -e "$quadlet_unit_dir/frank-large.container" ]]; then mkdir -p "$trash" - run_privileged mv -- /etc/systemd/system/frank-large.service \ - "$trash/frank-large.service" - echo "retired root unit frank-large.service -> $trash" - else - # This host's `systemctl disable` is allowed to delete the unit file - # outright (2026-08-26 incident), so there is often nothing left to - # move. Say so: previously no message meant no evidence either way. - echo "root unit frank-large.service was already removed by disable" + mv -- "$quadlet_unit_dir/frank-large.container" "$trash/frank-large.container" + echo "retired quadlet frank-large.container -> $trash" fi - run_privileged systemctl daemon-reload + systemctl --user daemon-reload } # Wait until a TCP port has no socket on it at all -- listening, accepted, or @@ -1085,58 +1076,41 @@ wait_port_free() { } deploy_llm_large() { - retire_root_llm_large - # The predecessor can also be THIS quadlet -- a redeploy of a running - # frank-large -- so stop the user unit before waiting on the port. On a - # first cutover it is not running and this is a no-op. - systemctl --user stop frank-large.service 2>/dev/null || true - # Pull once, then run the localhost/-aliased tag: this host's quadlet path - # cannot resolve registry-prefixed refs from local storage at create time - # (same workaround as anubis/hermes), and Pull=never keeps boots off the - # registry. - local image="ghcr.io/peonist-ai/halogen-flash-server:0.5.8" - local alias="localhost/peonist-ai/halogen-flash-server:0.5.8" - podman pull "$image" - podman tag "$image" "$alias" - # The weights volume must exist, and the container needs it read-write: - # that is where a first start downloads into. - mkdir -p /home/dank/models/halogen - # Nothing may start until the predecessor has released the port. On - # 2026-09-16 this sequence failed exactly here: the ik server logged - # "Deactivated successfully", pasta tried to bind 7770 a second later and - # reported "Address already in use", and the unit exited 126. Restart= - # on-failure recovered it minutes later, but a deploy that starts a doomed - # container has not deployed anything -- so gate on the port, not on hope. - wait_port_free 7770 || return 1 - install_quadlet frank-large dankclaw/frank-large/quadlet frank-large - # No enable: generated units cannot be enabled on this host (v1 quadlet - # generator); boot persistence comes from [Install] in the unit file. - systemctl --user restart frank-large.service - wait_for_llm_large_ready - wait_for_health frank-large http://127.0.0.1:7770/health 1200 -} - -# Bootstrap-aware gate: if the checkpoint is absent the container is fetching -# it; wait for the file (the fetch resumes, and the unit restarts it on -# failure) rather than gating on a /health that cannot answer yet. Reports -# success either way, so a first deploy does not look like a failure while the -# engine does exactly what it was told to. -wait_for_llm_large_ready() { - [[ -f "$LLM_LARGE_CHECKPOINT" ]] && return 0 - echo "frank-large: checkpoint absent at $LLM_LARGE_CHECKPOINT" - echo "frank-large: first start — the container is downloading ~127 GB from" - echo "frank-large: peonist-ai/halogen-qwen3.8-flash-next (resumes if interrupted)." - local i - for ((i = 1; i <= 1200; i++)); do - if [[ -f "$LLM_LARGE_CHECKPOINT" ]]; then - echo "frank-large: weights landed after approximately $((i * 2))s; gating on /health" - return 0 + # Refuse rather than restart-loop. A unit whose runtime or payload is + # missing never becomes healthy, and a green deploy would leave the + # endpoint dark with nothing in the deploy log to say why. + local missing=0 f + for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT" "$LLM_LARGE_PLE"; do + if [[ ! -f "$f" ]]; then + echo "ERROR: payload absent at $f" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large-orca.sh" >&2 + missing=1 fi - sleep 2 done - echo "WARN: frank-large download still in progress after 40m — NOT a failure." >&2 - echo "WARN: the unit keeps fetching; re-run this deploy (or re-check /health) later." >&2 - return 0 + [[ "$missing" -eq 0 ]] || return 1 + + retire_halogen_large_quadlet + # On a re-deploy the predecessor is THIS unit: stop it before waiting on + # the port, or the wait deadlocks on the server it just stopped serving. + # Then sweep a stray manual run of the SAME model -- that is how a bench + # harness leaves the port busy. Match the model path, not the binary name: + # frank-small on this host is also a llama-server and must not be killed. + # The pattern never appears in the deploying shell's own command line, so + # -f is safe here (it is not in `bash deploy-remote.sh`). + run_privileged systemctl stop frank-large.service 2>/dev/null || true + pkill -f 'Qwen3.8-Flash-CIRU-STRIX-Orca' 2>/dev/null || true + sleep 2 + # Nothing may start until the predecessor has released the port. On + # 2026-09-16 the cutover failed exactly here: the old server logged + # "Deactivated successfully", the next engine tried to bind 7770 a second + # later and reported "Address already in use". Gate on the port, not on + # hope. + wait_port_free 7770 || return 1 + relink_unit frank-large + run_privileged systemctl daemon-reload + run_privileged systemctl enable frank-large.service + run_privileged systemctl restart frank-large.service + wait_for_health frank-large http://127.0.0.1:7770/health 1200 } # deploy_llm_small — frank-small is mainline llama.cpp on the RTX 3090, as a @@ -1146,7 +1120,7 @@ wait_for_llm_large_ready() { # is that unit's [Install] section. # # It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is -# not shared: frank-large's halogen engine runs on the gfx1151 iGPU. +# not shared: frank-large's engine runs on the gfx1151 iGPU. LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/Gemma4-12B-QAT/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf}" deploy_llm_small() { diff --git a/scripts/frank-large.env b/scripts/frank-large.env new file mode 100644 index 0000000..562f6c0 --- /dev/null +++ b/scripts/frank-large.env @@ -0,0 +1,44 @@ +# frank-large — CIRU v4.4.1 (QSA window hotfix) serving +# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca on the Strix Halo iGPU (gfx1151). +# +# These are the launcher tunables read by the runtime's +# scripts/ciru/run-orca-server.sh (see the serve script). The 36-variable +# kernel bundle and the PLE pager defaults live in the runtime's own +# profiles/strix-halo-production.env, which the launcher sources AFTER this +# file; every profile line is a ${VAR:-default}, so a value set here wins. +# +# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP runtime +# reads those bare (the 2026-08-13 TEMP incident). HOST is the one exception +# because the launcher itself consumes it for --host; its value doubles as a +# JIT-cache directory name, which is why 0.0.0.0 is safe. + +# The assembled payload (scripts/download-frank-large-orca.sh): the launcher +# refuses to start without the main GGUF, the MTP head and the PLE sidecar. +MODEL_DIR=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca + +# Fleet endpoint: the same port and alias the halogen predecessor served, so +# no client, probe or models.yml entry changes address. +HOST=0.0.0.0 +PORT=7770 + +# One stream, full context (operator call, 2026-09-19). MTP requires exactly +# one slot in this runtime, so the drafter's constraint and the operator's +# ask are the same number. 262144 is the model's trained window; the GGUF +# carries no rope scaling. +CONTEXT_SIZE=262144 +PARALLEL_SLOTS=1 + +# MTP head: on. Depth 4 is the Orca variant's own default (mtp_depth_default +# in the launcher); the sidecar is the Q8_0 MTP file from the model repo. +ENABLE_MTP=1 + +# Text-only for now: the F16 projector is on disk at +# vision/mmproj-Qwen3.8-Flash-Orca-F16.mmproj; flip to 1 to enable it. The +# fleet's image path is frank-small either way, so models.yml stays +# input: [text] until this changes. +ENABLE_VISION=0 + +# PLE pager: the 52.4 GiB payload is mmap'd and paged by the engine's own +# workers; this is the hot window kept resident. The vendor's production +# profile default is 4096. +PLE_CACHE_MIB=4096 diff --git a/scripts/frank-large.service b/scripts/frank-large.service new file mode 100644 index 0000000..9f76cb6 --- /dev/null +++ b/scripts/frank-large.service @@ -0,0 +1,21 @@ +[Unit] +Description=CIRU v4.4.1 Orca Qwen3.8-Flash (dankclaw, frank-large, Strix Halo iGPU gfx1151) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=dank +Group=dank +# The engine pins staging buffers and the PLE pager's hot window; keep the +# memlock budget unbounded so an RLIMIT smaller than the pinned set cannot +# fail the load. Parity with the halogen quadlet's Ulimit=memlock=-1:-1 and +# the other LLM units on this fleet. +LimitMEMLOCK=infinity +Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env" +ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh new file mode 100755 index 0000000..3cbfda3 --- /dev/null +++ b/scripts/serve-frank-large.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Serve jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca as frank-large on Dankclaw's +# Strix Halo iGPU (gfx1151), using the CIRU v4.4.1 runtime with the QSA +# window hotfix. +# +# Runtime: the v4.4.1 source with the QSA window hotfix pre-applied +# (ciru-runtime-v4.4.1-qsa-hotfix-source.tar.gz, shipped in the model repo +# under runtime/v4.4.1/qsa-window-hotfix/), built ON the host against the +# private ROCm 10.0.0 SDK that scripts/ciru/setup-linux-amd.sh installs into +# the runtime checkout's own venv. The launcher below re-verifies on every +# start that the built libraries carry the v4.4.1 corrections (it greps the +# common and HIP libraries for the fix markers) before exec. +# +# The launcher owns the flag list: -sm none (one iGPU, no split), -ngl all, +# f16 KV, flash attention, cont-batching, the PLE sidecar, the MTP drafter +# (depth 4 for the Orca variant) and the production kernel-bundle profile. +# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides it; +# anything not set there falls through to the launcher's own defaults. +# +# It replaces the halogen-flash-server quadlet, which served +# peonist-ai/halogen-qwen3.8-flash-next on this same iGPU since 2026-09-16. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}" +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +# --- defaults (used when the env file does not set them) ------------------ +: "${RUNTIME_DIR:=/home/dank/GitPackages/ciru-runtime-v4.4.1-qsa-hotfix}" +: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca}" +: "${HOST:=0.0.0.0}" +: "${PORT:=7770}" +: "${CONTEXT_SIZE:=262144}" +: "${PARALLEL_SLOTS:=1}" + +LAUNCHER="$RUNTIME_DIR/scripts/ciru/run-orca-server.sh" +SERVER_BIN="$RUNTIME_DIR/build-gfx1151-sdk/bin/llama-server" + +if [[ ! -x "$LAUNCHER" ]]; then + echo "missing runtime launcher: $LAUNCHER" >&2 + echo "fetch the hotfix source tarball from the model repo" >&2 + echo "(runtime/v4.4.1/qsa-window-hotfix/) and build it:" >&2 + echo " bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2 + exit 1 +fi +if [[ ! -x "$SERVER_BIN" ]]; then + echo "missing built runtime: $SERVER_BIN" >&2 + echo "build it: bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2 + exit 1 +fi +if [[ ! -r "$MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" ]]; then + echo "missing target model: $MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" >&2 + echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2 + exit 1 +fi + +# The launcher checks the MTP head and the PLE sidecar itself, but a missing +# payload should be a deploy error with a pointer, not a runtime exit 2. +for required in \ + "$MODEL_DIR/mtp/Qwen3.8-Flash-CIRU-STRIX-Orca-MTP-Q8_0.gguf" \ + "$MODEL_DIR/ple/ple.payload.bin" \ + "$MODEL_DIR/ple/ple.manifest.json" \ + "$MODEL_DIR/ple/ple.scale.bf16"; do + if [[ ! -r "$required" ]]; then + echo "missing payload: $required" >&2 + echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2 + exit 1 + fi +done + +# The Qwen3.8 family template keeps prior thinking blocks across turns when +# preserve_thinking is set; the launcher passes --jinja but no template +# kwargs, so name them here. A request-level chat_template_kwargs still +# wins over this. +exec "$LAUNCHER" --chat-template-kwargs '{"preserve_thinking":true}' diff --git a/tests/deployment_scoping.bats b/tests/deployment_scoping.bats index 7028dbc..35146af 100644 --- a/tests/deployment_scoping.bats +++ b/tests/deployment_scoping.bats @@ -18,7 +18,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" @test "deployment planner maps large-model changes to the large model only" { run python3 "$PLANNER" --target dankclaw --paths \ - dankclaw/frank-large/quadlet/frank-large.container + scripts/frank-large.env [ "$status" -eq 0 ] [ "$output" = "frank-large" ] } diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index 5991dd8..f0a0c48 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -1,113 +1,121 @@ #!/usr/bin/env bats -# Seam: the frank-large halogen-flash-server quadlet on dankclaw. -# halogen-flash-server replaced ik_llama.cpp again on 2026-09-16, so this -# service is once more a rootless Quadlet container on the gfx1151 iGPU. This -# seam pins what a flag list cannot show: the boot-persistent podman posture -# (both device nodes, memlock, the create-time image contract), the -# matched ctx/pool/slots set that IS the two-conversation promise, the -# co-tenancy budget the arena and reserve are chosen for, the image handoff to -# frank-small, and a deploy handler that retires the previous ROOT unit without -# deleting it. +# Seam: the frank-large CIRU v4.4.1 root unit on dankclaw. +# +# The 2026-09-19 cutover moved this service out of the halogen-flash-server +# quadlet and onto the CIRU v4.4.1 runtime (QSA window hotfix) serving +# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca, as a ROOT unit on a host build -- the +# same shape as frank-small and frank-medium, because a private-SDK host +# binary has no container image to install and no generated unit to dodge. +# This seam pins what a flag list cannot show: the operator's one-stream / +# full-context shape, the payload the launcher refuses to start without, a +# deploy handler that retires the halogen quadlet without deleting it and +# gates on the port, and the fact that the sweep for a stray benchmark run +# cannot take frank-small down with it. load helpers -QUADLET="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" +ENV_FILE="$REPO_ROOT/scripts/frank-large.env" +UNIT="$REPO_ROOT/scripts/frank-large.service" +SERVE="$REPO_ROOT/scripts/serve-frank-large.sh" +FETCH="$REPO_ROOT/scripts/download-frank-large-orca.sh" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" -@test "the frank-large quadlet exists where the deployer installs it from" { - [ -f "$QUADLET" ] +@test "the frank-large unit, env and serve files exist where the deployer installs them from" { + [ -f "$ENV_FILE" ] + [ -f "$UNIT" ] + [ -x "$SERVE" ] } -@test "frank-large is boot-persistent, self-healing and container-shaped" { - grep -q '^WantedBy=default.target$' "$QUADLET" - grep -q '^Restart=on-failure$' "$QUADLET" - grep -q '^RestartSec=' "$QUADLET" - grep -q '^Pull=never$' "$QUADLET" - # The localhost/-aliased tag is what create-time resolution needs on this - # host; the deployer pulls the ghcr.io ref and re-tags it. - grep -q '^Image=localhost/peonist-ai/halogen-flash-server:' "$QUADLET" +@test "frank-large is boot-persistent, self-healing and unprivileged" { + grep -q '^WantedBy=multi-user.target$' "$UNIT" + grep -q '^Restart=on-failure$' "$UNIT" + grep -q '^RestartSec=' "$UNIT" + grep -q '^User=dank$' "$UNIT" + # The engine pins staging buffers and the PLE pager's hot window; the + # memlock budget must be unbounded or the load fails on the RLIMIT. + grep -q '^LimitMEMLOCK=infinity$' "$UNIT" + # The unit runs the repo's own serve script with the repo's env. + grep -q 'ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh' "$UNIT" + grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT" } -@test "frank-large has the gfx1151 devices and the image's documented posture" { - grep -q '^AddDevice=/dev/kfd$' "$QUADLET" - grep -q '^AddDevice=/dev/dri$' "$QUADLET" - grep -q '^PodmanArgs=--ipc=host --security-opt seccomp=unconfined$' "$QUADLET" - grep -q '^Ulimit=memlock=-1:-1$' "$QUADLET" - # The weights mount must stay read-write: the first start downloads into it. - grep -q '^Volume=/home/dank/models/halogen:/models$' "$QUADLET" +@test "the operator's shape is one stream of the full trained window" { + local ctx slots + ctx="$(sed -nE 's/^CONTEXT_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" + slots="$(sed -nE 's/^PARALLEL_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")" + # 262144 is the model's trained window and the GGUF carries no rope + # scaling; the operator asked for full context. + [ "$ctx" -eq 262144 ] + # One slot. This is also the drafter's constraint: the launcher refuses + # MTP with more than one slot, so the two cannot drift apart silently. + [ "$slots" -eq 1 ] + grep -q '^ENABLE_MTP=1$' "$ENV_FILE" } -@test "frank-large keeps the fleet alias and publishes only the API port" { - grep -q '^Environment=HALOGEN_MODEL_ID=frank-large$' "$QUADLET" - grep -q '^PublishPort=7770:8731$' "$QUADLET" - # The engine's own listener stays inside the container: it is unauthenticated. - ! grep -q 'PublishPort=.*8730' "$QUADLET" -} - -@test "the KV pool is exactly two conversations of the per-request ceiling" { - local ctx pool slots - ctx="$(sed -nE 's/^Environment=HALOGEN_CTX=([0-9]+)$/\1/p' "$QUADLET")" - pool="$(sed -nE 's/^Environment=HALOGEN_KV_POOL_POSITIONS=([0-9]+)$/\1/p' "$QUADLET")" - slots="$(sed -nE 's/^Environment=HALOGEN_KV_SLOTS=([0-9]+)$/\1/p' "$QUADLET")" - [ -n "$ctx" ] && [ -n "$pool" ] && [ -n "$slots" ] - # The deployed promise is two resident conversations: the pool is twice the - # ceiling, one slot per conversation. Slots still oversubscribe the pool for - # any real request, because admission reserves prompt + max_tokens. - [ "$pool" -eq $((ctx * 2)) ] - [ "$slots" -eq 2 ] -} - -@test "the prefill arena stays inside the range the vendor tested" { - local max_tok reserve - max_tok="$(sed -nE 's/^Environment=HALOGEN_MAX_TOK=([0-9]+)$/\1/p' "$QUADLET")" - reserve="$(sed -nE 's/^Environment=HALOGEN_HOST_RESERVE_GIB=([0-9]+)$/\1/p' "$QUADLET")" - [ -n "$max_tok" ] && [ -n "$reserve" ] - # 32768 is the vendor default; 65536 cannot boot on this host because the - # arena alone outgrows the device budget. The arena is a per-call prefill - # chunk, so a smaller value costs prefill speed, not context. - [ "$max_tok" -le 32768 ] - [ "$max_tok" -ge 16384 ] - [ "$reserve" -ge 15 ] -} - -@test "frank-large loads no vision tower: the fleet's image path is frank-small" { - ! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET" +@test "frank-large stays text-only until the projector is enabled" { + grep -q '^ENABLE_VISION=0$' "$ENV_FILE" + # The fleet's image path is frank-small; the projector is on disk and is + # one env flip away, but models.yml declares frank-large text-only. grep -q -- '--mmproj' "$REPO_ROOT/scripts/serve-frank-small.sh" } -@test "the deploy handler retires the previous ROOT unit without deleting it" { - grep -q 'retire_root_llm_large' "$REMOTE_DEPLOY" - grep -q 'systemctl stop frank-large.service' "$REMOTE_DEPLOY" - grep -q 'systemctl disable frank-large.service' "$REMOTE_DEPLOY" +@test "the serve script pins the hotfix runtime and refuses a missing build" { + grep -q 'ciru-runtime-v4.4.1-qsa-hotfix' "$SERVE" + grep -q 'build-gfx1151-sdk/bin/llama-server' "$SERVE" + grep -q 'run-orca-server.sh' "$SERVE" + # A missing runtime or payload is a deploy error with a pointer, not a + # restart loop. + grep -q 'missing built runtime' "$SERVE" + grep -q 'missing target model' "$SERVE" + # The Qwen3.8 template keeps prior thinking blocks across turns only + # when the kwarg is set; the launcher passes --jinja but no kwargs. + grep -q 'preserve_thinking' "$SERVE" +} + +@test "the payload is fetched resumably and verified by sha256" { + grep -q 'jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca' "$FETCH" + grep -q -- '-fL -C -' "$FETCH" + grep -q 'parts.json' "$FETCH" + grep -q 'sha256sum -c SHA256SUMS' "$FETCH" + # The PLE sidecar is the largest transport part; the script must fetch + # and verify it, not just the GGUF. + grep -q 'ple.payload.bin' "$FETCH" +} + +@test "the fleet endpoint and alias survive the cutover" { + grep -q '^HOST=0.0.0.0$' "$ENV_FILE" + grep -q '^PORT=7770$' "$ENV_FILE" + grep -q '^MODEL_DIR=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca$' "$ENV_FILE" +} + +@test "the deploy handler retires the halogen quadlet without deleting it" { + grep -q 'retire_halogen_large_quadlet' "$REMOTE_DEPLOY" grep -q 'HOME/.AGENTTRASH' "$REMOTE_DEPLOY" ! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?.*frank-large' "$REMOTE_DEPLOY" } -@test "the deploy handler installs the quadlet and gates the first start on weights" { - grep -q 'install_quadlet frank-large dankclaw/frank-large/quadlet frank-large' "$REMOTE_DEPLOY" - grep -q 'wait_for_llm_large_ready' "$REMOTE_DEPLOY" - grep -q 'qwen38-flash-next-w4b.hgn' "$REMOTE_DEPLOY" - grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY" -} - -@test "the ik_llama.cpp runtime that held this service is archived, not live" { - # Its files must not sit in scripts/ where a stale root unit could still be - # pointed at them, and must not be gone either: .AGENTTRASH is the rollback. - local retired - for retired in frank-large.env.ik-llama.cpp frank-large.service.ik-llama.cpp \ - serve-frank-large.sh.ik-llama.cpp; do - [ -f "$REPO_ROOT/scripts/.AGENTTRASH/$retired" ] - done - [ ! -e "$REPO_ROOT/scripts/frank-large.env" ] - [ ! -e "$REPO_ROOT/scripts/frank-large.service" ] - [ ! -e "$REPO_ROOT/scripts/serve-frank-large.sh" ] -} - -@test "the deploy handler waits for the predecessor to release the port" { - # The 2026-09-16 cutover failed at exactly this: the ik server's socket was - # still on 7770 when pasta tried to bind it ("Address already in use"), and - # the container died on it. Restart=on-failure recovered minutes later, but - # a deploy that starts a doomed container has not deployed anything. +@test "the deploy handler installs the root unit, gates the port and refuses a missing payload" { + grep -q 'relink_unit frank-large' "$REMOTE_DEPLOY" + grep -q 'systemctl enable frank-large.service' "$REMOTE_DEPLOY" grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY" - grep -q '^wait_port_free() {' "$REMOTE_DEPLOY" -} \ No newline at end of file + grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY" + grep -q 'LLM_LARGE_MODEL' "$REMOTE_DEPLOY" + grep -q 'LLM_LARGE_PLE' "$REMOTE_DEPLOY" +} + +@test "the stray-run sweep cannot take frank-small down" { + # frank-small on this host is ALSO a llama-server. The sweep must match + # the model path, not the binary name, or a frank-large deploy restarts + # the 3090 endpoint. + grep -q "pkill -f 'Qwen3.8-Flash-CIRU-STRIX-Orca'" "$REMOTE_DEPLOY" + # ...and only inside the large handler: extract it and check there is no + # bare binary-name pkill in it. + local large + large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")" + ! grep -q 'pkill -x llama-server' <<<"$large" +} + +@test "the halogen quadlet is archived, not live" { + [ ! -e "$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" ] + [ -f "$REPO_ROOT/dankclaw/.AGENTTRASH/frank-large.container.halogen" ] +} diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 01163b1..6b77526 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -14,7 +14,7 @@ UNIT="$REPO_ROOT/scripts/frank-small.service" SERVE="$REPO_ROOT/scripts/serve-frank-small.sh" ENV_FILE="$REPO_ROOT/scripts/frank-small.env" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" -QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" +ENV_LARGE="$REPO_ROOT/scripts/frank-large.env" @test "frank-small unit, serve script and env exist in scripts/" { [ -f "$UNIT" ] @@ -58,7 +58,7 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" @test "frank-small carries the fleet's image path, and only it" { grep -q '^MMPROJ_PATH=.*mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16\.gguf$' "$ENV_FILE" grep -q -- '--mmproj "$MMPROJ_PATH"' "$SERVE" - ! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET_LARGE" + ! grep -q '^ENABLE_VISION=1$' "$ENV_LARGE" } @test "frank-small's slot grid is full-window and fits the 24 GiB card" { @@ -94,10 +94,10 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" grep -q '^ALIAS=frank-small$' "$ENV_FILE" grep -q -- '--port "$LISTEN_PORT"' "$SERVE" grep -q -- '--host "$BIND_ADDR"' "$SERVE" - # The two endpoints cannot share a port, and only the halogen unit publishes - # 7770. - grep -q '^PublishPort=7770:8731$' "$QUADLET_LARGE" - ! grep -q 'PublishPort=.*7771' "$QUADLET_LARGE" + # The two endpoints cannot share a port: frank-large holds 7770, this one + # 7771. + grep -q '^PORT=7770$' "$ENV_LARGE" + ! grep -q '^PORT=7771$' "$ENV_LARGE" } @test "the deploy handler installs the root unit and refuses a missing model" {