diff --git a/dankclaw/frank-large/quadlet/frank-large.container b/dankclaw/frank-large/quadlet/frank-large.container deleted file mode 100644 index 16b6906..0000000 --- a/dankclaw/frank-large/quadlet/frank-large.container +++ /dev/null @@ -1,219 +0,0 @@ -# Quadlet unit: frank-large (rootless, boot-persistent). -# -# Install path on dankclaw: ~/.config/containers/systemd/frank-large.container -# Unit name: frank-large.service (container name: frank-large) -# -# The large inference service. The halogen-flash-server runtime (since -# 2026-09-11, restored 2026-09-16 after a two-day ik_llama.cpp hybrid -# interlude) serving peonist-ai/halogen-qwen3.8-flash-next (Qwen3.8-Flash-Next, -# w4b .hgn checkpoint + quality overlay sidecar). It replaced the llama.cpp / -# kingjones30-ROCmFPX STRIX deployment because the engine is ~4x faster on -# prefill (the dominant cost of any real prompt) on this exact SKU. The RTX -# 3090 it briefly shared with frank-large is now frank-small's card. -# -# SHAPE. The image runs both halves in one container: the engine on container -# loopback (:8730, UNAUTHENTICATED — never publish it) and the OpenAI -# front-end on :8731, published here as host 7770 so the fleet's clients, -# services.json probe and docs keep the port they have always used. Same for -# HALOGEN_MODEL_ID=frank-large: it is the id at /v1/models, in /health and in -# every response, so no client had to be renamed. Halogen never rejects a -# request for naming a different model, it just serves it. -# -# DEVICES. gfx1151 through the amdgpu/KFD stack: /dev/kfd plus the DRM render -# node. Both are 0666 on this host, so the rootless user namespace can open -# them (id dank is also in render/video). seccomp=unconfined and --ipc=host are -# the image's documented requirements; memlock must be unlimited. -# -# DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP -# runtime reads bare TEMP for its JIT cache path (2026-08-13 incident). -# -# MEMORY. Weights are pinned (HALOGEN_FLASH_PIN_TRUNK default 1) — that is the -# configuration the published decode numbers were measured with, and `free`, -# MemAvailable and every monitor built on them will overstate free memory by -# the size of the model (~68 GiB) because the kernel counts the locked weights -# as reclaimable page cache. Halogen prints the corrected figure at startup. -# The pinned trunk means this container holds most of the host; a co-tenant -# that then takes the remainder can stall both for minutes at full CPU with no -# disk activity and no output. That is documented behaviour, not a crash, and -# the escape hatch is HALOGEN_FLASH_PIN_TRUNK=0 (at several times the decode -# cost). Sizing is chosen for a SHARED 121 GiB host, and HALOGEN_KV_POOL_FIT -# stays on as the safety net: if the machine refuses the configured pool the -# engine lowers it itself and prints what it settled on. Read that ledger on -# every restart — it is the ground truth for sizing, not this comment. -# -# POOL RETUNED 2026-09-16: 400,384 positions -- exactly two 200,192-token -# conversations -- beside a per-conversation ceiling of 200,192 and two slots. -# It replaces the 401,408 the 2026-09-13 trim left behind, which held one full -# native-262k conversation beside a ~139k one, or three at 131k. -# -# MEASURED AT 401,408 (2026-09-13 deploy ledger): 68.0 weights + 11.0 KV + -# 21.0 working = 100.0 GiB held, 9.1 GiB / 1396 contiguous 2 MiB blocks left, -# engine need printed ~31.5 against the modelled ~33.4, and NO stall-warning -# NOTE. Fit-check budget here: 121.2 - 67.7 - 20 = 33.5. That boot is the -# baseline the 09-16 retune is measured against: the vision tower (1.97 GiB) -# and roughly 4.2 GiB of arena come back to the host, which is what pays for -# the co-tenant below. -# -# CO-TENANT since 2026-09-16: this host also serves frank-small on the RTX -# 3090 (llama.cpp, ~12.2 GiB of weights that load through the page cache and -# then live in VRAM, plus a CUDA context). It is the first real co-tenant this -# service has had, which is why the arena was halved and the reserve raised. -# -# BOOT NOTE: on the first request after a (re)start the engine can go silent -# for ~45-60 s ("has not answered PING") while it first-touches the freshly -# reserved pool; /health reads engine_unresponsive during it and recovers by -# itself. The quadlet health check's HealthStartPeriod=300s covers it; do not -# restart the unit over it. -# -# WEIGHTS. HALOGEN_DOWNLOAD fetches the whole weights repo on FIRST START ONLY -# (the entrypoint checks whether the checkpoint is absent) into the bind mount -# below, which must therefore be read-WRITE. huggingface_hub resumes partial -# files natively, and the quadlet runs the container with --replace --rm, so a -# restart mid-fetch continues rather than starting over. Roughly 127 GB: the -# 115.55 GiB checkpoint, the 2.31 GiB quality overlay, the 0.84 GiB vision -# tower and the tokenizer. The repo is public and ungated, so no token is -# needed; later starts open no outbound connections at all. -# -# VISION. Deliberately NOT set since 2026-09-16. The tower is a 1.97 GiB -# charge against the same device budget as the pool and the arena, and the -# fleet's image path moved to frank-small (llama.cpp on the RTX 3090, loaded -# with --mmproj mmproj-Qwen3.8-27B-BF16). This server refuses image content -# parts, and models.yml declares frank-large text-only to match. -# -# CLIENT CONTRACT (see serverAppConfigs-docs/LLM/halogen-flash-server.md). -# There are no server-side sampler defaults: a request that sends no -# temperature decodes GREEDY. The per-request budget defaults to 8,192 tokens -# and REASONING COUNTS AGAINST IT, so a budget that runs out mid-thought -# returns finish_reason "length" with empty content and the partial reasoning -# in reasoning_content. The cap is 65,536; above it a request is a 400, never a -# silent truncation. -# -# IMAGE. The unit runs the localhost/-aliased tag: this host's podman quadlet -# path cannot resolve registry-prefixed refs from local storage at create -# time. deploy-remote.sh pulls the ghcr.io ref and re-tags it before the -# restart. Pull=never keeps boots off the registry. -[Unit] -Description=frank-large — halogen-flash-server, Qwen3.8-Flash-Next (w4b) on gfx1151 -After=network-online.target - -[Container] -ContainerName=frank-large -Image=localhost/peonist-ai/halogen-flash-server:0.5.8 -# Published port stays 7770: Hermes, open-webui, the harness and the homepage -# probe all point here. 8730 (the engine) is deliberately not published. -PublishPort=7770:8731 -AddDevice=/dev/kfd -AddDevice=/dev/dri -# Read-write: the first start downloads into it. Contains only weights. -Volume=/home/dank/models/halogen:/models -# Fetch the weights on first start only (absent checkpoint gates it). -Environment=HALOGEN_DOWNLOAD=peonist-ai/halogen-qwen3.8-flash-next -# Fleet identity: the id at /v1/models, /health and every response body. -Environment=HALOGEN_MODEL_ID=frank-large -# Per-conversation ceiling: exactly half the pool, so two conversations of the -# full window are always resident. 200,192 rather than a round 200,000 is -# pool/2. The model's native window is 262,144; a conversation past this -# ceiling is refused, never silently truncated. Clients declare 200192 (see -# the frank-large contextWindow in models.yml). -Environment=HALOGEN_CTX=200192 -# The KV pool: 400,384 positions = 2 x 200,192 (the 2026-09-16 retune, down -# from 401,408). ~28 KiB per position including block scratch, so ~11.0 GiB -# held -- the 401,408 ledger entry measured exactly 11.0. The two slots still -# OVERSUBSCRIBE the pool for any real request, because admission reserves -# prompt + max_tokens: two full-window conversations each holding an output -# budget are pool-bound, and the second waits in arrival order for the first -# to release its region. Read the `kv pool:` lines on every boot; -# HALOGEN_KV_POOL_FIT lowers the pool itself if the budget refuses the -# configured one, and a green health check does not mean the pool you asked -# for is the pool you got. -Environment=HALOGEN_KV_POOL_POSITIONS=400384 -# Concurrency cap: two, one per half of the pool. Each slot is 111.0 MiB of -# O(1) state, so this hands ~111 MiB back against the three set on 2026-09-12 -# for the fleet size of the time. -Environment=HALOGEN_KV_SLOTS=2 -# Sizing is budget arithmetic, all of it. ONE budget covers the KV pool, the -# per-call prefill arena and the vision tower together: -# -# budget = 121.2 host - 67.7 resident weights - reserve -# need = KV (~28 KiB/position) + arena + vision -# arena = 3.80 GiB + 8.40 GiB x (HALOGEN_MAX_TOK / 16384) -# -# The arena term is linear in max_tok, and it reproduces five live restarts -# AND upstream's three published "on the device" figures to within 0.1 GiB — -# note those published figures are pool+arena totals, not pool alone, which is -# the misreading that first made a doubling look affordable here. Corners this -# box offered at a reserve of 24, at the old 262,144-position pool: -# -# 16384 -> need 28.6 vs 29.5 FITS, 0.9 spare (doubling, 8-9% prefill cost) -# 32768 -> need 37.0 vs 29.5 POOL CLAMPED to 262144, deploy still green -# 65536 -> need 46.5 vs 29.5 OOM, will not boot; unit still reads ACTIVE -# -# LOWERED 32768 -> 24576 on 2026-09-16 for the frank-small co-tenant on this -# same host. The arena's variable term goes 8.40 x 2 -> 8.40 x 1.5, handing -# ~4.2 GiB back to the host at roughly a 4-5 % prefill cost -- the same lever -# at 16,384 costs 8-9 %. The vendor's own notes name 24,576 as the largest -# arena that allocated with headroom to spare, so this stays inside the range -# it has tested. -# -# The arena is a PER-CALL PREFILL CHUNK, not a prompt ceiling: the -# 200,192-token conversation is prefilled in arena-sized pieces, so a smaller -# arena means more pieces and a slower prefill, not a smaller conversation. -# -# THE POOL, ARENA AND RESERVE ARE A MATCHED SET, NOT independent tunables -- -# the fit check's printed ledger on each boot is the arbiter, not these -# comments. -Environment=HALOGEN_MAX_TOK=24576 -# RAM the fit check refuses to count on, watched over by the engine's own -# n-gram lookup table (47.7 GiB, read through the page cache and never held) -# and, since 2026-09-16, by frank-small on the same host. -# -# RAISED 15 -> 20 with the 2026-09-16 retune. This knob feeds the fit-check -# budget and is NOT an allocation: raising it cannot hand memory back to the -# host, because the actual footprint is weights + KV + arena + vision, none of -# which it touches — it can only take the pool away. What it does is keep the -# fit check honest at the smaller arena. With the arena at 24,576 and no -# vision tower, the engine's printed need falls to roughly 25 GiB against a -# budget of 121.2 - 67.7 - 20 = 33.5, leaving ~8 GiB of margin, so the pool is -# not clamped. -# -# The 2026-09-13 profile for comparison: 103.5 GiB held and "7.1 GiB left for -# everything else" (310 contiguous 2 MiB blocks) at the doubled 524,288 pool, -# which is why the pool came down; at 401,408 the same ledger read 9.1 GiB / -# 1396 blocks. The 09-16 retune gives back the vision tower (1.97) and ~4.2 of -# arena on top of that for the co-tenant, so the expected figure at this boot -# is ~15 GiB left. Read the ledger; do not trust this arithmetic. -# -# To free host memory, change the POOL or the ARENA, and the reserve then -# becomes free to rise: HALOGEN_KV_POOL_POSITIONS=262144 hands back 3.6 GiB, -# HALOGEN_MAX_TOK=16384 hands back another 4.2. Both are one-line edits the -# fit check confirms on the next boot. -Environment=HALOGEN_HOST_RESERVE_GIB=20 -# Verbose engine logging: the vendor's first-line evidence for load, -# allocation and admission questions. Shipping it on means an incident report -# already has the full startup account instead of needing a restart-into-debug. -# HALOGEN_DMALLOC_LOG is deliberately NOT set — it logs every allocation -# over 64 MB, which is steady-state noise here; raise it only for a live -# OOM hunt. -Environment=HALOGEN_VERBOSE=1 -# Required by the image: the engine maps the checkpoint and registers it with -# the GPU in place, which needs host IPC and an unconfined seccomp profile. -PodmanArgs=--ipc=host --security-opt seccomp=unconfined -Ulimit=memlock=-1:-1 -# The image ships its own HEALTHCHECK; this pins it. /health asks the ENGINE -# for a PONG before answering, so 200 means "ready to serve", not "process -# alive". During the first-start download it stays unhealthy by design. -HealthCmd=halogen-healthcheck api -HealthInterval=30s -HealthTimeout=10s -HealthRetries=3 -HealthStartPeriod=300s -Pull=never - -[Service] -# The first start reads ~68 GiB off disk and pins it before serving. -TimeoutStartSec=900 -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=default.target diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py index 2fa5aea..ad2ae9d 100755 --- a/scripts/deploy-plan.py +++ b/scripts/deploy-plan.py @@ -314,11 +314,8 @@ def services_for_path(path: str, target: str) -> set[str]: if len(parts) >= 2 and parts[1] == "open-webui": _add(services, "open-webui") return services - if len(parts) >= 2 and parts[1] == "frank-large": - # The service is a rootless Quadlet container since 2026-09-11; - # its unit and fleet settings live in one .container file. - _add(services, "frank-large") - return services + # frank-large left the quadlet shape on 2026-09-19 (CIRU v4.4.1 host + # build, root unit): its files live in scripts/ and map there. if len(parts) >= 2 and parts[1] == "observability": # The five observability containers became Quadlet units on # 2026-09-11. A unit change selects exactly its own service; @@ -414,6 +411,12 @@ def services_for_path(path: str, target: str) -> set[str]: # runtime and payload are host-side one-offs: they change no # deployed state, so they select nothing. _add(services, "frank-medium") + if name in {"frank-large.env", "frank-large.service", "serve-frank-large.sh"}: + # CIRU v4.4.1 frank-large (2026-09-19) keeps its unit, env and + # serve files here the same way: a root unit on a host-built + # runtime. The download script fetches the payload, which is + # host-side state, so it selects nothing. + _add(services, "frank-large") # Deployment/test/benchmark scripts and other repo tooling do not # restart a runtime service. return services diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 15e1cf1..6444a85 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1028,41 +1028,32 @@ deploy_anubis() { wait_container_healthy anubis 60 || return 1 } -# frank-large runs as a rootless Quadlet unit again since 2026-09-16: the -# halogen-flash-server runtime is back (it replaced ik_llama.cpp, which had -# replaced it), and the RTX 3090 it shared during that interlude now belongs -# to frank-small. +# frank-large is the CIRU v4.4.1 runtime (QSA window hotfix) serving +# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca on the Strix Halo iGPU, as a ROOT +# unit since the 2026-09-19 cutover: the runtime is a host build against +# the private ROCm 10 SDK, so there is no container image to install and no +# generated unit to dodge. relink_unit points +# /etc/systemd/system/frank-large.service at the repo file and boot +# persistence is that unit's [Install] section. # -# TWO MANAGERS, ONE UNIT NAME. The ik incarnation ran as a ROOT unit whose -# file is a symlink into this checkout; the halogen unit is a USER (quadlet) -# unit. Both must never exist at once — they would race for port 7770 — so the -# root unit is retired first: stop, disable, move the /etc symlink into the -# home trash (never `rm`), reload the root manager. This host's `systemctl -# disable` has been observed to delete the unit file itself (2026-08-26 -# incident), so the move tolerates a file that is already gone. -# -# FIRST START fetches ~127 GB of weights, and /health cannot answer while it -# does. wait_for_llm_large_ready recognises that (checkpoint absent) and waits -# for the file instead of burning the health budget on a failure that is not -# one. -LLM_LARGE_CHECKPOINT="${LLM_LARGE_CHECKPOINT:-/home/dank/models/halogen/qwen38-flash-next-w4b.hgn}" +# It replaces the halogen-flash-server quadlet, which served +# peonist-ai/halogen-qwen3.8-flash-next on this same iGPU since 2026-09-16. +# The quadlet is retired HERE (the unit file moves into the home trash, +# never `rm`), and the port gate is the same as the 2026-09-16 cutover: +# nothing starts until 7770 is released. +LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf}" +LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/mtp/Qwen3.8-Flash-CIRU-STRIX-Orca-MTP-Q8_0.gguf}" +LLM_LARGE_PLE="${LLM_LARGE_PLE:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/ple/ple.payload.bin}" -retire_root_llm_large() { - local trash="$HOME/.AGENTTRASH/20260916-halogen-restore/host-units" - run_privileged systemctl stop frank-large.service 2>/dev/null || true - run_privileged systemctl disable frank-large.service 2>/dev/null || true - if [[ -e /etc/systemd/system/frank-large.service || -L /etc/systemd/system/frank-large.service ]]; then +retire_halogen_large_quadlet() { + local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" + systemctl --user stop frank-large.service 2>/dev/null || true + if [[ -e "$quadlet_unit_dir/frank-large.container" ]]; then mkdir -p "$trash" - run_privileged mv -- /etc/systemd/system/frank-large.service \ - "$trash/frank-large.service" - echo "retired root unit frank-large.service -> $trash" - else - # This host's `systemctl disable` is allowed to delete the unit file - # outright (2026-08-26 incident), so there is often nothing left to - # move. Say so: previously no message meant no evidence either way. - echo "root unit frank-large.service was already removed by disable" + mv -- "$quadlet_unit_dir/frank-large.container" "$trash/frank-large.container" + echo "retired quadlet frank-large.container -> $trash" fi - run_privileged systemctl daemon-reload + systemctl --user daemon-reload } # Wait until a TCP port has no socket on it at all -- listening, accepted, or @@ -1085,58 +1076,41 @@ wait_port_free() { } deploy_llm_large() { - retire_root_llm_large - # The predecessor can also be THIS quadlet -- a redeploy of a running - # frank-large -- so stop the user unit before waiting on the port. On a - # first cutover it is not running and this is a no-op. - systemctl --user stop frank-large.service 2>/dev/null || true - # Pull once, then run the localhost/-aliased tag: this host's quadlet path - # cannot resolve registry-prefixed refs from local storage at create time - # (same workaround as anubis/hermes), and Pull=never keeps boots off the - # registry. - local image="ghcr.io/peonist-ai/halogen-flash-server:0.5.8" - local alias="localhost/peonist-ai/halogen-flash-server:0.5.8" - podman pull "$image" - podman tag "$image" "$alias" - # The weights volume must exist, and the container needs it read-write: - # that is where a first start downloads into. - mkdir -p /home/dank/models/halogen - # Nothing may start until the predecessor has released the port. On - # 2026-09-16 this sequence failed exactly here: the ik server logged - # "Deactivated successfully", pasta tried to bind 7770 a second later and - # reported "Address already in use", and the unit exited 126. Restart= - # on-failure recovered it minutes later, but a deploy that starts a doomed - # container has not deployed anything -- so gate on the port, not on hope. - wait_port_free 7770 || return 1 - install_quadlet frank-large dankclaw/frank-large/quadlet frank-large - # No enable: generated units cannot be enabled on this host (v1 quadlet - # generator); boot persistence comes from [Install] in the unit file. - systemctl --user restart frank-large.service - wait_for_llm_large_ready - wait_for_health frank-large http://127.0.0.1:7770/health 1200 -} - -# Bootstrap-aware gate: if the checkpoint is absent the container is fetching -# it; wait for the file (the fetch resumes, and the unit restarts it on -# failure) rather than gating on a /health that cannot answer yet. Reports -# success either way, so a first deploy does not look like a failure while the -# engine does exactly what it was told to. -wait_for_llm_large_ready() { - [[ -f "$LLM_LARGE_CHECKPOINT" ]] && return 0 - echo "frank-large: checkpoint absent at $LLM_LARGE_CHECKPOINT" - echo "frank-large: first start — the container is downloading ~127 GB from" - echo "frank-large: peonist-ai/halogen-qwen3.8-flash-next (resumes if interrupted)." - local i - for ((i = 1; i <= 1200; i++)); do - if [[ -f "$LLM_LARGE_CHECKPOINT" ]]; then - echo "frank-large: weights landed after approximately $((i * 2))s; gating on /health" - return 0 + # Refuse rather than restart-loop. A unit whose runtime or payload is + # missing never becomes healthy, and a green deploy would leave the + # endpoint dark with nothing in the deploy log to say why. + local missing=0 f + for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT" "$LLM_LARGE_PLE"; do + if [[ ! -f "$f" ]]; then + echo "ERROR: payload absent at $f" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large-orca.sh" >&2 + missing=1 fi - sleep 2 done - echo "WARN: frank-large download still in progress after 40m — NOT a failure." >&2 - echo "WARN: the unit keeps fetching; re-run this deploy (or re-check /health) later." >&2 - return 0 + [[ "$missing" -eq 0 ]] || return 1 + + retire_halogen_large_quadlet + # On a re-deploy the predecessor is THIS unit: stop it before waiting on + # the port, or the wait deadlocks on the server it just stopped serving. + # Then sweep a stray manual run of the SAME model -- that is how a bench + # harness leaves the port busy. Match the model path, not the binary name: + # frank-small on this host is also a llama-server and must not be killed. + # The pattern never appears in the deploying shell's own command line, so + # -f is safe here (it is not in `bash deploy-remote.sh`). + run_privileged systemctl stop frank-large.service 2>/dev/null || true + pkill -f 'Qwen3.8-Flash-CIRU-STRIX-Orca' 2>/dev/null || true + sleep 2 + # Nothing may start until the predecessor has released the port. On + # 2026-09-16 the cutover failed exactly here: the old server logged + # "Deactivated successfully", the next engine tried to bind 7770 a second + # later and reported "Address already in use". Gate on the port, not on + # hope. + wait_port_free 7770 || return 1 + relink_unit frank-large + run_privileged systemctl daemon-reload + run_privileged systemctl enable frank-large.service + run_privileged systemctl restart frank-large.service + wait_for_health frank-large http://127.0.0.1:7770/health 1200 } # deploy_llm_small — frank-small is mainline llama.cpp on the RTX 3090, as a @@ -1146,7 +1120,7 @@ wait_for_llm_large_ready() { # is that unit's [Install] section. # # It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is -# not shared: frank-large's halogen engine runs on the gfx1151 iGPU. +# not shared: frank-large's engine runs on the gfx1151 iGPU. LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/Gemma4-12B-QAT/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf}" deploy_llm_small() { diff --git a/scripts/frank-large.env b/scripts/frank-large.env new file mode 100644 index 0000000..562f6c0 --- /dev/null +++ b/scripts/frank-large.env @@ -0,0 +1,44 @@ +# frank-large — CIRU v4.4.1 (QSA window hotfix) serving +# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca on the Strix Halo iGPU (gfx1151). +# +# These are the launcher tunables read by the runtime's +# scripts/ciru/run-orca-server.sh (see the serve script). The 36-variable +# kernel bundle and the PLE pager defaults live in the runtime's own +# profiles/strix-halo-production.env, which the launcher sources AFTER this +# file; every profile line is a ${VAR:-default}, so a value set here wins. +# +# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP runtime +# reads those bare (the 2026-08-13 TEMP incident). HOST is the one exception +# because the launcher itself consumes it for --host; its value doubles as a +# JIT-cache directory name, which is why 0.0.0.0 is safe. + +# The assembled payload (scripts/download-frank-large-orca.sh): the launcher +# refuses to start without the main GGUF, the MTP head and the PLE sidecar. +MODEL_DIR=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca + +# Fleet endpoint: the same port and alias the halogen predecessor served, so +# no client, probe or models.yml entry changes address. +HOST=0.0.0.0 +PORT=7770 + +# One stream, full context (operator call, 2026-09-19). MTP requires exactly +# one slot in this runtime, so the drafter's constraint and the operator's +# ask are the same number. 262144 is the model's trained window; the GGUF +# carries no rope scaling. +CONTEXT_SIZE=262144 +PARALLEL_SLOTS=1 + +# MTP head: on. Depth 4 is the Orca variant's own default (mtp_depth_default +# in the launcher); the sidecar is the Q8_0 MTP file from the model repo. +ENABLE_MTP=1 + +# Text-only for now: the F16 projector is on disk at +# vision/mmproj-Qwen3.8-Flash-Orca-F16.mmproj; flip to 1 to enable it. The +# fleet's image path is frank-small either way, so models.yml stays +# input: [text] until this changes. +ENABLE_VISION=0 + +# PLE pager: the 52.4 GiB payload is mmap'd and paged by the engine's own +# workers; this is the hot window kept resident. The vendor's production +# profile default is 4096. +PLE_CACHE_MIB=4096 diff --git a/scripts/frank-large.service b/scripts/frank-large.service new file mode 100644 index 0000000..9f76cb6 --- /dev/null +++ b/scripts/frank-large.service @@ -0,0 +1,21 @@ +[Unit] +Description=CIRU v4.4.1 Orca Qwen3.8-Flash (dankclaw, frank-large, Strix Halo iGPU gfx1151) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=dank +Group=dank +# The engine pins staging buffers and the PLE pager's hot window; keep the +# memlock budget unbounded so an RLIMIT smaller than the pinned set cannot +# fail the load. Parity with the halogen quadlet's Ulimit=memlock=-1:-1 and +# the other LLM units on this fleet. +LimitMEMLOCK=infinity +Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env" +ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh new file mode 100755 index 0000000..3cbfda3 --- /dev/null +++ b/scripts/serve-frank-large.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Serve jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca as frank-large on Dankclaw's +# Strix Halo iGPU (gfx1151), using the CIRU v4.4.1 runtime with the QSA +# window hotfix. +# +# Runtime: the v4.4.1 source with the QSA window hotfix pre-applied +# (ciru-runtime-v4.4.1-qsa-hotfix-source.tar.gz, shipped in the model repo +# under runtime/v4.4.1/qsa-window-hotfix/), built ON the host against the +# private ROCm 10.0.0 SDK that scripts/ciru/setup-linux-amd.sh installs into +# the runtime checkout's own venv. The launcher below re-verifies on every +# start that the built libraries carry the v4.4.1 corrections (it greps the +# common and HIP libraries for the fix markers) before exec. +# +# The launcher owns the flag list: -sm none (one iGPU, no split), -ngl all, +# f16 KV, flash attention, cont-batching, the PLE sidecar, the MTP drafter +# (depth 4 for the Orca variant) and the production kernel-bundle profile. +# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides it; +# anything not set there falls through to the launcher's own defaults. +# +# It replaces the halogen-flash-server quadlet, which served +# peonist-ai/halogen-qwen3.8-flash-next on this same iGPU since 2026-09-16. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}" +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +# --- defaults (used when the env file does not set them) ------------------ +: "${RUNTIME_DIR:=/home/dank/GitPackages/ciru-runtime-v4.4.1-qsa-hotfix}" +: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca}" +: "${HOST:=0.0.0.0}" +: "${PORT:=7770}" +: "${CONTEXT_SIZE:=262144}" +: "${PARALLEL_SLOTS:=1}" + +LAUNCHER="$RUNTIME_DIR/scripts/ciru/run-orca-server.sh" +SERVER_BIN="$RUNTIME_DIR/build-gfx1151-sdk/bin/llama-server" + +if [[ ! -x "$LAUNCHER" ]]; then + echo "missing runtime launcher: $LAUNCHER" >&2 + echo "fetch the hotfix source tarball from the model repo" >&2 + echo "(runtime/v4.4.1/qsa-window-hotfix/) and build it:" >&2 + echo " bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2 + exit 1 +fi +if [[ ! -x "$SERVER_BIN" ]]; then + echo "missing built runtime: $SERVER_BIN" >&2 + echo "build it: bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2 + exit 1 +fi +if [[ ! -r "$MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" ]]; then + echo "missing target model: $MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" >&2 + echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2 + exit 1 +fi + +# The launcher checks the MTP head and the PLE sidecar itself, but a missing +# payload should be a deploy error with a pointer, not a runtime exit 2. +for required in \ + "$MODEL_DIR/mtp/Qwen3.8-Flash-CIRU-STRIX-Orca-MTP-Q8_0.gguf" \ + "$MODEL_DIR/ple/ple.payload.bin" \ + "$MODEL_DIR/ple/ple.manifest.json" \ + "$MODEL_DIR/ple/ple.scale.bf16"; do + if [[ ! -r "$required" ]]; then + echo "missing payload: $required" >&2 + echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2 + exit 1 + fi +done + +# The Qwen3.8 family template keeps prior thinking blocks across turns when +# preserve_thinking is set; the launcher passes --jinja but no template +# kwargs, so name them here. A request-level chat_template_kwargs still +# wins over this. +exec "$LAUNCHER" --chat-template-kwargs '{"preserve_thinking":true}' diff --git a/tests/deployment_scoping.bats b/tests/deployment_scoping.bats index 7028dbc..35146af 100644 --- a/tests/deployment_scoping.bats +++ b/tests/deployment_scoping.bats @@ -18,7 +18,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" @test "deployment planner maps large-model changes to the large model only" { run python3 "$PLANNER" --target dankclaw --paths \ - dankclaw/frank-large/quadlet/frank-large.container + scripts/frank-large.env [ "$status" -eq 0 ] [ "$output" = "frank-large" ] } diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index 5991dd8..f0a0c48 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -1,113 +1,121 @@ #!/usr/bin/env bats -# Seam: the frank-large halogen-flash-server quadlet on dankclaw. -# halogen-flash-server replaced ik_llama.cpp again on 2026-09-16, so this -# service is once more a rootless Quadlet container on the gfx1151 iGPU. This -# seam pins what a flag list cannot show: the boot-persistent podman posture -# (both device nodes, memlock, the create-time image contract), the -# matched ctx/pool/slots set that IS the two-conversation promise, the -# co-tenancy budget the arena and reserve are chosen for, the image handoff to -# frank-small, and a deploy handler that retires the previous ROOT unit without -# deleting it. +# Seam: the frank-large CIRU v4.4.1 root unit on dankclaw. +# +# The 2026-09-19 cutover moved this service out of the halogen-flash-server +# quadlet and onto the CIRU v4.4.1 runtime (QSA window hotfix) serving +# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca, as a ROOT unit on a host build -- the +# same shape as frank-small and frank-medium, because a private-SDK host +# binary has no container image to install and no generated unit to dodge. +# This seam pins what a flag list cannot show: the operator's one-stream / +# full-context shape, the payload the launcher refuses to start without, a +# deploy handler that retires the halogen quadlet without deleting it and +# gates on the port, and the fact that the sweep for a stray benchmark run +# cannot take frank-small down with it. load helpers -QUADLET="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" +ENV_FILE="$REPO_ROOT/scripts/frank-large.env" +UNIT="$REPO_ROOT/scripts/frank-large.service" +SERVE="$REPO_ROOT/scripts/serve-frank-large.sh" +FETCH="$REPO_ROOT/scripts/download-frank-large-orca.sh" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" -@test "the frank-large quadlet exists where the deployer installs it from" { - [ -f "$QUADLET" ] +@test "the frank-large unit, env and serve files exist where the deployer installs them from" { + [ -f "$ENV_FILE" ] + [ -f "$UNIT" ] + [ -x "$SERVE" ] } -@test "frank-large is boot-persistent, self-healing and container-shaped" { - grep -q '^WantedBy=default.target$' "$QUADLET" - grep -q '^Restart=on-failure$' "$QUADLET" - grep -q '^RestartSec=' "$QUADLET" - grep -q '^Pull=never$' "$QUADLET" - # The localhost/-aliased tag is what create-time resolution needs on this - # host; the deployer pulls the ghcr.io ref and re-tags it. - grep -q '^Image=localhost/peonist-ai/halogen-flash-server:' "$QUADLET" +@test "frank-large is boot-persistent, self-healing and unprivileged" { + grep -q '^WantedBy=multi-user.target$' "$UNIT" + grep -q '^Restart=on-failure$' "$UNIT" + grep -q '^RestartSec=' "$UNIT" + grep -q '^User=dank$' "$UNIT" + # The engine pins staging buffers and the PLE pager's hot window; the + # memlock budget must be unbounded or the load fails on the RLIMIT. + grep -q '^LimitMEMLOCK=infinity$' "$UNIT" + # The unit runs the repo's own serve script with the repo's env. + grep -q 'ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh' "$UNIT" + grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT" } -@test "frank-large has the gfx1151 devices and the image's documented posture" { - grep -q '^AddDevice=/dev/kfd$' "$QUADLET" - grep -q '^AddDevice=/dev/dri$' "$QUADLET" - grep -q '^PodmanArgs=--ipc=host --security-opt seccomp=unconfined$' "$QUADLET" - grep -q '^Ulimit=memlock=-1:-1$' "$QUADLET" - # The weights mount must stay read-write: the first start downloads into it. - grep -q '^Volume=/home/dank/models/halogen:/models$' "$QUADLET" +@test "the operator's shape is one stream of the full trained window" { + local ctx slots + ctx="$(sed -nE 's/^CONTEXT_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" + slots="$(sed -nE 's/^PARALLEL_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")" + # 262144 is the model's trained window and the GGUF carries no rope + # scaling; the operator asked for full context. + [ "$ctx" -eq 262144 ] + # One slot. This is also the drafter's constraint: the launcher refuses + # MTP with more than one slot, so the two cannot drift apart silently. + [ "$slots" -eq 1 ] + grep -q '^ENABLE_MTP=1$' "$ENV_FILE" } -@test "frank-large keeps the fleet alias and publishes only the API port" { - grep -q '^Environment=HALOGEN_MODEL_ID=frank-large$' "$QUADLET" - grep -q '^PublishPort=7770:8731$' "$QUADLET" - # The engine's own listener stays inside the container: it is unauthenticated. - ! grep -q 'PublishPort=.*8730' "$QUADLET" -} - -@test "the KV pool is exactly two conversations of the per-request ceiling" { - local ctx pool slots - ctx="$(sed -nE 's/^Environment=HALOGEN_CTX=([0-9]+)$/\1/p' "$QUADLET")" - pool="$(sed -nE 's/^Environment=HALOGEN_KV_POOL_POSITIONS=([0-9]+)$/\1/p' "$QUADLET")" - slots="$(sed -nE 's/^Environment=HALOGEN_KV_SLOTS=([0-9]+)$/\1/p' "$QUADLET")" - [ -n "$ctx" ] && [ -n "$pool" ] && [ -n "$slots" ] - # The deployed promise is two resident conversations: the pool is twice the - # ceiling, one slot per conversation. Slots still oversubscribe the pool for - # any real request, because admission reserves prompt + max_tokens. - [ "$pool" -eq $((ctx * 2)) ] - [ "$slots" -eq 2 ] -} - -@test "the prefill arena stays inside the range the vendor tested" { - local max_tok reserve - max_tok="$(sed -nE 's/^Environment=HALOGEN_MAX_TOK=([0-9]+)$/\1/p' "$QUADLET")" - reserve="$(sed -nE 's/^Environment=HALOGEN_HOST_RESERVE_GIB=([0-9]+)$/\1/p' "$QUADLET")" - [ -n "$max_tok" ] && [ -n "$reserve" ] - # 32768 is the vendor default; 65536 cannot boot on this host because the - # arena alone outgrows the device budget. The arena is a per-call prefill - # chunk, so a smaller value costs prefill speed, not context. - [ "$max_tok" -le 32768 ] - [ "$max_tok" -ge 16384 ] - [ "$reserve" -ge 15 ] -} - -@test "frank-large loads no vision tower: the fleet's image path is frank-small" { - ! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET" +@test "frank-large stays text-only until the projector is enabled" { + grep -q '^ENABLE_VISION=0$' "$ENV_FILE" + # The fleet's image path is frank-small; the projector is on disk and is + # one env flip away, but models.yml declares frank-large text-only. grep -q -- '--mmproj' "$REPO_ROOT/scripts/serve-frank-small.sh" } -@test "the deploy handler retires the previous ROOT unit without deleting it" { - grep -q 'retire_root_llm_large' "$REMOTE_DEPLOY" - grep -q 'systemctl stop frank-large.service' "$REMOTE_DEPLOY" - grep -q 'systemctl disable frank-large.service' "$REMOTE_DEPLOY" +@test "the serve script pins the hotfix runtime and refuses a missing build" { + grep -q 'ciru-runtime-v4.4.1-qsa-hotfix' "$SERVE" + grep -q 'build-gfx1151-sdk/bin/llama-server' "$SERVE" + grep -q 'run-orca-server.sh' "$SERVE" + # A missing runtime or payload is a deploy error with a pointer, not a + # restart loop. + grep -q 'missing built runtime' "$SERVE" + grep -q 'missing target model' "$SERVE" + # The Qwen3.8 template keeps prior thinking blocks across turns only + # when the kwarg is set; the launcher passes --jinja but no kwargs. + grep -q 'preserve_thinking' "$SERVE" +} + +@test "the payload is fetched resumably and verified by sha256" { + grep -q 'jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca' "$FETCH" + grep -q -- '-fL -C -' "$FETCH" + grep -q 'parts.json' "$FETCH" + grep -q 'sha256sum -c SHA256SUMS' "$FETCH" + # The PLE sidecar is the largest transport part; the script must fetch + # and verify it, not just the GGUF. + grep -q 'ple.payload.bin' "$FETCH" +} + +@test "the fleet endpoint and alias survive the cutover" { + grep -q '^HOST=0.0.0.0$' "$ENV_FILE" + grep -q '^PORT=7770$' "$ENV_FILE" + grep -q '^MODEL_DIR=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca$' "$ENV_FILE" +} + +@test "the deploy handler retires the halogen quadlet without deleting it" { + grep -q 'retire_halogen_large_quadlet' "$REMOTE_DEPLOY" grep -q 'HOME/.AGENTTRASH' "$REMOTE_DEPLOY" ! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?.*frank-large' "$REMOTE_DEPLOY" } -@test "the deploy handler installs the quadlet and gates the first start on weights" { - grep -q 'install_quadlet frank-large dankclaw/frank-large/quadlet frank-large' "$REMOTE_DEPLOY" - grep -q 'wait_for_llm_large_ready' "$REMOTE_DEPLOY" - grep -q 'qwen38-flash-next-w4b.hgn' "$REMOTE_DEPLOY" - grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY" -} - -@test "the ik_llama.cpp runtime that held this service is archived, not live" { - # Its files must not sit in scripts/ where a stale root unit could still be - # pointed at them, and must not be gone either: .AGENTTRASH is the rollback. - local retired - for retired in frank-large.env.ik-llama.cpp frank-large.service.ik-llama.cpp \ - serve-frank-large.sh.ik-llama.cpp; do - [ -f "$REPO_ROOT/scripts/.AGENTTRASH/$retired" ] - done - [ ! -e "$REPO_ROOT/scripts/frank-large.env" ] - [ ! -e "$REPO_ROOT/scripts/frank-large.service" ] - [ ! -e "$REPO_ROOT/scripts/serve-frank-large.sh" ] -} - -@test "the deploy handler waits for the predecessor to release the port" { - # The 2026-09-16 cutover failed at exactly this: the ik server's socket was - # still on 7770 when pasta tried to bind it ("Address already in use"), and - # the container died on it. Restart=on-failure recovered minutes later, but - # a deploy that starts a doomed container has not deployed anything. +@test "the deploy handler installs the root unit, gates the port and refuses a missing payload" { + grep -q 'relink_unit frank-large' "$REMOTE_DEPLOY" + grep -q 'systemctl enable frank-large.service' "$REMOTE_DEPLOY" grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY" - grep -q '^wait_port_free() {' "$REMOTE_DEPLOY" -} \ No newline at end of file + grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY" + grep -q 'LLM_LARGE_MODEL' "$REMOTE_DEPLOY" + grep -q 'LLM_LARGE_PLE' "$REMOTE_DEPLOY" +} + +@test "the stray-run sweep cannot take frank-small down" { + # frank-small on this host is ALSO a llama-server. The sweep must match + # the model path, not the binary name, or a frank-large deploy restarts + # the 3090 endpoint. + grep -q "pkill -f 'Qwen3.8-Flash-CIRU-STRIX-Orca'" "$REMOTE_DEPLOY" + # ...and only inside the large handler: extract it and check there is no + # bare binary-name pkill in it. + local large + large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")" + ! grep -q 'pkill -x llama-server' <<<"$large" +} + +@test "the halogen quadlet is archived, not live" { + [ ! -e "$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" ] + [ -f "$REPO_ROOT/dankclaw/.AGENTTRASH/frank-large.container.halogen" ] +} diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 01163b1..6b77526 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -14,7 +14,7 @@ UNIT="$REPO_ROOT/scripts/frank-small.service" SERVE="$REPO_ROOT/scripts/serve-frank-small.sh" ENV_FILE="$REPO_ROOT/scripts/frank-small.env" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" -QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" +ENV_LARGE="$REPO_ROOT/scripts/frank-large.env" @test "frank-small unit, serve script and env exist in scripts/" { [ -f "$UNIT" ] @@ -58,7 +58,7 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" @test "frank-small carries the fleet's image path, and only it" { grep -q '^MMPROJ_PATH=.*mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16\.gguf$' "$ENV_FILE" grep -q -- '--mmproj "$MMPROJ_PATH"' "$SERVE" - ! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET_LARGE" + ! grep -q '^ENABLE_VISION=1$' "$ENV_LARGE" } @test "frank-small's slot grid is full-window and fits the 24 GiB card" { @@ -94,10 +94,10 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" grep -q '^ALIAS=frank-small$' "$ENV_FILE" grep -q -- '--port "$LISTEN_PORT"' "$SERVE" grep -q -- '--host "$BIND_ADDR"' "$SERVE" - # The two endpoints cannot share a port, and only the halogen unit publishes - # 7770. - grep -q '^PublishPort=7770:8731$' "$QUADLET_LARGE" - ! grep -q 'PublishPort=.*7771' "$QUADLET_LARGE" + # The two endpoints cannot share a port: frank-large holds 7770, this one + # 7771. + grep -q '^PORT=7770$' "$ENV_LARGE" + ! grep -q '^PORT=7771$' "$ENV_LARGE" } @test "the deploy handler installs the root unit and refuses a missing model" {