frank-large: swap halogen Quadlet for ik_llama.cpp (3090 + Strix Halo CPU path)

.gitignore

@@ -5,6 +5,9 @@
 # LLM fleet tunables (paths and numbers, never credentials)
 !scripts/*.env
 
+# LLM fleet runtime state (slot save/restore files written by the server)
+scripts/slot-state/
+
 # Playwright e2e: local creds, browser storage states, node artifacts
 tests/e2e/.env.local
 tests/e2e/.auth/

scripts/deploy-plan.py

@@ -378,6 +378,14 @@ def services_for_path(path: str, target: str) -> set[str]:
         name = p.name
         if name in {"homepage-status.py", "homepage-status.service", "homepage-status.timer"}:
             _add(services, "homepage")
+        if name in {"frank-large.env", "frank-large.service", "serve-frank-large.sh"}:
+            # The ik_llama.cpp runtime (2026-09-15 cutover) keeps its unit,
+            # env and serve files here; each one selects frank-large only.
+            _add(services, "frank-large")
+        if name == "chat-template.jinja":
+            # The shared chat template is read by frank-large's serve script
+            # again; the vllm-radiance runtime on Dankserver ships its own.
+            _add(services, "frank-large")
         # Deployment/test/benchmark scripts and other repo tooling do not
         # restart a runtime service.
         return services

scripts/deploy-remote.sh

@@ -912,85 +912,53 @@ deploy_anubis() {
     wait_container_healthy anubis 60 || return 1
 }
 
-# frank-large runs as a rootless Quadlet unit since 2026-09-11 (halogen-flash-
-# server replaced the llama.cpp/kingjones runtime). Unlike the sibling LLM
-# services it is therefore NOT a root unit: no relink_unit, no `systemctl
-# enable` (this host's quadlet generator produces units that cannot be
-# enabled — boot persistence comes from [Install] WantedBy=default.target in
-# the .container file plus linger).
+# frank-large runs as a ROOT unit again since 2026-09-15: ik_llama.cpp
+# (qwen4exp support merged upstream) replaced the halogen-flash-server
+# rootless Quadlet, and the ik server is a native binary — CUDA on the RTX
+# 3090 for dense/attention/KV, the Strix Halo CPU path for the MoE experts.
+# There is no container image to pull and no generated unit to avoid:
+# relink_unit points /etc/systemd/system/frank-large.service at the repo
+# file, and boot persistence is the unit's [Install] section.
 #
-# The llama.cpp-era ROOT unit is retired first. Its unit file is a symlink
-# into this checkout (the old relink_unit did `ln -sfn scripts/frank-large.
-# service /etc/systemd/system/`), so leaving it behind after the repo file
-# moves to the trash would leave systemd holding a dangling unit that can
-# still claim port 7770. Stop it, drop its enable symlinks, move the unit file
-# into the home trash (never rm), then reload the root manager.
-#
-# FIRST START: the container fetches ~127 GB of weights before it can serve,
-# so /health is legitimately down for the whole fetch. wait_for_health would
-# burn its budget and report a failure that is not one. The gate below
-# recognises the bootstrap (checkpoint absent) and waits for the checkpoint
-# instead, falling through to the normal health gate once it lands.
-LLM_LARGE_CHECKPOINT="${LLM_LARGE_CHECKPOINT:-/home/dank/models/halogen/qwen38-flash-next-w4b.hgn}"
+# The halogen Quadlet is retired first. Both managers must never know this
+# unit name at once (they would race for port 7770): stop the user unit,
+# poll until it is really gone, move the installed .container into the home
+# trash (never rm), then reload the user manager.
+LLM_LARGE_CHECKPOINT="${LLM_LARGE_CHECKPOINT:-/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M/Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf}"
 
-retire_root_llm_large() {
-    local trash="$HOME/.AGENTTRASH/20260911-halogen-swap/host-units"
-    run_privileged systemctl stop frank-default-large 2>/dev/null || true
-    run_privileged systemctl disable frank-default-large 2>/dev/null || true
-    run_privileged systemctl stop frank-large.service 2>/dev/null || true
-    # On this host `disable` has been observed to delete the unit file itself
-    # (2026-08-26 incident, see AGENTS.md); tolerate either behaviour and keep
-    # whatever remains by moving it to the trash.
-    run_privileged systemctl disable frank-large.service 2>/dev/null || true
-    if [[ -e /etc/systemd/system/frank-large.service || -L /etc/systemd/system/frank-large.service ]]; then
+retire_quadlet_llm_large() {
+    local trash="$HOME/.AGENTTRASH/20260915-ik-swap/halogen-quadlet"
+    systemctl --user stop frank-large.service 2>/dev/null || true
+    # A stopping quadlet runs podman stop (SIGTERM, then SIGKILL at the
+    # timeout); llama-family engines never preempt an in-flight generation.
+    # Poll until the user unit is really inactive before removing its file.
+    local i
+    for ((i = 0; i < 30; i++)); do
+        systemctl --user is-active --quiet frank-large.service 2>/dev/null || break
+        sleep 2
+    done
+    if [[ -e "$HOME/.config/containers/systemd/frank-large.container" || -L "$HOME/.config/containers/systemd/frank-large.container" ]]; then
         mkdir -p "$trash"
-        run_privileged mv -- /etc/systemd/system/frank-large.service \
-            "$trash/frank-large.service"
-        echo "retired root unit frank-large.service -> $trash"
+        mv -- "$HOME/.config/containers/systemd/frank-large.container" \
+            "$trash/frank-large.container"
+        echo "retired halogen quadlet -> $trash"
     fi
-    run_privileged systemctl daemon-reload
+    systemctl --user daemon-reload
 }
 
 deploy_llm_large() {
-    retire_root_llm_large
-    # Pull once, then run the localhost/-aliased tag: this host's quadlet path
-    # cannot resolve registry-prefixed refs from local storage at create time
-    # (same workaround as anubis/hermes).
-    local image="ghcr.io/peonist-ai/halogen-flash-server:0.5.8"
-    local alias="localhost/peonist-ai/halogen-flash-server:0.5.8"
-    podman pull "$image"
-    podman tag "$image" "$alias"
-    # The weights volume: read-write, because the container downloads into it.
-    mkdir -p /home/dank/models/halogen
-    install_quadlet frank-large dankclaw/frank-large/quadlet frank-large
-    systemctl --user restart frank-large.service
-    wait_for_llm_large_ready
+    retire_quadlet_llm_large
+    if [[ ! -f "$LLM_LARGE_CHECKPOINT" ]]; then
+        echo "ERROR: checkpoint absent at $LLM_LARGE_CHECKPOINT" >&2
+        echo "frank-large: download the mradermacher i1-Q4_K_M GGUF first; refusing to start a unit that cannot load." >&2
+        return 1
+    fi
+    relink_unit frank-large
+    run_privileged systemctl enable frank-large.service
+    run_privileged systemctl restart frank-large.service
     wait_for_health frank-large http://127.0.0.1:7770/health 1200
 }
 
-# wait_for_llm_large_ready — bootstrap-aware gate. If the checkpoint is absent
-# the container is downloading it; wait for the file (resumable, and the unit
-# restarts it on failure) rather than gating on a /health that cannot answer
-# yet. Reports success either way, so the first deploy does not look like a
-# failure while the engine does exactly what it was told to.
-wait_for_llm_large_ready() {
-    [[ -f "$LLM_LARGE_CHECKPOINT" ]] && return 0
-    echo "frank-large: checkpoint absent at $LLM_LARGE_CHECKPOINT"
-    echo "frank-large: first start — the container is downloading ~127 GB from"
-    echo "frank-large: peonist-ai/halogen-qwen3.8-flash-next (resumes if interrupted)."
-    local i
-    for ((i = 1; i <= 1200; i++)); do
-        if [[ -f "$LLM_LARGE_CHECKPOINT" ]]; then
-            echo "frank-large: weights landed after approximately $((i * 2))s; gating on /health"
-            return 0
-        fi
-        sleep 2
-    done
-    echo "WARN: frank-large download still in progress after 40m — NOT a failure."
-    echo "WARN: the unit keeps fetching; re-run this deploy (or re-check /health) later." >&2
-    return 0
-}
-
 # deploy_llm_medium — frank-medium runs as a rootless Quadlet since the
 # 2026-09-15 runtime swap to the vllm-radiance MXFP4 stack. It began the day
 # as a root llama.cpp unit (the ROCmFPX/Q8_0_ROCMFPX_AGENT deployment from

scripts/frank-large.env

@@ -0,0 +1,95 @@
+# Tunables for frank-large (mradermacher i1-Q4_K_M of
+# orcarouter/Qwen3.8-Flash-Next-Uncensored on ik_llama.cpp, Dankclaw,
+# RTX 3090 CUDA + Strix Halo CPU path, since 2026-09-15).
+#
+# Runtime: ik_llama.cpp master d85a1d7 (2026-09-15) with full qwen4exp
+# support (upstream PRs #2365/#2369/#2373 merged). Build recipe (CUDA for
+# the 3090 at sm_86, Zen5 AVX-512 IQK kernels for the CPU expert path):
+#   cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON \
+#     -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86 \
+#     -DGGML_AVX512=ON -DGGML_AVX512_VBMI=ON -DGGML_AVX512_VNNI=ON \
+#     -DGGML_AVX512_BF16=ON -DLLAMA_CURL=OFF
+# The startup banner must print HAVE_FANCY_SIMD is defined — without those
+# macros the quantized GEMM silently falls back to AVX2 (no warning).
+#
+# Model: mradermacher imatrix quant of the uncensored Flash-Next (qwen4exp
+# MoE: 512 experts / 10 used, 48 layers = 12 attention + 36 GDN/ssm, QSA
+# indexer, hyper-connections, PLE n-gram table ~320M rows). 111 GiB shipped
+# as 3 parts, concatenated on disk; each part's sha256 was verified against
+# its HF LFS oid at download time.
+#
+# Placement (ikawrakow's hybrid recipe, upstream issue #2402):
+#   - --cpu-moe: ALL expert tensors (121B elems, ~71 GB) live in the Strix
+#     Halo's LPDDR5X via the CPU path — ik's fastest known configuration
+#     for this shape (the author's 2x3090 + Threadripper reference box
+#     measures ~50 t/s on the same model family).
+#   - -ngl 999: dense/attention/output weights and the KV cache on the
+#     3090 (~3 GB of weights + ~6 GB KV at 262k f16; 24 GB card).
+#   - The ~35 GB PLE table (per_layer_token_embd) is intentionally
+#     RAM/mmap-resident by design — only a tiny slice is touched per token.
+#     NEVER add --no-mmap (the STRIX-era lesson transfers: the cgroup
+#     killer takes the process with nothing in the server log) and do not
+#     try to pin PLE to VRAM (it cannot fit a 24 GB card).
+#
+# Sampling: greedy-by-default to preserve the halogen-era client contract
+# (a request without temperature decodes greedy; temp 0.0 = argmax).
+# Clients that send their own temperature get their own values.
+#
+# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (see
+# the 2026-08-13 HIP incident in AGENTS.md). llama.cpp reads --host and
+# --temp as CLI flags, never as environment variables.
+
+# ik_llama.cpp runtime binary (master d85a1d7 build).
+LLAMA_BIN=/home/dank/GitPackages/ik_llama.cpp/build/bin/llama-server
+
+# Model package from
+# https://huggingface.co/mradermacher/Qwen3.8-Flash-Next-Uncensored-i1-GGUF
+# (parts concatenated). The mmproj is the matching vision tower from the
+# static repo mradermacher/Qwen3.8-Flash-Next-Uncensored-GGUF — ik_llama.cpp
+# links examples/mtmd, so image input survives the runtime swap.
+MODEL_DIR=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M
+MODEL_FILE=Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf
+MMPROJ_PATH=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M/Qwen3.8-Flash-Next-Uncensored.mmproj-f16.gguf
+
+# Qwen-Sharp chat template (tracked in the repo, staged on the host). The
+# server emits reasoning in DeepSeek format so clients receive
+# reasoning_content separately instead of literal thinking tags in content.
+CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
+
+# Alias + network.
+ALIAS=frank-large
+BIND_ADDR=0.0.0.0
+LISTEN_PORT=7770
+
+# Serving: single 262144-token slot (the model's full context window),
+# batch 4096 / ubatch 512, 16+16 threads (the Halo CPU has 16 cores, no
+# SMT), f16 KV. The KV cache scales with the 12 full-attention layers only
+# (~6 GB f16 at 262k); the GDN/ssm layers keep constant-size state. If
+# VRAM headroom is needed, --cache-type-k/v q8_0 halves the KV footprint.
+CTX_SIZE=262144
+PARALLEL=1
+BATCH_SIZE=4096
+UBATCH_SIZE=512
+NUM_THREADS=16
+THREAD_BATCH=16
+CACHE_TYPE_K=f16
+CACHE_TYPE_V=f16
+
+# GPU/MoE placement: everything on the 3090 except the experts, which all
+# live in host RAM (--cpu-moe). See the placement notes above.
+NGL=999
+CPU_MOE=on
+FLASH_ATTN=on
+
+# Prompt cache (RAM budget in MiB) and slot state dir.
+CACHE_RAM=8192
+SLOT_SAVE_PATH=/home/dank/serverAppConfigs/scripts/slot-state
+
+# Greedy-by-default (halogen contract). See sampling note above.
+TEMPERATURE=0.0
+
+# Reasoning on: DeepSeek format projects thinking into the API's
+# reasoning_content field instead of literal think tags in content;
+# unlimited reasoning budget (the model decides).
+REASONING_FORMAT=deepseek
+REASONING_BUDGET=-1

scripts/frank-large.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=ik_llama.cpp Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M 177B MoE (dankclaw, frank-large)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=dank
+Group=dank
+Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
+ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

scripts/serve-frank-large.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# Serve mradermacher/Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M as frank-large
+# via ik_llama.cpp on Dankclaw.
+#
+# Topology (ikawrakow's hybrid recipe, upstream issue #2402): the RTX 3090
+# (CUDA, sm_86) takes the dense/attention/output weights and the KV cache
+# (-ngl 999); ALL MoE expert tensors stay in the Strix Halo's LPDDR5X via
+# the CPU path (--cpu-moe) — ik's fastest known configuration for this
+# hybrid shape. The ~35 GB PLE n-gram table (per_layer_token_embd) is
+# mmap-resident BY DESIGN: only a tiny slice is touched per token, so it
+# streams from the OS page cache. NEVER add --no-mmap — the STRIX-era
+# lesson transfers directly (the cgroup killer takes the process with
+# nothing in the server log).
+#
+# Runtime: ik_llama.cpp master d85a1d7 built with CUDA (sm_86) + Zen5
+# AVX-512 IQK kernels; the startup banner must show HAVE_FANCY_SIMD
+# is defined. Flag set verified against this binary's --help before this
+# script was written. ik has no --cache-prompt/--cache-idle-slots (prompt
+# caching is its default behavior) and no --alias: it is a single-model
+# server and reports the GGUF's general.name at /v1/models.
+#
+# Sampling: greedy-by-default (halogen client contract) — clients that
+# send their own temperature get their own values.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -------------------
+: "${LLAMA_BIN:=/home/dank/GitPackages/ik_llama.cpp/build/bin/llama-server}"
+: "${MODEL_DIR:=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M}"
+: "${MODEL_FILE:=Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf}"
+: "${MMPROJ_PATH:=$MODEL_DIR/Qwen3.8-Flash-Next-Uncensored.mmproj-f16.gguf}"
+: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
+: "${BIND_ADDR:=0.0.0.0}"
+: "${LISTEN_PORT:=7770}"
+: "${CTX_SIZE:=262144}"
+: "${PARALLEL:=1}"
+: "${BATCH_SIZE:=4096}"
+: "${UBATCH_SIZE:=512}"
+: "${NUM_THREADS:=16}"
+: "${THREAD_BATCH:=16}"
+: "${CACHE_TYPE_K:=f16}"
+: "${CACHE_TYPE_V:=f16}"
+: "${NGL:=999}"
+: "${CPU_MOE:=on}"
+: "${FLASH_ATTN:=on}"
+: "${CACHE_RAM:=8192}"
+: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}"
+: "${TEMPERATURE:=0.0}"
+: "${REASONING:=on}"
+: "${REASONING_FORMAT:=deepseek}"
+: "${REASONING_BUDGET:=-1}"
+
+mkdir -p "$SLOT_SAVE_PATH"
+
+# The UVM device node is created lazily by nvidia-modprobe on the first
+# CUDA open after boot; make sure it exists before the server needs it.
+nvidia-modprobe -u -c=0 2>/dev/null || true
+
+CPU_MOE_ARGS=()
+if [[ "$CPU_MOE" == "on" ]]; then
+    CPU_MOE_ARGS=(--cpu-moe)
+fi
+
+MMPROJ_ARGS=()
+if [[ -n "${MMPROJ_PATH:-}" && -f "$MMPROJ_PATH" ]]; then
+    MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
+fi
+
+TEMPLATE_ARGS=()
+if [[ -n "${CHAT_TEMPLATE_FILE:-}" && -f "$CHAT_TEMPLATE_FILE" ]]; then
+    TEMPLATE_ARGS=(--chat-template-file "$CHAT_TEMPLATE_FILE")
+else
+    TEMPLATE_ARGS=(--jinja)
+fi
+
+exec "$LLAMA_BIN" \
+    --model "$MODEL_DIR/$MODEL_FILE" \
+    "${TEMPLATE_ARGS[@]}" \
+    --host "$BIND_ADDR" \
+    --port "$LISTEN_PORT" \
+    --gpu-layers "$NGL" \
+    "${CPU_MOE_ARGS[@]}" \
+    --flash-attn "$FLASH_ATTN" \
+    --parallel "$PARALLEL" \
+    --ctx-size "$CTX_SIZE" \
+    --batch-size "$BATCH_SIZE" \
+    --ubatch-size "$UBATCH_SIZE" \
+    --threads "$NUM_THREADS" \
+    --threads-batch "$THREAD_BATCH" \
+    --cache-type-k "$CACHE_TYPE_K" \
+    --cache-type-v "$CACHE_TYPE_V" \
+    --cache-ram "$CACHE_RAM" \
+    --slot-save-path "$SLOT_SAVE_PATH" \
+    --ctx-checkpoints 16 \
+    --metrics \
+    --temp "$TEMPERATURE" \
+    --reasoning "$REASONING" \
+    --reasoning-format "$REASONING_FORMAT" \
+    --reasoning-budget "$REASONING_BUDGET" \
+    "${MMPROJ_ARGS[@]}"

tests/deployment_scoping.bats

@@ -18,7 +18,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
 
 @test "deployment planner maps large-model changes to the large model only" {
     run python3 "$PLANNER" --target dankclaw --paths \
-        dankclaw/frank-large/quadlet/frank-large.container
+        scripts/frank-large.env scripts/frank-large.service scripts/serve-frank-large.sh
     [ "$status" -eq 0 ]
     [ "$output" = "frank-large" ]
 }
@@ -28,14 +28,14 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
     ! grep -q 'frank_npu\|frank-npu' "$REMOTE_DEPLOY"
 }
 
-@test "chat-template changes plan no service: the radiance runtime ships its own template" {
-    # The ROCmFPX llama.cpp deployment optionally read the tracked
-    # scripts/chat-template.jinja; the vllm-radiance runtime mounts its own
-    # qwen-fixed-v22.3.jinja out of the pinned clone instead. Nobody reads
-    # the tracked template any more, so an edit to it deploys nothing.
+@test "chat-template changes plan frank-large: the ik runtime reads the tracked template" {
+    # ik_llama.cpp (2026-09-15 cutover) serves the Qwen-Sharp template from
+    # the tracked scripts/chat-template.jinja again. The vllm-radiance
+    # runtime on Dankserver ships its own qwen-fixed-v22.3.jinja out of the
+    # pinned clone, so a template edit deploys nothing there.
     run python3 "$PLANNER" --target dankclaw --paths scripts/chat-template.jinja
     [ "$status" -eq 0 ]
-    [ -z "$output" ]
+    [ "$output" = "frank-large" ]
 
     run python3 "$PLANNER" --target dankserver --paths scripts/chat-template.jinja
     [ "$status" -eq 0 ]