From fe3fc62c4141fbb0209048fb1d05004861618e9b Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 15 Sep 2026 23:20:19 -0700 Subject: [PATCH] frank-large: swap halogen Quadlet for ik_llama.cpp (3090 + Strix Halo CPU path) --- .gitignore | 3 + scripts/deploy-plan.py | 8 +++ scripts/deploy-remote.sh | 106 ++++++++++++--------------------- scripts/frank-large.env | 95 ++++++++++++++++++++++++++++++ scripts/frank-large.service | 16 +++++ scripts/serve-frank-large.sh | 108 ++++++++++++++++++++++++++++++++++ tests/deployment_scoping.bats | 14 ++--- 7 files changed, 274 insertions(+), 76 deletions(-) create mode 100644 scripts/frank-large.env create mode 100644 scripts/frank-large.service create mode 100755 scripts/serve-frank-large.sh diff --git a/.gitignore b/.gitignore index 7175fa5..8656541 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ # LLM fleet tunables (paths and numbers, never credentials) !scripts/*.env +# LLM fleet runtime state (slot save/restore files written by the server) +scripts/slot-state/ + # Playwright e2e: local creds, browser storage states, node artifacts tests/e2e/.env.local tests/e2e/.auth/ diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py index fb8d394..3a03454 100755 --- a/scripts/deploy-plan.py +++ b/scripts/deploy-plan.py @@ -378,6 +378,14 @@ def services_for_path(path: str, target: str) -> set[str]: name = p.name if name in {"homepage-status.py", "homepage-status.service", "homepage-status.timer"}: _add(services, "homepage") + if name in {"frank-large.env", "frank-large.service", "serve-frank-large.sh"}: + # The ik_llama.cpp runtime (2026-09-15 cutover) keeps its unit, + # env and serve files here; each one selects frank-large only. + _add(services, "frank-large") + if name == "chat-template.jinja": + # The shared chat template is read by frank-large's serve script + # again; the vllm-radiance runtime on Dankserver ships its own. + _add(services, "frank-large") # Deployment/test/benchmark scripts and other repo tooling do not # restart a runtime service. return services diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 6ec0ec3..556e9f0 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -912,85 +912,53 @@ deploy_anubis() { wait_container_healthy anubis 60 || return 1 } -# frank-large runs as a rootless Quadlet unit since 2026-09-11 (halogen-flash- -# server replaced the llama.cpp/kingjones runtime). Unlike the sibling LLM -# services it is therefore NOT a root unit: no relink_unit, no `systemctl -# enable` (this host's quadlet generator produces units that cannot be -# enabled — boot persistence comes from [Install] WantedBy=default.target in -# the .container file plus linger). +# frank-large runs as a ROOT unit again since 2026-09-15: ik_llama.cpp +# (qwen4exp support merged upstream) replaced the halogen-flash-server +# rootless Quadlet, and the ik server is a native binary — CUDA on the RTX +# 3090 for dense/attention/KV, the Strix Halo CPU path for the MoE experts. +# There is no container image to pull and no generated unit to avoid: +# relink_unit points /etc/systemd/system/frank-large.service at the repo +# file, and boot persistence is the unit's [Install] section. # -# The llama.cpp-era ROOT unit is retired first. Its unit file is a symlink -# into this checkout (the old relink_unit did `ln -sfn scripts/frank-large. -# service /etc/systemd/system/`), so leaving it behind after the repo file -# moves to the trash would leave systemd holding a dangling unit that can -# still claim port 7770. Stop it, drop its enable symlinks, move the unit file -# into the home trash (never rm), then reload the root manager. -# -# FIRST START: the container fetches ~127 GB of weights before it can serve, -# so /health is legitimately down for the whole fetch. wait_for_health would -# burn its budget and report a failure that is not one. The gate below -# recognises the bootstrap (checkpoint absent) and waits for the checkpoint -# instead, falling through to the normal health gate once it lands. -LLM_LARGE_CHECKPOINT="${LLM_LARGE_CHECKPOINT:-/home/dank/models/halogen/qwen38-flash-next-w4b.hgn}" +# The halogen Quadlet is retired first. Both managers must never know this +# unit name at once (they would race for port 7770): stop the user unit, +# poll until it is really gone, move the installed .container into the home +# trash (never rm), then reload the user manager. +LLM_LARGE_CHECKPOINT="${LLM_LARGE_CHECKPOINT:-/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M/Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf}" -retire_root_llm_large() { - local trash="$HOME/.AGENTTRASH/20260911-halogen-swap/host-units" - run_privileged systemctl stop frank-default-large 2>/dev/null || true - run_privileged systemctl disable frank-default-large 2>/dev/null || true - run_privileged systemctl stop frank-large.service 2>/dev/null || true - # On this host `disable` has been observed to delete the unit file itself - # (2026-08-26 incident, see AGENTS.md); tolerate either behaviour and keep - # whatever remains by moving it to the trash. - run_privileged systemctl disable frank-large.service 2>/dev/null || true - if [[ -e /etc/systemd/system/frank-large.service || -L /etc/systemd/system/frank-large.service ]]; then +retire_quadlet_llm_large() { + local trash="$HOME/.AGENTTRASH/20260915-ik-swap/halogen-quadlet" + systemctl --user stop frank-large.service 2>/dev/null || true + # A stopping quadlet runs podman stop (SIGTERM, then SIGKILL at the + # timeout); llama-family engines never preempt an in-flight generation. + # Poll until the user unit is really inactive before removing its file. + local i + for ((i = 0; i < 30; i++)); do + systemctl --user is-active --quiet frank-large.service 2>/dev/null || break + sleep 2 + done + if [[ -e "$HOME/.config/containers/systemd/frank-large.container" || -L "$HOME/.config/containers/systemd/frank-large.container" ]]; then mkdir -p "$trash" - run_privileged mv -- /etc/systemd/system/frank-large.service \ - "$trash/frank-large.service" - echo "retired root unit frank-large.service -> $trash" + mv -- "$HOME/.config/containers/systemd/frank-large.container" \ + "$trash/frank-large.container" + echo "retired halogen quadlet -> $trash" fi - run_privileged systemctl daemon-reload + systemctl --user daemon-reload } deploy_llm_large() { - retire_root_llm_large - # Pull once, then run the localhost/-aliased tag: this host's quadlet path - # cannot resolve registry-prefixed refs from local storage at create time - # (same workaround as anubis/hermes). - local image="ghcr.io/peonist-ai/halogen-flash-server:0.5.8" - local alias="localhost/peonist-ai/halogen-flash-server:0.5.8" - podman pull "$image" - podman tag "$image" "$alias" - # The weights volume: read-write, because the container downloads into it. - mkdir -p /home/dank/models/halogen - install_quadlet frank-large dankclaw/frank-large/quadlet frank-large - systemctl --user restart frank-large.service - wait_for_llm_large_ready + retire_quadlet_llm_large + if [[ ! -f "$LLM_LARGE_CHECKPOINT" ]]; then + echo "ERROR: checkpoint absent at $LLM_LARGE_CHECKPOINT" >&2 + echo "frank-large: download the mradermacher i1-Q4_K_M GGUF first; refusing to start a unit that cannot load." >&2 + return 1 + fi + relink_unit frank-large + run_privileged systemctl enable frank-large.service + run_privileged systemctl restart frank-large.service wait_for_health frank-large http://127.0.0.1:7770/health 1200 } -# wait_for_llm_large_ready — bootstrap-aware gate. If the checkpoint is absent -# the container is downloading it; wait for the file (resumable, and the unit -# restarts it on failure) rather than gating on a /health that cannot answer -# yet. Reports success either way, so the first deploy does not look like a -# failure while the engine does exactly what it was told to. -wait_for_llm_large_ready() { - [[ -f "$LLM_LARGE_CHECKPOINT" ]] && return 0 - echo "frank-large: checkpoint absent at $LLM_LARGE_CHECKPOINT" - echo "frank-large: first start — the container is downloading ~127 GB from" - echo "frank-large: peonist-ai/halogen-qwen3.8-flash-next (resumes if interrupted)." - local i - for ((i = 1; i <= 1200; i++)); do - if [[ -f "$LLM_LARGE_CHECKPOINT" ]]; then - echo "frank-large: weights landed after approximately $((i * 2))s; gating on /health" - return 0 - fi - sleep 2 - done - echo "WARN: frank-large download still in progress after 40m — NOT a failure." - echo "WARN: the unit keeps fetching; re-run this deploy (or re-check /health) later." >&2 - return 0 -} - # deploy_llm_medium — frank-medium runs as a rootless Quadlet since the # 2026-09-15 runtime swap to the vllm-radiance MXFP4 stack. It began the day # as a root llama.cpp unit (the ROCmFPX/Q8_0_ROCMFPX_AGENT deployment from diff --git a/scripts/frank-large.env b/scripts/frank-large.env new file mode 100644 index 0000000..083a36e --- /dev/null +++ b/scripts/frank-large.env @@ -0,0 +1,95 @@ +# Tunables for frank-large (mradermacher i1-Q4_K_M of +# orcarouter/Qwen3.8-Flash-Next-Uncensored on ik_llama.cpp, Dankclaw, +# RTX 3090 CUDA + Strix Halo CPU path, since 2026-09-15). +# +# Runtime: ik_llama.cpp master d85a1d7 (2026-09-15) with full qwen4exp +# support (upstream PRs #2365/#2369/#2373 merged). Build recipe (CUDA for +# the 3090 at sm_86, Zen5 AVX-512 IQK kernels for the CPU expert path): +# cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON \ +# -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86 \ +# -DGGML_AVX512=ON -DGGML_AVX512_VBMI=ON -DGGML_AVX512_VNNI=ON \ +# -DGGML_AVX512_BF16=ON -DLLAMA_CURL=OFF +# The startup banner must print HAVE_FANCY_SIMD is defined — without those +# macros the quantized GEMM silently falls back to AVX2 (no warning). +# +# Model: mradermacher imatrix quant of the uncensored Flash-Next (qwen4exp +# MoE: 512 experts / 10 used, 48 layers = 12 attention + 36 GDN/ssm, QSA +# indexer, hyper-connections, PLE n-gram table ~320M rows). 111 GiB shipped +# as 3 parts, concatenated on disk; each part's sha256 was verified against +# its HF LFS oid at download time. +# +# Placement (ikawrakow's hybrid recipe, upstream issue #2402): +# - --cpu-moe: ALL expert tensors (121B elems, ~71 GB) live in the Strix +# Halo's LPDDR5X via the CPU path — ik's fastest known configuration +# for this shape (the author's 2x3090 + Threadripper reference box +# measures ~50 t/s on the same model family). +# - -ngl 999: dense/attention/output weights and the KV cache on the +# 3090 (~3 GB of weights + ~6 GB KV at 262k f16; 24 GB card). +# - The ~35 GB PLE table (per_layer_token_embd) is intentionally +# RAM/mmap-resident by design — only a tiny slice is touched per token. +# NEVER add --no-mmap (the STRIX-era lesson transfers: the cgroup +# killer takes the process with nothing in the server log) and do not +# try to pin PLE to VRAM (it cannot fit a 24 GB card). +# +# Sampling: greedy-by-default to preserve the halogen-era client contract +# (a request without temperature decodes greedy; temp 0.0 = argmax). +# Clients that send their own temperature get their own values. +# +# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (see +# the 2026-08-13 HIP incident in AGENTS.md). llama.cpp reads --host and +# --temp as CLI flags, never as environment variables. + +# ik_llama.cpp runtime binary (master d85a1d7 build). +LLAMA_BIN=/home/dank/GitPackages/ik_llama.cpp/build/bin/llama-server + +# Model package from +# https://huggingface.co/mradermacher/Qwen3.8-Flash-Next-Uncensored-i1-GGUF +# (parts concatenated). The mmproj is the matching vision tower from the +# static repo mradermacher/Qwen3.8-Flash-Next-Uncensored-GGUF — ik_llama.cpp +# links examples/mtmd, so image input survives the runtime swap. +MODEL_DIR=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M +MODEL_FILE=Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf +MMPROJ_PATH=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M/Qwen3.8-Flash-Next-Uncensored.mmproj-f16.gguf + +# Qwen-Sharp chat template (tracked in the repo, staged on the host). The +# server emits reasoning in DeepSeek format so clients receive +# reasoning_content separately instead of literal thinking tags in content. +CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja + +# Alias + network. +ALIAS=frank-large +BIND_ADDR=0.0.0.0 +LISTEN_PORT=7770 + +# Serving: single 262144-token slot (the model's full context window), +# batch 4096 / ubatch 512, 16+16 threads (the Halo CPU has 16 cores, no +# SMT), f16 KV. The KV cache scales with the 12 full-attention layers only +# (~6 GB f16 at 262k); the GDN/ssm layers keep constant-size state. If +# VRAM headroom is needed, --cache-type-k/v q8_0 halves the KV footprint. +CTX_SIZE=262144 +PARALLEL=1 +BATCH_SIZE=4096 +UBATCH_SIZE=512 +NUM_THREADS=16 +THREAD_BATCH=16 +CACHE_TYPE_K=f16 +CACHE_TYPE_V=f16 + +# GPU/MoE placement: everything on the 3090 except the experts, which all +# live in host RAM (--cpu-moe). See the placement notes above. +NGL=999 +CPU_MOE=on +FLASH_ATTN=on + +# Prompt cache (RAM budget in MiB) and slot state dir. +CACHE_RAM=8192 +SLOT_SAVE_PATH=/home/dank/serverAppConfigs/scripts/slot-state + +# Greedy-by-default (halogen contract). See sampling note above. +TEMPERATURE=0.0 + +# Reasoning on: DeepSeek format projects thinking into the API's +# reasoning_content field instead of literal think tags in content; +# unlimited reasoning budget (the model decides). +REASONING_FORMAT=deepseek +REASONING_BUDGET=-1 diff --git a/scripts/frank-large.service b/scripts/frank-large.service new file mode 100644 index 0000000..8b5d677 --- /dev/null +++ b/scripts/frank-large.service @@ -0,0 +1,16 @@ +[Unit] +Description=ik_llama.cpp Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M 177B MoE (dankclaw, frank-large) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=dank +Group=dank +Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env" +ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh new file mode 100755 index 0000000..dc8439f --- /dev/null +++ b/scripts/serve-frank-large.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Serve mradermacher/Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M as frank-large +# via ik_llama.cpp on Dankclaw. +# +# Topology (ikawrakow's hybrid recipe, upstream issue #2402): the RTX 3090 +# (CUDA, sm_86) takes the dense/attention/output weights and the KV cache +# (-ngl 999); ALL MoE expert tensors stay in the Strix Halo's LPDDR5X via +# the CPU path (--cpu-moe) — ik's fastest known configuration for this +# hybrid shape. The ~35 GB PLE n-gram table (per_layer_token_embd) is +# mmap-resident BY DESIGN: only a tiny slice is touched per token, so it +# streams from the OS page cache. NEVER add --no-mmap — the STRIX-era +# lesson transfers directly (the cgroup killer takes the process with +# nothing in the server log). +# +# Runtime: ik_llama.cpp master d85a1d7 built with CUDA (sm_86) + Zen5 +# AVX-512 IQK kernels; the startup banner must show HAVE_FANCY_SIMD +# is defined. Flag set verified against this binary's --help before this +# script was written. ik has no --cache-prompt/--cache-idle-slots (prompt +# caching is its default behavior) and no --alias: it is a single-model +# server and reports the GGUF's general.name at /v1/models. +# +# Sampling: greedy-by-default (halogen client contract) — clients that +# send their own temperature get their own values. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}" +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +# --- defaults (used when the env file does not set them) ------------------- +: "${LLAMA_BIN:=/home/dank/GitPackages/ik_llama.cpp/build/bin/llama-server}" +: "${MODEL_DIR:=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M}" +: "${MODEL_FILE:=Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf}" +: "${MMPROJ_PATH:=$MODEL_DIR/Qwen3.8-Flash-Next-Uncensored.mmproj-f16.gguf}" +: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}" +: "${BIND_ADDR:=0.0.0.0}" +: "${LISTEN_PORT:=7770}" +: "${CTX_SIZE:=262144}" +: "${PARALLEL:=1}" +: "${BATCH_SIZE:=4096}" +: "${UBATCH_SIZE:=512}" +: "${NUM_THREADS:=16}" +: "${THREAD_BATCH:=16}" +: "${CACHE_TYPE_K:=f16}" +: "${CACHE_TYPE_V:=f16}" +: "${NGL:=999}" +: "${CPU_MOE:=on}" +: "${FLASH_ATTN:=on}" +: "${CACHE_RAM:=8192}" +: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}" +: "${TEMPERATURE:=0.0}" +: "${REASONING:=on}" +: "${REASONING_FORMAT:=deepseek}" +: "${REASONING_BUDGET:=-1}" + +mkdir -p "$SLOT_SAVE_PATH" + +# The UVM device node is created lazily by nvidia-modprobe on the first +# CUDA open after boot; make sure it exists before the server needs it. +nvidia-modprobe -u -c=0 2>/dev/null || true + +CPU_MOE_ARGS=() +if [[ "$CPU_MOE" == "on" ]]; then + CPU_MOE_ARGS=(--cpu-moe) +fi + +MMPROJ_ARGS=() +if [[ -n "${MMPROJ_PATH:-}" && -f "$MMPROJ_PATH" ]]; then + MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH") +fi + +TEMPLATE_ARGS=() +if [[ -n "${CHAT_TEMPLATE_FILE:-}" && -f "$CHAT_TEMPLATE_FILE" ]]; then + TEMPLATE_ARGS=(--chat-template-file "$CHAT_TEMPLATE_FILE") +else + TEMPLATE_ARGS=(--jinja) +fi + +exec "$LLAMA_BIN" \ + --model "$MODEL_DIR/$MODEL_FILE" \ + "${TEMPLATE_ARGS[@]}" \ + --host "$BIND_ADDR" \ + --port "$LISTEN_PORT" \ + --gpu-layers "$NGL" \ + "${CPU_MOE_ARGS[@]}" \ + --flash-attn "$FLASH_ATTN" \ + --parallel "$PARALLEL" \ + --ctx-size "$CTX_SIZE" \ + --batch-size "$BATCH_SIZE" \ + --ubatch-size "$UBATCH_SIZE" \ + --threads "$NUM_THREADS" \ + --threads-batch "$THREAD_BATCH" \ + --cache-type-k "$CACHE_TYPE_K" \ + --cache-type-v "$CACHE_TYPE_V" \ + --cache-ram "$CACHE_RAM" \ + --slot-save-path "$SLOT_SAVE_PATH" \ + --ctx-checkpoints 16 \ + --metrics \ + --temp "$TEMPERATURE" \ + --reasoning "$REASONING" \ + --reasoning-format "$REASONING_FORMAT" \ + --reasoning-budget "$REASONING_BUDGET" \ + "${MMPROJ_ARGS[@]}" diff --git a/tests/deployment_scoping.bats b/tests/deployment_scoping.bats index fe849b4..d951ae5 100644 --- a/tests/deployment_scoping.bats +++ b/tests/deployment_scoping.bats @@ -18,7 +18,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" @test "deployment planner maps large-model changes to the large model only" { run python3 "$PLANNER" --target dankclaw --paths \ - dankclaw/frank-large/quadlet/frank-large.container + scripts/frank-large.env scripts/frank-large.service scripts/serve-frank-large.sh [ "$status" -eq 0 ] [ "$output" = "frank-large" ] } @@ -28,14 +28,14 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" ! grep -q 'frank_npu\|frank-npu' "$REMOTE_DEPLOY" } -@test "chat-template changes plan no service: the radiance runtime ships its own template" { - # The ROCmFPX llama.cpp deployment optionally read the tracked - # scripts/chat-template.jinja; the vllm-radiance runtime mounts its own - # qwen-fixed-v22.3.jinja out of the pinned clone instead. Nobody reads - # the tracked template any more, so an edit to it deploys nothing. +@test "chat-template changes plan frank-large: the ik runtime reads the tracked template" { + # ik_llama.cpp (2026-09-15 cutover) serves the Qwen-Sharp template from + # the tracked scripts/chat-template.jinja again. The vllm-radiance + # runtime on Dankserver ships its own qwen-fixed-v22.3.jinja out of the + # pinned clone, so a template edit deploys nothing there. run python3 "$PLANNER" --target dankclaw --paths scripts/chat-template.jinja [ "$status" -eq 0 ] - [ -z "$output" ] + [ "$output" = "frank-large" ] run python3 "$PLANNER" --target dankserver --paths scripts/chat-template.jinja [ "$status" -eq 0 ]