diff --git a/autoresearch.sh b/autoresearch.sh index 7c532a1..0e98e42 100755 --- a/autoresearch.sh +++ b/autoresearch.sh @@ -71,7 +71,7 @@ fi ASI_ARGS="$(ssh "$HOST" 'podman inspect frank-large-mimo \ --format "{{join .Config.Cmd \" \"}}" 2>/dev/null' | tr -d "\n" | head -c 400 || true)" if [ -z "$ASI_ARGS" ]; then - ASI_ARGS="$(ssh "$HOST" "tr '\\0' ' ' < /proc/\\$(pgrep -x llama-server | head -1)/cmdline 2>/dev/null" | head -c 400 || true)" + ASI_ARGS="$(ssh "$HOST" "sh -c 'tr \"\\0\" \" \" < /proc/\\$(pgrep -x llama-server | head -1)/cmdline 2>/dev/null || true'" | head -c 400 || true)" fi echo "ASI engine_args=${ASI_ARGS:0:400}" diff --git a/scripts/autoresearch-bench.py b/scripts/autoresearch-bench.py index de9468e..3c1fb15 100755 --- a/scripts/autoresearch-bench.py +++ b/scripts/autoresearch-bench.py @@ -84,7 +84,8 @@ def stream_completion(url: str, model: str, prompt: str): d = ch.get("delta") or {} # reasoning AND content deltas both arrive token by token; # count either so greedy text length drives the rate. - if d.get("content") or d.get("reasoning"): + # (llama.cpp names the thinking field reasoning_content.) + if d.get("content") or d.get("reasoning") or d.get("reasoning_content"): now = time.perf_counter() if first is None: first = now diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index f88fd67..5dd493e 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1077,20 +1077,27 @@ deploy_anubis() { wait_container_healthy anubis 60 || return 1 } -# frank-large is vLLM (the digest-pinned kyuz0 TheRock gfx1151 image) serving -# the BF16 MiMo-V2.6-Distill-Qwen-9B payload with cross-model MTP spec decode -# (Qwen/Qwen3.5-9B head, 2 draft tokens) on the Strix Halo iGPU. 2026-10-01 -# cutover from gufo; the gufo-era gates (built-from-source binary, GGUF -# target + DFlash2 file) are retired with it — rollback is git tag -# frank-large-gufo-q4kxl-20261001, and the gufo runtime and payloads stay on -# disk. The service drives `podman run` directly from its serve script, so -# the gates here mirror deploy_llm_primary's: image present by REPO DIGEST in -# dank's rootless storage, payload complete (config.json + index + 4 shards), -# and the spec draft present whenever FRANK_LARGE_SPEC names one. The port -# gate is the same as every cutover since 2026-09-16: nothing starts until -# 7770 is released. -LLM_LARGE_CONTAINER=frank-large-mimo -LLM_LARGE_LEGACY_CONTAINERS="vllm-mimo" +# frank-large is gufo (gufo-org/gufo), the Strix Halo inference engine built +# from the pinned v0.1.1 tag by scripts/build-gufo.sh, serving +# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL with the DFlash2 drafter +# (z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M) on the Strix Halo iGPU. It is a +# ROOT unit: the runtime is a native binary linked against the host ROCm, +# so there is no container image to install and no generated unit to dodge. +# relink_unit points /etc/systemd/system/frank-large.service at the repo +# file and boot persistence is that unit's [Install] section. +# +# The runtime gate matters as much as the payload gate: gufo is built from +# source on this host (there is no prebuilt release), so a machine whose +# /home/dank/GitPackages tree was never built has a unit that can never +# become healthy, and a green deploy would leave the endpoint dark. +# +# It replaces the stock llama.cpp prebuilt (unsloth b11030), which had +# replaced the Kairic Edge runtime (ciru-ai/ROCmFPX). The port gate is the +# same as every cutover since 2026-09-16: nothing starts until 7770 is +# released. +LLM_LARGE_RUNTIME="${LLM_LARGE_RUNTIME:-/home/dank/GitPackages/gufo-v0.1.1/bin/gufo}" +LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-27B-UD-Q4_K_XL/Qwen3.8-27B-UD-Q4_K_XL.gguf}" +LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}" retire_halogen_large_quadlet() { local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" @@ -1123,127 +1130,97 @@ wait_port_free() { } deploy_llm_large() { - # Refuse rather than restart-loop: a unit whose image or payload is + # Refuse rather than restart-loop. A unit whose runtime or payload is # missing never becomes healthy, and a green deploy would leave the - # endpoint dark. Gates mirror deploy_llm_primary. - local env_file="/home/dank/serverAppConfigs/scripts/frank-large.env" - local image model_dir shards count spec spec_dir digests - if [[ ! -f "$env_file" ]]; then - echo "ERROR: frank-large env file absent: $env_file" >&2 + # endpoint dark with nothing in the deploy log to say why. + if [[ ! -x "$LLM_LARGE_RUNTIME" ]]; then + echo "ERROR: gufo runtime absent at $LLM_LARGE_RUNTIME" >&2 + echo "frank-large: build it: bash scripts/build-gufo.sh" >&2 return 1 fi - image="$(sed -n 's/^FRANK_LARGE_IMAGE=//p' "$env_file" | tail -1)" - model_dir="$(sed -n 's/^FRANK_LARGE_MODEL_DIR=//p' "$env_file" | tail -1)" - shards="$(sed -n 's/^FRANK_LARGE_MODEL_SHARDS=//p' "$env_file" | tail -1)" - spec="$(sed -n 's/^FRANK_LARGE_SPEC=//p' "$env_file" | tail -1)" - spec_dir="$(sed -n 's/^FRANK_LARGE_SPEC_DIR=//p' "$env_file" | tail -1)" - if [[ -z "$image" || -z "$model_dir" || -z "$shards" ]]; then - echo "ERROR: frank-large.env missing FRANK_LARGE_IMAGE/MODEL_DIR/MODEL_SHARDS" >&2 - return 1 - fi - # Capture podman's output BEFORE grepping (the SIGPIPE race, 2026-09-30). - digests="$(podman images --format '{{.RepoDigests}}')" - if ! grep -qF "${image#*@}" <<<"$digests"; then - echo "ERROR: pinned runtime image absent from rootless storage: $image" >&2 - echo "frank-large: fetch it (podman pull $image)" >&2 - return 1 - fi - if [[ ! -f "$model_dir/config.json" || ! -f "$model_dir/model.safetensors.index.json" ]]; then - echo "ERROR: payload incomplete: $model_dir (config.json / weight-map index)" >&2 - echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2 - return 1 - fi - count="$(find "$model_dir" -maxdepth 1 -name 'model-*-of-*.safetensors' 2>/dev/null | wc -l)" - if [[ "$count" != "$shards" ]]; then - echo "ERROR: payload shard count wrong: $model_dir (found $count, want $shards)" >&2 - echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2 - return 1 - fi - if [[ -n "$spec" ]]; then - # Both draft weight layouts are real: dflash ships a single - # model.safetensors, the Qwen3.5-9B MTP head ships sharded behind - # an index. Mirrors the serve script's gate (2026-10-01: the - # single-file check rejected the deployed mtp arm). - if [[ -z "$spec_dir" || ! -f "$spec_dir/config.json" ]] || - { [[ ! -f "$spec_dir/model.safetensors" ]] && - [[ ! -f "$spec_dir/model.safetensors.index.json" ]]; }; then - echo "ERROR: spec draft absent or partial for FRANK_LARGE_SPEC=$spec: $spec_dir" >&2 - echo "frank-large: re-run scripts/download-frank-large-dflash.sh (or the Qwen3.5-9B fetch for mtp)" >&2 - return 1 + local missing=0 f + for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT"; do + if [[ ! -f "$f" ]]; then + echo "ERROR: payload absent at $f" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large.sh" >&2 + missing=1 fi - fi + done + [[ "$missing" -eq 0 ]] || return 1 retire_halogen_large_quadlet # On a re-deploy the predecessor is THIS unit: stop it before waiting on # the port, or the wait deadlocks on the server it just stopped serving. + # Then sweep a stray manual run -- that is how a bench harness leaves the + # port busy. Match the DRAFTER path, which every frank-large command + # line carries via --dflash-model and which is UNCHANGED by the 2026-09-28 + # payload swap: the target moved to a different repository while the + # drafter stayed put, so keying on the target filename would have quietly + # stopped matching. It is also the right key in principle — it names this + # endpoint's speculative drafter, not a binary. Match that, not a binary + # name: frank-small on this host serves a different GGUF over llama-server + # with no DFlash2 drafter and must not be killed. The pattern never + # appears in the deploying shell's own command line, so -f is safe here + # (it is not in `bash deploy-remote.sh`). run_privileged systemctl stop frank-large.service 2>/dev/null || true - # A stopped unit can leave its container running (the cgroup-kill vs - # conmon --rm race); the orphan holds the port and the GPU. The - # bench-window identity (vllm-mimo) dies here too — that is how a bench - # harness leaves the port busy. - local ctr - for ctr in $LLM_LARGE_LEGACY_CONTAINERS "$LLM_LARGE_CONTAINER"; do - podman rm -f "$ctr" >/dev/null 2>&1 || true - done - # Nothing may start until the predecessor has released the port - # (2026-09-16 and 2026-09-26 cutovers both failed exactly here). + pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M' 2>/dev/null || true + # Nothing may start until the predecessor has released the port. On + # 2026-09-16 the cutover failed exactly here: the old server logged + # "Deactivated successfully", the next engine tried to bind 7770 a second + # later and reported "Address already in use". Gate on the port, not on + # hope. wait_port_free 7770 || return 1 relink_unit frank-large run_privileged systemctl daemon-reload run_privileged systemctl enable frank-large.service run_privileged systemctl restart frank-large.service - # Warm boot is ~200-240 s (measured 2026-10-01); cold torch compile is - # longer. Parity with frank-primary's 1800 s allowance. - wait_for_health frank-large http://127.0.0.1:7770/health 1800 || return 1 - # False-green hardening: the port answering proves nothing about WHO - # answers — the unit must be active with a live MainPID. - local main_pid - main_pid="$(run_privileged systemctl show -p MainPID --value frank-large.service)" - if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then - echo "FAIL: :7770 answered but frank-large.service has no MainPID — a stray process holds the port" >&2 - return 1 - fi - echo "frank-large: health answered by unit MainPID $main_pid" + wait_for_health frank-large http://127.0.0.1:7770/health 1200 } -# deploy_llm_small — frank-small is SGLang (python3.12 venv) on the RTX 3090, -# as a ROOT unit: the runtime is a self-contained venv, so there is no -# container image to install and no generated unit to avoid. relink_unit -# points /etc/systemd/system/frank-small.service at the repo file and boot +# deploy_llm_small — frank-small is llama.cpp (master build, CUDA sm_86) on +# the RTX 3090 as a ROOT unit: the runtime is a native binary tree under +# ~/GitPackages/llama-master/build-cuda, so there is no container image and +# no generated unit to avoid. relink_unit points +# /etc/systemd/system/frank-small.service at the repo file and boot # persistence is that unit's [Install] section. # # It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is # not shared: frank-large's engine runs on the gfx1151 iGPU. -LLM_SMALL_MODEL_DIR="${LLM_SMALL_MODEL_DIR:-/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound}" -LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/sglang-cu128/bin/sglang}" +LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}" +LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" deploy_llm_small() { # Refuse rather than restart-loop: a unit whose runtime or payload is # missing never becomes healthy, and a green deploy would leave the # endpoint dark with nothing in the log to say why. if [[ ! -x "$LLM_SMALL_RUNTIME" ]]; then - echo "ERROR: SGLang runtime absent at $LLM_SMALL_RUNTIME" >&2 - echo "frank-small: build the venv (python3.12 -m venv /home/dank/GitPackages/sglang-cu128" >&2 - echo "frank-small: and pip install 'sglang[all]' into it) first." >&2 + echo "ERROR: llama.cpp runtime absent at $LLM_SMALL_RUNTIME" >&2 + echo "frank-small: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2 + echo "frank-small: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2 + echo "frank-small: && cmake --build build-cuda --target llama-server) first." >&2 return 1 fi - if [[ ! -f "$LLM_SMALL_MODEL_DIR/model.safetensors.index.json" ]]; then - echo "ERROR: payload index absent at $LLM_SMALL_MODEL_DIR" >&2 - echo "frank-small: run scripts/download-frank-small.sh first;" >&2 + if [[ ! -f "$LLM_SMALL_MODEL_FILE" ]]; then + echo "ERROR: payload GGUF absent at $LLM_SMALL_MODEL_FILE" >&2 + echo "frank-small: fetch bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF Q6_K first;" >&2 echo "frank-small: refusing to start a unit that cannot load." >&2 return 1 fi - local count - count="$(find "$LLM_SMALL_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)" - if [[ "$count" -ne 8 ]]; then - echo "ERROR: payload shard count wrong: $LLM_SMALL_MODEL_DIR (found $count, want 8)" >&2 - return 1 - fi relink_unit frank-small run_privileged systemctl daemon-reload run_privileged systemctl enable frank-small.service run_privileged systemctl restart frank-small.service - wait_for_health frank-small http://127.0.0.1:7771/health 900 + wait_for_health frank-small http://127.0.0.1:7771/health 900 || return 1 + # False-green hardening (frank-large's lesson): the port answering + # proves nothing about WHO answers — the unit must be active with a + # live MainPID, or a stray SGLang holdover is masquerading as deployed. + local main_pid + main_pid="$(run_privileged systemctl show -p MainPID --value frank-small.service)" + if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then + echo "FAIL: :7771 answered but frank-small.service has no MainPID — a stray process holds the port" >&2 + return 1 + fi + echo "frank-small: health answered by unit MainPID $main_pid" } # deploy_llm_primary — frank-primary is the tcclaviger RDNA4 vLLM fork diff --git a/scripts/frank-large.env b/scripts/frank-large.env index 5f30acb..9422e08 100644 --- a/scripts/frank-large.env +++ b/scripts/frank-large.env @@ -1,110 +1,63 @@ -# frank-large.env — vLLM production config for frank-large (Dankclaw iGPU :7770). +# frank-large — gufo (gufo-org/gufo) serving +# Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) on the Strix Halo iGPU. # -# 2026-10-01 cutover from gufo (git tag frank-large-gufo-q4kxl-20261001). -# Target: XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B in FULL BF16 precision -# (17.53 GiB, 4 shards) — the same architecture frank-small serves as an -# INT8 W8A16 AutoRound conversion on the 3090, here unquantized. The engine -# is the kyuz0 TheRock vLLM build, the only fleet-carried runtime path for -# gfx1151 (official vLLM images are CUDA-only). +# These are the tunables read by scripts/serve-frank-large.sh. The flag +# list is owned by the serve script; this file only sets fleet values. # -# MEASURED 2026-10-01 bench window on this host (identical client, one -# session, temperature 0, 28,647-token prefill / 512-token decode): -# -# arm prefill decode acceptance (per position) -# BF16 base 17,876 t/s 13.1 t/s — -# BF16 + DFlash 8 tok 7,976 t/s 17.4 t/s pos0 59% pos1 26%; 14.8% token rate -# BF16 + MTP 2 tok 7,380 t/s 19.8 t/s pos0 71% pos1 44%; 57.4% token rate -# -# Decode laws: 13.1 t/s is ~94% of the Strix Halo LPDDR5x roof for BF16 -# weights (17.5 GiB/token against ~256 GB/s) — a quant, not a bug; the same -# INT8 payload decodes 83.9 t/s on the 3090 because it moves ~12 GiB -# through ~936 GB/s. Speculative arms are the ONLY way past the wall, and -# both drafters pay ~55-59% of prefill (draft passes share the compute -# queues). Frank's call enabled MTP: the thinking-heavy traffic on this -# endpoint spends its wall clock in decode. -# -# MTP IS CROSS-MODEL: the head comes from Qwen/Qwen3.5-9B (the distill's -# source model, 15 mtp.* tensors, identical 4096-hidden/248320-vocab -# geometry) — MiMo itself ships ZERO mtp tensors, which is why frank-small's -# SGLang NEXTN is a 0.00-acceptance ghost (SGLang reads the head from the -# TARGET checkpoint; vLLM reads the draft from a separate path). The 71%/44% -# per-position acceptance is the distillation preserving Qwen3.5 hidden-state -# structure; it is measured, not assumed. -# -# NUMERICS NOTE (measured): with a spec drafter enabled, temperature-0 text -# DIFFERS from the no-spec run (greedy sha 1318aeb6 base vs 0208b45d dflash -# vs 20c41c38 mtp): batched verification changes the bf16 reduction order, -# so argmax near-ties diverge and cascade. Spec decode on vLLM is -# distribution-preserving, not bitwise-greedy-preserving. The gufo -# greedy==AR law does not carry over. -# -# HIP reads bare env vars TEMP, HOST, GPU_MAX_HEAP_SIZE — every key here is -# FRANK_LARGE_-prefixed so none can leak. +# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP +# runtime reads those bare (the 2026-08-13 TEMP incident). HOST is the one +# exception because the serve script itself consumes it for --host. -# --- runtime ---------------------------------------------------------------- -# Digest-pinned (the :latest tag of this fork moves with no notice): -# image ID 2d24a8c5a3fd, vLLM 0.30.0 / torch 2.11.0 / ROCm 10.0.0 (TheRock), -# verified to ship vllm/v1/spec_decode/dflash.py, the qwen3_5 model registry -# entry (Qwen3_5ForConditionalGeneration), and the Triton GDN backend. -FRANK_LARGE_IMAGE=docker.io/kyuz0/vllm-therock-gfx1151@sha256:98c1f3ac9833e7fa45e9a82fb7a4315e6d95daaf2548a211c8da3a871b17533a +# The target payload (scripts/download-frank-large-q4.sh): the serve script +# refuses to start without the main GGUF and the DFlash2 drafter. +# +# 2026-10-01 swap (Frank): unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL, 16.35 GiB, +# replacing ajgazin's abliterated Swift-1.5 UD-Q8_K_XL (29.3 GiB). Two +# consequences worth knowing at a glance: +# * this is the VANILLA (non-abliterated) Qwen3.8-27B — the endpoint is no +# longer uncensored, which was one of the reasons the Swift payload was +# chosen on 2026-09-28; +# * ~13 GiB of weights became available for KV, which is what makes four +# full-sized f16 sessions fit. +# gufo needs no template for either: its Qwen renderer is compiled in. +MODEL_DIR=/home/dank/models/Qwen3.8-27B-UD-Q4_K_XL +MODEL_FILE=/home/dank/models/Qwen3.8-27B-UD-Q4_K_XL/Qwen3.8-27B-UD-Q4_K_XL.gguf -# --- payload ---------------------------------------------------------------- -FRANK_LARGE_MODEL_DIR=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-BF16 -FRANK_LARGE_MODEL_SHARDS=4 +# The DFlash2 drafter is UNCHANGED by the model swap and still lives beside +# the previous target, so it carries its own directory. It has a sliding +# window of 2048, which is what keeps its KV small (~40 MiB). +DRAFTER_DIR=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL +DRAFTER_FILE=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf -# --- identity --------------------------------------------------------------- -# Same port and served alias as every frank-large predecessor; the harness -# models.yml id is unchanged. The bench-window container (vllm-mimo) is -# swept before every start. -FRANK_LARGE_CONTAINER=frank-large-mimo -FRANK_LARGE_LEGACY_CONTAINERS="vllm-mimo" -FRANK_LARGE_SERVED_NAME=frank-large -FRANK_LARGE_PUBLISH=7770:7770 +# The gufo runtime, built from source by scripts/build-gufo.sh. +RUNTIME_DIR=/home/dank/GitPackages/gufo-v0.1.1 -# --- engine shape (2026-10-01 window) ---------------------------------------- -# Full native window 262,144 at UTIL 0.45 booted clean on gfx1151 in all -# three arms (~200-240 s warm start). KV grows only on the 8 full-attention -# layers (32 KiB/token bf16: 8 x 2 x 4 heads x 256 dim x 2 B); the 24 GDN -# layers keep a FIXED ~25 MiB state per running sequence, so the pool is -# generous and sessions are cheap. SEQS 4 preserves the four-session -# contract the gufo shape ran; vLLM shares the pool across them by -# scheduler, unlike gufo's per-session carve. -FRANK_LARGE_MAXLEN=262144 -FRANK_LARGE_SEQS=4 -FRANK_LARGE_UTIL=0.45 -# First-deploy boot (2026-10-01): GPU KV cache size 819,332 tokens, -# Maximum concurrency for 262,144 tokens per request 3.13x; healthy at 230 s. -# Change MAXLEN/SEQS/UTIL or the image and this line moves — re-read it. +# Fleet endpoint: the same port and alias every frank-large has served, so +# no client, probe or models.yml entry changes address. +ALIAS=frank-large +HOST=0.0.0.0 +PORT=7770 -# --- speculative decode: cross-model MTP (measured +51% decode) --------------- -# mtp beats dflash here (19.8 vs 17.4 t/s) and needs 2 draft tokens, not 8: -# acceptance past position 1 is 44% and falling (DFlash's own curve decays -# 59/26/14/7). num_speculative_tokens beyond 2 is UNMEASURED — do not raise -# it without a fresh A/B. To flip drafters, set FRANK_LARGE_SPEC=dflash and -# repoint SPEC_DIR (the DFlash drafter stays on disk). -# The gufo-era law "spec decode is absent" was a frank-primary finding; this -# endpoint's traffic is decode-dominated thinking, which is the regime where -# spec wins (prefill costs ~55% in every arm). -FRANK_LARGE_SPEC= -FRANK_LARGE_SPEC_DIR=/home/dank/models/Qwen3.5-9B -FRANK_LARGE_SPEC_TOKENS=2 +# ONE session at the full native window (Frank, 2026-10-01 night: "roll back +# to qwen3.8 27b, one single max context"). The KV is UNQUANTIZED (f16) — +# gufo has no KV quantization lever at all (execution_policy.hpp pins +# kv_cache_storage to kFp16), so "unquantized" is inherent rather than +# configured. +# +# The arithmetic: f16 KV costs 64 KiB/token on this model (16 full-attention +# layers x 4 KV heads x 256 head_dim x 2 (K,V) x 2 bytes; the other 48 layers +# are linear-attention with a per-SEQUENCE fp32 state of ~144 MiB, not a +# per-token cache). 262,144 tokens = 16.0 GiB of KV. Budget: weights 16.35 + +# KV 16.0 + drafter 1.07 + linear state 0.14 + runtime ~4 = ~37.6 GiB against +# the 114 GiB GTT — enormous headroom; the session count or context policy +# can grow from here without pressure. +CONTEXT_SIZE=262144 +SESSIONS=1 -# --- AUTORESEARCH arm: llama.cpp runtime ------------------------------------- -# master build (qwen35 GDN kernels, HIP gfx1151) on Q6_K GGUF: 7.79 GiB -# weights = 0.51x the BF16 bytes, so the LPDDR roof doubles. Cross-model -# DFlash draft (qwen3.5-9b, Q8_0). --reasoning-format deepseek emits -# reasoning_content, NOT vLLM's reasoning. -FRANK_LARGE_RUNTIME=llama -FRANK_LARGE_LLAMA_BIN=/home/dank/GitPackages/llama-master/build-hip/bin/llama-server -FRANK_LARGE_GGUF=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf -FRANK_LARGE_DRAFT_GGUF=/home/dank/models/DFlash-Qwen35/qwen3.5-9b-dflash-Q8_0.gguf -FRANK_LARGE_LLAMA_CTX=1048576 -# --- client contract ---------------------------------------------------------- -# vLLM emits the thinking text in `reasoning` (NOT llama.cpp/gufo's -# `reasoning_content`) — the harness models.yml compat block for this -# provider must set reasoningContentField accordingly. The qwen3 parser is -# what keeps thinking out of `content`. Tools ride the qwen3_coder parser -# with auto tool choice. --language-model-only drops the vision tower (the -# endpoint stays text-only; the tower can come back by dropping the flag). -FRANK_LARGE_REASONING_PARSER=qwen3 -FRANK_LARGE_TOOL_PARSER=qwen3_coder +# DFlash2 spec decode (a block-diffusion drafter). gufo sizes the draft +# block itself; DRAFT_TOKENS caps the proposals per step (gufo's default is +# 7, chosen over llama.cpp's fixed depth 4 because gufo's adaptive policy +# shortens the block when acceptance drops) and DRAFT_POLICY is the block +# length controller. +DRAFT_TOKENS=7 +DRAFT_POLICY=adaptive diff --git a/scripts/frank-large.service b/scripts/frank-large.service index d35a828..7c9e9a4 100644 --- a/scripts/frank-large.service +++ b/scripts/frank-large.service @@ -1,46 +1,20 @@ -# frank-large.service — vLLM MiMo-V2.6-Distill-Qwen-9B (BF16) on the Strix -# Halo gfx1151 iGPU, :7770. 2026-10-01 cutover from gufo (tag -# frank-large-gufo-q4kxl-20261001 restores the Q4_K_XL + DFlash2 shape). -# -# WHY A ROOT UNIT: parity with frank-primary/frank-small — the unit is -# relink_unit'd from this repo into /etc/systemd/system/, PID1 grants -# LimitMEMLOCK=infinity (vLLM pins host buffers), and OOM protection lands -# on the engine instead of the co-tenants. The container runs in dank's -# ROOTLESS podman storage. -# -# The serve script drives `podman run` directly (image entrypoint is -# overridden to `vllm serve`) and blocks on the container, so a container -# death is a non-zero ExecStart. NOTE from frank-primary's experience: -# vLLM's API server can drain and exit 0 after an engine fault, so the -# restart policy MUST be always, not on-failure. [Unit] -Description=frank-large — vLLM MiMo-V2.6-Distill-Qwen-9B (BF16) on the gfx1151 iGPU +Description=gufo (gufo-org/gufo, ROCm gfx1151) Qwen3.8-27B UD-Q8_K_XL, DFlash2 spec decode, 262144 ctx / 1 session, root systemd unit (dankclaw, frank-large, Strix Halo iGPU) After=network-online.target Wants=network-online.target -# Never let the start-rate limiter give up after a crash loop. -StartLimitIntervalSec=0 [Service] +Type=simple User=dank Group=dank +# The engine pins staging buffers; keep the memlock budget unbounded so an +# RLIMIT smaller than the pinned set cannot fail the load. Parity with the +# other LLM units on this fleet. LimitMEMLOCK=infinity Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env" ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh -# systemd stop: podman wait alone cannot stop the container child; stop it -# explicitly or the engine outlives the unit holding the GPU and port. -ExecStop=/usr/bin/podman stop -t 120 frank-large-mimo -# Boot cost: ~200-240 s warm (measured 2026-10-01 bench window); cold torch -# compile is longer. Health stays 503 until the engine is up — do not kill. -TimeoutStartSec=1800 -# ExecStop allows 120 s of drain; the stop timeout must exceed it (the -# frank-primary 2026-10-01 lesson: a short TimeoutStopSec records 'Failed -# with result timeout' on every stop). -TimeoutStopSec=180 -Restart=always +Restart=on-failure RestartSec=10 -# The engine holds 17.5 GiB of weights plus the KV pool in the GTT -# aperture; it is the last process on this host the kernel should pick. -OOMScoreAdjust=-500 [Install] WantedBy=multi-user.target diff --git a/scripts/frank-small.env b/scripts/frank-small.env index b235c2f..362932c 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -1,86 +1,37 @@ -# Tunables for frank-small (letechlead/MiMo-V2.6-Distill-Qwen-9B-INT8- -# W8A16-AutoRound, qwen3_5 hybrid, in-checkpoint vision) on SGLang, -# Dankclaw's RTX 3090, since 2026-10-01. +# frank-small — llama.cpp (master, CUDA sm_86) serving +# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K on the RTX 3090, :7771. # -# Runtime: SGLang in a python3.12 venv (pip "sglang[all]"): -# python3.12 -m venv /home/dank/GitPackages/sglang-cu128 -# /home/dank/GitPackages/sglang-cu128/bin/pip install "sglang[all]" -# The venv carries its own CUDA torch + sgl-kernel + flashinfer; it is -# self-contained and does not touch the host Python (3.14) or the ROCm -# stack that serves frank-large. +# 2026-10-01 night (Frank): swapped from SGLang INT8-W8A16-AutoRound to the +# GGUF path specifically for the ASYMMETRIC KV contract — K unquantized +# (f16), V q8_0 — which only llama.cpp exposes per-tensor. Q6_K is 7.79 GiB +# of weights (above the fleet's 6-bit accuracy floor). # -# Model: MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound — a Qwen3.5-9B -# distillation at INT8 W8A16 (AutoRound, group 128, symmetric). 8 -# safetensors shards, 12.0 GiB total. The vision tower -# (qwen3_5_vision, 27 layers) and embed_tokens stay fp16 inside the -# checkpoint, so the image path needs no separate projector file. -# Architecture: 32 layers, 8 full-attention (4 KV heads x 256 head_dim) -# + 24 Gated DeltaNet, native 262144 window. text_config declares -# mtp_num_hidden_layers = 1, but the shipped safetensors carry NO mtp.* -# tensors (verified against model.safetensors.index.json 2026-10-01 — -# base BF16 and this INT8 conversion both omit the head), so there is no -# spec decode: SGLang would draft with an uninitialised 4.24 GiB ghost -# model at 0% acceptance and pay the verify tax for nothing. -# -# SIZING (RTX 3090), measured 2026-10-01: the 0.98 budget holds the -# checkpoint (11,354 MiB) + the DeltaNet ssm state pool (795 MiB at 32 -# bf16 slots) + the KV pool. The engine's autofit picks a 687,388-token -# pool and leaves ~0.53 GB — too little: Triton's deferred kernel loads -# take most of it after boot, and eager-prefill activations then OOM on -# the first long request. The pool is therefore PINNED at 606,208 tokens -# (2 x 262,144 native slots since the 2026-10-01 reslice; 4 x 151,552 -# before), which leaves ~0.8 GB for the prefill workspace; a 170k single -# request and 4 concurrent 54k requests were verified at the 4-slot -# shape. -# The ssm pool is the 4.5 GiB the weights-only arithmetic misses. -# -# Sampling: greedy by default (the fleet contract); a client that sends a -# temperature gets its own, and the harness applies per-model overrides -# from models.yml. -# -# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (HIP -# reads those bare vars on the ROCm hosts -- see the 2026-08-13 incident in -# AGENTS.md). This box is CUDA, but the rule keeps env files portable. +# KV arithmetic: attention KV exists only in the 8 full-attention layers +# (4 KV heads x 256 dim). K f16 = 16 KiB/token; V q8_0 ~ 8.5 KiB/token +# (34 B per 32-element block) => ~24.5 KiB/token. Two full native windows +# (2 x 262,144 = 524,288 tokens) cost ~12.6 GiB; weights 7.8 + KV 12.6 + +# compute buffers and CUDA graphs ~1.5-2 => ~22-23 GiB against the card's +# 24 GiB — this is the "two contexts with room for checkpoints" fit. If the +# graph capture OOMs, lower CTX_SIZE (262144 = two 131072 windows) rather +# than quantizing K. +LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server +MODEL_FILE=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf -# SGLang runtime: python3.12 venv, self-contained CUDA stack. -SGLANG_BIN=/home/dank/GitPackages/sglang-cu128/bin/sglang - -# Model package (8 shards + configs, sha256-verified by -# scripts/download-frank-small.sh). -MODEL_DIR=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound - -# Fleet identity and network. 7771 is this service's own endpoint on -# Dankclaw; 7770 stays frank-large's. The alias is the id at /v1/models -# and what clients ask for -- the harness entry in models.yml names it. +# Fleet endpoint: same port and alias as the SGLang era; the harness +# models.yml id does not move. ALIAS=frank-small BIND_ADDR=0.0.0.0 LISTEN_PORT=7771 -# Two slots at the model's native 262,144 window on fp8 KV. The pool is -# pinned (see SIZING above): 606,208 tokens, and 2 x 262,144 = 524,288 -# leaves ~82k tokens of pool slack. Resliced 2026-10-01 from 4 x 151,552 -# so a compaction request (frank-primary's trigger 183,501 tokens + a -# 32,768-token summary budget = 216,269) fits ONE request; the grid is -# the two-way concurrency contract. A single request may still grow to -# the whole pool (a 170k prompt was verified at the 4-slot shape). -MEM_FRACTION=0.98 -MAX_TOTAL_TOKENS=606208 -CONTEXT_LENGTH=262144 -MAX_RUNNING=2 -KV_DTYPE=fp8_e4m3 -QUANTIZATION=auto-round -# DeltaNet (mamba) recurrent states: the ssm state pool is the single -# biggest hidden consumer on this card. The engine autofills the slot -# COUNT to spend a fixed ~5.1 GB budget regardless of dtype (fp32 at 110 -# slots, bf16 at 217 — same 5.1 GB), so BOTH dials are pinned below: -# bf16 dtype + 32 slots = 795 MiB. -MAMBA_SSM_DTYPE=bfloat16 -# 32 slots is 16x the 2-way concurrency: enough for the radix mamba -# cache to keep shared-prefix states, small enough to leave the KV pool -# room. -MAX_MAMBA_CACHE_SIZE=32 +# Two contexts at the model's native window each. llama.cpp divides +# --ctx-size across --parallel slots (n_ctx_slot = CTX_SIZE/PARALLEL), and +# without unified KV each slot gets its own cache. +CTX_SIZE=524288 +PARALLEL=2 -# Reasoning is extracted into reasoning_content; tools use the qwen3_coder -# grammar (both parsers per the SGLang Qwen3.5 cookbook). -REASONING_PARSER=qwen3 -TOOLCALL_PARSER=qwen3_coder +# The KV contract (Frank, 2026-10-01 night): unquantized K, q8_0 V. +# q8_0 on V is only served by the flash-attention path; the serve script +# pins --flash-attn on and must never lose it. +CACHE_TYPE_K=f16 +CACHE_TYPE_V=q8_0 +NGL=99 diff --git a/scripts/frank-small.service b/scripts/frank-small.service index eec44cc..ac0807f 100644 --- a/scripts/frank-small.service +++ b/scripts/frank-small.service @@ -1,5 +1,5 @@ [Unit] -Description=SGLang MiMo-V2.6-Distill-Qwen-9B-INT8 (qwen3_5 hybrid, in-checkpoint vision, no MTP) 4x151k fp8 KV (dankclaw, frank-small) +Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q6_K GGUF (qwen35 hybrid, 2x262144, f16 K + q8_0 V) on the RTX 3090 (dankclaw, frank-small) After=network-online.target Wants=network-online.target diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 45165f8..caf524c 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -1,199 +1,121 @@ #!/usr/bin/env bash -# serve-frank-large.sh — frank-large: MiMo-V2.6-Distill-Qwen-9B (BF16) under -# vLLM on the Strix Halo gfx1151 iGPU, :7770. +# Serve Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) as frank-large on +# Dankclaw's Strix Halo iGPU (gfx1151), using gufo (gufo-org/gufo) — the +# inference engine built specifically for this hardware. # -# 2026-10-01 cutover from gufo (Qwen3.8-27B GGUF + DFlash2). The runtime is -# the digest-pinned kyuz0 TheRock image (the fork that carries the gfx1151 -# build: ROCm 10 / torch 2.11 / vLLM 0.30 with the Triton Gated DeltaNet -# kernels); like frank-primary, the service drives `podman run` directly and -# then blocks on the container, so a container death is a non-zero ExecStart -# and the unit's Restart=always is what self-heals. +# Runtime: /home/dank/GitPackages/gufo-v0.1.1/bin/gufo, built from source +# against the host ROCm by scripts/build-gufo.sh. It replaced the stock +# llama.cpp prebuilt (unsloth b11030) on 2026-09-28; port, alias and client +# contract did not move. # -# The engine flags are the ones measured in the 2026-10-01 bench window on -# this host (see frank-large.env for the numbers). Every tunable arrives via -# frank-large.env; keys are FRANK_LARGE_-prefixed because HIP reads bare -# TEMP/HOST/GPU_MAX_HEAP_SIZE from the environment. +# Payload: ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF's +# UD-Q8_K_XL, fetched by scripts/download-frank-large-swift.sh. It replaced +# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL on 2026-09-28. The two are structurally +# interchangeable for this engine — same `qwen35` architecture, the same 866 +# tensors in the same order, the same per-tensor Q8_0/F32/BF16 layout, the +# same MTP head at blk.64 — so the swap needed no engine change. The fine-tune +# is abliterated (uncensored) and carries the SWIFT OPEN LICENSE 1.0, which +# is not an OSI licence. +# +# Shape: one session, full 262144 context (the model's native window). The +# 27B dense weights (16.35 GiB at UD-Q4_K_XL) + the DFlash2 drafter (1.07) + KV + +# compute buffers fit the iGPU's GTT aperture (~114 GiB); a load at this +# context measured 66.5 GiB of the 116.7 GiB device. +# +# Spec method: DFlash2 (a block-diffusion drafter), gufo's adaptive block +# policy, capped at DRAFT_TOKENS proposals per step. The drafter is the +# z-lab Qwen3.8-27B-DFlash2-Q4_K_M and it has a sliding window of 2048 +# (small draft KV). It is UNCHANGED by the model swap and still lives beside +# the previous target, which is why DRAFTER_DIR is its own tunable. +# +# Sampling: greedy (--temperature 0) server-side, as every frank-large has +# served. The harness overrides it per request via extraBody (1.0 / 0.95 / +# top-k 20). +# +# Template: gufo compiles the official Qwen Jinja into the binary and takes +# NO --chat-template-file — and it is not a Jinja engine at all +# (src/models/qwen/chat_template.cpp is a hand-written C++ renderer), so the +# fleet's scripts/qwen-sharp-v22.4.0.jinja cannot be supplied to it. That law +# was given up when this endpoint moved to gufo on 2026-09-28, by decision: +# the terse system-prompt injection, the in-band <|think_*|> control, the +# json|xml tool_call_format switch and the tool-argument truncation are all +# absent. gufo's own renderer keeps the effort instruction strings and +# preserve_thinking, which are the parts the harness depends on. Accepted +# 2026-09-28; recovering the terse prompt from the client side is the +# cheapest partial restore. +# +# Text-only: gufo auto-discovers a mmproj beside the model and this payload +# has none (the base repo ships one separately, unused), so the endpoint +# stays text-only. +# +# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides +# it; anything not set there falls through to the defaults below. set -euo pipefail -ENV_FILE="${FRANK_LARGE_ENV_FILE:-/home/dank/serverAppConfigs/scripts/frank-large.env}" +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}" if [[ -f "$ENV_FILE" ]]; then set -a - # shellcheck disable=SC1090 source "$ENV_FILE" set +a fi -# --- runtime switch ----------------------------------------------------------- -# llama: bare llama-server (master build with qwen35 GDN kernels) on GGUF -# weights. vllm (default): the digest-pinned TheRock container below. -FRANK_LARGE_RUNTIME="${FRANK_LARGE_RUNTIME:-vllm}" -if [[ "$FRANK_LARGE_RUNTIME" == "llama" ]]; then - : "${FRANK_LARGE_LLAMA_BIN:?FRANK_LARGE_LLAMA_BIN not set}" - : "${FRANK_LARGE_GGUF:?FRANK_LARGE_GGUF not set (target GGUF)}" - : "${FRANK_LARGE_DRAFT_GGUF:=}" - : "${FRANK_LARGE_LLAMA_CTX:=1048576}" # 4 slots x 262,144, f16 KV - for f in "$FRANK_LARGE_LLAMA_BIN" "$FRANK_LARGE_GGUF"; do - [[ -x "$f" || -f "$f" ]] || { echo "ERROR: missing $f" >&2; exit 1; } - done - DRAFT_ARGS=() - if [[ -n "$FRANK_LARGE_DRAFT_GGUF" ]]; then - [[ -f "$FRANK_LARGE_DRAFT_GGUF" ]] || { echo "ERROR: missing draft $FRANK_LARGE_DRAFT_GGUF" >&2; exit 1; } - DRAFT_ARGS=(--model-draft "$FRANK_LARGE_DRAFT_GGUF" --spec-type draft-dflash) - fi - # ComfyUI co-tenancy on this iGPU: gufo-era law, 2 resident queues. - export GPU_MAX_HW_QUEUES=2 - FRANK_LARGE_PORT="${FRANK_LARGE_PUBLISH:-7770:7770}" - FRANK_LARGE_PORT="${FRANK_LARGE_PORT##*:}" - exec "$FRANK_LARGE_LLAMA_BIN" \ - --model "$FRANK_LARGE_GGUF" \ - "${DRAFT_ARGS[@]}" \ - --alias "${FRANK_LARGE_SERVED_NAME:=frank-large}" \ - --host 0.0.0.0 --port "$FRANK_LARGE_PORT" \ - --parallel 4 --ctx-size "$FRANK_LARGE_LLAMA_CTX" \ - --n-gpu-layers 99 --flash-attn on --jinja \ - --reasoning auto --reasoning-format deepseek \ - --cont-batching -fi +# --- defaults (used when the env file does not set them) ------------------ +: "${RUNTIME_DIR:=/home/dank/GitPackages/gufo-v0.1.1}" +: "${MODEL_DIR:=/home/dank/models/Swift-1.5-Qwen3.8-27B-Uncensored-MTP-UD-Q8_K_XL}" +: "${ALIAS:=frank-large}" +: "${HOST:=0.0.0.0}" +: "${PORT:=7770}" +: "${CONTEXT_SIZE:=262144}" +: "${SESSIONS:=1}" +: "${DRAFT_TOKENS:=7}" +: "${DRAFT_POLICY:=adaptive}" -# --- below: vllm runtime ------------------------------------------------------ -: "${FRANK_LARGE_IMAGE:?FRANK_LARGE_IMAGE not set (digest-pinned kyuz0 vllm-therock image)}" -: "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR not set}" -: "${FRANK_LARGE_MODEL_SHARDS:?FRANK_LARGE_MODEL_SHARDS not set}" -: "${FRANK_LARGE_CONTAINER:=frank-large-mimo}" -: "${FRANK_LARGE_LEGACY_CONTAINERS:=vllm-mimo}" -: "${FRANK_LARGE_SERVED_NAME:=frank-large}" -: "${FRANK_LARGE_PUBLISH:=7770:7770}" -: "${FRANK_LARGE_MAXLEN:=262144}" -: "${FRANK_LARGE_SEQS:=4}" -: "${FRANK_LARGE_UTIL:=0.45}" -: "${FRANK_LARGE_REASONING_PARSER:=qwen3}" -: "${FRANK_LARGE_TOOL_PARSER:=qwen3_coder}" -: "${FRANK_LARGE_SPEC:=}" -: "${FRANK_LARGE_SPEC_TOKENS:=8}" -: "${FRANK_LARGE_MODELS_DIR:=${FRANK_LARGE_MODEL_DIR%/*}}" -FRANK_LARGE_QUANT="${FRANK_LARGE_QUANT:-}" -FRANK_LARGE_QUANT_CONFIG="${FRANK_LARGE_QUANT_CONFIG:-}" +GUFO_BIN="$RUNTIME_DIR/bin/gufo" -# --- gates (fail FAST, before touching the running stack) ------------------ -# Image gate by REPO DIGEST, not the human-readable ref: rootless podman -# resolves the tag from the network, and the whole point of the pin is to -# boot exactly these bytes offline. Capture first, grep second — piping -# straight into `grep -q` dies of the SIGPIPE race under pipefail. -if ! digests="$(podman images --format '{{.RepoDigests}}')" || - ! grep -qF "${FRANK_LARGE_IMAGE#*@}" <<<"$digests"; then - echo "ERROR: pinned runtime image absent from rootless storage: $FRANK_LARGE_IMAGE" >&2 - echo "frank-large: fetch it (podman pull $FRANK_LARGE_IMAGE)" >&2 +# The target GGUF and the drafter are named explicitly because they do not +# share a directory any more: the target is a different repository's artifact +# and keeps its own long filename, while the DFlash2 drafter is unchanged and +# still lives beside the previous target. MODEL_FILE/DRAFTER_FILE default to +# the deployed payload, so the env file only has to state them when it moves. +: "${MODEL_FILE:=$MODEL_DIR/Qwen3.8-27B-UD-Q4_K_XL.gguf}" +: "${DRAFTER_DIR:=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL}" +: "${DRAFTER_FILE:=$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}" + +if [[ ! -x "$GUFO_BIN" ]]; then + echo "runtime not found at $GUFO_BIN" >&2 + echo "frank-large: build it: bash scripts/build-gufo.sh" >&2 exit 1 fi - -# Payload gate: config + weight-map index + the exact shard count. A partial -# payload fails the load minutes later, after the port has been freed and -# the old server stopped. -if [[ ! -f "$FRANK_LARGE_MODEL_DIR/config.json" ]]; then - echo "ERROR: payload config missing: $FRANK_LARGE_MODEL_DIR/config.json" >&2 - echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2 - exit 1 -fi -if [[ ! -f "$FRANK_LARGE_MODEL_DIR/model.safetensors.index.json" ]]; then - echo "ERROR: weight-map index missing: $FRANK_LARGE_MODEL_DIR/model.safetensors.index.json" >&2 - echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2 - exit 1 -fi -shards_found=$(find "$FRANK_LARGE_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l) -if [[ "$shards_found" != "$FRANK_LARGE_MODEL_SHARDS" ]]; then - echo "ERROR: payload shard count wrong: $FRANK_LARGE_MODEL_DIR (found $shards_found, want $FRANK_LARGE_MODEL_SHARDS)" >&2 - echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2 - exit 1 -fi -if [[ -n "$FRANK_LARGE_SPEC" ]]; then - : "${FRANK_LARGE_SPEC_DIR:?FRANK_LARGE_SPEC_DIR not set (host dir holding the draft weights)}" - # dflash ships ONE model.safetensors; the Qwen3.5-9B MTP head ships as - # shards behind model.safetensors.index.json. Either shape is a real - # draft; config.json must be present in both, and a directory with - # neither weight layout is not a draft at all. - if [[ ! -f "$FRANK_LARGE_SPEC_DIR/config.json" ]] || - { [[ ! -f "$FRANK_LARGE_SPEC_DIR/model.safetensors" ]] && - [[ ! -f "$FRANK_LARGE_SPEC_DIR/model.safetensors.index.json" ]]; }; then - echo "ERROR: spec draft absent or partial: $FRANK_LARGE_SPEC_DIR (FRANK_LARGE_SPEC=$FRANK_LARGE_SPEC)" >&2 - echo "frank-large: re-run scripts/download-frank-large-dflash.sh (or the Qwen3.5-9B fetch for mtp)" >&2 - exit 1 - fi - # The models tree is mounted read-only at /models; the engine must be - # given the CONTAINER path to the draft, not the host path. - FRANK_LARGE_SPEC_CONTAINER_DIR="/models/${FRANK_LARGE_SPEC_DIR##*/}" -fi - -# The engine needs the RENDER nodes too, not just /dev/kfd: mounting the -# whole /dev/dri directory (never individual renderD nodes) is the -# verified-working shape (Dankserver 2026-09-29: individual render nodes -# enumerate the device but cannot create a context). -for dev in /dev/kfd /dev/dri; do - if [[ ! -e "$dev" ]]; then - echo "ERROR: $dev missing; gfx1151 cannot serve without it" >&2 +for f in "$MODEL_FILE" "$DRAFTER_FILE"; do + if [[ ! -f "$f" ]]; then + echo "missing asset $f (run the matching scripts/download-frank-large*.sh)" >&2 exit 1 fi done -SPEC_ARGS=() -if [[ -n "$FRANK_LARGE_SPEC" ]]; then - SPEC_ARGS=(--speculative-config "{\"method\":\"$FRANK_LARGE_SPEC\",\"model\":\"$FRANK_LARGE_SPEC_CONTAINER_DIR\",\"num_speculative_tokens\":$FRANK_LARGE_SPEC_TOKENS}") -fi +# --- env ------------------------------------------------------------------- +# gufo links against the host ROCm through its RPATH (/opt/rocm -> +# /etc/alternatives/rocm), so no LD_LIBRARY_PATH is needed. The device is +# already gfx1151 — `gufo diagnose` reports it natively — so no +# HSA_OVERRIDE_GFX_VERSION either; the override existed for the prebuilt +# llama.cpp runtime and can only misreport the arch here. +export HIP_VISIBLE_DEVICES=0 -QUANT_ARGS=() -if [[ -n "$FRANK_LARGE_QUANT" ]]; then - # Online quantization: vLLM quantizes the BF16 checkpoint at load time - # (weights >= 6-bit only; KV is untouched — the context stays bf16). - QUANT_ARGS=(--quantization "$FRANK_LARGE_QUANT") - if [[ -n "$FRANK_LARGE_QUANT_CONFIG" ]]; then - QUANT_ARGS+=(--quantization-config "$FRANK_LARGE_QUANT_CONFIG") - fi -fi - - -# Remove stale container identities BEFORE starting: a crashed unit can -# leave the name registered forever after, and the bench-window identity -# (vllm-mimo) holds the GPU and port when a window ends without teardown. -for name in $FRANK_LARGE_LEGACY_CONTAINERS "$FRANK_LARGE_CONTAINER"; do - if podman ps -a --format '{{.Names}}' | grep -qx "$name"; then - echo "removing stale container: $name" - podman rm -f "$name" >/dev/null 2>&1 || true - fi -done - -# --device /dev/mem exists for the fleet's other GPU containers; the GTT -# aperture (114 GiB) needs no size knob. Models mount read-only wholesale so -# spec drafts resolve from disk. THE PATH HANDED TO vllm IS THE CONTAINER -# PATH: the engine runs inside this container, and a host path that is not -# mounted at the same location dies as "Repo id must be in the form -# 'namespace/repo_name'" (2026-10-01 first boot, 51 crash-loop restarts). -MODEL_CONTAINER_DIR="/models/${FRANK_LARGE_MODEL_DIR##*/}" -podman run -d \ - --name "$FRANK_LARGE_CONTAINER" \ - --device /dev/kfd --device /dev/dri --device /dev/mem \ - --security-opt seccomp=unconfined --group-add keep-groups \ - --ulimit memlock=-1:-1 \ - -v "$FRANK_LARGE_MODELS_DIR:/models:ro" \ - -v /home/dank/models/hf-cache:/root/.cache/huggingface \ - -e HSA_ENABLE_SDMA=0 \ - -p "$FRANK_LARGE_PUBLISH" \ - --entrypoint vllm \ - "$FRANK_LARGE_IMAGE" \ - serve "$MODEL_CONTAINER_DIR" \ - --served-model-name "$FRANK_LARGE_SERVED_NAME" \ - --host 0.0.0.0 --port "${FRANK_LARGE_PUBLISH##*:}" \ - --dtype bfloat16 \ - --max-model-len "$FRANK_LARGE_MAXLEN" \ - --max-num-seqs "$FRANK_LARGE_SEQS" \ - --gpu-memory-utilization "$FRANK_LARGE_UTIL" \ - --reasoning-parser "$FRANK_LARGE_REASONING_PARSER" \ - --enable-auto-tool-choice \ - --tool-call-parser "$FRANK_LARGE_TOOL_PARSER" \ - --language-model-only \ - --trust-remote-code \ - "${QUANT_ARGS[@]}" \ - "${SPEC_ARGS[@]}" - -# Block on the container so a container death returns non-zero from ExecStart -# and the unit's Restart=always fires. systemd stop goes through the unit's -# ExecStop (podman stop -t); podman wait unblocks when the container is gone. -exec podman wait "$FRANK_LARGE_CONTAINER" +# --- the serve command --------------------------------------------------- +# Server options precede the subcommand, model options follow it. +exec "$GUFO_BIN" serve \ + --host "$HOST" \ + --port "$PORT" \ + --sessions "$SESSIONS" \ + llm \ + --model "$MODEL_FILE" \ + --served-model-name "$ALIAS" \ + --context "$CONTEXT_SIZE" \ + --speculative dflash2 \ + --dflash-model "$DRAFTER_FILE" \ + --draft-tokens "$DRAFT_TOKENS" \ + --draft-policy "$DRAFT_POLICY" \ + --temperature 0 \ + --think auto \ + --preserve-thinking on diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 00b41c9..bef244c 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -1,30 +1,24 @@ #!/usr/bin/env bash -# Serve letechlead/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound as -# frank-small on Dankclaw's RTX 3090 via SGLang. +# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q6_K) as frank-small on +# Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86). # -# Topology: the whole checkpoint (12.0 GiB, INT8 W8A16 AutoRound) lives in -# VRAM; the 3090's 24 GiB holds weights + a ~11.8 GiB fp8 KV pool at -# --mem-fraction-static 0.98. The host CPU only tokenizes. +# 2026-10-01 night (Frank): swapped from SGLang + INT8-W8A16-AutoRound +# safetensors to the llama.cpp GGUF path, because llama.cpp is the only +# runtime on this host that can serve the MiMo qwen35 hybrid with +# ASYMMETRIC KV: --cache-type-k f16 (unquantized keys) + --cache-type-v +# q8_0 (quantized values), which is Frank's explicit KV contract for this +# endpoint. SGLang's --kv-cache-dtype only moves K and V together. # # Architecture: qwen3_5 hybrid — 32 layers, 8 full-attention + 24 Gated -# DeltaNet, native 262144 window, and a vision tower (qwen3_5_vision) -# baked into the safetensors: no separate projector file, the image path -# is in the checkpoint. The config declares an MTP layer, but the shipped -# weights carry none (no mtp.* tensors in the index), so this service runs -# WITHOUT speculative decoding — a NEXTN draft would be an uninitialised -# 4.24 GiB ghost model drafting at 0% acceptance. +# DeltaNet, native 262144 window. KV lives only in the 8 attention layers; +# the DeltaNet state is per-slot fixed size. GGUF path is TEXT-ONLY: no +# mmproj, the in-checkpoint vision tower does not exist here (models.yml +# declares frank-small text-only since this swap). # -# Context: CONTEXT_LENGTH x MAX_RUNNING slots must fit the fp8 KV pool, -# which the engine autofits around the weights AND the DeltaNet ssm state -# pool (see frank-small.env). The mamba pool defaults to ~5.1 GB, so it is -# pinned (bf16, 32 slots). With the grid at its measured fit the card has -# no headroom for prefill CUDA-graph capture, so prefill runs eager -# (--disable-prefill-cuda-graph); decode graphs are untouched. +# Sampling: llama.cpp takes it per request (the fleet harness sends its +# recipe from models.yml); greedy is the harness default. # -# Sampling: greedy by default (the fleet contract); clients that send their -# own temperature get their own values. -# -# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides it. +# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides. set -euo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" @@ -37,61 +31,44 @@ if [[ -f "$ENV_FILE" ]]; then fi # --- defaults (used when the env file does not set them) ------------------- -: "${SGLANG_BIN:=/home/dank/GitPackages/sglang-cu128/bin/sglang}" -: "${MODEL_DIR:=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound}" +: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" +: "${MODEL_FILE:=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}" : "${ALIAS:=frank-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7771}" -: "${MEM_FRACTION:=0.98}" -: "${CONTEXT_LENGTH:=151552}" -: "${MAX_TOTAL_TOKENS:=606208}" -: "${MAX_RUNNING:=4}" -: "${KV_DTYPE:=fp8_e4m3}" -: "${QUANTIZATION:=auto-round}" -: "${REASONING_PARSER:=qwen3}" -: "${TOOLCALL_PARSER:=qwen3_coder}" -: "${MAMBA_SSM_DTYPE:=bfloat16}" -: "${MAX_MAMBA_CACHE_SIZE:=32}" +# ctx-size is the POOL llama.cpp divides across slots: 524288 / 2 = TWO +# full native 262144 windows. +: "${CTX_SIZE:=524288}" +: "${PARALLEL:=2}" +: "${CACHE_TYPE_K:=f16}" +: "${CACHE_TYPE_V:=q8_0}" +: "${NGL:=99}" -# Fail loudly rather than half-loading a model into VRAM: a missing file here -# is a download that has not finished, and the unit's restart policy would -# otherwise turn that into a crash loop that reads like a runtime fault. -for path in "$SGLANG_BIN" "$MODEL_DIR/model.safetensors.index.json" \ - "$(cd -- "$(dirname -- "$SGLANG_BIN")/.." && pwd)/cuda-nvcc/usr/local/cuda-13.0/bin/nvcc"; do +# Fail loudly rather than half-loading: a missing file is a download that +# has not finished, and Restart=on-failure would turn it into a crash loop +# that reads like a runtime fault. +for path in "$LLAMA_BIN" "$MODEL_FILE"; do [[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; } done -shard_count=$(find "$MODEL_DIR" -maxdepth 1 -name 'model-*.safetensors' | wc -l) -if [[ "$shard_count" -ne 8 ]]; then - echo "frank-small: expected 8 safetensors shards in $MODEL_DIR, found $shard_count" >&2 - exit 1 -fi # The UVM device node is created lazily by nvidia-modprobe on the first CUDA # open after boot; make sure it exists before the server needs it. nvidia-modprobe -u -c=0 2>/dev/null || true -# FlashInfer JITs its attention kernels with nvcc at boot. The host's -# /usr/bin/nvcc is CUDA 12.4 and dies on `--compress-mode` (a 12.8+ -# option), so the venv carries a matching CUDA 13.0 nvcc (extracted from -# the cuda-nvcc-13-0 deb); it must shadow the system one on PATH. -export PATH="$(cd -- "$(dirname -- "$SGLANG_BIN")/.." && pwd)/cuda-nvcc/usr/local/cuda-13.0/bin:$PATH" - -# Sampling: SGLang has no server-wide sampling defaults; the harness sends -# its per-model recipe from models.yml (compat.extraBody), and other clients -# get the engine's own defaults. -exec "$SGLANG_BIN" serve \ - --model-path "$MODEL_DIR" \ - --served-model-name "$ALIAS" \ +# q8_0 on V REQUIRES flash attention (the quantized-value kernels only exist +# on the FA path); an FA-less build silently falls back and the KV contract +# in frank-small.env becomes a lie, so pin the flag pair here. +exec "$LLAMA_BIN" \ + --model "$MODEL_FILE" \ + --alias "$ALIAS" \ --host "$BIND_ADDR" \ --port "$LISTEN_PORT" \ - --mem-fraction-static "$MEM_FRACTION" \ - --context-length "$CONTEXT_LENGTH" \ - --max-running-requests "$MAX_RUNNING" \ - --kv-cache-dtype "$KV_DTYPE" \ - --quantization "$QUANTIZATION" \ - --reasoning-parser "$REASONING_PARSER" \ - --tool-call-parser "$TOOLCALL_PARSER" \ - --mamba-ssm-dtype "$MAMBA_SSM_DTYPE" \ - --max-mamba-cache-size "$MAX_MAMBA_CACHE_SIZE" \ - --disable-prefill-cuda-graph \ - --max-total-tokens "$MAX_TOTAL_TOKENS" + --ctx-size "$CTX_SIZE" \ + --parallel "$PARALLEL" \ + --n-gpu-layers "$NGL" \ + --flash-attn on \ + --cache-type-k "$CACHE_TYPE_K" \ + --cache-type-v "$CACHE_TYPE_V" \ + --jinja \ + --reasoning auto \ + --reasoning-format deepseek diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index 9589a96..6e812eb 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -1,206 +1,338 @@ #!/usr/bin/env bats -# Seam: frank-large — the Dankclaw iGPU endpoint on :7770. +# Seam: the frank-large root unit on dankclaw. # -# This file asserts the vLLM ROOT-UNIT shape of the 2026-10-01 cutover: the -# BF16 MiMo-V2.6-Distill-Qwen-9B payload (the same arch frank-small serves -# INT8 on the 3090) under the digest-pinned kyuz0 TheRock gfx1151 image, -# with CROSS-MODEL MTP spec decode from the Qwen3.5-9B head. The gufo-era -# laws (built-from-source runtime, GGUF schema gates, DFlash2-on-27B) retired -# with that runtime; rollback is the tag frank-large-gufo-q4kxl-20261001 and -# scripts/build-gufo.sh stays on disk for it. +# Since 2026-09-28 this endpoint is served by gufo (gufo-org/gufo), the +# Strix Halo inference engine built from the pinned v0.1.1 tag, serving +# Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) UD-Q8_K_XL with the z-lab +# DFlash2 Q4_K_M drafter. Two changes landed on this endpoint that day: the +# engine (from the stock llama.cpp prebuilt) and the payload (from +# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL). Port, alias and client contract did +# not move. # -# What the seam pins that the flags alone do not say: the measured laws of -# the 2026-10-01 window (BF16 decode 13.1 t/s is ~94% of the LPDDR5x roof, -# so spec is the only way past the wall; MTP-2 measured +51% decode against -# DFlash-8's +33%; every spec arm pays ~55% of prefill; spec arms do NOT -# reproduce the no-spec greedy text on vLLM), the roof-free shape (KV grows -# only on the 8 full-attention layers, so 4 x 262,144 fits the GTT -# aperture), the client contract difference (vLLM emits `reasoning`, not -# llama.cpp's `reasoning_content`), and the deployer's image/payload/spec -# gates with the false-green MainPID check. - +# Most of this seam runs the serve script against a stub engine and asserts +# on the argv it actually execs, so what is pinned is the command the +# operator gets rather than the text of the script: a tunable renamed, +# dropped, or silently defaulted away fails here the way it would fail a +# deployment. load helpers +ENV_FILE="$REPO_ROOT/scripts/frank-large.env" UNIT="$REPO_ROOT/scripts/frank-large.service" SERVE="$REPO_ROOT/scripts/serve-frank-large.sh" -ENV_FILE="$REPO_ROOT/scripts/frank-large.env" +FETCH="$REPO_ROOT/scripts/download-frank-large-q4.sh" +DRAFTER_FETCH="$REPO_ROOT/scripts/download-frank-large.sh" +BUILD="$REPO_ROOT/scripts/build-gufo.sh" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" -FETCH_TARGET="$REPO_ROOT/scripts/download-frank-large-mimo.sh" -FETCH_DRAFT="$REPO_ROOT/scripts/download-frank-large-dflash.sh" -FETCH_MTPSRC="$REPO_ROOT/scripts/download-frank-large-qwen35.sh" +# The deployed target's filename, kept here so a rename in the env file and a +# rename in this stub cannot drift apart unnoticed. +TARGET_STUB="Qwen3.8-27B-UD-Q4_K_XL.gguf" +setup() { + STAGE="$(mktemp -d)" + RUNTIME_DIR="$STAGE/gufo" + MODEL_DIR="$STAGE/models" + DRAFTER_DIR="$STAGE/draft" + mkdir -p "$RUNTIME_DIR/bin" "$MODEL_DIR" "$DRAFTER_DIR" + # A stub engine: the serve script execs it, so $@ is exactly what gufo + # would be invoked with. + cat > "$RUNTIME_DIR/bin/gufo" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$@" +STUB + chmod +x "$RUNTIME_DIR/bin/gufo" + # The target and the drafter live in SEPARATE directories since the + # 2026-09-28 payload swap: the target came from a different repository, + # the DFlash2 drafter did not move. A stub that colocated them would + # never notice DRAFTER_DIR going stale. + : > "$MODEL_DIR/$TARGET_STUB" + : > "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" + STUB_ENV="$STAGE/stub.env" + cat > "$STUB_ENV" < — exact token match, so --port never satisfies a test +# looking for a different flag. +argv_has() { + grep -qxF -- "$1" <<<"$output" +} + +# argv_value — the token following , empty if the flag is absent. +argv_value() { + local i + i="$(grep -nx -- "$1" <<<"$output" | cut -d: -f1)" + [ -n "$i" ] || return 0 + sed -n "$((i + 1))p" <<<"$output" +} + +@test "the frank-large unit, env, serve and build files exist where the deployer installs them from" { + [ -f "$ENV_FILE" ] [ -f "$UNIT" ] [ -x "$SERVE" ] - [ -f "$ENV_FILE" ] + [ -x "$BUILD" ] } -@test "frank-large is boot-persistent and self-healing" { +@test "frank-large is boot-persistent, self-healing and unprivileged" { grep -q '^WantedBy=multi-user.target$' "$UNIT" - # Restart=always, not on-failure: vLLM's API server can drain and exit 0 - # after an engine fault (the frank-primary 2026-10-01 lesson), so - # on-failure would leave the endpoint dark until a human notices. - grep -q '^Restart=always$' "$UNIT" - ! grep -q '^Restart=on-failure$' "$UNIT" + grep -q '^Restart=on-failure$' "$UNIT" grep -q '^RestartSec=' "$UNIT" - # ExecStop runs `podman stop -t 120`; TimeoutStopSec MUST exceed it or - # every stop records 'Failed with result timeout' (frank-primary law). - grep -q '^ExecStop=.*podman stop -t 120 frank-large-mimo' "$UNIT" - stop_t=$(awk -F= '/^TimeoutStopSec=/{print $2}' "$UNIT") - [ -n "$stop_t" ] && [ "$stop_t" -gt 120 ] - # StartLimitIntervalSec belongs to [Unit]; a [Service] copy is an - # 'Unknown key' warning on every load. - [ "$(grep -c '^StartLimitIntervalSec=' "$UNIT")" = "1" ] - awk '/^\[Service\]/{in_svc=1} /^StartLimitIntervalSec=/{if(in_svc) exit 1}' "$UNIT" grep -q '^User=dank$' "$UNIT" + # The engine pins buffers; the memlock budget must be unbounded or the + # load fails on the RLIMIT. grep -q '^LimitMEMLOCK=infinity$' "$UNIT" - grep -q '^OOMScoreAdjust=-500$' "$UNIT" -} - -@test "the unit runs the repo's own serve script with the repo's env" { - grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh$' "$UNIT" + # The unit runs the repo's own serve script with the repo's env. + grep -q 'ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh' "$UNIT" grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT" } -@test "the serve shape is direct podman run, not the retired gufo or llama shapes" { - grep -q 'podman run' "$SERVE" - grep -q 'exec podman wait "$FRANK_LARGE_CONTAINER"' "$SERVE" - # The gufo runtime and the llama.cpp prebuilt are both retired engines; - # nothing in this trio may reach for them. - ! grep -q 'gufo serve' "$SERVE" - ! grep -q 'llama-server' "$SERVE" - ! grep -q -- '--sessions' "$SERVE" - ! grep -q -- '--model-draft' "$SERVE" - ! grep -q -- '--dflash-model' "$SERVE" +@test "the fleet endpoint and alias survive the cutover" { + serve_argv + [ "$status" -eq 0 ] + # frank-large at :7770, whatever engine answers it. + [ "$(argv_value --served-model-name)" = "frank-large" ] + [ "$(argv_value --port)" = "7770" ] + [ "$(argv_value --host)" = "0.0.0.0" ] } -@test "the serve script gates the digest-pinned image and the complete payload" { - grep -q 'FRANK_LARGE_IMAGE' "$SERVE" - grep -q 'RepoDigests' "$SERVE" - grep -q 'FRANK_LARGE_MODEL_SHARDS' "$SERVE" - grep -q 'config.json' "$SERVE" - grep -q 'model.safetensors.index.json' "$SERVE" - # Whole /dev/dri, never individual render nodes (Dankserver 2026-09-29: - # renderD-only mounts enumerate the GPU but cannot create a context). - grep -q -- '--device /dev/dri' "$SERVE" - ! grep -q 'renderD' "$SERVE" - # The env file pins by digest, not by the floating tag. - ! grep -qE '^FRANK_LARGE_IMAGE=.*:latest$' "$ENV_FILE" - grep -q '^FRANK_LARGE_IMAGE=docker.io/kyuz0/vllm-therock-gfx1151@sha256:' "$ENV_FILE" +@test "the built-in defaults are one session of the full 262144 window with DFlash2" { + serve_argv + [ "$status" -eq 0 ] + # 262144 is the model's native window (the operator's call, 2026-09-21). + [ "$(argv_value --context)" -eq 262144 ] + # One session is the single stream every prior frank-large served. + [ "$(argv_value --sessions)" -eq 1 ] + [ "$(argv_value --speculative)" = "dflash2" ] + # A draft ceiling of 0 would silently serve the autoregressive rate. + [ "$(argv_value --draft-tokens)" -gt 0 ] } -@test "the qualified engine shape is in the env file" { - # 2026-10-01 window: 262144 is the model's native window and the GTT - # aperture swallows it — KV grows ONLY on the 8 full-attention layers - # (32 KiB/token bf16); the 24 GDN layers hold fixed ~25 MiB state per - # running sequence. SEQS 4 keeps the four-session contract; the pool is - # shared by the scheduler instead of carved per session like gufo. - grep -q '^FRANK_LARGE_MAXLEN=262144$' "$ENV_FILE" - grep -q '^FRANK_LARGE_SEQS=4$' "$ENV_FILE" - grep -q '^FRANK_LARGE_UTIL=0.45$' "$ENV_FILE" - grep -q '^FRANK_LARGE_MODEL_DIR=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-BF16$' "$ENV_FILE" - grep -q '^FRANK_LARGE_MODEL_SHARDS=4$' "$ENV_FILE" +@test "the shipped env file, not just the defaults, serves the deployed shape" { + # The defaults are a floor; systemd sources the env file. Only the + # on-host paths are redirected at the stub, so every fleet value in the + # shipped file is the one under test. MODEL_FILE and DRAFTER_FILE are + # absolute in the shipped file, so they need redirecting too — a sed that + # only rewrote MODEL_DIR would leave them pointing at /home/dank and the + # test would pass for the wrong reason (or fail on a box that has them). + sed -e "s#^RUNTIME_DIR=.*#RUNTIME_DIR=$RUNTIME_DIR#" \ + -e "s#^MODEL_DIR=.*#MODEL_DIR=$MODEL_DIR#" \ + -e "s#^MODEL_FILE=.*#MODEL_FILE=$MODEL_DIR/$TARGET_STUB#" \ + -e "s#^DRAFTER_DIR=.*#DRAFTER_DIR=$DRAFTER_DIR#" \ + -e "s#^DRAFTER_FILE=.*#DRAFTER_FILE=$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf#" \ + "$ENV_FILE" > "$STAGE/shipped.env" + serve_argv "$STAGE/shipped.env" + [ "$status" -eq 0 ] + [ "$(argv_value --served-model-name)" = "frank-large" ] + [ "$(argv_value --host)" = "0.0.0.0" ] + [ "$(argv_value --port)" = "7770" ] + # 2026-10-01 night: Frank rolled frank-large back to ONE session at the + # full native window (gufo Q4_K_XL, f16 KV — gufo has no KV quant lever). + [ "$(argv_value --context)" -eq 262144 ] + [ "$(argv_value --sessions)" -eq 1 ] + [ "$(argv_value --speculative)" = "dflash2" ] + [ "$(argv_value --draft-tokens)" -gt 0 ] + [ "$(argv_value --draft-policy)" = "adaptive" ] + # The shipped file must name the target the fleet actually serves, not a + # path that merely happens to exist. + [ "$(argv_value --model)" = "$MODEL_DIR/$TARGET_STUB" ] + [ "$(argv_value --dflash-model)" = "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" ] } -@test "the serve script wires the engine flags from the env" { - grep -q -- '--max-model-len "$FRANK_LARGE_MAXLEN"' "$SERVE" - grep -q -- '--max-num-seqs "$FRANK_LARGE_SEQS"' "$SERVE" - grep -q -- '--gpu-memory-utilization "$FRANK_LARGE_UTIL"' "$SERVE" - grep -q -- '--dtype bfloat16' "$SERVE" - grep -q -- '--trust-remote-code' "$SERVE" - grep -q -- '--ulimit memlock=-1:-1' "$SERVE" - # vllm runs INSIDE the container: the model path handed to `serve` is - # the /models mount path, never the host path (2026-10-01: the host - # path crash-looped 51 times on "Repo id must be in the form…"). - grep -q 'serve "$MODEL_CONTAINER_DIR"' "$SERVE" - grep -q 'MODEL_CONTAINER_DIR="/models/' "$SERVE" - ! grep -qE '^\s+serve "\$FRANK_LARGE_MODEL_DIR"' "$SERVE" +@test "the engine is pointed at the payload that is actually on disk" { + serve_argv + [ "$status" -eq 0 ] + [ -f "$(argv_value --model)" ] + # A drafter path that does not exist would load the target and decode at + # the autoregressive rate, which reads as "the new runtime is slow". + [ -f "$(argv_value --dflash-model)" ] + [ "$(argv_value --model)" != "$(argv_value --dflash-model)" ] } -@test "speculative decode is cross-model MTP BY MEASUREMENT" { - # The 2026-10-01 window on this host: base 13.1 t/s, DFlash-8 17.4, - # MTP-2 19.8 (+51%). MTP acceptance is 71%/44% per position (token rate - # 57%) because the head trained on the distill's SOURCE (Qwen/Qwen3.5-9B, - # the only place mtp.* tensors exist for this family) still reads MiMo's - # hidden states. Beyond 2 draft tokens is UNMEASURED; the DFlash drafter - # stays on disk as the alternative method. - grep -q '^FRANK_LARGE_SPEC=mtp$' "$ENV_FILE" - grep -q '^FRANK_LARGE_SPEC_DIR=/home/dank/models/Qwen3.5-9B$' "$ENV_FILE" - grep -q '^FRANK_LARGE_SPEC_TOKENS=2$' "$ENV_FILE" - # The JSON is built ONLY when SPEC is set, and the draft path handed to - # the engine is the CONTAINER mount path, never the host path. - grep -q -- '--speculative-config' "$SERVE" - grep -q 'FRANK_LARGE_SPEC_CONTAINER_DIR="/models/' "$SERVE" - # The draft gate accepts BOTH weight layouts: a single - # model.safetensors (dflash) and a sharded index (the Qwen3.5-9B MTP - # head). A gate keyed only to the single file rejects the deployed arm. - grep -q 'FRANK_LARGE_SPEC_DIR/model.safetensors.index.json' "$SERVE" - # itself carries ZERO mtp.* tensors (that absence is what makes SGLang's - # NEXTN on frank-small a 0.00-acceptance ghost). - grep -q 'ZERO mtp' "$ENV_FILE" +@test "frank-large stays text-only" { + # The payload has no mmproj, so an --mmproj would point at nothing. The + # fleet's image path is frank-small: its SGLang MiMo payload carries the + # vision tower inside the checkpoint (no projector file at all), and + # models.yml declares frank-large text-only. + serve_argv + ! argv_has --mmproj + grep -q 'MiMo-Q6K' "$REPO_ROOT/scripts/frank-small.env" } -@test "tools and reasoning are wired — the client contract is served" { - grep -q '^FRANK_LARGE_REASONING_PARSER=qwen3$' "$ENV_FILE" - grep -q '^FRANK_LARGE_TOOL_PARSER=qwen3_coder$' "$ENV_FILE" - grep -q -- '--reasoning-parser "$FRANK_LARGE_REASONING_PARSER"' "$SERVE" - grep -q -- '--enable-auto-tool-choice' "$SERVE" - grep -q -- '--tool-call-parser "$FRANK_LARGE_TOOL_PARSER"' "$SERVE" - # vLLM's field name is `reasoning`, NOT llama.cpp/gufo's - # `reasoning_content` — the harness models.yml compat block must match. - grep -q 'reasoning' "$ENV_FILE" +@test "the engine is asked for greedy decoding and for preserved prior reasoning" { + # Greedy is what this endpoint has always served; a silent switch to + # sampled output would be a model-behaviour change disguised as a + # runtime swap. Prior thinking blocks must survive across turns or a + # multi-turn agent loses its own reasoning between tool calls. + serve_argv + [ "$(argv_value --temperature)" = "0" ] + [ "$(argv_value --preserve-thinking)" = "on" ] } -@test "the serve script clears stale containers including the bench identity" { - grep -q '^FRANK_LARGE_LEGACY_CONTAINERS="vllm-mimo"$' "$ENV_FILE" - grep -q '^FRANK_LARGE_CONTAINER=frank-large-mimo$' "$ENV_FILE" - grep -q 'FRANK_LARGE_LEGACY_CONTAINERS' "$SERVE" +@test "the serve script refuses a missing runtime with the build pointer" { + rm -f "$RUNTIME_DIR/bin/gufo" + serve_argv + [ "$status" -ne 0 ] + [[ "$output" == *"runtime not found"* ]] + [[ "$output" == *"build-gufo.sh"* ]] + # A restart loop would be the alternative; nothing may be exec'd. + ! argv_has serve } -@test "frank-large holds the single fleet port under its own alias" { - grep -q '^FRANK_LARGE_PUBLISH=7770:7770$' "$ENV_FILE" - grep -q '^FRANK_LARGE_SERVED_NAME=frank-large$' "$ENV_FILE" +@test "the serve script refuses a missing drafter with a fetch pointer" { + # The drafter lives in its own directory now, so the target can be present + # and this still has to be caught. + rm -f "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" + serve_argv + [ "$status" -ne 0 ] + [[ "$output" == *"missing asset"* ]] + [[ "$output" == *"Qwen3.8-27B-DFlash2-Q4_K_M.gguf"* ]] + [[ "$output" == *"download-frank-large"* ]] + ! argv_has serve } -@test "no bare HIP-reserved env var is defined" { - # HIP reads bare TEMP/HOST/GPU_MAX_HEAP_SIZE; the serve script sources - # this file with set -a, so an unprefixed key leaks into the engine. - ! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE" - ! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE" +@test "the serve script refuses a missing target and names the file" { + rm -f "$MODEL_DIR/$TARGET_STUB" + serve_argv + [ "$status" -ne 0 ] + [[ "$output" == *"missing asset"* ]] + [[ "$output" == *"$TARGET_STUB"* ]] } -@test "the fetch scripts carry the verified payload and draft manifests" { - grep -q 'XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B' "$FETCH_TARGET" - [ "$(grep -c 'safetensors.*[0-9a-f]\{64\}' "$FETCH_TARGET")" = "4" ] - grep -q -- '--verify-only' "$FETCH_TARGET" - grep -q 'z-lab/Qwen3.5-9B-DFlash' "$FETCH_DRAFT" - grep -q -- '--verify-only' "$FETCH_DRAFT" - # The env file's payload/draft dirs must be where the fetch scripts write. - grep -q 'MiMo-V2.6-Distill-Qwen-9B-BF16' "$FETCH_TARGET" - grep -q 'Qwen3.5-9B-DFlash' "$FETCH_DRAFT" - grep -q 'Qwen/Qwen3.5-9B' "$FETCH_MTPSRC" - grep -q '/home/dank/models/Qwen3.5-9B' "$FETCH_MTPSRC" +@test "both payload files are fetched resumably and verified against declared sizes + sha256" { + # The target and the drafter come from DIFFERENT repositories, fetched by + # two scripts. Anchor the repo checks to the URL ASSIGNMENT, not to the + # file: the same repo string also appears in each script's header comment, + # so a plain grep stayed green after the URL was repointed at another + # repository -- the script would then download something the serve script + # never asked for. + grep -qE '^URL="https://huggingface\.co/unsloth/Qwen3\.8-27B-GGUF/resolve/main/' "$FETCH" + grep -qE 'https://huggingface\.co/z-lab/Qwen3\.8-27B-DFlash2-GGUF/resolve/main/' "$DRAFTER_FETCH" + for f in "$FETCH" "$DRAFTER_FETCH"; do + # -f so an HTTP error is a failure rather than a small "artifact"; + # -C - so an interrupted 29 GiB run resumes. + grep -q -- '-fL -C -' "$f" + # The size and the sha256 are compared after the transfer, not trusted. + grep -q 'SHA256 MISMATCH' "$f" + grep -q 'SIZE MISMATCH' "$f" + done } -@test "the gufo deploy gates are retired with the gufo runtime" { - ! grep -q 'LLM_LARGE_RUNTIME' "$REMOTE_DEPLOY" - ! grep -q 'LLM_LARGE_MODEL' "$REMOTE_DEPLOY" - ! grep -q 'LLM_LARGE_DRAFT' "$REMOTE_DEPLOY" - ! grep -q "pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M'" "$REMOTE_DEPLOY" - # build-gufo.sh stays on disk as the rollback path to the tagged engine. - [ -f "$REPO_ROOT/scripts/build-gufo.sh" ] +@test "the serve script and the fetch scripts agree on the payload filenames" { + # The serve script's MODEL_FILE and the fetch script's REL are two + # independent spellings of one filename, and a repo is not required to + # keep them in step: a re-upload under a new name, or an env edited + # without the fetch, leaves the endpoint unable to start with a + # "missing asset" that no test would have predicted. Bind them here. + local served fetched drafter_served drafter_fetched + served="$(basename "$(grep '^MODEL_FILE=' "$ENV_FILE" | cut -d= -f2-)")" + fetched="$(sed -n 's/^REL="\(.*\)"$/\1/p' "$FETCH")" + [ -n "$served" ] + [ "$served" = "$fetched" ] + drafter_served="$(basename "$(grep '^DRAFTER_FILE=' "$ENV_FILE" | cut -d= -f2-)")" + drafter_fetched="$(sed -n 's/^ *"\(Qwen3\.8-27B-DFlash2[^ ]*\.gguf\) .*/\1/p' "$DRAFTER_FETCH")" + [ -n "$drafter_served" ] + [ "$drafter_served" = "$drafter_fetched" ] } -@test "the deployer gates image payload and spec then health-checks with MainPID" { - grep -q 'LLM_LARGE_CONTAINER=frank-large-mimo' "$REMOTE_DEPLOY" - grep -q 'LLM_LARGE_LEGACY_CONTAINERS="vllm-mimo"' "$REMOTE_DEPLOY" - grep -q 'FRANK_LARGE_IMAGE' "$REMOTE_DEPLOY" - grep -q "podman images --format '{{.RepoDigests}}'" "$REMOTE_DEPLOY" - grep -q 'FRANK_LARGE_MODEL_SHARDS' "$REMOTE_DEPLOY" - # The deployer's draft gate must accept the SHARDED MTP layout too — - # the 2026-10-01 first deploy died on exactly this single-file check. - grep -q 'spec_dir/model.safetensors.index.json' "$REMOTE_DEPLOY" - grep -q 'FRANK_LARGE_SPEC=' "$REMOTE_DEPLOY" +@test "the drafter is still fetched by the script that carries its Hub repo" { + # Guards a future "simplification" that folds the drafter into the + # payload-fetch script and silently orphans the artifact the serve script + # still points at via DRAFTER_DIR. The literal drafter name must not + # appear in that script AT ALL, comments included. + grep -q 'Qwen3.8-27B-UD-Q8_K_XL' "$DRAFTER_FETCH" + ! grep -q 'DFlash2' "$FETCH" +} + +@test "the build script refuses a source tree that is not the pinned commit" { + # gufo ships no prebuilt Linux binaries, so the runtime is built here. + # A tag that moves under us would silently install different kernels + # under the same RUNTIME_DIR, so the pin is enforced, not documented. + local src="$STAGE/not-the-pinned-commit" + mkdir -p "$src" + git -C "$src" init -q + git -C "$src" -c user.email=t@t -c user.name=t commit -q --allow-empty -m drift + run env GUFO_SRC_DIR="$src" GUFO_PREFIX="$STAGE/absent-prefix" bash "$BUILD" --verify + [ "$status" -ne 0 ] + [[ "$output" == *"GUFO_COMMIT mismatch"* ]] +} + +@test "the build script reports a runtime that was never built" { + # --verify on a host with no install must say so rather than exit 0 on + # an empty check. + run env GUFO_SRC_DIR="$STAGE/no-source-tree" GUFO_PREFIX="$STAGE/absent-prefix" bash "$BUILD" --verify + [ "$status" -ne 0 ] + [[ "$output" == *"no gufo at"* ]] +} + +@test "the deploy handler gates on the runtime, the payload and the port" { + grep -q 'relink_unit frank-large' "$REMOTE_DEPLOY" + grep -q 'systemctl enable frank-large.service' "$REMOTE_DEPLOY" grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY" - grep -q 'MainPID --value frank-large.service' "$REMOTE_DEPLOY" + grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY" + grep -q 'LLM_LARGE_RUNTIME' "$REMOTE_DEPLOY" + grep -q 'LLM_LARGE_MODEL' "$REMOTE_DEPLOY" + grep -q 'LLM_LARGE_DRAFT' "$REMOTE_DEPLOY" +} + +@test "the deploy handler will not stop a serving endpoint it cannot replace" { + # The runtime gate must be live (a real conditional on an executable + # path, not a branch that never fires) and it must return BEFORE the + # stop, or a host whose gufo was never built takes frank-large down on + # its way to reporting the failure. + local large gate bail stop + large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")" + [[ "$large" == *'build-gufo.sh'* ]] + gate="$(grep -n 'if \[\[ ! -x "\$LLM_LARGE_RUNTIME" \]\]' <<<"$large" | head -1 | cut -d: -f1)" + stop="$(grep -n 'systemctl stop frank-large' <<<"$large" | head -1 | cut -d: -f1)" + [ -n "$gate" ] + [ -n "$stop" ] + bail="$(awk -v g="$gate" 'NR > g && /return 1/ { print NR; exit }' <<<"$large")" + [ -n "$bail" ] + [ "$bail" -lt "$stop" ] +} + +@test "the stray-run sweep keys on the drafter and cannot take frank-small down" { + # The sweep used to key on the TARGET filename, which the 2026-09-28 + # payload swap moved to a different repository. It would have stopped + # matching the live process, and a stray run would then hold 7770 past the + # port gate and fail the deploy. The drafter is the part that did NOT + # move, and every frank-large command line carries it via --dflash-model. + grep -q "pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M'" "$REMOTE_DEPLOY" + # frank-small serves a different GGUF on the 3090 with no DFlash2 drafter, + # so a binary-name sweep is the only thing that would endanger it. + local large + large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")" + ! grep -q 'pkill -x llama-server' <<<"$large" + ! grep -q 'pkill -x gufo' <<<"$large" +} + +@test "the deploy handler retires the halogen quadlet without deleting it" { + grep -q 'retire_halogen_large_quadlet' "$REMOTE_DEPLOY" + grep -q 'HOME/.AGENTTRASH' "$REMOTE_DEPLOY" + ! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?.*frank-large' "$REMOTE_DEPLOY" + # Nothing live ships for this endpoint as a container any more. The + # archived copy is host state in an ignored .AGENTTRASH — deliberately + # untracked, and present only in a primary checkout — so search the + # tracked tree, not the filesystem. + [ ! -e "$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" ] + ! git -C "$REPO_ROOT" grep -q 'Image=.*halogen' -- dankclaw } diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index b57a2e0..488f11c 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -1,30 +1,65 @@ #!/usr/bin/env bats -# Seam: the frank-small SGLang root unit on dankclaw. -# frank-small is the RTX 3090 endpoint; since 2026-10-01 it serves -# letechlead/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound (qwen3_5 hybrid, -# INT8 W8A16 AutoRound, in-checkpoint vision) via SGLang in a -# python3.12 venv. This seam pins what the flags alone do not say: boot -# persistence for a NATIVE runtime (no generated quadlet is involved), the -# repo-relative paths the unit executes, the separate client port, the -# no-spec-decode gate (the checkpoint ships NO mtp.* tensors, so a NEXTN -# draft would be an uninitialised ghost model), the 8-shard payload gate, -# and the KV budget: fp8 KV at 0.98 utilization whose 4-slot grid must fit -# the card's 24 GiB. +# Seam: the frank-small llama.cpp root unit on dankclaw. +# frank-small is the RTX 3090 endpoint; since 2026-10-01 night it serves +# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K via a CUDA (sm_86) build of +# llama.cpp master, chosen for the ASYMMETRIC KV Frank asked for: K f16 +# (unquantized) + V q8_0, which only llama.cpp exposes per-tensor. This +# seam runs the serve script against a stub engine and asserts on the argv +# it actually execs, so what is pinned is the command the operator gets. load helpers UNIT="$REPO_ROOT/scripts/frank-small.service" SERVE="$REPO_ROOT/scripts/serve-frank-small.sh" ENV_FILE="$REPO_ROOT/scripts/frank-small.env" -DOWNLOAD="$REPO_ROOT/scripts/download-frank-small.sh" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" ENV_LARGE="$REPO_ROOT/scripts/frank-large.env" +TARGET_STUB="MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf" -@test "frank-small unit, serve script, env and download script exist in scripts/" { +setup() { + STAGE="$(mktemp -d)" + RUNTIME_DIR="$STAGE/llama" + MODEL_DIR="$STAGE/models" + mkdir -p "$RUNTIME_DIR/bin" "$MODEL_DIR" + cat > "$RUNTIME_DIR/bin/llama-server" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$@" +STUB + chmod +x "$RUNTIME_DIR/bin/llama-server" "$RUNTIME_DIR/bin" 2>/dev/null || chmod +x "$RUNTIME_DIR/bin/llama-server" + touch "$MODEL_DIR/$TARGET_STUB" +} + +teardown() { + rm -rf "$STAGE" +} + +# serve_argv [env-file] — run the serve script and capture the engine's +# argv, one token per line, in $output. +serve_argv() { + local envf="${1:-$STAGE/test.env}" + cat > "$envf" </dev/null + grep -qxF -- '--jinja' <<<"$output" } -@test "frank-small's slot grid is fp8 KV and fits the measured card budget" { - local ctx running kv ssmdtype mambasize required - ctx="$(sed -nE 's/^CONTEXT_LENGTH=([0-9]+)$/\1/p' "$ENV_FILE")" - running="$(sed -nE 's/^MAX_RUNNING=([0-9]+)$/\1/p' "$ENV_FILE")" - kv="$(sed -nE 's/^KV_DTYPE=(.+)$/\1/p' "$ENV_FILE")" - ssmdtype="$(sed -nE 's/^MAMBA_SSM_DTYPE=(.+)$/\1/p' "$ENV_FILE")" - mambasize="$(sed -nE 's/^MAX_MAMBA_CACHE_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" - [ -n "$ctx" ] && [ -n "$running" ] && [ -n "$kv" ] - # fp8 KV is the only dtype whose 16 KiB/token makes four slots fit; - # 64 tokens per MiB of pool. - case "$kv" in fp8*) ;; *) return 1 ;; esac - # The DeltaNet ssm state pool is the hidden fit cost: the engine - # autofills mamba slots to spend a fixed ~5.1 GB budget, so dtype - # alone does not shrink it — the SLOT COUNT must be pinned (bf16, - # bounded). These two gates are what make the measured pool below - # exist; change either and re-measure the pool at boot. - [ "$ssmdtype" = "bfloat16" ] - [ -n "$mambasize" ] && [ "$mambasize" -ge "$running" ] && [ "$mambasize" -le 64 ] - grep -q -- '--mamba-ssm-dtype "$MAMBA_SSM_DTYPE"' "$SERVE" - grep -q -- '--max-mamba-cache-size "$MAX_MAMBA_CACHE_SIZE"' "$SERVE" - # The slot window is inside the model's native 262144 window. - [ "$ctx" -le 262144 ] - # The grid must equal the pinned pool. The engine's autofit picks a - # bigger pool (687,388 tokens) but leaves ~0.53 GB — Triton's - # deferred kernel loads take most of it and eager-prefill activations - # then OOM on the first long request. 606,208 (4 x 151,552) is the - # verified ceiling: 170k single request and 4 concurrent 54k requests - # run against it (2026-10-01). A weights or mamba-flag change moves - # this number; re-measure before trusting the grid. - pinned_pool=606208 - required=$((running * ctx)) - [ "$required" -le "$pinned_pool" ] - grep -q -- '--max-total-tokens "$MAX_TOTAL_TOKENS"' "$SERVE" - grep -q -- '--disable-prefill-cuda-graph' "$SERVE" +@test "the serve script refuses a missing runtime or payload" { + rm -f "$RUNTIME_DIR/bin/llama-server" + serve_argv + [ "$status" -ne 0 ] + [[ "$output" == *"frank-small: missing"* ]] } -@test "frank-small runs on its own port beside frank-large's" { - grep -q '^LISTEN_PORT=7771$' "$ENV_FILE" - grep -q '^ALIAS=frank-small$' "$ENV_FILE" - grep -q -- '--port "$LISTEN_PORT"' "$SERVE" - grep -q -- '--host "$BIND_ADDR"' "$SERVE" - # The two endpoints cannot share a port: frank-large publishes - # 7770:7770 through podman, this one binds 7771 natively. (The - # frank-large vLLM cutover renamed PORT to FRANK_LARGE_PUBLISH.) - grep -q '^FRANK_LARGE_PUBLISH=7770:7770$' "$ENV_LARGE" - ! grep -q '^FRANK_LARGE_PUBLISH=7771' "$ENV_LARGE" -} - -@test "the deploy handler installs the root unit and refuses a missing payload" { +@test "the deploy handler gates on the binary and the GGUF, then MainPID-hardens health" { grep -q 'deploy_llm_small' "$REMOTE_DEPLOY" + grep -q 'LLM_SMALL_RUNTIME' "$REMOTE_DEPLOY" + grep -q 'LLM_SMALL_MODEL_FILE' "$REMOTE_DEPLOY" grep -q 'relink_unit frank-small' "$REMOTE_DEPLOY" grep -q 'systemctl enable frank-small.service' "$REMOTE_DEPLOY" - grep -q 'systemctl restart frank-small.service' "$REMOTE_DEPLOY" - grep -q 'MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound' "$REMOTE_DEPLOY" - grep -q 'refusing to start a unit that cannot load' "$REMOTE_DEPLOY" grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY" + grep -q 'frank-small.service has no MainPID' "$REMOTE_DEPLOY" } -@test "frank-small's serve script refuses to start on a half-finished download" { - # The payload is 8 safetensors shards; the serve script counts them and - # the download script pins each one's LFS sha256. - grep -q 'frank-small: missing' "$SERVE" - grep -q 'expected 8 safetensors shards' "$SERVE" - grep -q 'model-00001-of-00008.safetensors' "$DOWNLOAD" - grep -q 'model-00008-of-00008.safetensors' "$DOWNLOAD" +@test "the two endpoints keep separate ports and the large handler never pkills llama-server" { + # frank-large's gufo sweep must not take down frank-small, whose + # process is llama-server on the SAME host since this swap. + grep -q '^LISTEN_PORT=7771$' "$ENV_FILE" + local large + large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")" + ! grep -q 'pkill -x llama-server' <<<"$large" }