diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 97dca2c..6133dde 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1077,27 +1077,20 @@ deploy_anubis() { wait_container_healthy anubis 60 || return 1 } -# frank-large is gufo (gufo-org/gufo), the Strix Halo inference engine built -# from the pinned v0.1.1 tag by scripts/build-gufo.sh, serving -# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL with the DFlash2 drafter -# (z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M) on the Strix Halo iGPU. It is a -# ROOT unit: the runtime is a native binary linked against the host ROCm, -# so there is no container image to install and no generated unit to dodge. -# relink_unit points /etc/systemd/system/frank-large.service at the repo -# file and boot persistence is that unit's [Install] section. -# -# The runtime gate matters as much as the payload gate: gufo is built from -# source on this host (there is no prebuilt release), so a machine whose -# /home/dank/GitPackages tree was never built has a unit that can never -# become healthy, and a green deploy would leave the endpoint dark. -# -# It replaces the stock llama.cpp prebuilt (unsloth b11030), which had -# replaced the Kairic Edge runtime (ciru-ai/ROCmFPX). The port gate is the -# same as every cutover since 2026-09-16: nothing starts until 7770 is -# released. -LLM_LARGE_RUNTIME="${LLM_LARGE_RUNTIME:-/home/dank/GitPackages/gufo-v0.1.1/bin/gufo}" -LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Swift-1.5-Qwen3.8-27B-Uncensored-MTP-UD-Q8_K_XL/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}" -LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}" +# frank-large is vLLM (the digest-pinned kyuz0 TheRock gfx1151 image) serving +# the BF16 MiMo-V2.6-Distill-Qwen-9B payload with cross-model MTP spec decode +# (Qwen/Qwen3.5-9B head, 2 draft tokens) on the Strix Halo iGPU. 2026-10-01 +# cutover from gufo; the gufo-era gates (built-from-source binary, GGUF +# target + DFlash2 file) are retired with it — rollback is git tag +# frank-large-gufo-q4kxl-20261001, and the gufo runtime and payloads stay on +# disk. The service drives `podman run` directly from its serve script, so +# the gates here mirror deploy_llm_primary's: image present by REPO DIGEST in +# dank's rootless storage, payload complete (config.json + index + 4 shards), +# and the spec draft present whenever FRANK_LARGE_SPEC names one. The port +# gate is the same as every cutover since 2026-09-16: nothing starts until +# 7770 is released. +LLM_LARGE_CONTAINER=frank-large-mimo +LLM_LARGE_LEGACY_CONTAINERS="vllm-mimo" retire_halogen_large_quadlet() { local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" @@ -1130,51 +1123,81 @@ wait_port_free() { } deploy_llm_large() { - # Refuse rather than restart-loop. A unit whose runtime or payload is + # Refuse rather than restart-loop: a unit whose image or payload is # missing never becomes healthy, and a green deploy would leave the - # endpoint dark with nothing in the deploy log to say why. - if [[ ! -x "$LLM_LARGE_RUNTIME" ]]; then - echo "ERROR: gufo runtime absent at $LLM_LARGE_RUNTIME" >&2 - echo "frank-large: build it: bash scripts/build-gufo.sh" >&2 + # endpoint dark. Gates mirror deploy_llm_primary. + local env_file="/home/dank/serverAppConfigs/scripts/frank-large.env" + local image model_dir shards count spec spec_dir digests + if [[ ! -f "$env_file" ]]; then + echo "ERROR: frank-large env file absent: $env_file" >&2 return 1 fi - local missing=0 f - for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT"; do - if [[ ! -f "$f" ]]; then - echo "ERROR: payload absent at $f" >&2 - echo "frank-large: fetch it: bash scripts/download-frank-large.sh" >&2 - missing=1 + image="$(sed -n 's/^FRANK_LARGE_IMAGE=//p' "$env_file" | tail -1)" + model_dir="$(sed -n 's/^FRANK_LARGE_MODEL_DIR=//p' "$env_file" | tail -1)" + shards="$(sed -n 's/^FRANK_LARGE_MODEL_SHARDS=//p' "$env_file" | tail -1)" + spec="$(sed -n 's/^FRANK_LARGE_SPEC=//p' "$env_file" | tail -1)" + spec_dir="$(sed -n 's/^FRANK_LARGE_SPEC_DIR=//p' "$env_file" | tail -1)" + if [[ -z "$image" || -z "$model_dir" || -z "$shards" ]]; then + echo "ERROR: frank-large.env missing FRANK_LARGE_IMAGE/MODEL_DIR/MODEL_SHARDS" >&2 + return 1 + fi + # Capture podman's output BEFORE grepping (the SIGPIPE race, 2026-09-30). + digests="$(podman images --format '{{.RepoDigests}}')" + if ! grep -qF "${image#*@}" <<<"$digests"; then + echo "ERROR: pinned runtime image absent from rootless storage: $image" >&2 + echo "frank-large: fetch it (podman pull $image)" >&2 + return 1 + fi + if [[ ! -f "$model_dir/config.json" || ! -f "$model_dir/model.safetensors.index.json" ]]; then + echo "ERROR: payload incomplete: $model_dir (config.json / weight-map index)" >&2 + echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2 + return 1 + fi + count="$(find "$model_dir" -maxdepth 1 -name 'model-*-of-*.safetensors' 2>/dev/null | wc -l)" + if [[ "$count" != "$shards" ]]; then + echo "ERROR: payload shard count wrong: $model_dir (found $count, want $shards)" >&2 + echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2 + return 1 + fi + if [[ -n "$spec" ]]; then + if [[ -z "$spec_dir" || ! -f "$spec_dir/model.safetensors" ]]; then + echo "ERROR: spec draft absent for FRANK_LARGE_SPEC=$spec: $spec_dir/model.safetensors" >&2 + echo "frank-large: re-run scripts/download-frank-large-dflash.sh (or the Qwen3.5-9B fetch for mtp)" >&2 + return 1 fi - done - [[ "$missing" -eq 0 ]] || return 1 + fi retire_halogen_large_quadlet # On a re-deploy the predecessor is THIS unit: stop it before waiting on # the port, or the wait deadlocks on the server it just stopped serving. - # Then sweep a stray manual run -- that is how a bench harness leaves the - # port busy. Match the DRAFTER path, which every frank-large command - # line carries via --dflash-model and which is UNCHANGED by the 2026-09-28 - # payload swap: the target moved to a different repository while the - # drafter stayed put, so keying on the target filename would have quietly - # stopped matching. It is also the right key in principle — it names this - # endpoint's speculative drafter, not a binary. Match that, not a binary - # name: frank-small on this host serves a different GGUF over llama-server - # with no DFlash2 drafter and must not be killed. The pattern never - # appears in the deploying shell's own command line, so -f is safe here - # (it is not in `bash deploy-remote.sh`). run_privileged systemctl stop frank-large.service 2>/dev/null || true - pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M' 2>/dev/null || true - # Nothing may start until the predecessor has released the port. On - # 2026-09-16 the cutover failed exactly here: the old server logged - # "Deactivated successfully", the next engine tried to bind 7770 a second - # later and reported "Address already in use". Gate on the port, not on - # hope. + # A stopped unit can leave its container running (the cgroup-kill vs + # conmon --rm race); the orphan holds the port and the GPU. The + # bench-window identity (vllm-mimo) dies here too — that is how a bench + # harness leaves the port busy. + local ctr + for ctr in $LLM_LARGE_LEGACY_CONTAINERS "$LLM_LARGE_CONTAINER"; do + podman rm -f "$ctr" >/dev/null 2>&1 || true + done + # Nothing may start until the predecessor has released the port + # (2026-09-16 and 2026-09-26 cutovers both failed exactly here). wait_port_free 7770 || return 1 relink_unit frank-large run_privileged systemctl daemon-reload run_privileged systemctl enable frank-large.service run_privileged systemctl restart frank-large.service - wait_for_health frank-large http://127.0.0.1:7770/health 1200 + # Warm boot is ~200-240 s (measured 2026-10-01); cold torch compile is + # longer. Parity with frank-primary's 1800 s allowance. + wait_for_health frank-large http://127.0.0.1:7770/health 1800 || return 1 + # False-green hardening: the port answering proves nothing about WHO + # answers — the unit must be active with a live MainPID. + local main_pid + main_pid="$(run_privileged systemctl show -p MainPID --value frank-large.service)" + if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then + echo "FAIL: :7770 answered but frank-large.service has no MainPID — a stray process holds the port" >&2 + return 1 + fi + echo "frank-large: health answered by unit MainPID $main_pid" } # deploy_llm_small — frank-small is SGLang (python3.12 venv) on the RTX 3090, diff --git a/scripts/frank-large.env b/scripts/frank-large.env index d823d47..05dab9e 100644 --- a/scripts/frank-large.env +++ b/scripts/frank-large.env @@ -1,63 +1,97 @@ -# frank-large — gufo (gufo-org/gufo) serving -# Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) on the Strix Halo iGPU. +# frank-large.env — vLLM production config for frank-large (Dankclaw iGPU :7770). # -# These are the tunables read by scripts/serve-frank-large.sh. The flag -# list is owned by the serve script; this file only sets fleet values. +# 2026-10-01 cutover from gufo (git tag frank-large-gufo-q4kxl-20261001). +# Target: XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B in FULL BF16 precision +# (17.53 GiB, 4 shards) — the same architecture frank-small serves as an +# INT8 W8A16 AutoRound conversion on the 3090, here unquantized. The engine +# is the kyuz0 TheRock vLLM build, the only fleet-carried runtime path for +# gfx1151 (official vLLM images are CUDA-only). # -# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP -# runtime reads those bare (the 2026-08-13 TEMP incident). HOST is the one -# exception because the serve script itself consumes it for --host. - -# The target payload (scripts/download-frank-large-q4.sh): the serve script -# refuses to start without the main GGUF and the DFlash2 drafter. +# MEASURED 2026-10-01 bench window on this host (identical client, one +# session, temperature 0, 28,647-token prefill / 512-token decode): # -# 2026-10-01 swap (Frank): unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL, 16.35 GiB, -# replacing ajgazin's abliterated Swift-1.5 UD-Q8_K_XL (29.3 GiB). Two -# consequences worth knowing at a glance: -# * this is the VANILLA (non-abliterated) Qwen3.8-27B — the endpoint is no -# longer uncensored, which was one of the reasons the Swift payload was -# chosen on 2026-09-28; -# * ~13 GiB of weights became available for KV, which is what makes four -# full-sized f16 sessions fit. -# gufo needs no template for either: its Qwen renderer is compiled in. -MODEL_DIR=/home/dank/models/Qwen3.8-27B-UD-Q4_K_XL -MODEL_FILE=/home/dank/models/Qwen3.8-27B-UD-Q4_K_XL/Qwen3.8-27B-UD-Q4_K_XL.gguf - -# The DFlash2 drafter is UNCHANGED by the model swap and still lives beside -# the previous target, so it carries its own directory. It has a sliding -# window of 2048, which is what keeps its KV small (~40 MiB). -DRAFTER_DIR=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL -DRAFTER_FILE=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf - -# The gufo runtime, built from source by scripts/build-gufo.sh. -RUNTIME_DIR=/home/dank/GitPackages/gufo-v0.1.1 - -# Fleet endpoint: the same port and alias every frank-large has served, so -# no client, probe or models.yml entry changes address. -ALIAS=frank-large -HOST=0.0.0.0 -PORT=7770 - -# Four sessions, 163840 context each (Frank, 2026-10-01: "4x 160k"). The -# context is per session and the KV is UNQUANTIZED (f16) — gufo has no KV -# quantization lever at all (execution_policy.hpp pins kv_cache_storage to -# kFp16), so "unquantized" is inherent rather than configured. +# arm prefill decode acceptance (per position) +# BF16 base 17,876 t/s 13.1 t/s — +# BF16 + DFlash 8 tok 7,976 t/s 17.4 t/s pos0 59% pos1 26%; 14.8% token rate +# BF16 + MTP 2 tok 7,380 t/s 19.8 t/s pos0 71% pos1 44%; 57.4% token rate # -# The arithmetic that makes this fit: f16 KV costs 64 KiB/token on this model -# (16 full-attention layers x 4 KV heads x 256 head_dim x 2 (K,V) x 2 bytes; -# the other 48 layers are linear-attention with a per-SEQUENCE fp32 state of -# ~144 MiB, not a per-token cache). 4 x 163840 = 655,360 tokens = 40.0 GiB of -# KV, and each session's 163,840 tokens is exactly 10 GiB. Budget: weights -# 16.35 + KV 40.0 + drafter 1.07 + linear state 0.56 + runtime ~4 = ~62 GiB -# against the 114 GiB GTT — roughly 50 GiB of headroom, so the session count -# or the per-session context can both grow from here. -CONTEXT_SIZE=163840 -SESSIONS=4 +# Decode laws: 13.1 t/s is ~94% of the Strix Halo LPDDR5x roof for BF16 +# weights (17.5 GiB/token against ~256 GB/s) — a quant, not a bug; the same +# INT8 payload decodes 83.9 t/s on the 3090 because it moves ~12 GiB +# through ~936 GB/s. Speculative arms are the ONLY way past the wall, and +# both drafters pay ~55-59% of prefill (draft passes share the compute +# queues). Frank's call enabled MTP: the thinking-heavy traffic on this +# endpoint spends its wall clock in decode. +# +# MTP IS CROSS-MODEL: the head comes from Qwen/Qwen3.5-9B (the distill's +# source model, 15 mtp.* tensors, identical 4096-hidden/248320-vocab +# geometry) — MiMo itself ships ZERO mtp tensors, which is why frank-small's +# SGLang NEXTN is a 0.00-acceptance ghost (SGLang reads the head from the +# TARGET checkpoint; vLLM reads the draft from a separate path). The 71%/44% +# per-position acceptance is the distillation preserving Qwen3.5 hidden-state +# structure; it is measured, not assumed. +# +# NUMERICS NOTE (measured): with a spec drafter enabled, temperature-0 text +# DIFFERS from the no-spec run (greedy sha 1318aeb6 base vs 0208b45d dflash +# vs 20c41c38 mtp): batched verification changes the bf16 reduction order, +# so argmax near-ties diverge and cascade. Spec decode on vLLM is +# distribution-preserving, not bitwise-greedy-preserving. The gufo +# greedy==AR law does not carry over. +# +# HIP reads bare env vars TEMP, HOST, GPU_MAX_HEAP_SIZE — every key here is +# FRANK_LARGE_-prefixed so none can leak. -# DFlash2 spec decode (a block-diffusion drafter). gufo sizes the draft -# block itself; DRAFT_TOKENS caps the proposals per step (gufo's default is -# 7, chosen over llama.cpp's fixed depth 4 because gufo's adaptive policy -# shortens the block when acceptance drops) and DRAFT_POLICY is the block -# length controller. -DRAFT_TOKENS=7 -DRAFT_POLICY=adaptive +# --- runtime ---------------------------------------------------------------- +# Digest-pinned (the :latest tag of this fork moves with no notice): +# image ID 2d24a8c5a3fd, vLLM 0.30.0 / torch 2.11.0 / ROCm 10.0.0 (TheRock), +# verified to ship vllm/v1/spec_decode/dflash.py, the qwen3_5 model registry +# entry (Qwen3_5ForConditionalGeneration), and the Triton GDN backend. +FRANK_LARGE_IMAGE=docker.io/kyuz0/vllm-therock-gfx1151@sha256:98c1f3ac9833e7fa45e9a82fb7a4315e6d95daaf2548a211c8da3a871b17533a + +# --- payload ---------------------------------------------------------------- +FRANK_LARGE_MODEL_DIR=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-BF16 +FRANK_LARGE_MODEL_SHARDS=4 + +# --- identity --------------------------------------------------------------- +# Same port and served alias as every frank-large predecessor; the harness +# models.yml id is unchanged. The bench-window container (vllm-mimo) is +# swept before every start. +FRANK_LARGE_CONTAINER=frank-large-mimo +FRANK_LARGE_LEGACY_CONTAINERS="vllm-mimo" +FRANK_LARGE_SERVED_NAME=frank-large +FRANK_LARGE_PUBLISH=7770:7770 + +# --- engine shape (2026-10-01 window) ---------------------------------------- +# Full native window 262,144 at UTIL 0.45 booted clean on gfx1151 in all +# three arms (~200-240 s warm start). KV grows only on the 8 full-attention +# layers (32 KiB/token bf16: 8 x 2 x 4 heads x 256 dim x 2 B); the 24 GDN +# layers keep a FIXED ~25 MiB state per running sequence, so the pool is +# generous and sessions are cheap. SEQS 4 preserves the four-session +# contract the gufo shape ran; vLLM shares the pool across them by +# scheduler, unlike gufo's per-session carve. +FRANK_LARGE_MAXLEN=262144 +FRANK_LARGE_SEQS=4 +FRANK_LARGE_UTIL=0.45 + +# --- speculative decode: cross-model MTP (measured +51% decode) --------------- +# mtp beats dflash here (19.8 vs 17.4 t/s) and needs 2 draft tokens, not 8: +# acceptance past position 1 is 44% and falling (DFlash's own curve decays +# 59/26/14/7). num_speculative_tokens beyond 2 is UNMEASURED — do not raise +# it without a fresh A/B. To flip drafters, set FRANK_LARGE_SPEC=dflash and +# repoint SPEC_DIR (the DFlash drafter stays on disk). +# The gufo-era law "spec decode is absent" was a frank-primary finding; this +# endpoint's traffic is decode-dominated thinking, which is the regime where +# spec wins (prefill costs ~55% in every arm). +FRANK_LARGE_SPEC=mtp +FRANK_LARGE_SPEC_DIR=/home/dank/models/Qwen3.5-9B +FRANK_LARGE_SPEC_TOKENS=2 + +# --- client contract ---------------------------------------------------------- +# vLLM emits the thinking text in `reasoning` (NOT llama.cpp/gufo's +# `reasoning_content`) — the harness models.yml compat block for this +# provider must set reasoningContentField accordingly. The qwen3 parser is +# what keeps thinking out of `content`. Tools ride the qwen3_coder parser +# with auto tool choice. --language-model-only drops the vision tower (the +# endpoint stays text-only; the tower can come back by dropping the flag). +FRANK_LARGE_REASONING_PARSER=qwen3 +FRANK_LARGE_TOOL_PARSER=qwen3_coder diff --git a/scripts/frank-large.service b/scripts/frank-large.service index 7c9e9a4..d35a828 100644 --- a/scripts/frank-large.service +++ b/scripts/frank-large.service @@ -1,20 +1,46 @@ +# frank-large.service — vLLM MiMo-V2.6-Distill-Qwen-9B (BF16) on the Strix +# Halo gfx1151 iGPU, :7770. 2026-10-01 cutover from gufo (tag +# frank-large-gufo-q4kxl-20261001 restores the Q4_K_XL + DFlash2 shape). +# +# WHY A ROOT UNIT: parity with frank-primary/frank-small — the unit is +# relink_unit'd from this repo into /etc/systemd/system/, PID1 grants +# LimitMEMLOCK=infinity (vLLM pins host buffers), and OOM protection lands +# on the engine instead of the co-tenants. The container runs in dank's +# ROOTLESS podman storage. +# +# The serve script drives `podman run` directly (image entrypoint is +# overridden to `vllm serve`) and blocks on the container, so a container +# death is a non-zero ExecStart. NOTE from frank-primary's experience: +# vLLM's API server can drain and exit 0 after an engine fault, so the +# restart policy MUST be always, not on-failure. [Unit] -Description=gufo (gufo-org/gufo, ROCm gfx1151) Qwen3.8-27B UD-Q8_K_XL, DFlash2 spec decode, 262144 ctx / 1 session, root systemd unit (dankclaw, frank-large, Strix Halo iGPU) +Description=frank-large — vLLM MiMo-V2.6-Distill-Qwen-9B (BF16) on the gfx1151 iGPU After=network-online.target Wants=network-online.target +# Never let the start-rate limiter give up after a crash loop. +StartLimitIntervalSec=0 [Service] -Type=simple User=dank Group=dank -# The engine pins staging buffers; keep the memlock budget unbounded so an -# RLIMIT smaller than the pinned set cannot fail the load. Parity with the -# other LLM units on this fleet. LimitMEMLOCK=infinity Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env" ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh -Restart=on-failure +# systemd stop: podman wait alone cannot stop the container child; stop it +# explicitly or the engine outlives the unit holding the GPU and port. +ExecStop=/usr/bin/podman stop -t 120 frank-large-mimo +# Boot cost: ~200-240 s warm (measured 2026-10-01 bench window); cold torch +# compile is longer. Health stays 503 until the engine is up — do not kill. +TimeoutStartSec=1800 +# ExecStop allows 120 s of drain; the stop timeout must exceed it (the +# frank-primary 2026-10-01 lesson: a short TimeoutStopSec records 'Failed +# with result timeout' on every stop). +TimeoutStopSec=180 +Restart=always RestartSec=10 +# The engine holds 17.5 GiB of weights plus the KV pool in the GTT +# aperture; it is the last process on this host the kernel should pick. +OOMScoreAdjust=-500 [Install] WantedBy=multi-user.target diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index caf524c..69bc00f 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -1,121 +1,149 @@ #!/usr/bin/env bash -# Serve Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) as frank-large on -# Dankclaw's Strix Halo iGPU (gfx1151), using gufo (gufo-org/gufo) — the -# inference engine built specifically for this hardware. +# serve-frank-large.sh — frank-large: MiMo-V2.6-Distill-Qwen-9B (BF16) under +# vLLM on the Strix Halo gfx1151 iGPU, :7770. # -# Runtime: /home/dank/GitPackages/gufo-v0.1.1/bin/gufo, built from source -# against the host ROCm by scripts/build-gufo.sh. It replaced the stock -# llama.cpp prebuilt (unsloth b11030) on 2026-09-28; port, alias and client -# contract did not move. +# 2026-10-01 cutover from gufo (Qwen3.8-27B GGUF + DFlash2). The runtime is +# the digest-pinned kyuz0 TheRock image (the fork that carries the gfx1151 +# build: ROCm 10 / torch 2.11 / vLLM 0.30 with the Triton Gated DeltaNet +# kernels); like frank-primary, the service drives `podman run` directly and +# then blocks on the container, so a container death is a non-zero ExecStart +# and the unit's Restart=always is what self-heals. # -# Payload: ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF's -# UD-Q8_K_XL, fetched by scripts/download-frank-large-swift.sh. It replaced -# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL on 2026-09-28. The two are structurally -# interchangeable for this engine — same `qwen35` architecture, the same 866 -# tensors in the same order, the same per-tensor Q8_0/F32/BF16 layout, the -# same MTP head at blk.64 — so the swap needed no engine change. The fine-tune -# is abliterated (uncensored) and carries the SWIFT OPEN LICENSE 1.0, which -# is not an OSI licence. -# -# Shape: one session, full 262144 context (the model's native window). The -# 27B dense weights (16.35 GiB at UD-Q4_K_XL) + the DFlash2 drafter (1.07) + KV + -# compute buffers fit the iGPU's GTT aperture (~114 GiB); a load at this -# context measured 66.5 GiB of the 116.7 GiB device. -# -# Spec method: DFlash2 (a block-diffusion drafter), gufo's adaptive block -# policy, capped at DRAFT_TOKENS proposals per step. The drafter is the -# z-lab Qwen3.8-27B-DFlash2-Q4_K_M and it has a sliding window of 2048 -# (small draft KV). It is UNCHANGED by the model swap and still lives beside -# the previous target, which is why DRAFTER_DIR is its own tunable. -# -# Sampling: greedy (--temperature 0) server-side, as every frank-large has -# served. The harness overrides it per request via extraBody (1.0 / 0.95 / -# top-k 20). -# -# Template: gufo compiles the official Qwen Jinja into the binary and takes -# NO --chat-template-file — and it is not a Jinja engine at all -# (src/models/qwen/chat_template.cpp is a hand-written C++ renderer), so the -# fleet's scripts/qwen-sharp-v22.4.0.jinja cannot be supplied to it. That law -# was given up when this endpoint moved to gufo on 2026-09-28, by decision: -# the terse system-prompt injection, the in-band <|think_*|> control, the -# json|xml tool_call_format switch and the tool-argument truncation are all -# absent. gufo's own renderer keeps the effort instruction strings and -# preserve_thinking, which are the parts the harness depends on. Accepted -# 2026-09-28; recovering the terse prompt from the client side is the -# cheapest partial restore. -# -# Text-only: gufo auto-discovers a mmproj beside the model and this payload -# has none (the base repo ships one separately, unused), so the endpoint -# stays text-only. -# -# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides -# it; anything not set there falls through to the defaults below. +# The engine flags are the ones measured in the 2026-10-01 bench window on +# this host (see frank-large.env for the numbers). Every tunable arrives via +# frank-large.env; keys are FRANK_LARGE_-prefixed because HIP reads bare +# TEMP/HOST/GPU_MAX_HEAP_SIZE from the environment. set -euo pipefail -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}" +ENV_FILE="${FRANK_LARGE_ENV_FILE:-/home/dank/serverAppConfigs/scripts/frank-large.env}" if [[ -f "$ENV_FILE" ]]; then set -a + # shellcheck disable=SC1090 source "$ENV_FILE" set +a fi -# --- defaults (used when the env file does not set them) ------------------ -: "${RUNTIME_DIR:=/home/dank/GitPackages/gufo-v0.1.1}" -: "${MODEL_DIR:=/home/dank/models/Swift-1.5-Qwen3.8-27B-Uncensored-MTP-UD-Q8_K_XL}" -: "${ALIAS:=frank-large}" -: "${HOST:=0.0.0.0}" -: "${PORT:=7770}" -: "${CONTEXT_SIZE:=262144}" -: "${SESSIONS:=1}" -: "${DRAFT_TOKENS:=7}" -: "${DRAFT_POLICY:=adaptive}" +: "${FRANK_LARGE_IMAGE:?FRANK_LARGE_IMAGE not set (digest-pinned kyuz0 vllm-therock image)}" +: "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR not set}" +: "${FRANK_LARGE_MODEL_SHARDS:?FRANK_LARGE_MODEL_SHARDS not set}" +: "${FRANK_LARGE_CONTAINER:=frank-large-mimo}" +: "${FRANK_LARGE_LEGACY_CONTAINERS:=vllm-mimo}" +: "${FRANK_LARGE_SERVED_NAME:=frank-large}" +: "${FRANK_LARGE_PUBLISH:=7770:7770}" +: "${FRANK_LARGE_MAXLEN:=262144}" +: "${FRANK_LARGE_SEQS:=4}" +: "${FRANK_LARGE_UTIL:=0.45}" +: "${FRANK_LARGE_REASONING_PARSER:=qwen3}" +: "${FRANK_LARGE_TOOL_PARSER:=qwen3_coder}" +: "${FRANK_LARGE_SPEC:=}" +: "${FRANK_LARGE_SPEC_TOKENS:=8}" +: "${FRANK_LARGE_MODELS_DIR:=${FRANK_LARGE_MODEL_DIR%/*}}" -GUFO_BIN="$RUNTIME_DIR/bin/gufo" - -# The target GGUF and the drafter are named explicitly because they do not -# share a directory any more: the target is a different repository's artifact -# and keeps its own long filename, while the DFlash2 drafter is unchanged and -# still lives beside the previous target. MODEL_FILE/DRAFTER_FILE default to -# the deployed payload, so the env file only has to state them when it moves. -: "${MODEL_FILE:=$MODEL_DIR/Qwen3.8-27B-UD-Q4_K_XL.gguf}" -: "${DRAFTER_DIR:=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL}" -: "${DRAFTER_FILE:=$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}" - -if [[ ! -x "$GUFO_BIN" ]]; then - echo "runtime not found at $GUFO_BIN" >&2 - echo "frank-large: build it: bash scripts/build-gufo.sh" >&2 +# --- gates (fail FAST, before touching the running stack) ------------------ +# Image gate by REPO DIGEST, not the human-readable ref: rootless podman +# resolves the tag from the network, and the whole point of the pin is to +# boot exactly these bytes offline. Capture first, grep second — piping +# straight into `grep -q` dies of the SIGPIPE race under pipefail. +if ! digests="$(podman images --format '{{.RepoDigests}}')" || + ! grep -qF "${FRANK_LARGE_IMAGE#*@}" <<<"$digests"; then + echo "ERROR: pinned runtime image absent from rootless storage: $FRANK_LARGE_IMAGE" >&2 + echo "frank-large: fetch it (podman pull $FRANK_LARGE_IMAGE)" >&2 exit 1 fi -for f in "$MODEL_FILE" "$DRAFTER_FILE"; do - if [[ ! -f "$f" ]]; then - echo "missing asset $f (run the matching scripts/download-frank-large*.sh)" >&2 + +# Payload gate: config + weight-map index + the exact shard count. A partial +# payload fails the load minutes later, after the port has been freed and +# the old server stopped. +if [[ ! -f "$FRANK_LARGE_MODEL_DIR/config.json" ]]; then + echo "ERROR: payload config missing: $FRANK_LARGE_MODEL_DIR/config.json" >&2 + echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2 + exit 1 +fi +if [[ ! -f "$FRANK_LARGE_MODEL_DIR/model.safetensors.index.json" ]]; then + echo "ERROR: weight-map index missing: $FRANK_LARGE_MODEL_DIR/model.safetensors.index.json" >&2 + echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2 + exit 1 +fi +shards_found=$(find "$FRANK_LARGE_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l) +if [[ "$shards_found" != "$FRANK_LARGE_MODEL_SHARDS" ]]; then + echo "ERROR: payload shard count wrong: $FRANK_LARGE_MODEL_DIR (found $shards_found, want $FRANK_LARGE_MODEL_SHARDS)" >&2 + echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2 + exit 1 +fi +if [[ -n "$FRANK_LARGE_SPEC" ]]; then + : "${FRANK_LARGE_SPEC_DIR:?FRANK_LARGE_SPEC_DIR not set (host dir holding the draft weights)}" + # dflash ships ONE model.safetensors; the Qwen3.5-9B MTP head ships as + # shards behind model.safetensors.index.json. Either shape is a real + # draft; config.json must be present in both, and a directory with + # neither weight layout is not a draft at all. + if [[ ! -f "$FRANK_LARGE_SPEC_DIR/config.json" ]] || + { [[ ! -f "$FRANK_LARGE_SPEC_DIR/model.safetensors" ]] && + [[ ! -f "$FRANK_LARGE_SPEC_DIR/model.safetensors.index.json" ]]; }; then + echo "ERROR: spec draft absent or partial: $FRANK_LARGE_SPEC_DIR (FRANK_LARGE_SPEC=$FRANK_LARGE_SPEC)" >&2 + echo "frank-large: re-run scripts/download-frank-large-dflash.sh (or the Qwen3.5-9B fetch for mtp)" >&2 + exit 1 + fi + # The models tree is mounted read-only at /models; the engine must be + # given the CONTAINER path to the draft, not the host path. + FRANK_LARGE_SPEC_CONTAINER_DIR="/models/${FRANK_LARGE_SPEC_DIR##*/}" +fi + +# The engine needs the RENDER nodes too, not just /dev/kfd: mounting the +# whole /dev/dri directory (never individual renderD nodes) is the +# verified-working shape (Dankserver 2026-09-29: individual render nodes +# enumerate the device but cannot create a context). +for dev in /dev/kfd /dev/dri; do + if [[ ! -e "$dev" ]]; then + echo "ERROR: $dev missing; gfx1151 cannot serve without it" >&2 exit 1 fi done -# --- env ------------------------------------------------------------------- -# gufo links against the host ROCm through its RPATH (/opt/rocm -> -# /etc/alternatives/rocm), so no LD_LIBRARY_PATH is needed. The device is -# already gfx1151 — `gufo diagnose` reports it natively — so no -# HSA_OVERRIDE_GFX_VERSION either; the override existed for the prebuilt -# llama.cpp runtime and can only misreport the arch here. -export HIP_VISIBLE_DEVICES=0 +SPEC_ARGS=() +if [[ -n "$FRANK_LARGE_SPEC" ]]; then + SPEC_ARGS=(--speculative-config "{\"method\":\"$FRANK_LARGE_SPEC\",\"model\":\"$FRANK_LARGE_SPEC_CONTAINER_DIR\",\"num_speculative_tokens\":$FRANK_LARGE_SPEC_TOKENS}") +fi -# --- the serve command --------------------------------------------------- -# Server options precede the subcommand, model options follow it. -exec "$GUFO_BIN" serve \ - --host "$HOST" \ - --port "$PORT" \ - --sessions "$SESSIONS" \ - llm \ - --model "$MODEL_FILE" \ - --served-model-name "$ALIAS" \ - --context "$CONTEXT_SIZE" \ - --speculative dflash2 \ - --dflash-model "$DRAFTER_FILE" \ - --draft-tokens "$DRAFT_TOKENS" \ - --draft-policy "$DRAFT_POLICY" \ - --temperature 0 \ - --think auto \ - --preserve-thinking on +# Remove stale container identities BEFORE starting: a crashed unit can +# leave the name registered forever after, and the bench-window identity +# (vllm-mimo) holds the GPU and port when a window ends without teardown. +for name in $FRANK_LARGE_LEGACY_CONTAINERS "$FRANK_LARGE_CONTAINER"; do + if podman ps -a --format '{{.Names}}' | grep -qx "$name"; then + echo "removing stale container: $name" + podman rm -f "$name" >/dev/null 2>&1 || true + fi +done + +# --device /dev/mem exists for the fleet's other GPU containers; the GTT +# aperture (114 GiB) needs no size knob. Models mount read-only wholesale so +# spec drafts resolve from disk, and the HF cache mount keeps tokenizer +# metadata out of the container's ephemeral layer. +podman run -d \ + --name "$FRANK_LARGE_CONTAINER" \ + --device /dev/kfd --device /dev/dri --device /dev/mem \ + --security-opt seccomp=unconfined --group-add keep-groups \ + --ulimit memlock=-1:-1 \ + -v "$FRANK_LARGE_MODELS_DIR:/models:ro" \ + -v /home/dank/models/hf-cache:/root/.cache/huggingface \ + -e HSA_ENABLE_SDMA=0 \ + -p "$FRANK_LARGE_PUBLISH" \ + --entrypoint vllm \ + "$FRANK_LARGE_IMAGE" \ + serve "$FRANK_LARGE_MODEL_DIR" \ + --served-model-name "$FRANK_LARGE_SERVED_NAME" \ + --host 0.0.0.0 --port "${FRANK_LARGE_PUBLISH##*:}" \ + --dtype bfloat16 \ + --max-model-len "$FRANK_LARGE_MAXLEN" \ + --max-num-seqs "$FRANK_LARGE_SEQS" \ + --gpu-memory-utilization "$FRANK_LARGE_UTIL" \ + --reasoning-parser "$FRANK_LARGE_REASONING_PARSER" \ + --enable-auto-tool-choice \ + --tool-call-parser "$FRANK_LARGE_TOOL_PARSER" \ + --language-model-only \ + --trust-remote-code \ + "${SPEC_ARGS[@]}" + +# Block on the container so a container death returns non-zero from ExecStart +# and the unit's Restart=always fires. systemd stop goes through the unit's +# ExecStop (podman stop -t); podman wait unblocks when the container is gone. +exec podman wait "$FRANK_LARGE_CONTAINER" diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index db607d9..37d23b2 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -1,338 +1,197 @@ #!/usr/bin/env bats -# Seam: the frank-large root unit on dankclaw. +# Seam: frank-large — the Dankclaw iGPU endpoint on :7770. # -# Since 2026-09-28 this endpoint is served by gufo (gufo-org/gufo), the -# Strix Halo inference engine built from the pinned v0.1.1 tag, serving -# Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) UD-Q8_K_XL with the z-lab -# DFlash2 Q4_K_M drafter. Two changes landed on this endpoint that day: the -# engine (from the stock llama.cpp prebuilt) and the payload (from -# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL). Port, alias and client contract did -# not move. +# This file asserts the vLLM ROOT-UNIT shape of the 2026-10-01 cutover: the +# BF16 MiMo-V2.6-Distill-Qwen-9B payload (the same arch frank-small serves +# INT8 on the 3090) under the digest-pinned kyuz0 TheRock gfx1151 image, +# with CROSS-MODEL MTP spec decode from the Qwen3.5-9B head. The gufo-era +# laws (built-from-source runtime, GGUF schema gates, DFlash2-on-27B) retired +# with that runtime; rollback is the tag frank-large-gufo-q4kxl-20261001 and +# scripts/build-gufo.sh stays on disk for it. # -# Most of this seam runs the serve script against a stub engine and asserts -# on the argv it actually execs, so what is pinned is the command the -# operator gets rather than the text of the script: a tunable renamed, -# dropped, or silently defaulted away fails here the way it would fail a -# deployment. +# What the seam pins that the flags alone do not say: the measured laws of +# the 2026-10-01 window (BF16 decode 13.1 t/s is ~94% of the LPDDR5x roof, +# so spec is the only way past the wall; MTP-2 measured +51% decode against +# DFlash-8's +33%; every spec arm pays ~55% of prefill; spec arms do NOT +# reproduce the no-spec greedy text on vLLM), the roof-free shape (KV grows +# only on the 8 full-attention layers, so 4 x 262,144 fits the GTT +# aperture), the client contract difference (vLLM emits `reasoning`, not +# llama.cpp's `reasoning_content`), and the deployer's image/payload/spec +# gates with the false-green MainPID check. + load helpers -ENV_FILE="$REPO_ROOT/scripts/frank-large.env" UNIT="$REPO_ROOT/scripts/frank-large.service" SERVE="$REPO_ROOT/scripts/serve-frank-large.sh" -FETCH="$REPO_ROOT/scripts/download-frank-large-q4.sh" -DRAFTER_FETCH="$REPO_ROOT/scripts/download-frank-large.sh" -BUILD="$REPO_ROOT/scripts/build-gufo.sh" +ENV_FILE="$REPO_ROOT/scripts/frank-large.env" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" -# The deployed target's filename, kept here so a rename in the env file and a -# rename in this stub cannot drift apart unnoticed. -TARGET_STUB="Qwen3.8-27B-UD-Q4_K_XL.gguf" -setup() { - STAGE="$(mktemp -d)" - RUNTIME_DIR="$STAGE/gufo" - MODEL_DIR="$STAGE/models" - DRAFTER_DIR="$STAGE/draft" - mkdir -p "$RUNTIME_DIR/bin" "$MODEL_DIR" "$DRAFTER_DIR" - # A stub engine: the serve script execs it, so $@ is exactly what gufo - # would be invoked with. - cat > "$RUNTIME_DIR/bin/gufo" <<'STUB' -#!/usr/bin/env bash -printf '%s\n' "$@" -STUB - chmod +x "$RUNTIME_DIR/bin/gufo" - # The target and the drafter live in SEPARATE directories since the - # 2026-09-28 payload swap: the target came from a different repository, - # the DFlash2 drafter did not move. A stub that colocated them would - # never notice DRAFTER_DIR going stale. - : > "$MODEL_DIR/$TARGET_STUB" - : > "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" - STUB_ENV="$STAGE/stub.env" - cat > "$STUB_ENV" < — exact token match, so --port never satisfies a test -# looking for a different flag. -argv_has() { - grep -qxF -- "$1" <<<"$output" -} - -# argv_value — the token following , empty if the flag is absent. -argv_value() { - local i - i="$(grep -nx -- "$1" <<<"$output" | cut -d: -f1)" - [ -n "$i" ] || return 0 - sed -n "$((i + 1))p" <<<"$output" -} - -@test "the frank-large unit, env, serve and build files exist where the deployer installs them from" { - [ -f "$ENV_FILE" ] +@test "frank-large unit, serve script and env exist in scripts/" { [ -f "$UNIT" ] [ -x "$SERVE" ] - [ -x "$BUILD" ] + [ -f "$ENV_FILE" ] } -@test "frank-large is boot-persistent, self-healing and unprivileged" { +@test "frank-large is boot-persistent and self-healing" { grep -q '^WantedBy=multi-user.target$' "$UNIT" - grep -q '^Restart=on-failure$' "$UNIT" + # Restart=always, not on-failure: vLLM's API server can drain and exit 0 + # after an engine fault (the frank-primary 2026-10-01 lesson), so + # on-failure would leave the endpoint dark until a human notices. + grep -q '^Restart=always$' "$UNIT" + ! grep -q '^Restart=on-failure$' "$UNIT" grep -q '^RestartSec=' "$UNIT" + # ExecStop runs `podman stop -t 120`; TimeoutStopSec MUST exceed it or + # every stop records 'Failed with result timeout' (frank-primary law). + grep -q '^ExecStop=.*podman stop -t 120 frank-large-mimo' "$UNIT" + stop_t=$(awk -F= '/^TimeoutStopSec=/{print $2}' "$UNIT") + [ -n "$stop_t" ] && [ "$stop_t" -gt 120 ] + # StartLimitIntervalSec belongs to [Unit]; a [Service] copy is an + # 'Unknown key' warning on every load. + [ "$(grep -c '^StartLimitIntervalSec=' "$UNIT")" = "1" ] + awk '/^\[Service\]/{in_svc=1} /^StartLimitIntervalSec=/{if(in_svc) exit 1}' "$UNIT" grep -q '^User=dank$' "$UNIT" - # The engine pins buffers; the memlock budget must be unbounded or the - # load fails on the RLIMIT. grep -q '^LimitMEMLOCK=infinity$' "$UNIT" - # The unit runs the repo's own serve script with the repo's env. - grep -q 'ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh' "$UNIT" + grep -q '^OOMScoreAdjust=-500$' "$UNIT" +} + +@test "the unit runs the repo's own serve script with the repo's env" { + grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh$' "$UNIT" grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT" } -@test "the fleet endpoint and alias survive the cutover" { - serve_argv - [ "$status" -eq 0 ] - # frank-large at :7770, whatever engine answers it. - [ "$(argv_value --served-model-name)" = "frank-large" ] - [ "$(argv_value --port)" = "7770" ] - [ "$(argv_value --host)" = "0.0.0.0" ] +@test "the serve shape is direct podman run, not the retired gufo or llama shapes" { + grep -q 'podman run' "$SERVE" + grep -q 'exec podman wait "$FRANK_LARGE_CONTAINER"' "$SERVE" + # The gufo runtime and the llama.cpp prebuilt are both retired engines; + # nothing in this trio may reach for them. + ! grep -q 'gufo serve' "$SERVE" + ! grep -q 'llama-server' "$SERVE" + ! grep -q -- '--sessions' "$SERVE" + ! grep -q -- '--model-draft' "$SERVE" + ! grep -q -- '--dflash-model' "$SERVE" } -@test "the built-in defaults are one session of the full 262144 window with DFlash2" { - serve_argv - [ "$status" -eq 0 ] - # 262144 is the model's native window (the operator's call, 2026-09-21). - [ "$(argv_value --context)" -eq 262144 ] - # One session is the single stream every prior frank-large served. - [ "$(argv_value --sessions)" -eq 1 ] - [ "$(argv_value --speculative)" = "dflash2" ] - # A draft ceiling of 0 would silently serve the autoregressive rate. - [ "$(argv_value --draft-tokens)" -gt 0 ] +@test "the serve script gates the digest-pinned image and the complete payload" { + grep -q 'FRANK_LARGE_IMAGE' "$SERVE" + grep -q 'RepoDigests' "$SERVE" + grep -q 'FRANK_LARGE_MODEL_SHARDS' "$SERVE" + grep -q 'config.json' "$SERVE" + grep -q 'model.safetensors.index.json' "$SERVE" + # Whole /dev/dri, never individual render nodes (Dankserver 2026-09-29: + # renderD-only mounts enumerate the GPU but cannot create a context). + grep -q -- '--device /dev/dri' "$SERVE" + ! grep -q 'renderD' "$SERVE" + # The env file pins by digest, not by the floating tag. + ! grep -qE '^FRANK_LARGE_IMAGE=.*:latest$' "$ENV_FILE" + grep -q '^FRANK_LARGE_IMAGE=docker.io/kyuz0/vllm-therock-gfx1151@sha256:' "$ENV_FILE" } -@test "the shipped env file, not just the defaults, serves the deployed shape" { - # The defaults are a floor; systemd sources the env file. Only the - # on-host paths are redirected at the stub, so every fleet value in the - # shipped file is the one under test. MODEL_FILE and DRAFTER_FILE are - # absolute in the shipped file, so they need redirecting too — a sed that - # only rewrote MODEL_DIR would leave them pointing at /home/dank and the - # test would pass for the wrong reason (or fail on a box that has them). - sed -e "s#^RUNTIME_DIR=.*#RUNTIME_DIR=$RUNTIME_DIR#" \ - -e "s#^MODEL_DIR=.*#MODEL_DIR=$MODEL_DIR#" \ - -e "s#^MODEL_FILE=.*#MODEL_FILE=$MODEL_DIR/$TARGET_STUB#" \ - -e "s#^DRAFTER_DIR=.*#DRAFTER_DIR=$DRAFTER_DIR#" \ - -e "s#^DRAFTER_FILE=.*#DRAFTER_FILE=$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf#" \ - "$ENV_FILE" > "$STAGE/shipped.env" - serve_argv "$STAGE/shipped.env" - [ "$status" -eq 0 ] - [ "$(argv_value --served-model-name)" = "frank-large" ] - [ "$(argv_value --host)" = "0.0.0.0" ] - [ "$(argv_value --port)" = "7770" ] - # 2026-10-01: four sessions x 163840 context (Frank's "4x 160k"), f16 KV. - # 163840 tokens x 64 KiB/token = 10 GiB of KV per session exactly. - [ "$(argv_value --context)" -eq 163840 ] - [ "$(argv_value --sessions)" -eq 4 ] - [ "$(argv_value --speculative)" = "dflash2" ] - [ "$(argv_value --draft-tokens)" -gt 0 ] - [ "$(argv_value --draft-policy)" = "adaptive" ] - # The shipped file must name the target the fleet actually serves, not a - # path that merely happens to exist. - [ "$(argv_value --model)" = "$MODEL_DIR/$TARGET_STUB" ] - [ "$(argv_value --dflash-model)" = "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" ] +@test "the qualified engine shape is in the env file" { + # 2026-10-01 window: 262144 is the model's native window and the GTT + # aperture swallows it — KV grows ONLY on the 8 full-attention layers + # (32 KiB/token bf16); the 24 GDN layers hold fixed ~25 MiB state per + # running sequence. SEQS 4 keeps the four-session contract; the pool is + # shared by the scheduler instead of carved per session like gufo. + grep -q '^FRANK_LARGE_MAXLEN=262144$' "$ENV_FILE" + grep -q '^FRANK_LARGE_SEQS=4$' "$ENV_FILE" + grep -q '^FRANK_LARGE_UTIL=0.45$' "$ENV_FILE" + grep -q '^FRANK_LARGE_MODEL_DIR=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-BF16$' "$ENV_FILE" + grep -q '^FRANK_LARGE_MODEL_SHARDS=4$' "$ENV_FILE" } -@test "the engine is pointed at the payload that is actually on disk" { - serve_argv - [ "$status" -eq 0 ] - [ -f "$(argv_value --model)" ] - # A drafter path that does not exist would load the target and decode at - # the autoregressive rate, which reads as "the new runtime is slow". - [ -f "$(argv_value --dflash-model)" ] - [ "$(argv_value --model)" != "$(argv_value --dflash-model)" ] +@test "the serve script wires the engine flags from the env" { + grep -q -- '--max-model-len "$FRANK_LARGE_MAXLEN"' "$SERVE" + grep -q -- '--max-num-seqs "$FRANK_LARGE_SEQS"' "$SERVE" + grep -q -- '--gpu-memory-utilization "$FRANK_LARGE_UTIL"' "$SERVE" + grep -q -- '--dtype bfloat16' "$SERVE" + grep -q -- '--trust-remote-code' "$SERVE" + grep -q -- '--ulimit memlock=-1:-1' "$SERVE" } -@test "frank-large stays text-only" { - # The payload has no mmproj, so an --mmproj would point at nothing. The - # fleet's image path is frank-small: its SGLang MiMo payload carries the - # vision tower inside the checkpoint (no projector file at all), and - # models.yml declares frank-large text-only. - serve_argv - ! argv_has --mmproj - grep -q 'MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound' "$REPO_ROOT/scripts/frank-small.env" +@test "speculative decode is cross-model MTP BY MEASUREMENT" { + # The 2026-10-01 window on this host: base 13.1 t/s, DFlash-8 17.4, + # MTP-2 19.8 (+51%). MTP acceptance is 71%/44% per position (token rate + # 57%) because the head trained on the distill's SOURCE (Qwen/Qwen3.5-9B, + # the only place mtp.* tensors exist for this family) still reads MiMo's + # hidden states. Beyond 2 draft tokens is UNMEASURED; the DFlash drafter + # stays on disk as the alternative method. + grep -q '^FRANK_LARGE_SPEC=mtp$' "$ENV_FILE" + grep -q '^FRANK_LARGE_SPEC_DIR=/home/dank/models/Qwen3.5-9B$' "$ENV_FILE" + grep -q '^FRANK_LARGE_SPEC_TOKENS=2$' "$ENV_FILE" + # The JSON is built ONLY when SPEC is set, and the draft path handed to + # the engine is the CONTAINER mount path, never the host path. + grep -q -- '--speculative-config' "$SERVE" + grep -q 'FRANK_LARGE_SPEC_CONTAINER_DIR="/models/' "$SERVE" + # The draft gate accepts BOTH weight layouts: a single + # model.safetensors (dflash) and a sharded index (the Qwen3.5-9B MTP + # head). A gate keyed only to the single file rejects the deployed arm. + grep -q 'FRANK_LARGE_SPEC_DIR/model.safetensors.index.json' "$SERVE" + # itself carries ZERO mtp.* tensors (that absence is what makes SGLang's + # NEXTN on frank-small a 0.00-acceptance ghost). + grep -q 'ZERO mtp' "$ENV_FILE" } -@test "the engine is asked for greedy decoding and for preserved prior reasoning" { - # Greedy is what this endpoint has always served; a silent switch to - # sampled output would be a model-behaviour change disguised as a - # runtime swap. Prior thinking blocks must survive across turns or a - # multi-turn agent loses its own reasoning between tool calls. - serve_argv - [ "$(argv_value --temperature)" = "0" ] - [ "$(argv_value --preserve-thinking)" = "on" ] +@test "tools and reasoning are wired — the client contract is served" { + grep -q '^FRANK_LARGE_REASONING_PARSER=qwen3$' "$ENV_FILE" + grep -q '^FRANK_LARGE_TOOL_PARSER=qwen3_coder$' "$ENV_FILE" + grep -q -- '--reasoning-parser "$FRANK_LARGE_REASONING_PARSER"' "$SERVE" + grep -q -- '--enable-auto-tool-choice' "$SERVE" + grep -q -- '--tool-call-parser "$FRANK_LARGE_TOOL_PARSER"' "$SERVE" + # vLLM's field name is `reasoning`, NOT llama.cpp/gufo's + # `reasoning_content` — the harness models.yml compat block must match. + grep -q 'reasoning' "$ENV_FILE" } -@test "the serve script refuses a missing runtime with the build pointer" { - rm -f "$RUNTIME_DIR/bin/gufo" - serve_argv - [ "$status" -ne 0 ] - [[ "$output" == *"runtime not found"* ]] - [[ "$output" == *"build-gufo.sh"* ]] - # A restart loop would be the alternative; nothing may be exec'd. - ! argv_has serve +@test "the serve script clears stale containers including the bench identity" { + grep -q '^FRANK_LARGE_LEGACY_CONTAINERS="vllm-mimo"$' "$ENV_FILE" + grep -q '^FRANK_LARGE_CONTAINER=frank-large-mimo$' "$ENV_FILE" + grep -q 'FRANK_LARGE_LEGACY_CONTAINERS' "$SERVE" } -@test "the serve script refuses a missing drafter with a fetch pointer" { - # The drafter lives in its own directory now, so the target can be present - # and this still has to be caught. - rm -f "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" - serve_argv - [ "$status" -ne 0 ] - [[ "$output" == *"missing asset"* ]] - [[ "$output" == *"Qwen3.8-27B-DFlash2-Q4_K_M.gguf"* ]] - [[ "$output" == *"download-frank-large"* ]] - ! argv_has serve +@test "frank-large holds the single fleet port under its own alias" { + grep -q '^FRANK_LARGE_PUBLISH=7770:7770$' "$ENV_FILE" + grep -q '^FRANK_LARGE_SERVED_NAME=frank-large$' "$ENV_FILE" } -@test "the serve script refuses a missing target and names the file" { - rm -f "$MODEL_DIR/$TARGET_STUB" - serve_argv - [ "$status" -ne 0 ] - [[ "$output" == *"missing asset"* ]] - [[ "$output" == *"$TARGET_STUB"* ]] +@test "no bare HIP-reserved env var is defined" { + # HIP reads bare TEMP/HOST/GPU_MAX_HEAP_SIZE; the serve script sources + # this file with set -a, so an unprefixed key leaks into the engine. + ! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE" + ! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE" } -@test "both payload files are fetched resumably and verified against declared sizes + sha256" { - # The target and the drafter come from DIFFERENT repositories, fetched by - # two scripts. Anchor the repo checks to the URL ASSIGNMENT, not to the - # file: the same repo string also appears in each script's header comment, - # so a plain grep stayed green after the URL was repointed at another - # repository -- the script would then download something the serve script - # never asked for. - grep -qE '^URL="https://huggingface\.co/unsloth/Qwen3\.8-27B-GGUF/resolve/main/' "$FETCH" - grep -qE 'https://huggingface\.co/z-lab/Qwen3\.8-27B-DFlash2-GGUF/resolve/main/' "$DRAFTER_FETCH" - for f in "$FETCH" "$DRAFTER_FETCH"; do - # -f so an HTTP error is a failure rather than a small "artifact"; - # -C - so an interrupted 29 GiB run resumes. - grep -q -- '-fL -C -' "$f" - # The size and the sha256 are compared after the transfer, not trusted. - grep -q 'SHA256 MISMATCH' "$f" - grep -q 'SIZE MISMATCH' "$f" - done +@test "the fetch scripts carry the verified payload and draft manifests" { + grep -q 'XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B' "$FETCH_TARGET" + [ "$(grep -c 'safetensors.*[0-9a-f]\{64\}' "$FETCH_TARGET")" = "4" ] + grep -q -- '--verify-only' "$FETCH_TARGET" + grep -q 'z-lab/Qwen3.5-9B-DFlash' "$FETCH_DRAFT" + grep -q -- '--verify-only' "$FETCH_DRAFT" + # The env file's payload/draft dirs must be where the fetch scripts write. + grep -q 'MiMo-V2.6-Distill-Qwen-9B-BF16' "$FETCH_TARGET" + grep -q 'Qwen3.5-9B-DFlash' "$FETCH_DRAFT" + grep -q 'Qwen/Qwen3.5-9B' "$FETCH_MTPSRC" + grep -q '/home/dank/models/Qwen3.5-9B' "$FETCH_MTPSRC" } -@test "the serve script and the fetch scripts agree on the payload filenames" { - # The serve script's MODEL_FILE and the fetch script's REL are two - # independent spellings of one filename, and a repo is not required to - # keep them in step: a re-upload under a new name, or an env edited - # without the fetch, leaves the endpoint unable to start with a - # "missing asset" that no test would have predicted. Bind them here. - local served fetched drafter_served drafter_fetched - served="$(basename "$(grep '^MODEL_FILE=' "$ENV_FILE" | cut -d= -f2-)")" - fetched="$(sed -n 's/^REL="\(.*\)"$/\1/p' "$FETCH")" - [ -n "$served" ] - [ "$served" = "$fetched" ] - drafter_served="$(basename "$(grep '^DRAFTER_FILE=' "$ENV_FILE" | cut -d= -f2-)")" - drafter_fetched="$(sed -n 's/^ *"\(Qwen3\.8-27B-DFlash2[^ ]*\.gguf\) .*/\1/p' "$DRAFTER_FETCH")" - [ -n "$drafter_served" ] - [ "$drafter_served" = "$drafter_fetched" ] +@test "the gufo deploy gates are retired with the gufo runtime" { + ! grep -q 'LLM_LARGE_RUNTIME' "$REMOTE_DEPLOY" + ! grep -q 'LLM_LARGE_MODEL' "$REMOTE_DEPLOY" + ! grep -q 'LLM_LARGE_DRAFT' "$REMOTE_DEPLOY" + ! grep -q "pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M'" "$REMOTE_DEPLOY" + # build-gufo.sh stays on disk as the rollback path to the tagged engine. + [ -f "$REPO_ROOT/scripts/build-gufo.sh" ] } -@test "the drafter is still fetched by the script that carries its Hub repo" { - # Guards a future "simplification" that folds the drafter into the - # payload-fetch script and silently orphans the artifact the serve script - # still points at via DRAFTER_DIR. The literal drafter name must not - # appear in that script AT ALL, comments included. - grep -q 'Qwen3.8-27B-UD-Q8_K_XL' "$DRAFTER_FETCH" - ! grep -q 'DFlash2' "$FETCH" -} - -@test "the build script refuses a source tree that is not the pinned commit" { - # gufo ships no prebuilt Linux binaries, so the runtime is built here. - # A tag that moves under us would silently install different kernels - # under the same RUNTIME_DIR, so the pin is enforced, not documented. - local src="$STAGE/not-the-pinned-commit" - mkdir -p "$src" - git -C "$src" init -q - git -C "$src" -c user.email=t@t -c user.name=t commit -q --allow-empty -m drift - run env GUFO_SRC_DIR="$src" GUFO_PREFIX="$STAGE/absent-prefix" bash "$BUILD" --verify - [ "$status" -ne 0 ] - [[ "$output" == *"GUFO_COMMIT mismatch"* ]] -} - -@test "the build script reports a runtime that was never built" { - # --verify on a host with no install must say so rather than exit 0 on - # an empty check. - run env GUFO_SRC_DIR="$STAGE/no-source-tree" GUFO_PREFIX="$STAGE/absent-prefix" bash "$BUILD" --verify - [ "$status" -ne 0 ] - [[ "$output" == *"no gufo at"* ]] -} - -@test "the deploy handler gates on the runtime, the payload and the port" { - grep -q 'relink_unit frank-large' "$REMOTE_DEPLOY" - grep -q 'systemctl enable frank-large.service' "$REMOTE_DEPLOY" +@test "the deployer gates image payload and spec then health-checks with MainPID" { + grep -q 'LLM_LARGE_CONTAINER=frank-large-mimo' "$REMOTE_DEPLOY" + grep -q 'LLM_LARGE_LEGACY_CONTAINERS="vllm-mimo"' "$REMOTE_DEPLOY" + grep -q 'FRANK_LARGE_IMAGE' "$REMOTE_DEPLOY" + grep -q "podman images --format '{{.RepoDigests}}'" "$REMOTE_DEPLOY" + grep -q 'FRANK_LARGE_MODEL_SHARDS' "$REMOTE_DEPLOY" + grep -q 'FRANK_LARGE_SPEC=' "$REMOTE_DEPLOY" grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY" - grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY" - grep -q 'LLM_LARGE_RUNTIME' "$REMOTE_DEPLOY" - grep -q 'LLM_LARGE_MODEL' "$REMOTE_DEPLOY" - grep -q 'LLM_LARGE_DRAFT' "$REMOTE_DEPLOY" -} - -@test "the deploy handler will not stop a serving endpoint it cannot replace" { - # The runtime gate must be live (a real conditional on an executable - # path, not a branch that never fires) and it must return BEFORE the - # stop, or a host whose gufo was never built takes frank-large down on - # its way to reporting the failure. - local large gate bail stop - large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")" - [[ "$large" == *'build-gufo.sh'* ]] - gate="$(grep -n 'if \[\[ ! -x "\$LLM_LARGE_RUNTIME" \]\]' <<<"$large" | head -1 | cut -d: -f1)" - stop="$(grep -n 'systemctl stop frank-large' <<<"$large" | head -1 | cut -d: -f1)" - [ -n "$gate" ] - [ -n "$stop" ] - bail="$(awk -v g="$gate" 'NR > g && /return 1/ { print NR; exit }' <<<"$large")" - [ -n "$bail" ] - [ "$bail" -lt "$stop" ] -} - -@test "the stray-run sweep keys on the drafter and cannot take frank-small down" { - # The sweep used to key on the TARGET filename, which the 2026-09-28 - # payload swap moved to a different repository. It would have stopped - # matching the live process, and a stray run would then hold 7770 past the - # port gate and fail the deploy. The drafter is the part that did NOT - # move, and every frank-large command line carries it via --dflash-model. - grep -q "pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M'" "$REMOTE_DEPLOY" - # frank-small serves a different GGUF on the 3090 with no DFlash2 drafter, - # so a binary-name sweep is the only thing that would endanger it. - local large - large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")" - ! grep -q 'pkill -x llama-server' <<<"$large" - ! grep -q 'pkill -x gufo' <<<"$large" -} - -@test "the deploy handler retires the halogen quadlet without deleting it" { - grep -q 'retire_halogen_large_quadlet' "$REMOTE_DEPLOY" - grep -q 'HOME/.AGENTTRASH' "$REMOTE_DEPLOY" - ! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?.*frank-large' "$REMOTE_DEPLOY" - # Nothing live ships for this endpoint as a container any more. The - # archived copy is host state in an ignored .AGENTTRASH — deliberately - # untracked, and present only in a primary checkout — so search the - # tracked tree, not the filesystem. - [ ! -e "$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" ] - ! git -C "$REPO_ROOT" grep -q 'Image=.*halogen' -- dankclaw + grep -q 'MainPID --value frank-large.service' "$REMOTE_DEPLOY" } diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 028f38a..b57a2e0 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -108,10 +108,11 @@ ENV_LARGE="$REPO_ROOT/scripts/frank-large.env" grep -q '^ALIAS=frank-small$' "$ENV_FILE" grep -q -- '--port "$LISTEN_PORT"' "$SERVE" grep -q -- '--host "$BIND_ADDR"' "$SERVE" - # The two endpoints cannot share a port: frank-large holds 7770, this one - # 7771. - grep -q '^PORT=7770$' "$ENV_LARGE" - ! grep -q '^PORT=7771$' "$ENV_LARGE" + # The two endpoints cannot share a port: frank-large publishes + # 7770:7770 through podman, this one binds 7771 natively. (The + # frank-large vLLM cutover renamed PORT to FRANK_LARGE_PUBLISH.) + grep -q '^FRANK_LARGE_PUBLISH=7770:7770$' "$ENV_LARGE" + ! grep -q '^FRANK_LARGE_PUBLISH=7771' "$ENV_LARGE" } @test "the deploy handler installs the root unit and refuses a missing payload" {