From 31b1ec397bdba140ba24851df821cf875b4b7e05 Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 6 Oct 2026 03:07:40 -0700 Subject: [PATCH] frank-large: cutover to flash-server v2 (abliterated Swift1.5) Engine: halogen-flash-server 0.16.2, digest-pinned (pulled 0c61bf84). Payload: Quat3rnion qwen38-flash-next-v2-swift15-abliterated.hgn (66,687,678,464 B, sha256-verified) + REQUIRED n-gram table (51,200,246,144 B @peonist 5cc17cea) + vision tower (897,916,416 B @peonist e053f488) + shared tokenizer (6 files). Shape: 2 slots x 200192 (KV pool 400384 positions, ~10.7 GiB), MTP always-on (HALOGEN_SPEC_ADAPT=0, card-mandated), server-side sampler at the card's 1.0/0.95/20, MAX_TOK 8192 with the 65536/7200 coupled pair, vision ON. Rollback = revert this commit (27B payload stays on disk). - serve script: byte-exact gates on all three .hgn files + flat tokenizer, foreground podman run (ledger to the log file) - deploy-remote: twin gates for n-gram/vision, flashnext fetch pointer - seam: 19 tests rewritten (spec-adapt law, pool product, compaction math, render shape); full suite 362/362 green --- scripts/deploy-remote.sh | 72 ++++--- scripts/download-frank-large-flashnext.sh | 119 ++++++++++++ scripts/frank-large.env | 222 ++++++++++++---------- scripts/frank-large.service | 4 +- scripts/serve-frank-large.sh | 150 ++++++++------- tests/frank_large_unit.bats | 194 ++++++++++++------- 6 files changed, 495 insertions(+), 266 deletions(-) create mode 100755 scripts/download-frank-large-flashnext.sh diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index c26d95e..ab8091c 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1086,33 +1086,37 @@ deploy_anubis() { wait_container_healthy anubis 60 || return 1 } -# frank-large is the halogen CONTAINER (ghcr.io/peonist-ai/halogen, -# digest-pinned through scripts/frank-large.env) serving -# peonist-ai/halogen-qwen3.8-27b — qwen3.8-27b-p1w4d-d2.hgn, 33.4 GiB, -# DFlash2 spec decode, 1 slot x 200700, text-only — on the Strix Halo -# iGPU. It stays a ROOT unit because the serve script drives `podman run` -# directly (the frank-primary pattern): relink_unit points -# /etc/systemd/system/frank-large.service at the repo file, boot +# frank-large is the halogen-flash-server CONTAINER (0.16.2, digest-pinned +# through scripts/frank-large.env) serving the Quat3rnion ABLITERATED +# Swift1.5 finetune of peonist's Qwen3.8-Flash-Next v2 checkpoint — +# qwen38-flash-next-v2-swift15-abliterated.hgn (66,687,678,464 B, ~62 GiB +# resident) + the REQUIRED n-gram lookup table + the vision tower — +# MTP spec decode (HALOGEN_SPEC_ADAPT=0), 2 slots x 200192, image input — +# on the Strix Halo iGPU. It stays a ROOT unit because the serve script +# drives `podman run` directly (the frank-primary pattern): relink_unit +# points /etc/systemd/system/frank-large.service at the repo file, boot # persistence is that unit's [Install], and LimitMEMLOCK=infinity is -# fleet parity. 2026-10-05 cutover from the ROCmFPX llama.cpp shape — -# rollback = revert the swap commit's trio; that payload, its draft head -# and the fork build all stay on disk -# (~/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP, -# ~/GitPackages/kingjones-ROCmFPX). +# fleet parity. 2026-10-06 cutover from the halogen 27B-dense shape — +# rollback = revert the swap commit's trio; the 27B payload stays on +# disk (~/models/halogen-qwen3.8-27b). # # The gates mirror serve-frank-large.sh's predicates EXACTLY (the # 2026-10-01 crash-loop lesson): image digest in dank's podman storage, -# the .hgn checkpoint at the exact byte count, the tokenizer directory -# holding every named file as a regular (not symlinked) file — a green -# deploy against a missing artifact would -# leave a restart-looping unit and a dark endpoint. The port gate is the -# same as every cutover since 2026-09-16: nothing starts until 7770 is -# released. -LLM_LARGE_IMAGE="${LLM_LARGE_IMAGE:-ghcr.io/peonist-ai/halogen@sha256:dc0a39a0016d6cfc58a197978febaafdf8d28403f724ded6111d98b5fb7ac0ea}" -LLM_LARGE_CKPT="${LLM_LARGE_CKPT:-/home/dank/models/halogen-qwen3.8-27b/qwen3.8-27b-p1w4d-d2.hgn}" -LLM_LARGE_CKPT_BYTES="${LLM_LARGE_CKPT_BYTES:-35865565184}" -LLM_LARGE_TOK_DIR="${LLM_LARGE_TOK_DIR:-/home/dank/models/halogen-qwen3.8-27b/tokenizer}" -LLM_LARGE_TOK_FILES="${LLM_LARGE_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json}" +# each .hgn payload (checkpoint, n-gram table, vision tower) at the exact +# byte count, the tokenizer directory holding every named file as a +# regular (not symlinked) file — a green deploy against a missing +# artifact would leave a restart-looping unit and a dark endpoint. The +# port gate is the same as every cutover since 2026-09-16: nothing +# starts until 7770 is released. +LLM_LARGE_IMAGE="${LLM_LARGE_IMAGE:-ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a}" +LLM_LARGE_CKPT="${LLM_LARGE_CKPT:-/home/dank/models/halogen-flashnext-v2/qwen38-flash-next-v2-swift15-abliterated.hgn}" +LLM_LARGE_CKPT_BYTES="${LLM_LARGE_CKPT_BYTES:-66687678464}" +LLM_LARGE_NGRAM="${LLM_LARGE_NGRAM:-/home/dank/models/halogen-flashnext-v2/qwen38-flash-next-ngram.hgn}" +LLM_LARGE_NGRAM_BYTES="${LLM_LARGE_NGRAM_BYTES:-51200246144}" +LLM_LARGE_VISION="${LLM_LARGE_VISION:-/home/dank/models/halogen-flashnext-v2/qwen38-flash-next-vision.hgn}" +LLM_LARGE_VISION_BYTES="${LLM_LARGE_VISION_BYTES:-897916416}" +LLM_LARGE_TOK_DIR="${LLM_LARGE_TOK_DIR:-/home/dank/models/halogen-flashnext-v2/tokenizer}" +LLM_LARGE_TOK_FILES="${LLM_LARGE_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json}" retire_halogen_large_quadlet() { local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" @@ -1156,23 +1160,37 @@ deploy_llm_large() { fi if [[ ! -f "$LLM_LARGE_CKPT" ]]; then echo "ERROR: halogen checkpoint absent at $LLM_LARGE_CKPT" >&2 - echo "frank-large: fetch it: bash scripts/download-frank-large-halogen.sh" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large-flashnext.sh" >&2 return 1 fi if [[ "$(stat -c %s "$LLM_LARGE_CKPT")" != "$LLM_LARGE_CKPT_BYTES" ]]; then echo "ERROR: halogen checkpoint is not $LLM_LARGE_CKPT_BYTES bytes (partial download?)" >&2 - echo "frank-large: re-check: bash scripts/download-frank-large-halogen.sh --verify-only" >&2 + echo "frank-large: re-check: bash scripts/download-frank-large-flashnext.sh --verify-only" >&2 return 1 fi + for gate in "$LLM_LARGE_NGRAM:$LLM_LARGE_NGRAM_BYTES:n-gram table" \ + "$LLM_LARGE_VISION:$LLM_LARGE_VISION_BYTES:vision tower"; do + IFS=: read -r gpath gbytes gname <<<"$gate" + if [[ ! -f "$gpath" ]]; then + echo "ERROR: halogen $gname absent at $gpath" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large-flashnext.sh" >&2 + return 1 + fi + if [[ "$(stat -c %s "$gpath")" != "$gbytes" ]]; then + echo "ERROR: halogen $gname is not $gbytes bytes (partial download?)" >&2 + echo "frank-large: re-check: bash scripts/download-frank-large-flashnext.sh --verify-only" >&2 + return 1 + fi + done if [[ -L "$LLM_LARGE_TOK_DIR" || ! -d "$LLM_LARGE_TOK_DIR" ]]; then echo "ERROR: tokenizer dir missing or a symlink at $LLM_LARGE_TOK_DIR" >&2 - echo "frank-large: fetch it: bash scripts/download-frank-large-halogen.sh" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large-flashnext.sh" >&2 return 1 fi for tf in $LLM_LARGE_TOK_FILES; do if [[ ! -f "$LLM_LARGE_TOK_DIR/$tf" || -L "$LLM_LARGE_TOK_DIR/$tf" ]]; then echo "ERROR: tokenizer file missing or a symlink: $LLM_LARGE_TOK_DIR/$tf" >&2 - echo "frank-large: fetch it: bash scripts/download-frank-large-halogen.sh" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large-flashnext.sh" >&2 return 1 fi done diff --git a/scripts/download-frank-large-flashnext.sh b/scripts/download-frank-large-flashnext.sh new file mode 100755 index 0000000..35e13dc --- /dev/null +++ b/scripts/download-frank-large-flashnext.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# download-frank-large-flashnext.sh — fetch frank-large's flash-server v2 +# payload: the Quat3rnion abliterated Swift1.5 checkpoint of peonist's v2 +# .hgn, the REQUIRED lookup table and the optional vision tower from +# peonist-ai/halogen-qwen3.8-flash-next (both at the revisions the model +# card pins), and the flat tokenizer the two repos share. All public, no +# token — and per the fleet secret law, none would ever touch argv. +# +# The fetch law, same as every sibling script: curl -f (an access error +# must not become a "successful" 153-byte artifact), exact byte count from +# the Hub tree API, LFS sha256 verification, --verify-only to re-check an +# existing payload without downloading. +# +# File manifest (verified against the Hub tree API on 2026-10-06): +# qwen38-flash-next-v2-swift15-abliterated.hgn 66,687,678,464 B sha256 +# 9c880b8ba48deabfa06172eff1e7572e4fe465e09e97faec6e50c3f6fdcac037 +# (Quat3rnion main; compacted r2 of the card's 89,775,868,032 B +# append-only original — byte-identical live tensors, dropped padding) +# qwen38-flash-next-ngram.hgn 51,200,246,144 B sha256 +# 3450acada94e19aabad88bd49b45eb70304820949c2240fc178067b20ae5dffe +# (peonist @5cc17cea; REQUIRED — the engine reads it beside the +# checkpoint) +# qwen38-flash-next-vision.hgn 897,916,416 B sha256 +# d62e0ae553fe88afd3833733d4a4c669f34d20fd8dfce4b9610525bed2134b10 +# (peonist @e053f488; optional, enables image input) +# tokenizer/* @e053f488 — byte-identical to the 27B-era tokenizer +# already on disk (tokenizer.json LFS +# 0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3); +# fetched fresh rather than symlinked, per the no-symlink law. +set -euo pipefail + +DEST="${FRANK_LARGE_MODEL_DIR:-$HOME/models/halogen-flashnext-v2}" +ABLIT="https://huggingface.co/Quat3rnion/halogen-swift1.5-qwen3.8-flash-next-v2-abliterated/resolve/main" +PEONIST="https://huggingface.co/peonist-ai/halogen-qwen3.8-flash-next/resolve" +NGRAM_REV="5cc17cea1a10b502b4a5db41d8d2b1a0f22dba26" +TOK_REV="e053f488b120b99ed2525e6ac99f68c51b3b6179" + +FILES=(qwen38-flash-next-v2-swift15-abliterated.hgn + qwen38-flash-next-ngram.hgn + qwen38-flash-next-vision.hgn + tokenizer/tokenizer.json + tokenizer/chat_template.jinja + tokenizer/generation_config.json + tokenizer/merges.txt + tokenizer/vocab.json + tokenizer/tokenizer_config.json) +SIZES=(66687678464 + 51200246144 + 897916416 + 12809320 + 8952 + 202 + 3353259 + 6722759 + 17928) +SHAS=(9c880b8ba48deabfa06172eff1e7572e4fe465e09e97faec6e50c3f6fdcac037 + 3450acada94e19aabad88bd49b45eb70304820949c2240fc178067b20ae5dffe + d62e0ae553fe88afd3833733d4a4c669f34d20fd8dfce4b9610525bed2134b10 + 0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3 + - - - - - -) + +# Fetch source per file: repo base + revision (empty = ABLIT @main). +SRCS=("$ABLIT" + "$PEONIST/$NGRAM_REV" + "$PEONIST/$TOK_REV" + "$PEONIST/$TOK_REV" + "$PEONIST/$TOK_REV" + "$PEONIST/$TOK_REV" + "$PEONIST/$TOK_REV" + "$PEONIST/$TOK_REV" + "$PEONIST/$TOK_REV") + +VERIFY_ONLY=0 +[ "${1:-}" = "--verify-only" ] && VERIFY_ONLY=1 + +mkdir -p "$DEST" + +for i in "${!FILES[@]}"; do + f="${FILES[$i]}"; want="${SIZES[$i]}"; sha="${SHAS[$i]}"; base="${SRCS[$i]}" + path="$DEST/$f" + mkdir -p "$(dirname "$path")" + if [ "$VERIFY_ONLY" -eq 0 ]; then + # A file already at the published size skips the fetch entirely: + # resuming it would ask the CDN for a range past EOF (416 -> curl + # error under set -e), and re-pulling 62 GiB to prove a point is + # not what --verify-only is for. sha256 below still decides. + if [ -f "$path" ] && [ "$(stat -c %s "$path")" = "$want" ]; then + echo "HAVE $f ($want bytes) — skipping fetch" + else + echo "FETCH $f -> $path" + # HF answers /resolve/ with a 302 to the Xet CDN and curl DROPS + # the Range header across redirects — a naive -C - then + # "resumes" by pulling the ~1 KB redirect body (measured + # 2026-10-05 on the 27B fetch: file frozen 14 MiB short, curl + # exit 0). Resolve the effective URL first, then resume + # against the CDN, which honors Range. + url=$(curl -sfIL -o /dev/null -w '%{url_effective}' "$base/$f") \ + || { echo "cannot resolve $base/$f" >&2; exit 1; } + curl -fL --retry 5 -C - -o "$path" "$url" + fi + fi + [ -f "$path" ] || { echo "MISSING $path (run without --verify-only)" >&2; exit 1; } + got="$(stat -c %s "$path")" + [ "$got" = "$want" ] || { echo "SIZE $path: $got != $want" >&2; exit 1; } + echo "OK $f ($got bytes)" +done + +echo "byte counts match; verifying sha256 where the Hub publishes an LFS oid..." +for i in "${!FILES[@]}"; do + f="${FILES[$i]}"; sha="${SHAS[$i]}" + if [ "$sha" = "-" ]; then + echo "OK $f (plain git blob, size-checked)" + continue + fi + got="$(sha256sum "$DEST/$f" | cut -d' ' -f1)" + [ "$got" = "$sha" ] || { echo "SHA256 $f: $got != $sha" >&2; exit 1; } + echo "OK $f sha256" +done +echo "DOWNLOAD_DONE $DEST" diff --git a/scripts/frank-large.env b/scripts/frank-large.env index 2678555..4092685 100644 --- a/scripts/frank-large.env +++ b/scripts/frank-large.env @@ -1,75 +1,88 @@ # frank-large.env — tunables for the frank-large endpoint on Dankclaw's # Strix Halo iGPU (gfx1151), host port 7770. # -# RUNTIME (2026-10-05, Frank): halogen — the closed-source C++/HIP engine -# (github.com/peonist-ai/halogen-server, image ghcr.io/peonist-ai/halogen, -# digest-pinned below) serving peonist-ai/halogen-qwen3.8-27b, a ~6.32 bpw -# NVFP4-derivative quant of the BASE Qwen/Qwen3.8-27B (35,865,565,184 B, -# 1352 tensors), as a podman container under a root unit, driven by -# scripts/serve-frank-large.sh. This CUTS OVER the -# 2026-10-02..10-03 ROCmFPX llama.cpp shape (uncensored -# Q6_0_ROCMFPX_AGENT + MTP, 27.6-28.1 t/s): rollback = revert the swap -# commit's trio; the ROCmFPX payload, draft head and build all stay on -# disk at /home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/. +# RUNTIME (2026-10-06, Frank): halogen-flash-server 0.16.2 — the +# closed-source C++/HIP engine (image ghcr.io/peonist-ai/halogen-flash-server, +# digest-pinned below) serving the Quat3rnion ABLITERATED Swift1.5 finetune +# of peonist's Qwen3.8-Flash-Next v2 checkpoint, as a podman container under +# a root unit, driven by scripts/serve-frank-large.sh. This CUTS OVER the +# 2026-10-05 halogen 27B-dense shape (censored base, DFlash2, 31.7 t/s): +# rollback = revert the swap commit's trio; the 27B payload stays on disk at +# /home/dank/models/halogen-qwen3.8-27b/. # # WHY THE RUNTIME MOVED WITH THE MODEL: the payload is .hgn — per its own -# card it "will not load in transformers, vLLM, or llama.cpp". Accepted -# consequences (Frank, 2026-10-05): TEXT-ONLY (halogen ignores the model's -# vision encoder; the fleet's vision path is frank-small) and the CENSORED -# base (this is a finetune of Qwen/Qwen3.8-27B, not the abliterated file — -# refusals may return). What it buys: vendor-measured decode DFlash2 31.7 -# t/s mean (vs this endpoint's MTP 27.6-28.1 and the old halogen-era MTP -# 26.9), prefill ~566 t/s at 32k (was ~400 on the sibling engine), 1.4-2.3x -# wall-clock on prefill-heavy agent traffic, at HIGHER effective precision -# (~6.32 bpw vs ~6.6 with the old W4A4 promotion — see FLAGS.md). +# card it "will not load in transformers, vLLM, or llama.cpp" — and the v2 +# .hgn loads ONLY in the flash-server line (the old halogen engine is for +# the 27B p1w4d file; the v2 quickstart pins flash-server). What the move +# buys (card + live probe evidence): MTP spec decode (53-57 t/s greedy, +# 3-10x serial), ~1,640 t/s prefill at 34k, ~72% fewer thinking tokens than +# the censored 27B (905 mean), the ABLITERATED base (refusals should NOT +# return), IMAGE INPUT (the v2 vision tower loads beside the checkpoint — +# red-PNG probe verified on the card; the 27B engine ignored its encoder), +# tool calls verified, and the native 262,144 context. # -# SHAPE — the two knobs that are NOT free: -# HALOGEN_KV_SLOTS=1 KEEPS SPECULATION ON. halogen makes batching and -# speculation mutually exclusive: with >1 slot the drafter is disabled -# and per-stream decode falls to serial speed (the sibling engine -# measured 41 t/s -> 14 t/s when slots were raised, and halogen's own -# FLAGS.md prints the same law). frank-large was already a one-session -# endpoint under ROCmFPX (PARALLEL=1), so slots=1 is the same contract, -# not a regression. -# HALOGEN_SLOT_CTX=200700 (Frank's window) is the per-conversation hard +# SHAPE — the knobs that are NOT free, with the arithmetic: +# HALOGEN_CTX=200192 is the PER-SLOT window and the per-conversation hard # ceiling — a prompt over it is a 400 naming the limit, never a silent -# truncation. KV pool = slots x slot_ctx x 64 KiB = 12.2 GiB. Budget -# against the 121 GiB of unified memory: weights 33.4 GiB (mmap'd — page -# cache, not anon), KV 12.2, engine working set + auto prompt cache -# ~20-30, host baseline ~10 -> ~75-85 GiB in use. The 2026-09-16 halogen -# era ran the same engine class on this box with LESS headroom. -# HALOGEN_DRAFTER=2 (DFlash2) is the shipped default and the fastest -# exact drafter on this checkpoint; the .hgn BAKES BOTH drafters (MTP + -# DFlash2), so unlike the Flash-Next era there is no sidecar to mount -# and no --dflash flag — the drafter is an int knob, byte-identical -# output to greedy verification either way. +# truncation. Harness compaction fires at 70% = 140,134, and 140,134 + +# 57,344 (harness maxTokens) = 197,478 <= 200,192 keeps prompt + budget +# inside the slot. models.yml contextWindow must quote 200192 or the +# harness discovers the wall as a 400. +# HALOGEN_KV_POOL_POSITIONS=400384 = KV_SLOTS(2) x CTX — the explicit +# form of the engine's default, so the seam test can pin the product. +# KV costs ~28 KiB/position (FLAGS.md) => 400,384 x 28 KiB = 10.7 GiB. +# HALOGEN_KV_SLOTS=2 restores the w4b-era shape on this box: the MTP +# drafter is NOT slot-locked on this engine (batching and speculation +# coexist — DFlash-era slots>1 law does NOT carry over; the card's own +# quickstart ships 2 slots WITH the head), so the second slot is +# concurrent capacity, not a speculation tax. +# Memory ledger against the box's 121.2 GiB unified (the n-gram table is +# mmap'd page cache beside the checkpoint, not a pinned charge — the +# card's own quickstart arithmetic does not bill it): +# weights ~62 GiB resident + KV 10.7 + arena (HALOGEN_MAX_TOK=8192 => +# ~8 GiB per FLAGS.md arena formula 3.8+8.4*(MT/16384)) + vision ~2 +# = ~82.7 GiB engine-side; HALOGEN_HOST_RESERVE_GIB=20 keeps the host +# honest; baseline co-tenancy (frank-small, ComfyUI, Caddy/observ) +# ~10-15. ~18 GiB margin. The boot ledger is the referee — read it +# after deploy and re-size if the engine prints a clamp. +# HALOGEN_SPEC_ADAPT=0 is MANDATORY (card): the default adaptive policy +# turns the MTP head off for whole requests after low acceptance and +# HALVES decode as a result. Frank accepted the head always-on trade +# (temp 1.0 traffic costs acceptance; the card still ships the recipe). # -# CLIENT CONTRACT deltas vs the ROCmFPX shape (the harness side lives in +# CLIENT CONTRACT deltas vs the 27B shape (harness side lives in # ~/.omp/agent/models.yml): -# * NO server-side sampler defaults: a request without temperature -# decodes GREEDY. The harness sends temp 1.0 / top_p 0.95 via -# extraBody, so that wire recipe is load-bearing now, not cosmetic. -# * max_tokens defaults to 8192 PER REQUEST and INCLUDES thinking; a -# reasoning reply over a small budget returns empty content with -# finish_reason "length". The harness maxTokens is 57344: compaction -# fires at 70% of 200700 = 140,490 and 140,490 + 57,344 = 197,834 <= -# 200,700 keeps prompt + budget inside the pool (the 262144-era 65536 -# would overflow). Requests over HALOGEN_MAX_TOKENS_CAP are a 400 -# naming the cap; cap and queue timeout are COUPLED (see FLAGS.md -# table) — do not move one without the other. -# * response_format json_object/json_schema is a 400 (no constrained -# decoding in this build). -# * Thinking lands in reasoning_content — same field as the llama.cpp -# era, so the harness compat block does NOT change. reasoning_effort -# accepts none|low|xhigh on this build (template default xhigh); the -# models.yml fold (high->xhigh) still applies. +# * SERVER-SIDE SAMPLER EXISTS NOW: HALOGEN_TEMPERATURE/TOP_P/TOP_K set +# the card's thinking recipe (1.0 / 0.95 / 20) as engine defaults, so +# an absent temperature is no longer greedy. models.yml extraBody +# carries the same values — consistent, not load-bearing. +# * HALOGEN_MAX_TOK=8192 is only the DEFAULT for requests that omit +# max_tokens; the harness always sends 57344. Requests over +# HALOGEN_MAX_TOKENS_CAP (65536) are a 400 naming the cap. Cap and +# queue timeout are COUPLED (FLAGS.md table): 65536 at ~10 t/s serial +# worst case needs > 6550 s; 7200 honors it. Do not move one alone. +# * max_tokens INCLUDES thinking; a reasoning reply over a small budget +# returns empty content with finish_reason "length". +# * Prompt cache is ON by default (PROMPT_CACHE=2): a follow-up turn at +# 100k context re-prefills ~2 s instead of ~88 s cold. No pin needed — +# unlike the 27B engine, auto-sizing fits this box's ledger. +# * Thinking lands in reasoning_content (same as the 27B engine); the +# v2 chat template rejects reasoning_effort outside xhigh|medium|low, +# so the models.yml fold (high->xhigh) still applies. +# * Vision: OpenAI image_url content parts only (the fleet's standard +# contract). >=256 px probe images or the model answers "video". +# * response_format json_object/json_schema unsupported in this line +# (no constrained decoding) — unchanged from the 27B engine. # * The engine's token protocol has no auth and binds loopback INSIDE -# the container; only the frontend port is published. Keep it that way. +# the container; only the frontend port (8731) is published. Same as +# before. Kernel needs CONFIG_HSA_AMD_SVM (CachyOS ships it) and +# amdgpu.noretry=0 (check at deploy). # Image: digest-pinned (fetch law: tag pulls silently re-point; the digest -# is what was qualified). 0.1.4 = the version the model card names. -FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen@sha256:dc0a39a0016d6cfc58a197978febaafdf8d28403f724ded6111d98b5fb7ac0ea -# Container identity: the halogen container name. +# is what was qualified). 0.16.2 = the version the model card names as its +# quickstart pin; RepoDigest captured at pull on 2026-10-06. +FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a +# Container identity: kept from the 27B era (unit ExecStop names it). FRANK_LARGE_CONTAINER=frank-large-halogen # Old quadlet-era container identity; the serve script clears stale names. FRANK_LARGE_LEGACY_CONTAINERS=frank-large @@ -77,51 +90,56 @@ FRANK_LARGE_LEGACY_CONTAINERS=frank-large # default); the engine port (8730) is never published. FRANK_LARGE_PUBLISH=7770:8731 -FRANK_LARGE_MODEL_DIR=/home/dank/models/halogen-qwen3.8-27b -FRANK_LARGE_CHECKPOINT=qwen3.8-27b-p1w4d-d2.hgn -FRANK_LARGE_CHECKPOINT_BYTES=35865565184 -FRANK_LARGE_CHECKPOINT_SHA256=274c3fc767fb57faf025dd03c76376b1fc1b32448aef45aaf02d1898fa962089 -# The tokenizer is a FLAT DIRECTORY, $FRANK_LARGE_MODEL_DIR/tokenizer, mounted -# whole at /tokenizer — the model card ships `tokenizer/` and the engine's -# HALOGEN_TOKENIZER default is that directory (the card's own docker run sets -# no tokenizer flag). The Hub stores it as 5 files (sizes from the Hub API, -# 2026-10-05): tokenizer.json 12809320 (LFS sha256 -# 0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3), -# vocab.json 6722759, merges.txt 3353259, chat_template.jinja 8952, -# tokenizer_config.json 17928 (plain git blobs — size-checked, no LFS oid). -# The serve gate and the deployer refuse a missing file or ANY symlink: an -# HF cache snapshot symlinks each file into blobs/, and those links dangle +FRANK_LARGE_MODEL_DIR=/home/dank/models/halogen-flashnext-v2 +# The checkpoint: Quat3rnion/halogen-swift1.5-qwen3.8-flash-next-v2-abliterated +# qwen38-flash-next-v2-swift15-abliterated.hgn — compacted r2 of the card's +# append-only original (byte-identical live tensors, padding dropped: +# 66,687,678,464 B vs 89,775,868,032 B). LFS sha256 verified at fetch. +FRANK_LARGE_CHECKPOINT=qwen38-flash-next-v2-swift15-abliterated.hgn +FRANK_LARGE_CHECKPOINT_BYTES=66687678464 +FRANK_LARGE_CHECKPOINT_SHA256=9c880b8ba48deabfa06172eff1e7572e4fe465e09e97faec6e50c3f6fdcac037 +# The n-gram lookup table: REQUIRED (the engine reads it beside the +# checkpoint; the card's quickstart names it explicitly). From +# peonist-ai/halogen-qwen3.8-flash-next @ 5cc17cea. +FRANK_LARGE_NGRAM_TABLE=qwen38-flash-next-ngram.hgn +FRANK_LARGE_NGRAM_BYTES=51200246144 +FRANK_LARGE_NGRAM_SHA256=3450acada94e19aabad88bd49b45eb70304820949c2240fc178067b20ae5dffe +# The vision tower: OPTIONAL on the engine, ENABLED here (image input). +# From peonist-ai/halogen-qwen3.8-flash-next @ e053f488. +FRANK_LARGE_VISION_TOWER=qwen38-flash-next-vision.hgn +FRANK_LARGE_VISION_BYTES=897916416 +FRANK_LARGE_VISION_SHA256=d62e0ae553fe88afd3833733d4a4c669f34d20fd8dfce4b9610525bed2134b10 +# The tokenizer is a FLAT DIRECTORY, $FRANK_LARGE_MODEL_DIR/tokenizer, +# mounted whole at /tokenizer. The two repos ship the same one; fetched +# fresh from peonist @ e053f488 (tokenizer.json is LFS, byte-identical to +# the 27B-era copy; the rest are plain git blobs — size-checked only). The +# serve gate and the deployer refuse a missing file or ANY symlink: an HF +# cache snapshot symlinks each file into blobs/, and those links dangle # inside the container mount. -FRANK_LARGE_TOKENIZER_FILES="tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json" +FRANK_LARGE_TOKENIZER_FILES="tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json" -# --- engine flags (docs/FLAGS.md of the engine repo; anything unset stays -# at the image default, which is what was qualified) ------------------------ +# --- engine flags (docs/FLAGS.md of the flash-server repo; anything unset +# stays at the image default, which is what was qualified) ------------------ HALOGEN_MODEL_ID=frank-large -HALOGEN_SLOT_CTX=200700 -HALOGEN_KV_SLOTS=1 -HALOGEN_DRAFTER=2 +HALOGEN_CTX=200192 +HALOGEN_KV_POOL_POSITIONS=400384 +HALOGEN_KV_SLOTS=2 +HALOGEN_HOST_RESERVE_GIB=20 +# Spec decode: MTP head is INSIDE the checkpoint (do NOT set HALOGEN_MTP_HEAD); +# depth 2 is the documented "best all-round" default. HALOGEN_DRAFTER / +# HALOGEN_DFLASH_* from the 27B engine DO NOT EXIST here. +HALOGEN_SPEC_ADAPT=0 # Coupled pair (FLAGS.md table): the cap bounds how long ONE request holds -# the GPU, the timeout bounds how long the NEXT client waits. 65536 at -# ~10 t/s serial worst case needs > 6550 s queue timeout; the defaults -# honor that. Do not lower the timeout alone. +# the GPU, the timeout bounds how long the NEXT client waits. Do not move +# one alone (see the client-contract note above). +HALOGEN_MAX_TOK=8192 HALOGEN_MAX_TOKENS_CAP=65536 HALOGEN_QUEUE_TIMEOUT=7200 -# Prompt cache — PINNED (2026-10-06). Auto is OFF on this box: the boot -# ledger refused it — "serve: prompt cache OFF - 35.7 GB free after the -# mapped checkpoint, 8.6 GB reserved for the system, which leaves less -# than the 36.9 GB floor (2 x a 18.4 GB full-context entry). Set -# HALOGEN_CACHE_MB to override." Auto demands TWO full-context entries; -# the documented override is an explicit byte budget. Measured cost of -# leaving it off: every deep agent request re-prefills its whole -# context — 84.9-179.8 s for 46-81k prompts that grow by only ~500-650 -# tokens per turn (serve_api lines, 2026-10-06) — while the README -# prices the cache at ~20x time-to-first-token on a warm turn. -# -# Sizing: an entry costs ~70 KB/token (engine figure), so ONE full-slot -# entry (200,700 tok) is ~13.7 GB; 16384 MB holds it with margin, and -# several ordinary 60-80k conversations under LRU-by-bytes eviction. -# The reserve (HALOGEN_CACHE_RESERVE_MB, default 8192) is re-checked -# against live MemAvailable on every store, so a pin cannot starve the -# host silently. HALOGEN_CACHE_ALIGN stays at its shipped 2048 — that -# value is what makes warm answers byte-identical to cold ones. -HALOGEN_CACHE_MB=16384 +# Server-side sampler = the card's thinking recipe (the engine now HAS one; +# requests may still override per-call). Matches models.yml extraBody. +HALOGEN_TEMPERATURE=1.0 +HALOGEN_TOP_P=0.95 +HALOGEN_TOP_K=20 +# Card's belt-and-braces: the engine pins weights by default +# (HALOGEN_FLASH_PIN_TRUNK=1; only the retired w4b build honored 0). +HALOGEN_WEIGHTS_LOCK=1 diff --git a/scripts/frank-large.service b/scripts/frank-large.service index 8db85c1..dc9aba7 100644 --- a/scripts/frank-large.service +++ b/scripts/frank-large.service @@ -1,5 +1,5 @@ [Unit] -Description=halogen container (peonist-ai/halogen-qwen3.8-27b, DFlash2 spec decode, 1 slot x 200700, text-only), root systemd unit (dankclaw, frank-large, Strix Halo iGPU) +Description=halogen-flash-server container (Quat3rnion abliterated Swift1.5 of Qwen3.8-Flash-Next v2, MTP spec decode, 2 slots x 200192, vision ON), root systemd unit (dankclaw, frank-large, Strix Halo iGPU) After=network-online.target Wants=network-online.target @@ -22,7 +22,7 @@ TimeoutStopSec=120 # would otherwise leave the endpoint down until someone notices. Restart=always RestartSec=10 -# The engine mmaps 33.4 GiB of weights and keeps a multi-GiB working set +# The engine pins ~62 GiB of v2 weights and keeps a multi-GiB working set # on a box that also hosts frank-small, ComfyUI and the Caddy/observability # stack; make it a late OOM candidate, not the first. OOMScoreAdjust=-500 diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index d5a8c5a..3a342f9 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -1,30 +1,32 @@ #!/usr/bin/env bash -# serve-frank-large.sh — frank-large endpoint: halogen (closed-source -# C++/HIP engine, image ghcr.io/peonist-ai/halogen, digest-pinned in -# frank-large.env) serving peonist-ai/halogen-qwen3.8-27b on Dankclaw's -# Strix Halo iGPU, host :7770. 2026-10-05 cutover from the ROCmFPX -# llama.cpp shape — revert the swap commit's trio to roll back (payload -# and fork build stay on disk). +# serve-frank-large.sh — frank-large endpoint: halogen-flash-server 0.16.2 +# (closed-source engine, image ghcr.io/peonist-ai/halogen-flash-server, +# digest-pinned in frank-large.env) serving the Quat3rnion abliterated +# Swift1.5 finetune of peonist's Qwen3.8-Flash-Next v2 checkpoint on +# Dankclaw's Strix Halo iGPU, host :7770. 2026-10-06 cutover from the +# halogen 27B-dense shape — revert the swap commit's trio to roll back +# (the 27B payload stays on disk). # # The engine is driven by `podman run` DIRECTLY (the fleet's frank-primary -# pattern): the image entrypoint `halogen-docker` in default "all" mode -# starts engine + OpenAI frontend in one container. The frontend listens on -# 8731 inside (HALOGEN_API_PORT default) and is published as host 7770; the -# engine's own token port (8730, no auth) stays unpublished on the -# container's loopback. The image ships NO default checkpoint: -# HALOGEN_CHECKPOINT selects the file inside the /models mount. +# pattern): the image starts engine + OpenAI frontend in one container. +# The frontend listens on 8731 inside (HALOGEN_API_PORT default) and is +# published as host 7770; the engine's own token port (8730, no auth) +# stays unpublished on the container's loopback. HALOGEN_CHECKPOINT +# selects the file inside the /models mount; the n-gram lookup table +# (HALOGEN_NGRAM_TABLE, the card sets it explicitly though the documented +# fallback is "beside the checkpoint") and the vision tower +# (HALOGEN_VISION_TOWER) ride in the same directory. # # Gates mirror the deployer EXACTLY (the 2026-10-01 crash-loop lesson — # serve gate and deploy gate disagreeing = green deploy, red unit): -# tokenizer directory present with every named file a FLAT REGULAR FILE — -# the model card ships `tokenizer/` as a flat directory and the engine -# defaults HALOGEN_TOKENIZER to /tokenizer, so a HF cache snapshot's -# symlinks into blobs/ dangle inside the mount; GPU device nodes present. +# checkpoint/n-gram/vision byte-exact, tokenizer directory present with +# every named file a FLAT REGULAR FILE (a HF cache snapshot's symlinks +# into blobs/ dangle inside the mount), GPU device nodes present. # -# Why containers instead of the bare binary (the old frank-large shape): -# the payload is .hgn and only halogen reads it; halogen ships as a -# container; the root unit keeps LimitMEMLOCK=infinity for parity and to -# outlive the login session, exactly like frank-primary's. +# Why containers instead of a bare binary: the payload is .hgn and only +# the flash-server line reads it; it ships as a container; the root unit +# keeps LimitMEMLOCK=infinity for parity and to outlive the login session, +# exactly like frank-primary's. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -44,11 +46,16 @@ set +a : "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR must name the payload dir}" : "${FRANK_LARGE_CHECKPOINT:?FRANK_LARGE_CHECKPOINT must name the .hgn inside it}" : "${FRANK_LARGE_CHECKPOINT_BYTES:?FRANK_LARGE_CHECKPOINT_BYTES must pin the size}" +: "${FRANK_LARGE_NGRAM_TABLE:?FRANK_LARGE_NGRAM_TABLE must name the lookup table}" +: "${FRANK_LARGE_NGRAM_BYTES:?FRANK_LARGE_NGRAM_BYTES must pin the table size}" +: "${FRANK_LARGE_VISION_TOWER:?FRANK_LARGE_VISION_TOWER must name the tower}" +: "${FRANK_LARGE_VISION_BYTES:?FRANK_LARGE_VISION_BYTES must pin the tower size}" : "${FRANK_LARGE_TOKENIZER_FILES:?FRANK_LARGE_TOKENIZER_FILES must list the tokenizer dir contents}" : "${HALOGEN_MODEL_ID:?HALOGEN_MODEL_ID must be the served alias (frank-large)}" -: "${HALOGEN_SLOT_CTX:?HALOGEN_SLOT_CTX must be set}" -: "${HALOGEN_KV_SLOTS:?HALOGEN_KV_SLOTS must be set (1 keeps the drafter on)}" -: "${HALOGEN_DRAFTER:?HALOGEN_DRAFTER must be set (2 = DFlash2)}" +: "${HALOGEN_CTX:?HALOGEN_CTX must be set (per-slot window)}" +: "${HALOGEN_KV_POOL_POSITIONS:?HALOGEN_KV_POOL_POSITIONS must be set}" +: "${HALOGEN_KV_SLOTS:?HALOGEN_KV_SLOTS must be set}" +: "${HALOGEN_SPEC_ADAPT:?HALOGEN_SPEC_ADAPT must be set (0 = head always on)}" # Capture podman's output BEFORE grepping: under `set -o pipefail` the # `podman images | grep -q` pattern races (grep exits on first match, @@ -61,36 +68,41 @@ if ! grep -qF "${FRANK_LARGE_IMAGE#*@}" <<<"$DIGESTS"; then exit 1 fi -# The checkpoint gate is size-exact: a truncated curl exits 0 on some -# failures and a partial .hgn loads far enough to look alive, so the byte -# count from the model card is the contract (the fleet fetch law). -CKPT="$FRANK_LARGE_MODEL_DIR/$FRANK_LARGE_CHECKPOINT" -if [ ! -f "$CKPT" ]; then - echo "frank-large: checkpoint $CKPT missing;" >&2 - echo " fetch it: bash scripts/download-frank-large-halogen.sh" >&2 - exit 1 -fi -CKPT_SIZE="$(stat -c %s "$CKPT")" -if [ "$CKPT_SIZE" != "$FRANK_LARGE_CHECKPOINT_BYTES" ]; then - echo "frank-large: checkpoint is $CKPT_SIZE bytes, expected $FRANK_LARGE_CHECKPOINT_BYTES (partial download?)" >&2 - exit 1 -fi +# Size-exact gates on all three payload files: a truncated curl exits 0 on +# some failures and a partial .hgn loads far enough to look alive, so the +# byte counts from the Hub tree API are the contract (fleet fetch law). +ck() { + # ck + if [ ! -f "$2" ]; then + echo "frank-large: $1 $2 missing;" >&2 + echo " fetch it: bash scripts/download-frank-large-flashnext.sh" >&2 + exit 1 + fi + local sz + sz="$(stat -c %s "$2")" + if [ "$sz" != "$3" ]; then + echo "frank-large: $1 is $sz bytes, expected $3 (partial download?)" >&2 + exit 1 + fi +} +ck checkpoint "$FRANK_LARGE_MODEL_DIR/$FRANK_LARGE_CHECKPOINT" "$FRANK_LARGE_CHECKPOINT_BYTES" +ck n-gram-table "$FRANK_LARGE_MODEL_DIR/$FRANK_LARGE_NGRAM_TABLE" "$FRANK_LARGE_NGRAM_BYTES" +ck vision-tower "$FRANK_LARGE_MODEL_DIR/$FRANK_LARGE_VISION_TOWER" "$FRANK_LARGE_VISION_BYTES" -# The payload ships the tokenizer as a FLAT DIRECTORY (model card: -# "tokenizer/ flat tokenizer directory"). A HuggingFace cache snapshot -# symlinks each file into a sibling blobs/, and those symlinks dangle -# inside the container mount — so the directory must be real and every -# named file must be a regular file, not a link. +# The payload ships the tokenizer as a FLAT DIRECTORY. A HuggingFace cache +# snapshot symlinks each file into a sibling blobs/, and those symlinks +# dangle inside the container mount — so the directory must be real and +# every named file must be a regular file, not a link. TOK_DIR="$FRANK_LARGE_MODEL_DIR/tokenizer" if [ -L "$TOK_DIR" ] || [ ! -d "$TOK_DIR" ]; then echo "frank-large: tokenizer dir $TOK_DIR missing or a symlink (HF cache snapshot?);" >&2 - echo " fetch it: bash scripts/download-frank-large-halogen.sh" >&2 + echo " fetch it: bash scripts/download-frank-large-flashnext.sh" >&2 exit 1 fi for TF in $FRANK_LARGE_TOKENIZER_FILES; do if [ ! -f "$TOK_DIR/$TF" ] || [ -L "$TOK_DIR/$TF" ]; then echo "frank-large: tokenizer file $TOK_DIR/$TF missing or a symlink;" >&2 - echo " fetch it: bash scripts/download-frank-large-halogen.sh" >&2 + echo " fetch it: bash scripts/download-frank-large-flashnext.sh" >&2 exit 1 fi done @@ -109,16 +121,15 @@ done LOG="${FRANK_LARGE_LOG:-/home/dank/logs/frank_large.log}" mkdir -p "$(dirname "$LOG")" -# for the udev-granted supplementary group, seccomp unconfined for the ROCm -# ioctl surface, and unbounded memlock (the unit's LimitMEMLOCK=infinity is -# inherited, so runc never has to RAISE it). NO --ulimit nofile: Dankclaw's -# hard limit is 524288, rootless runc cannot raise a hard limit and dies -# with "error setting rlimit type 7: operation not permitted" (measured -# 2026-10-05 cutover), and 524288 fds is already ~100x what the engine -# wants; the model card and both sibling serve scripts pass no nofile -# either. No comment lines inside the backslash -# chain — a mid-chain # terminates the continuation (the 2026-10-05 -# exit-127 crash loop on the sibling serve script). +# --ipc=host for the udev-granted supplementary group, seccomp unconfined +# for the ROCm ioctl surface, and unbounded memlock (the unit's +# LimitMEMLOCK=infinity is inherited, so runc never has to RAISE it). NO +# --ulimit nofile: Dankclaw's hard limit is 524288, rootless runc cannot +# raise a hard limit and dies with "error setting rlimit type 7: +# operation not permitted" (measured 2026-10-05 cutover); the model card +# and both sibling serve scripts pass no nofile either. No comment lines +# inside the backslash chain — a mid-chain # terminates the continuation +# (the 2026-10-05 exit-127 crash loop on the sibling serve script). podman run --name "$FRANK_LARGE_CONTAINER" \ --publish "$FRANK_LARGE_PUBLISH" \ --ipc=host \ @@ -128,24 +139,29 @@ podman run --name "$FRANK_LARGE_CONTAINER" \ --security-opt seccomp=unconfined \ -e HALOGEN_MODEL_ID="$HALOGEN_MODEL_ID" \ -e HALOGEN_CHECKPOINT="/models/$FRANK_LARGE_CHECKPOINT" \ - -e HALOGEN_SLOT_CTX="$HALOGEN_SLOT_CTX" \ - -e HALOGEN_KV_SLOTS="${HALOGEN_KV_SLOTS:-1}" \ - -e HALOGEN_DRAFTER="$HALOGEN_DRAFTER" \ + -e HALOGEN_NGRAM_TABLE="/models/$FRANK_LARGE_NGRAM_TABLE" \ + -e HALOGEN_VISION_TOWER="/models/$FRANK_LARGE_VISION_TOWER" \ + -e HALOGEN_TOKENIZER=/tokenizer \ + -e HALOGEN_CTX="$HALOGEN_CTX" \ + -e HALOGEN_KV_POOL_POSITIONS="$HALOGEN_KV_POOL_POSITIONS" \ + -e HALOGEN_KV_SLOTS="$HALOGEN_KV_SLOTS" \ + -e HALOGEN_HOST_RESERVE_GIB="${HALOGEN_HOST_RESERVE_GIB:-20}" \ + -e HALOGEN_SPEC_ADAPT="$HALOGEN_SPEC_ADAPT" \ + -e HALOGEN_MAX_TOK="${HALOGEN_MAX_TOK:-8192}" \ -e HALOGEN_MAX_TOKENS_CAP="${HALOGEN_MAX_TOKENS_CAP:-65536}" \ -e HALOGEN_QUEUE_TIMEOUT="${HALOGEN_QUEUE_TIMEOUT:-7200}" \ - ${HALOGEN_CACHE_MB:+-e HALOGEN_CACHE_MB="$HALOGEN_CACHE_MB"} \ + -e HALOGEN_TEMPERATURE="${HALOGEN_TEMPERATURE:-1.0}" \ + -e HALOGEN_TOP_P="${HALOGEN_TOP_P:-0.95}" \ + -e HALOGEN_TOP_K="${HALOGEN_TOP_K:-20}" \ + ${HALOGEN_WEIGHTS_LOCK:+-e HALOGEN_WEIGHTS_LOCK="$HALOGEN_WEIGHTS_LOCK"} \ -v "$FRANK_LARGE_MODEL_DIR:/models:ro" \ -v "$FRANK_LARGE_MODEL_DIR/tokenizer:/tokenizer:ro" \ "$FRANK_LARGE_IMAGE" \ > "$LOG" 2>&1 # FOREGROUND `podman run`: the engine's stdout/stderr are this script's -# output, so the boot ledger (kv pool, prompt cache, drafter state, any -# clamp) lands in $LOG as documented. The previous `-d` + `exec podman -# wait` shape captured only the container ID — the ledger existed -# nowhere on disk and the runbook's post-deploy grep read a one-line -# hex file (found 2026-10-06 while tuning). podman run returns the -# container's exit code, so a crash is a nonzero ExecStart and the -# unit's Restart=always stays real; systemd stop goes through the -# unit's ExecStop and the run client forwards SIGTERM to the engine, -# which drains on it. +# output, so the boot ledger (kv pool, prompt cache, MTP head, any clamp) +# lands in $LOG as documented. podman run returns the container's exit +# code, so a crash is a nonzero ExecStart and the unit's Restart=always +# stays real; systemd stop goes through the unit's ExecStop and the run +# client forwards SIGTERM to the engine, which drains on it. diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index 3b164ff..cd503c6 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -1,18 +1,20 @@ #!/usr/bin/env bats -# Seam: frank-large serves peonist-ai/halogen-qwen3.8-27b (a .hgn halogen -# checkpoint) on the Strix Halo iGPU through the halogen CONTAINER -# (ghcr.io/peonist-ai/halogen, digest-pinned), driven by the root unit's -# serve script — one conversation of 200,700 tokens with the DFlash2 -# drafter ON. 2026-10-05 cutover from the ROCmFPX llama.cpp shape. +# Seam: frank-large serves the Quat3rnion ABLITERATED Swift1.5 finetune of +# peonist's Qwen3.8-Flash-Next v2 checkpoint (a .hgn flash-server payload) +# on the Strix Halo iGPU through the halogen-flash-server CONTAINER (0.16.2, +# digest-pinned), driven by the root unit's serve script — two 200,192-token +# slots with the MTP head always on (HALOGEN_SPEC_ADAPT=0) and the vision +# tower mounted. 2026-10-06 cutover from the halogen 27B-dense shape. # # This seam exists because the artifacts are a triangle that must agree on -# exact strings the engine reads: the image DIGEST, the checkpoint file -# name AND byte count inside the /models mount, and the tokenizer's -# flat-file shape (a HF cache snapshot's symlinks dangle inside the -# container). A serve gate and a deployer gate that disagree is the +# exact strings the engine reads: the image DIGEST, the checkpoint/n-gram/ +# vision file names AND byte counts inside the /models mount, and the +# tokenizer's flat-file shape (a HF cache snapshot's symlinks dangle inside +# the container). A serve gate and a deployer gate that disagree is the # 2026-10-01 crash-loop; both sides are pinned here, plus the one engine -# law the fleet must never "optimize" away: HALOGEN_KV_SLOTS=1 is what -# keeps speculative decoding on. +# law the card is emphatic about: HALOGEN_SPEC_ADAPT=0 — the default +# adaptive policy silently disables the MTP head per request and HALVES +# decode. # # Everything is tested from what an operator or supervisor can observe — # the command line podman would run, the exit code, the deployer's error @@ -36,7 +38,7 @@ teardown() { # podman_stub — a fake podman on PATH that prints the argv of `run` and # `wait` and answers `images` with whatever digests the test names, so # "what would the supervisor run" is observable output and the digest -# gate is testable without a 3.5 GB image. +# gate is testable without a 3.6 GB image. podman_stub() { mkdir -p "$TEST_TMP/bin" cat > "$TEST_TMP/bin/podman" <<'STUB' @@ -52,14 +54,14 @@ STUB } # good_env — the minimum env the serve script demands, with the digest -# stub agrees on and zero-byte artifacts passing the size gate. +# stub agrees on and zero-byte artifacts passing the size gates. good_env() { : > "$TEST_TMP/empty.env" - touch "$TEST_TMP/qwen.hgn" + touch "$TEST_TMP/qwen.hgn" "$TEST_TMP/ngram.hgn" "$TEST_TMP/vision.hgn" mkdir -p "$TEST_TMP/tokenizer" && touch "$TEST_TMP/tokenizer/tok.json" GOOD_ENV=( "FRANK_LARGE_ENV_FILE=$TEST_TMP/empty.env" - "FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen@sha256:cafe" + "FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:cafe" "FRANK_LARGE_CONTAINER=frank-large-halogen" "FRANK_LARGE_LEGACY_CONTAINERS=frank-large" "FRANK_LARGE_DEV_KFD=$TEST_TMP/qwen.hgn" @@ -69,42 +71,56 @@ good_env() { "FRANK_LARGE_MODEL_DIR=$TEST_TMP" "FRANK_LARGE_CHECKPOINT=qwen.hgn" "FRANK_LARGE_CHECKPOINT_BYTES=0" + "FRANK_LARGE_NGRAM_TABLE=ngram.hgn" + "FRANK_LARGE_NGRAM_BYTES=0" + "FRANK_LARGE_VISION_TOWER=vision.hgn" + "FRANK_LARGE_VISION_BYTES=0" "FRANK_LARGE_TOKENIZER_FILES=tok.json" "HALOGEN_MODEL_ID=frank-large" - "HALOGEN_SLOT_CTX=200700" - "HALOGEN_KV_SLOTS=1" - "HALOGEN_DRAFTER=2" - "HALOGEN_CACHE_MB=16384" - "STUB_DIGESTS=localhost/x ghcr.io/peonist-ai/halogen@sha256:cafe" + "HALOGEN_CTX=200192" + "HALOGEN_KV_POOL_POSITIONS=400384" + "HALOGEN_KV_SLOTS=2" + "HALOGEN_SPEC_ADAPT=0" + "HALOGEN_MAX_TOK=8192" + "STUB_DIGESTS=localhost/x ghcr.io/peonist-ai/halogen-flash-server@sha256:cafe" ) } -@test "the env file defines the halogen deployment values" { - grep -qE '^FRANK_LARGE_IMAGE=ghcr\.io/peonist-ai/halogen@sha256:[0-9a-f]{64}$' "$ENV_FILE" +@test "the env file defines the flash-server deployment values" { + grep -qE '^FRANK_LARGE_IMAGE=ghcr\.io/peonist-ai/halogen-flash-server@sha256:[0-9a-f]{64}$' "$ENV_FILE" grep -qE '^FRANK_LARGE_CONTAINER=frank-large-halogen$' "$ENV_FILE" grep -qE '^FRANK_LARGE_PUBLISH=7770:8731$' "$ENV_FILE" grep -qE '^FRANK_LARGE_CHECKPOINT=.*\.hgn$' "$ENV_FILE" grep -qE '^FRANK_LARGE_CHECKPOINT_BYTES=[0-9]+$' "$ENV_FILE" grep -qE '^FRANK_LARGE_CHECKPOINT_SHA256=[0-9a-f]{64}$' "$ENV_FILE" + grep -qE '^FRANK_LARGE_NGRAM_TABLE=.*\.hgn$' "$ENV_FILE" + grep -qE '^FRANK_LARGE_NGRAM_BYTES=[0-9]+$' "$ENV_FILE" + grep -qE '^FRANK_LARGE_NGRAM_SHA256=[0-9a-f]{64}$' "$ENV_FILE" + grep -qE '^FRANK_LARGE_VISION_TOWER=.*\.hgn$' "$ENV_FILE" + grep -qE '^FRANK_LARGE_VISION_BYTES=[0-9]+$' "$ENV_FILE" + grep -qE '^FRANK_LARGE_VISION_SHA256=[0-9a-f]{64}$' "$ENV_FILE" grep -qE '^FRANK_LARGE_TOKENIZER_FILES="tokenizer\.json( [a-z_.]+)+"$' "$ENV_FILE" grep -qE '^HALOGEN_MODEL_ID=frank-large$' "$ENV_FILE" - grep -qE '^HALOGEN_SLOT_CTX=[0-9]+$' "$ENV_FILE" + grep -qE '^HALOGEN_CTX=[0-9]+$' "$ENV_FILE" } -@test "the drafter stays on: slots must stay 1" { - # halogen makes batching and speculation mutually exclusive — with - # more than one slot the drafter is DISABLED and per-stream decode - # falls to serial speed (the sibling engine measured 41 -> 14 t/s - # when slots went up). frank-large was a one-session endpoint before - # this swap too (PARALLEL=1), so slots=1 preserves the contract; - # raising it is a product decision, never a default change. - grep -qE '^HALOGEN_KV_SLOTS=1$' "$ENV_FILE" - grep -qE '^HALOGEN_DRAFTER=2$' "$ENV_FILE" +@test "the head stays on: adaptive spec must stay 0 and the window shape pinned" { + # The card is emphatic: the default adaptive policy turns the MTP head + # off for whole requests after low acceptance and HALVES decode. The + # two-slot law does NOT carry over from the 27B engine — on this line + # batching and speculation coexist (the card's quickstart ships 2 + # slots WITH the head), so 2 is concurrent capacity, not a tax. + grep -qE '^HALOGEN_SPEC_ADAPT=0$' "$ENV_FILE" + grep -qE '^HALOGEN_KV_SLOTS=2$' "$ENV_FILE" # The cap/timeout pair is coupled (FLAGS.md table): a request at the # cap must not outlast the queue timeout, or everyone queued behind - # it eats a 503. + # it eats a 503. The per-request default must stay under the cap. + grep -qE '^HALOGEN_MAX_TOK=[0-9]+$' "$ENV_FILE" grep -qE '^HALOGEN_MAX_TOKENS_CAP=[0-9]+$' "$ENV_FILE" grep -qE '^HALOGEN_QUEUE_TIMEOUT=[0-9]+$' "$ENV_FILE" + mt="$(sed -nE 's/^HALOGEN_MAX_TOK=([0-9]+)$/\1/p' "$ENV_FILE")" + cap="$(sed -nE 's/^HALOGEN_MAX_TOKENS_CAP=([0-9]+)$/\1/p' "$ENV_FILE")" + [ "$mt" -le "$cap" ] } @test "the serve script refuses when there is no GPU to serve on" { @@ -116,20 +132,25 @@ good_env() { [[ "$output" == *"no gfx1151 to serve on"* ]] } -@test "the KV pool plus pinned cache fit unified memory with margin" { - # Pool arithmetic is the engine's documented product: slots x - # slot_ctx x 64 KiB. The prompt cache is PINNED (HALOGEN_CACHE_MB, - # ~70 KB/token per entry — 16384 MB >= one full-slot entry). Against - # both: 33.4 GiB of mapped weights, ~20 GiB engine working set, host - # baseline and frank-small co-tenancy. The box has 121 GiB unified; - # the 2026-09-16 halogen era ran the same engine class here on a - # LARGER pool with less co-tenancy slack. - ctx="$(sed -nE 's/^HALOGEN_SLOT_CTX=([0-9]+)$/\1/p' "$ENV_FILE")" +@test "the pool product and memory ledger fit unified memory with margin" { + # Pool arithmetic is the engine's documented product: the explicit + # HALOGEN_KV_POOL_POSITIONS must equal slots x ctx, and KV costs + # ~28 KiB/position (FLAGS.md). Against it: ~62 GiB resident weights, + # the arena scratch 3.8 + 8.4*(MAX_TOK/16384) GiB, the vision tower + # ~2 GiB pool charge, the 20 GiB host reserve and the host baseline + # + co-tenancy (frank-small, ComfyUI, Caddy/observability) ~15. The + # box has 121 GiB unified. The n-gram table rides as mmap'd page + # cache beside the checkpoint and is not a pinned charge (the card's + # own quickstart arithmetic does not bill it). + ctx="$(sed -nE 's/^HALOGEN_CTX=([0-9]+)$/\1/p' "$ENV_FILE")" slots="$(sed -nE 's/^HALOGEN_KV_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")" - cache_mb="$(sed -nE 's/^HALOGEN_CACHE_MB=([0-9]+)$/\1/p' "$ENV_FILE")" - pool_gib=$(( slots * ctx * 65536 / 1073741824 )) - cache_gib=$(( ${cache_mb:-0} / 1024 )) - total_gib=$(( 34 + pool_gib + cache_gib + 20 + 15 )) + pool="$(sed -nE 's/^HALOGEN_KV_POOL_POSITIONS=([0-9]+)$/\1/p' "$ENV_FILE")" + mt="$(sed -nE 's/^HALOGEN_MAX_TOK=([0-9]+)$/\1/p' "$ENV_FILE")" + reserve="$(sed -nE 's/^HALOGEN_HOST_RESERVE_GIB=([0-9]+)$/\1/p' "$ENV_FILE")" + [ "$pool" -eq $(( slots * ctx )) ] + kv_gib=$(( pool * 28 * 1024 / 1073741824 )) + arena_tenths=$(( 38 + 84 * mt / 16384 )) + total_gib=$(( 62 + kv_gib + arena_tenths / 10 + 2 + reserve + 15 )) [ "$total_gib" -le 121 ] } @@ -149,12 +170,12 @@ good_env() { FRANK_LARGE_CHECKPOINT=absent.hgn "$SERVE" [ "$status" -ne 0 ] [[ "$output" == *"checkpoint"*"missing"* ]] - [[ "$output" == *"download-frank-large-halogen.sh"* ]] + [[ "$output" == *"download-frank-large-flashnext.sh"* ]] } @test "the serve script refuses a checkpoint of the wrong byte count" { - # A truncated curl exit-0 artifact must not boot; the model card's - # byte count is the contract (fleet fetch law). + # A truncated curl exit-0 artifact must not boot; the Hub's byte + # count is the contract (fleet fetch law). podman_stub good_env run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ @@ -163,6 +184,38 @@ good_env() { [[ "$output" == *"expected 42"* ]] } +@test "the serve script refuses a missing or wrong-size n-gram table" { + # The lookup table is REQUIRED: the engine reads it beside the + # checkpoint (the card names it explicitly), and prompt-lookup + # drafting dies without it. + podman_stub + good_env + run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ + FRANK_LARGE_NGRAM_TABLE=absent.hgn "$SERVE" + [ "$status" -ne 0 ] + [[ "$output" == *"n-gram-table"*"missing"* ]] + run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ + FRANK_LARGE_NGRAM_BYTES=42 "$SERVE" + [ "$status" -ne 0 ] + [[ "$output" == *"expected 42"* ]] +} + +@test "the serve script refuses a missing or wrong-size vision tower" { + # Optional on the engine, ENABLED here (image input is a fleet + # contract) — a silent absence would degrade the endpoint to + # text-only with no error anywhere. + podman_stub + good_env + run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ + FRANK_LARGE_VISION_TOWER=absent.hgn "$SERVE" + [ "$status" -ne 0 ] + [[ "$output" == *"vision-tower"*"missing"* ]] + run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ + FRANK_LARGE_VISION_BYTES=42 "$SERVE" + [ "$status" -ne 0 ] + [[ "$output" == *"expected 42"* ]] +} + @test "the serve script refuses a symlinked tokenizer file" { # A HuggingFace cache snapshot's entries are symlinks into blobs/ and # dangle inside the container mount — even when the directory itself @@ -189,28 +242,32 @@ $(cat "$TEST_TMP/serve.log")" [[ "$output" == *"--name frank-large-halogen"* ]] [[ "$output" == *"--publish 7770:8731"* ]] [[ "$output" == *"-e HALOGEN_MODEL_ID=frank-large"* ]] - [[ "$output" == *"-e HALOGEN_SLOT_CTX=200700"* ]] - [[ "$output" == *"-e HALOGEN_KV_SLOTS=1"* ]] - [[ "$output" == *"-e HALOGEN_DRAFTER=2"* ]] + [[ "$output" == *"-e HALOGEN_CTX=200192"* ]] + [[ "$output" == *"-e HALOGEN_KV_POOL_POSITIONS=400384"* ]] + [[ "$output" == *"-e HALOGEN_KV_SLOTS=2"* ]] + [[ "$output" == *"-e HALOGEN_SPEC_ADAPT=0"* ]] [[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]] + [[ "$output" == *"-e HALOGEN_NGRAM_TABLE=/models/ngram.hgn"* ]] + [[ "$output" == *"-e HALOGEN_VISION_TOWER=/models/vision.hgn"* ]] [[ "$output" == *"--device $TEST_TMP/qwen.hgn"* ]] [[ "$output" == *"$TEST_TMP/tokenizer:/tokenizer:ro"* ]] - [[ "$output" == *"ghcr.io/peonist-ai/halogen@sha256:cafe"* ]] + [[ "$output" == *"ghcr.io/peonist-ai/halogen-flash-server@sha256:cafe"* ]] # Foreground run, never -d: the engine ledger must land in the log # file (`podman run -d` captured only the container ID and the # runbook's ledger grep read a one-line hex file, found 2026-10-06). ! grep -qE '(^|[[:space:]])-d([[:space:]]|$)' "$TEST_TMP/serve.log" ! grep -q 'PODMAN wait' "$TEST_TMP/serve.log" - [[ "$output" == *"-e HALOGEN_CACHE_MB=16384"* ]] } @test "the shipped env pins the fleet's window and shape" { - # 200700 is the per-conversation hard ceiling (a longer prompt is a - # 400 naming the limit, never a silent truncation), and the harness + # 200192 is the per-slot hard ceiling (a longer prompt is a 400 + # naming the limit, never a silent truncation), and the harness # compaction math must fit prompt + budget under it: OMP compacts at - # 70% = 140490, and 140490 + 57344 = 197834 <= 200700. - grep -qE '^HALOGEN_SLOT_CTX=200700$' "$ENV_FILE" - ctx=200700 + # 70% = 140134, and 140134 + 57344 = 197478 <= 200192. models.yml + # contextWindow must quote the same number or the harness discovers + # the wall as a 400. + grep -qE '^HALOGEN_CTX=200192$' "$ENV_FILE" + ctx=200192 [ $(( ctx * 70 / 100 + 57344 )) -le "$ctx" ] } @@ -230,22 +287,23 @@ $(cat "$TEST_TMP/serve.log")" grep -qF "LLM_LARGE_IMAGE=\"\${LLM_LARGE_IMAGE:-$img}\"" "$DEPLOYER" dir="$(sed -nE 's/^FRANK_LARGE_MODEL_DIR=(.*)$/\1/p' "$ENV_FILE")" ckpt="$(sed -nE 's/^FRANK_LARGE_CHECKPOINT=(.*)$/\1/p' "$ENV_FILE")" - tokf="$(sed -nE 's/^FRANK_LARGE_TOKENIZER_FILES="(.*)"$/\1/p' "$ENV_FILE")" + tokf="$(sed -nE 's/^FRANK_LARGE_TOKENIZER_FILES=\"(.*)\"$/\1/p' "$ENV_FILE")" grep -qF "LLM_LARGE_TOK_DIR=\"\${LLM_LARGE_TOK_DIR:-$dir/tokenizer}\"" "$DEPLOYER" grep -qF "LLM_LARGE_TOK_FILES=\"\${LLM_LARGE_TOK_FILES:-$tokf}\"" "$DEPLOYER" bytes="$(sed -nE 's/^FRANK_LARGE_CHECKPOINT_BYTES=([0-9]+)$/\1/p' "$ENV_FILE")" grep -qF "LLM_LARGE_CKPT_BYTES=\"\${LLM_LARGE_CKPT_BYTES:-$bytes}\"" "$DEPLOYER" } -@test "the deployer points a missing artifact at the halogen fetch script" { - grep -qE 'download-frank-large-halogen\.sh' "$DEPLOYER" +@test "the deployer points a missing artifact at the flashnext fetch script" { + grep -qE 'download-frank-large-flashnext\.sh' "$DEPLOYER" grep -qE 'podman pull' "$DEPLOYER" } @test "the stray sweep cannot take frank-small or the deployer down" { # frank-small also listens on this host (7771, llama-server) and the # deployer's own command line contains "frank-large"; only the FULL - # container name matches a stray manual halogen run and nothing else. + # container name matches a stray manual flash-server run and nothing + # else. grep -q "pkill -f 'frank-large-halogen'" "$DEPLOYER" ! grep -qE "pkill -f 'llama-server'" "$DEPLOYER" ! grep -qE "pkill -f 'frank-large'" "$DEPLOYER" @@ -262,8 +320,8 @@ $(cat "$TEST_TMP/serve.log")" grep -qE '^ExecStop=.*podman stop .* frank-large-halogen$' "$UNIT" } -@test "the unit runs the halogen runtime and no retired path" { - grep -q 'halogen' "$UNIT" +@test "the unit runs the flash-server runtime and no retired path" { + grep -q 'halogen-flash-server' "$UNIT" # Every retired payload/build stays on the host, so a stale path in # a unit still resolves — to a runtime nobody has measured in weeks. ! grep -qE 'GitPackages/(ROCmFPX-kairic|llama.cpp-ik_llama|llama-builds|ciru-orca)' "$UNIT" @@ -271,9 +329,9 @@ $(cat "$TEST_TMP/serve.log")" } @test "the rollback door stays qualified: the ROCmFPX build pin survives" { - # Rolling back this swap means rebuilding the fork the old payload - # needs; the pin (real commit, gfx1151, no webui fetch) is what makes - # that door trustworthy months later. + # Rolling back to the pre-halogen shape means rebuilding the fork the + # old payload needs; the pin (real commit, gfx1151, no webui fetch) + # is what makes that door trustworthy months later. grep -qE 'ROCMFPX_COMMIT=.*[0-9a-f]{40}' "$BUILD" grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD" grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD"