dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(frank-large): unsloth llama.cpp Qwen3.8-Flash-Next UD-IQ4_XS (200k/1 slot, MTP depth 2)

Browse Source 
Replace the CIRU v4.4.1 Orca runtime with the unsloth llama.cpp prebuilt
(b11030-mix-5ff778e, ROCm gfx1151, bundles its own ROCm) serving
unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (87.3 GiB, three HF split parts
concatenated) on the Strix Halo iGPU. One conversation, 200,000 ctx,
PARALLEL_SLOTS=1, MTP depth 2 via the explicit -md shared-Q8_0 head
(sidecar auto-discovery does not search the mtp/ subfolder), text-only.
Fleet contract unchanged: port 7770, alias frank-large.

- serve-frank-large.sh: unsloth binary, -m/-md, --spec-type draft-mtp
  --spec-draft-n-max 2, --jinja, preserve_thinking; gates on binary,
  GGUF, MTP head; conditional --mmproj on ENABLE_VISION=1
- download-frank-large-unsloth.sh: resumable fetch of 3 parts + MTP head
  + mmproj, each verified against its LFS sha256 fetched live from the
  Hub tree API; concatenates parts, records/verifies SHA256SUMS
- deploy-remote.sh: LLM_LARGE_MODEL/LLM_LARGE_DRAFT paths, pkill pattern
  on the model path (not the binary name), wait_port_free gate
- tests: seams for 200k/1-slot, explicit -md, text-only, handler gates,
  sweep safety
This commit is contained in:
Frank committed 2026-09-21 12:30:30 -07:00
1 parent e5a1f8e767
commit b149d02187
6 files changed
+272 -157
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/frank_large_unit.bats
+41 -44
View File
@@ -1,23 +1,24 @@
#!/usr/bin/env bats
# Seam: the frank-large CIRU v4.4.1 root unit on dankclaw.
# Seam: the frank-large unsloth-llama.cpp root unit on dankclaw.
#
# The 2026-09-19 cutover moved this service out of the halogen-flash-server
# quadlet and onto the CIRU v4.4.1 runtime (QSA window hotfix) serving
# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca, as a ROOT unit on a host build -- the
# same shape as frank-small and frank-medium, because a private-SDK host
# binary has no container image to install and no generated unit to dodge.
# This seam pins what a flag list cannot show: the operator's one-stream /
# full-context shape, the payload the launcher refuses to start without, a
# deploy handler that retires the halogen quadlet without deleting it and
# gates on the port, and the fact that the sweep for a stray benchmark run
# cannot take frank-small down with it.
# The 2026-09-21 cutover moved this service from the CIRU v4.4.1 runtime
# (jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca) onto the unsloth llama.cpp prebuilt
# (b11030-mix-5ff778e, ROCm gfx1151) serving
# unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS, still as a ROOT unit: a
# prebuilt tarball with its own bundled ROCm has no container image to
# install and no generated unit to dodge. This seam pins what a flag list
# cannot show: the operator's one-stream / 200k shape, the payload the
# serve script refuses to start without, the explicit -md MTP wiring
# (sidecar auto-discovery does not search the mtp/ subfolder), a deploy
# handler that gates on the port, and the fact that the sweep for a stray
# benchmark run cannot take frank-small down with it.
load helpers
ENV_FILE="$REPO_ROOT/scripts/frank-large.env"
UNIT="$REPO_ROOT/scripts/frank-large.service"
SERVE="$REPO_ROOT/scripts/serve-frank-large.sh"
FETCH="$REPO_ROOT/scripts/download-frank-large-orca.sh"
FETCH="$REPO_ROOT/scripts/download-frank-large-unsloth.sh"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
@test "the frank-large unit, env and serve files exist where the deployer installs them from" {
@@ -31,23 +32,22 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
grep -q '^Restart=on-failure$' "$UNIT"
grep -q '^RestartSec=' "$UNIT"
grep -q '^User=dank$' "$UNIT"
# The engine pins staging buffers and the PLE pager's hot window; the
# memlock budget must be unbounded or the load fails on the RLIMIT.
# The engine pins staging buffers; the memlock budget must be unbounded
# or the load fails on the RLIMIT.
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
# The unit runs the repo's own serve script with the repo's env.
grep -q 'ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh' "$UNIT"
grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT"
}
@test "the operator's shape is one stream of the full trained window" {
@test "the operator's shape is one stream of 200k context with MTP on" {
local ctx slots
ctx="$(sed -nE 's/^CONTEXT_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
slots="$(sed -nE 's/^PARALLEL_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")"
# 262144 is the model's trained window and the GGUF carries no rope
# scaling; the operator asked for full context.
[ "$ctx" -eq 262144 ]
# One slot. This is also the drafter's constraint: the launcher refuses
# MTP with more than one slot, so the two cannot drift apart silently.
# 200000 is the operator's requested serving shape (2026-09-21); the
# model's trained window is 262144.
[ "$ctx" -eq 200000 ]
# One stream.
[ "$slots" -eq 1 ]
grep -q '^ENABLE_MTP=1$' "$ENV_FILE"
}
@@ -59,44 +59,41 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
grep -q -- '--mmproj' "$REPO_ROOT/scripts/serve-frank-small.sh"
}
@test "the serve script pins the hotfix runtime and refuses a missing build" {
grep -q 'ciru-runtime-v4.4.1-qsa-hotfix' "$SERVE"
grep -q 'build-gfx1151-sdk/bin/llama-server' "$SERVE"
grep -q 'run-orca-server.sh' "$SERVE"
@test "the serve script pins the prebuilt runtime and refuses a missing build" {
grep -q 'unsloth-llama-b11030' "$SERVE"
grep -q 'llama-server' "$SERVE"
# A missing runtime or payload is a deploy error with a pointer, not a
# restart loop.
grep -q 'missing built runtime' "$SERVE"
grep -q 'missing runtime' "$SERVE"
grep -q 'missing target model' "$SERVE"
grep -q 'missing MTP head' "$SERVE"
# The Qwen3.8 template keeps prior thinking blocks across turns only
# when the kwarg is set; the launcher passes --jinja but no kwargs.
# when the kwarg is set.
grep -q 'preserve_thinking' "$SERVE"
}
@test "the serve script feeds the launcher its runtime root from the SDK venv" {
# The launcher exits 2 without hip/lib/libamdhip64.so +
# rocr/lib/libhsa-runtime64.so under its runtime root; the build links
# the private venv, so the serve script must point those entries there
# or the unit crash-loops on the gate (seen 2026-09-19).
grep -q '_rocm_sdk_devel/lib' "$SERVE"
grep -q 'runtime/hip/lib/libamdhip64.so' "$SERVE"
grep -q 'runtime/rocr/lib/libhsa-runtime64.so' "$SERVE"
grep -q 'missing SDK venv runtime' "$SERVE"
@test "the serve script wires the MTP head explicitly and speculates" {
# Sidecar auto-discovery does not search the mtp/ subfolder, so the head
# must be passed with -md or the server runs at the main model's speed
# with no error saying why.
grep -q -- '-md "$MTP_FILE"' "$SERVE"
grep -q -- '--spec-type draft-mtp' "$SERVE"
grep -q 'mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf' "$SERVE"
}
@test "the payload is fetched resumably and verified by sha256" {
grep -q 'jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca' "$FETCH"
@test "the payload is fetched resumably and verified against the live Hub tree" {
grep -q 'unsloth/Qwen3.8-Flash-Next-GGUF' "$FETCH"
grep -q -- '-fL -C -' "$FETCH"
grep -q 'parts.json' "$FETCH"
# The LFS oids come from the Hub tree API at verify time, not from a
# file that could drift.
grep -q 'tree/main' "$FETCH"
grep -q 'sha256sum -c SHA256SUMS' "$FETCH"
# The PLE sidecar is the largest transport part; the script must fetch
# and verify it, not just the GGUF.
grep -q 'ple.payload.bin' "$FETCH"
}
@test "the fleet endpoint and alias survive the cutover" {
grep -q '^HOST=0.0.0.0$' "$ENV_FILE"
grep -q '^PORT=7770$' "$ENV_FILE"
grep -q '^MODEL_DIR=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca$' "$ENV_FILE"
grep -q '^MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS$' "$ENV_FILE"
}
@test "the deploy handler retires the halogen quadlet without deleting it" {
@@ -111,14 +108,14 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY"
grep -q 'LLM_LARGE_MODEL' "$REMOTE_DEPLOY"
grep -q 'LLM_LARGE_PLE' "$REMOTE_DEPLOY"
grep -q 'LLM_LARGE_DRAFT' "$REMOTE_DEPLOY"
}
@test "the stray-run sweep cannot take frank-small down" {
# frank-small on this host is ALSO a llama-server. The sweep must match
# the model path, not the binary name, or a frank-large deploy restarts
# the 3090 endpoint.
grep -q "pkill -f 'Qwen3.8-Flash-CIRU-STRIX-Orca'" "$REMOTE_DEPLOY"
grep -q "pkill -f 'Qwen3.8-Flash-Next-UD-IQ4_XS'" "$REMOTE_DEPLOY"
# ...and only inside the large handler: extract it and check there is no
# bare binary-name pkill in it.
local large