fix(frank-large): load the model as its native 3-way split, not a concatenated file
The first cutover crash-looped the unit with 'invalid split file name': the fetch script concatenated the three split parts into a single GGUF, but part 1's header carries ggml.file_count=3, so the byte-correct concatenation still advertises itself as a split and the loader demands a -NNNNN-of-NNNNN filename. - serve-frank-large.sh: -m points at part 1; llama.cpp loads the split natively (parts 2-3 auto-discovered in the same directory) - deploy-remote.sh: LLM_LARGE_MODEL gate path follows - download-frank-large-unsloth.sh: assembly removed; the parts ARE the canonical payload, verified against live Hub LFS oids - docs: runbook model section + troubleshooting note, AGENTS.md, log - the redundant 87 GiB assembled file is staged on dankclaw at ~/.AGENTTRASH/20260921-frank-large-assembled/
This commit is contained in:
Frank
@@ -279,8 +279,8 @@ The unsloth llama.cpp prebuilt `app-b11030-mix-5ff778e` (ROCm gfx1151,
|
||||
`/home/dank/GitPackages/unsloth-llama-b11030/`, bundles its own ROCm runtime —
|
||||
no host SDK gate, no venv) as a **root systemd unit** (`scripts/
|
||||
frank-large.service`, `User=dank`), serving
|
||||
`unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS` (87.3 GiB main GGUF, three HF
|
||||
split parts concatenated, + a 2.6 GiB MTP head, all sha256-verified) on port
|
||||
`unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS` (87.3 GiB main GGUF as three HF
|
||||
split parts, loaded natively as a split via `-m` on part 1, + a 2.6 GiB MTP head, all sha256-verified) on port
|
||||
7770 from the gfx1151 iGPU. One stream, 200,000 ctx (`PARALLEL_SLOTS=1`,
|
||||
operator's requested shape; the trained window is 262,144), MTP depth 2
|
||||
(`SPEC_DRAFT_N_MAX=2`, the model card's default), reasoning extracted to
|
||||
|
||||
@@ -1084,7 +1084,7 @@ deploy_anubis() {
|
||||
# STRIX-Orca), which had replaced the halogen-flash-server quadlet. The
|
||||
# port gate is the same as every cutover since 2026-09-16: nothing starts
|
||||
# until 7770 is released.
|
||||
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf}"
|
||||
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf}"
|
||||
LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf}"
|
||||
|
||||
retire_halogen_large_quadlet() {
|
||||
|
||||
@@ -1,29 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download and assemble unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (the
|
||||
# Download unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (the
|
||||
# frank-large model) on Dankclaw.
|
||||
#
|
||||
# bash scripts/download-frank-large-unsloth.sh
|
||||
# bash scripts/download-frank-large-unsloth.sh --verify-only
|
||||
#
|
||||
# The UD-IQ4_XS GGUF ships as three llama.cpp split parts (HF caps uploads at
|
||||
# 50 GB); concatenating them in numeric order reconstructs the single GGUF.
|
||||
# 50 GB). The parts ARE the canonical payload: the serve script points -m at
|
||||
# part 1 and llama.cpp loads the 3-way split natively. Do NOT concatenate
|
||||
# them into a single file to serve — part 1's header carries
|
||||
# ggml.file_count=3, so a byte-correct concatenation still advertises itself
|
||||
# as a split and the loader rejects the non-split filename.
|
||||
# The MTP shared-Q8_0 head borrows the token embedding and the output
|
||||
# projection from the main model at runtime (it is the 2.6 GB remainder of
|
||||
# the 4B MTP layer). The F16 projector is fetched for a future vision flip;
|
||||
# the serve script does not load it while ENABLE_VISION=0.
|
||||
#
|
||||
# Integrity: every part is checked against its LFS sha256 (fetched live from
|
||||
# the Hub tree API, not trusted from disk), then the assembled GGUF against
|
||||
# the recorded SHA256SUMS. The repo is public (gated: false); an HF token at
|
||||
# ~/.cache/huggingface/token is used when present to clear the anonymous rate
|
||||
# limit on a 94 GB transfer.
|
||||
# the Hub tree API, not trusted from disk). The repo is public (gated:
|
||||
# false); an HF token at ~/.cache/huggingface/token is used when present to
|
||||
# clear the anonymous rate limit on a 94 GB transfer.
|
||||
#
|
||||
# Layout after a successful run (MODEL_DIR):
|
||||
# Qwen3.8-Flash-Next-UD-IQ4_XS.gguf 93,682,584,224 main model
|
||||
# UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-0000{1,2,3}-of-00003.gguf the 3-way split
|
||||
# mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf 2,786,568,256 draft head
|
||||
# vision/mmproj-F16.gguf 904,004,000 projector
|
||||
# UD-IQ4_XS/ the three verified transport parts (kept for --verify-only)
|
||||
# SHA256SUMS
|
||||
set -euo pipefail
|
||||
|
||||
REPO="unsloth/Qwen3.8-Flash-Next-GGUF"
|
||||
@@ -104,19 +105,11 @@ for rel in sorted(expected):
|
||||
sys.exit(0 if ok else 1)
|
||||
PY
|
||||
|
||||
FINAL="$MODEL_DIR/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf"
|
||||
if [[ "$mode" == "download" ]]; then
|
||||
echo "assembling the GGUF from the split parts"
|
||||
cat "$MODEL_DIR"/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-0000{1,2,3}-of-00003.gguf > "$FINAL"
|
||||
[[ "$(stat -c %s "$FINAL")" -eq 93682584224 ]] || {
|
||||
echo "assembled size wrong: $(stat -c %s "$FINAL")" >&2; exit 1; }
|
||||
mkdir -p "$MODEL_DIR/mtp" "$MODEL_DIR/vision"
|
||||
[[ -e "$MODEL_DIR/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" ]] ||
|
||||
mv "$MODEL_DIR/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" "$MODEL_DIR/mtp/"
|
||||
[[ -e "$MODEL_DIR/vision/mmproj-F16.gguf" ]] || mv "$MODEL_DIR/mmproj-F16.gguf" "$MODEL_DIR/vision/"
|
||||
(cd "$MODEL_DIR" && sha256sum Qwen3.8-Flash-Next-UD-IQ4_XS.gguf > SHA256SUMS)
|
||||
fi
|
||||
|
||||
echo "verifying assembled files against SHA256SUMS"
|
||||
(cd "$MODEL_DIR" && sha256sum -c SHA256SUMS)
|
||||
echo "frank-large payload complete: $MODEL_DIR"
|
||||
|
||||
@@ -51,10 +51,16 @@ if [[ ! -x "$SERVER_BIN" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
MODEL_FILE="$MODEL_DIR/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf"
|
||||
# Point -m at the FIRST split part, not a concatenated file: the parts
|
||||
# carry ggml.file_count=3 metadata in part 1's header, so even a
|
||||
# byte-correct concatenation still advertises itself as a 3-way split and
|
||||
# the loader rejects a non-split filename ("invalid split file name").
|
||||
# With -m on part 1, llama.cpp loads the 3-way split natively (parts 2-3
|
||||
# are auto-discovered in the same directory).
|
||||
MODEL_FILE="$MODEL_DIR/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf"
|
||||
if [[ ! -r "$MODEL_FILE" ]]; then
|
||||
echo "missing target model: $MODEL_FILE" >&2
|
||||
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
|
||||
echo "fetch it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
@@ -73,18 +73,24 @@ One root unit, one process, one published port:
|
||||
`:UD-IQ4_XS` tag is a directory in the repo, not a submodel).
|
||||
- **Transport**: three llama.cpp split parts (HF caps uploads at 50 GB):
|
||||
`00001` (10,946,624 B) + `00002` (49,835,229,856 B) + `00003`
|
||||
(43,836,407,744 B) = **93,682,584,224 B** concatenated in numeric order.
|
||||
- **Assembled at** `/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/`:
|
||||
- `Qwen3.8-Flash-Next-UD-IQ4_XS.gguf` — main model (87.3 GiB)
|
||||
(43,836,407,744 B) = **93,682,584,224 B**. The parts ARE the canonical
|
||||
payload — the serve script points `-m` at part 1 and llama.cpp loads the
|
||||
3-way split natively (parts 2–3 auto-discovered in the same directory).
|
||||
Do **not** concatenate them into a single file to serve: part 1's header
|
||||
carries `ggml.file_count=3`, so a byte-correct concatenation still
|
||||
advertises itself as a split and the loader rejects the non-split
|
||||
filename with `invalid split file name` (caught on 2026-09-21 — the
|
||||
assembled file crash-looped the unit ~110 times before the diagnosis).
|
||||
- **On disk** at `/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/`:
|
||||
- `UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-0000{1,2,3}-of-00003.gguf` — the
|
||||
main model, 3-way split (87.3 GiB)
|
||||
- `mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf` — MTP head (2.6 GiB)
|
||||
- `vision/mmproj-F16.gguf` — projector (904 MB, not loaded while
|
||||
`ENABLE_VISION=0`)
|
||||
- `UD-IQ4_XS/` + `SHA256SUMS` — transport parts and the assembled digest
|
||||
- **Fetch**: `scripts/download-frank-large-unsloth.sh` — resumable
|
||||
(`curl -fL -C -`), each part verified against its **LFS sha256 fetched
|
||||
live from the Hub tree API** (not a file that could drift), then the
|
||||
assembled GGUF against `SHA256SUMS`. `--verify-only` re-checks without
|
||||
downloading.
|
||||
live from the Hub tree API** (not a file that could drift).
|
||||
`--verify-only` re-checks without downloading.
|
||||
- The n-gram embedding (PLE) is **inside the GGUF** as ordinary tensors —
|
||||
there is no external PLE payload and no pager, unlike the CIRU Orca
|
||||
packaging (79.4 GiB main + 52.4 GiB paged PLE).
|
||||
@@ -160,6 +166,14 @@ swap file is the backstop.
|
||||
holds the port. The deploy handler sweeps `pkill -f
|
||||
'Qwen3.8-Flash-Next-UD-IQ4_XS'` (model path, not binary name — frank-small
|
||||
is also a `llama-server`) and gates on `wait_port_free 7770`.
|
||||
- **`invalid split file name` at model load** (fails in ~50 ms, before any
|
||||
weights are read): `-m` points at a file whose GGUF header advertises a
|
||||
split (`ggml.file_count > 1`) but whose name is not a
|
||||
`-NNNNN-of-NNNNN` part. This is what a concatenated split file does —
|
||||
the concatenation is byte-correct but part 1's header still says
|
||||
"3-way split". Fix: point `-m` at part 1
|
||||
(`UD-IQ4_XS/...-00001-of-00003.gguf`) and let the loader pick up parts
|
||||
2–3 from the same directory.
|
||||
- **Host memory pressure**: this endpoint is the largest LPDDR consumer on
|
||||
the box. If the co-tenants start thrashing, the levers are KV quant
|
||||
(`--cache-type-k/v q8_0` halves the ~5 GiB KV) and `SPEC_DRAFT_N_MAX`
|
||||
|
||||
@@ -52,7 +52,7 @@ timestamp: 2026-09-14
|
||||
Qwen3.8-Flash-CIRU-STRIX-Orca, 262144 ctx) to the unsloth llama.cpp
|
||||
prebuilt (b11030-mix-5ff778e, ROCm gfx1151, bundles its own ROCm 7.16)
|
||||
serving `unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS` (87.3 GiB, three HF
|
||||
split parts concatenated), one conversation at 200,000 ctx, MTP depth 2.
|
||||
split parts), one conversation at 200,000 ctx, MTP depth 2.
|
||||
The fleet contract is unchanged: port 7770, alias `frank-large`,
|
||||
text-only.
|
||||
- **Why the unsloth prebuilt.** The qwen4exp MTP graph exists only in the
|
||||
@@ -80,6 +80,19 @@ timestamp: 2026-09-14
|
||||
- **Files.** `scripts/serve-frank-large.sh`, `scripts/frank-large.env`,
|
||||
`scripts/frank-large.service`, `scripts/download-frank-large-unsloth.sh`,
|
||||
`scripts/deploy-remote.sh`, `tests/frank_large_unit.bats`.
|
||||
- **The split-filename trap (caught at cutover).** The first cutover
|
||||
crash-looped the unit ~110 times with `invalid split file name` at model
|
||||
load (~50 ms, before any weights were read): the fetch script
|
||||
concatenated the three parts into a single GGUF, but part 1's header
|
||||
carries `ggml.file_count=3`, so the byte-correct concatenation still
|
||||
advertises itself as a 3-way split and the loader demands a
|
||||
`-NNNNN-of-NNNNN` filename. Fix: the parts ARE the canonical payload —
|
||||
the serve script points `-m` at part 1 and llama.cpp loads the split
|
||||
natively (parts 2–3 auto-discovered in the same directory). A manual
|
||||
part-1 load confirmed the fix (model loaded in ~34 s from a warm page
|
||||
cache, health 200) before the redeploy. The redundant 87 GiB assembled
|
||||
file is staged at `~/.AGENTTRASH/20260921-frank-large-assembled/`
|
||||
(dankclaw) awaiting purge approval.
|
||||
|
||||
## 2026-09-21 — node-dankclaw Prometheus target fixed (netavark host-bind quirk)
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
<a id="bare-toggle" class="toggle" href="?bare=1">bare</a>
|
||||
</header>
|
||||
<div id="summary" class="summary"></div>
|
||||
<main id="grid" class="grid"></main>
|
||||
<main id="grid" class="sections"></main>
|
||||
<footer class="foot">
|
||||
<span>last master deployment: <span id="deployed-at"></span></span>
|
||||
<a href="/">home</a>
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
<a id="bare-toggle" class="toggle" href="?bare=1">bare</a>
|
||||
</header>
|
||||
<div id="summary" class="summary"></div>
|
||||
<main id="grid" class="grid"></main>
|
||||
<main id="grid" class="sections"></main>
|
||||
<footer class="foot">
|
||||
<span>last master deployment: <span id="deployed-at"></span></span>
|
||||
<a href="/docs">docs</a>
|
||||
|
||||
@@ -49,7 +49,10 @@ body::before {
|
||||
.title { color: var(--head); }
|
||||
.summary { color: var(--dim); margin: 12px 0; letter-spacing: 0.05em; }
|
||||
|
||||
.section { margin-bottom: 20px; }
|
||||
/* Sections render side by side: one column per section, panels stacked
|
||||
vertically inside. auto-fit wraps to fewer columns on narrow viewports. */
|
||||
.sections { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 16px; align-items: start; }
|
||||
.section { min-width: 0; }
|
||||
.section-title {
|
||||
color: var(--head);
|
||||
font-size: 1rem;
|
||||
@@ -63,7 +66,7 @@ body::before {
|
||||
|
||||
.grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(260px, 1fr));
|
||||
grid-template-columns: 1fr;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
|
||||
@@ -85,9 +85,10 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
grep -q 'unsloth/Qwen3.8-Flash-Next-GGUF' "$FETCH"
|
||||
grep -q -- '-fL -C -' "$FETCH"
|
||||
# The LFS oids come from the Hub tree API at verify time, not from a
|
||||
# file that could drift.
|
||||
# file that could drift; each part's local sha256 is compared to its
|
||||
# oid and a mismatch fails the run.
|
||||
grep -q 'tree/main' "$FETCH"
|
||||
grep -q 'sha256sum -c SHA256SUMS' "$FETCH"
|
||||
grep -q 'SHA256 MISMATCH' "$FETCH"
|
||||
}
|
||||
|
||||
@test "the fleet endpoint and alias survive the cutover" {
|
||||
|
||||
@@ -31,6 +31,12 @@ setup() {
|
||||
# renderSection()/renderPanel() components, not in per-page markup.
|
||||
grep -q "renderSection" "$BASE/app.js"
|
||||
grep -q "renderPanel" "$BASE/app.js"
|
||||
|
||||
@test "sections render horizontally (one column per section)" {
|
||||
grep -q 'class="sections"' "$BASE/index.html"
|
||||
grep -q 'class="sections"' "$BASE/admin.html"
|
||||
grep -q "repeat(auto-fit" "$BASE/style.css"
|
||||
}
|
||||
grep -q "SECTION_ORDER" "$BASE/app.js"
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user