fix(frank-large): load the model as its native 3-way split, not a concatenated file

The first cutover crash-looped the unit with 'invalid split file name':
the fetch script concatenated the three split parts into a single GGUF,
but part 1's header carries ggml.file_count=3, so the byte-correct
concatenation still advertises itself as a split and the loader demands
a -NNNNN-of-NNNNN filename.

- serve-frank-large.sh: -m points at part 1; llama.cpp loads the split
  natively (parts 2-3 auto-discovered in the same directory)
- deploy-remote.sh: LLM_LARGE_MODEL gate path follows
- download-frank-large-unsloth.sh: assembly removed; the parts ARE the
  canonical payload, verified against live Hub LFS oids
- docs: runbook model section + troubleshooting note, AGENTS.md, log
- the redundant 87 GiB assembled file is staged on dankclaw at
  ~/.AGENTTRASH/20260921-frank-large-assembled/
This commit is contained in:
Frank
2026-09-21 13:58:54 -07:00
parent 384747a72f
commit d6f9b8539a
11 changed files with 72 additions and 36 deletions
+1 -1
View File
@@ -1084,7 +1084,7 @@ deploy_anubis() {
# STRIX-Orca), which had replaced the halogen-flash-server quadlet. The
# port gate is the same as every cutover since 2026-09-16: nothing starts
# until 7770 is released.
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf}"
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf}"
LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf}"
retire_halogen_large_quadlet() {
+10 -17
View File
@@ -1,29 +1,30 @@
#!/usr/bin/env bash
# Download and assemble unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (the
# Download unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (the
# frank-large model) on Dankclaw.
#
# bash scripts/download-frank-large-unsloth.sh
# bash scripts/download-frank-large-unsloth.sh --verify-only
#
# The UD-IQ4_XS GGUF ships as three llama.cpp split parts (HF caps uploads at
# 50 GB); concatenating them in numeric order reconstructs the single GGUF.
# 50 GB). The parts ARE the canonical payload: the serve script points -m at
# part 1 and llama.cpp loads the 3-way split natively. Do NOT concatenate
# them into a single file to serve — part 1's header carries
# ggml.file_count=3, so a byte-correct concatenation still advertises itself
# as a split and the loader rejects the non-split filename.
# The MTP shared-Q8_0 head borrows the token embedding and the output
# projection from the main model at runtime (it is the 2.6 GB remainder of
# the 4B MTP layer). The F16 projector is fetched for a future vision flip;
# the serve script does not load it while ENABLE_VISION=0.
#
# Integrity: every part is checked against its LFS sha256 (fetched live from
# the Hub tree API, not trusted from disk), then the assembled GGUF against
# the recorded SHA256SUMS. The repo is public (gated: false); an HF token at
# ~/.cache/huggingface/token is used when present to clear the anonymous rate
# limit on a 94 GB transfer.
# the Hub tree API, not trusted from disk). The repo is public (gated:
# false); an HF token at ~/.cache/huggingface/token is used when present to
# clear the anonymous rate limit on a 94 GB transfer.
#
# Layout after a successful run (MODEL_DIR):
# Qwen3.8-Flash-Next-UD-IQ4_XS.gguf 93,682,584,224 main model
# UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-0000{1,2,3}-of-00003.gguf the 3-way split
# mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf 2,786,568,256 draft head
# vision/mmproj-F16.gguf 904,004,000 projector
# UD-IQ4_XS/ the three verified transport parts (kept for --verify-only)
# SHA256SUMS
set -euo pipefail
REPO="unsloth/Qwen3.8-Flash-Next-GGUF"
@@ -104,19 +105,11 @@ for rel in sorted(expected):
sys.exit(0 if ok else 1)
PY
FINAL="$MODEL_DIR/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf"
if [[ "$mode" == "download" ]]; then
echo "assembling the GGUF from the split parts"
cat "$MODEL_DIR"/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-0000{1,2,3}-of-00003.gguf > "$FINAL"
[[ "$(stat -c %s "$FINAL")" -eq 93682584224 ]] || {
echo "assembled size wrong: $(stat -c %s "$FINAL")" >&2; exit 1; }
mkdir -p "$MODEL_DIR/mtp" "$MODEL_DIR/vision"
[[ -e "$MODEL_DIR/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" ]] ||
mv "$MODEL_DIR/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" "$MODEL_DIR/mtp/"
[[ -e "$MODEL_DIR/vision/mmproj-F16.gguf" ]] || mv "$MODEL_DIR/mmproj-F16.gguf" "$MODEL_DIR/vision/"
(cd "$MODEL_DIR" && sha256sum Qwen3.8-Flash-Next-UD-IQ4_XS.gguf > SHA256SUMS)
fi
echo "verifying assembled files against SHA256SUMS"
(cd "$MODEL_DIR" && sha256sum -c SHA256SUMS)
echo "frank-large payload complete: $MODEL_DIR"
+8 -2
View File
@@ -51,10 +51,16 @@ if [[ ! -x "$SERVER_BIN" ]]; then
exit 1
fi
MODEL_FILE="$MODEL_DIR/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf"
# Point -m at the FIRST split part, not a concatenated file: the parts
# carry ggml.file_count=3 metadata in part 1's header, so even a
# byte-correct concatenation still advertises itself as a 3-way split and
# the loader rejects a non-split filename ("invalid split file name").
# With -m on part 1, llama.cpp loads the 3-way split natively (parts 2-3
# are auto-discovered in the same directory).
MODEL_FILE="$MODEL_DIR/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf"
if [[ ! -r "$MODEL_FILE" ]]; then
echo "missing target model: $MODEL_FILE" >&2
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
echo "fetch it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
exit 1
fi