#!/usr/bin/env bash
# Download unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (the
# frank-dragoon model) on Dankclaw.
#
#   bash scripts/download-frank-dragoon-unsloth.sh
#   bash scripts/download-frank-dragoon-unsloth.sh --verify-only
#
# The UD-IQ4_XS GGUF ships as three llama.cpp split parts (HF caps uploads at
# 50 GB). The parts ARE the canonical payload: the serve script points -m at
# part 1 and llama.cpp loads the 3-way split natively. Do NOT concatenate
# them into a single file to serve — part 1's header carries
# ggml.file_count=3, so a byte-correct concatenation still advertises itself
# as a split and the loader rejects the non-split filename.
# The MTP shared-Q8_0 head borrows the token embedding and the output
# projection from the main model at runtime (it is the 2.6 GB remainder of
# the 4B MTP layer). The F16 projector is fetched for a future vision flip;
# the serve script does not load it while ENABLE_VISION=0.
#
# Integrity: every part is checked against its LFS sha256 (fetched live from
# the Hub tree API, not trusted from disk). The repo is public (gated:
# false); an HF token at ~/.cache/huggingface/token is used when present to
# clear the anonymous rate limit on a 94 GB transfer.
#
# Layout after a successful run (MODEL_DIR):
#   UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-0000{1,2,3}-of-00003.gguf  the 3-way split
#   mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf   2,786,568,256  draft head
#   vision/mmproj-F16.gguf                         904,004,000  projector
set -euo pipefail

REPO="unsloth/Qwen3.8-Flash-Next-GGUF"
MODEL_DIR="/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS"
BASE="https://huggingface.co/$REPO/resolve/main"

mode="download"
[[ "${1:-}" == "--verify-only" ]] && mode="verify"

token=""
auth=()
if [[ -r "$HOME/.cache/huggingface/token" ]]; then
    token="$(tr -d '[:space:]' < "$HOME/.cache/huggingface/token")"
    auth=(-H "Authorization: Bearer $token")
fi

dl() { # dl <relpath>
    local rel="$1" dest="$MODEL_DIR/$rel"
    [[ -s "$dest" ]] && return 0
    mkdir -p "$(dirname "$dest")"
    echo "fetching $rel"
    curl -fL -C - "${auth[@]}" --retry 5 --retry-delay 3 -o "$dest" "$BASE/$rel"
}

# relpath size sha256 — the LFS oids from the Hub tree API (2026-09-21)
FILES=(
    "UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf 10946624 5ce89370720f8bf90890f439361282104c1aa1482d4013bb9a50923e758e71a4"
    "UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00002-of-00003.gguf 49835229856 577a38a2392b40ca2193cea502e1d92f60b8cd370675d308e0ec21885d9daaa7"
    "UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00003-of-00003.gguf 43836407744 d4634e6d84f0ebb0940be15c90d3790bf6464e3dea3a1cddc567dc0e83ad8833"
    "MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf 2786568256 5ff54097406a905cf3a724c709124ceb0e3e10235ee862298969e91c96fa96e6"
    "mmproj-F16.gguf 904004000 1f7b7f0b984cf065c604360c29c8098362ed61b290db0ff12c6f360bb1a8a980"
)

RELS=()
for line in "${FILES[@]}"; do
    read -r rel size _ <<<"$line"
    RELS+=("$rel")
    if [[ "$mode" == "download" ]]; then
        dl "$rel"
    fi
done

echo "verifying parts against the live Hub tree API"
python3 - "$MODEL_DIR" "$REPO" "${RELS[@]}" <<'PY'
import hashlib, json, os, sys, urllib.request

model_dir, repo = sys.argv[1], sys.argv[2]
tree = json.load(urllib.request.urlopen(
    f"https://huggingface.co/api/models/{repo}/tree/main?recursive=true"))
want = {}
for f in tree:
    if f["type"] == "file" and f.get("lfs"):
        want[f["path"]] = (f["lfs"]["size"], f["lfs"]["oid"])

def sha256(path):
    h = hashlib.sha256()
    with open(path, "rb") as fh:
        for chunk in iter(lambda: fh.read(1 << 24), b""):
            h.update(chunk)
    return h.hexdigest()

ok = True
expected = set(sys.argv[3:])
for rel in sorted(expected):
    if rel not in want:
        print(f"NOT IN HUB TREE {rel}"); ok = False; continue
    size, oid = want[rel]
    path = os.path.join(model_dir, rel)
    if not os.path.isfile(path):
        print(f"MISSING {rel}"); ok = False; continue
    actual_size = os.path.getsize(path)
    if actual_size != size:
        print(f"SIZE MISMATCH {rel}: {actual_size} != {size}"); ok = False; continue
    digest = sha256(path)
    if digest != oid:
        print(f"SHA256 MISMATCH {rel}"); ok = False; continue
    print(f"ok {rel}")
sys.exit(0 if ok else 1)
PY

if [[ "$mode" == "download" ]]; then
    mkdir -p "$MODEL_DIR/mtp" "$MODEL_DIR/vision"
    [[ -e "$MODEL_DIR/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" ]] ||
        mv "$MODEL_DIR/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" "$MODEL_DIR/mtp/"
    [[ -e "$MODEL_DIR/vision/mmproj-F16.gguf" ]] || mv "$MODEL_DIR/mmproj-F16.gguf" "$MODEL_DIR/vision/"
fi

echo "frank-dragoon payload complete: $MODEL_DIR"