#!/usr/bin/env bash
# download-frank-archon.sh — payload fetch/verify for frank-archon:
# the OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint (131 safetensors
# shards + weight-map index + tokenizer + vision preprocessing configs,
# ~172.8 GB). 2026-10-05: replaces the orcarouter uncensored payload of the
# FreeToken era (that script lived here; its repo was gated and needed
# HF_TOKEN — this one is UNGATED and needs no token at all).
#
#   bash scripts/download-frank-archon.sh --fetch         resume/complete
#   bash scripts/download-frank-archon.sh                 verify only
#
# Fetch uses the vendored scripts/hfget.sh (StillDeadcode's Apache-2.0
# listing/resume downloader, curl+jq only): per-file size checks against
# the Hub listing, `-C -` resume, pickle-format duplicates skipped.
# Verify (default) is the AGENTS.md law in three parts: every listed file
# present with the listed size, the shard count matches the weight-map
# index, and every shard's sha256 equals the Hub's LFS oid. Verify needs
# only the public API.
set -euo pipefail

REPO="Qwen/Qwen3.8-Flash-Next-FP8"
DEST="/home/dank/models/Qwen/Qwen3.8-Flash-Next-FP8"
EXPECTED_SHARDS=131

mode="verify"
[[ "${1:-}" == "--fetch" ]] && mode="fetch"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

if [[ "$mode" == "fetch" ]]; then
    # HFGET_JOBS=4 is hfget's default (a few streams saturate the link).
    bash "$SCRIPT_DIR/hfget.sh" "$REPO" "$DEST" main
fi

echo "verifying local payload: $DEST"
fail=0

INDEX="$DEST/model.safetensors.index.json"
if [[ ! -f "$INDEX" ]]; then
    echo "  BAD  weight-map index missing: $INDEX" >&2
    exit 1
fi

python3 - "$REPO" "$DEST" <<'PY' || fail=1
import hashlib, json, os, sys, urllib.request

repo, dest = sys.argv[1], sys.argv[2]

def listing():
    """The tree API, following the Link cursor exactly like hfget.sh."""
    out, api, pages = [], (
        f"https://huggingface.co/api/models/{repo}/tree/main"
        "?recursive=true&expand=true"), 0
    while api:
        with urllib.request.urlopen(api) as r:
            out += json.load(r)
            api = None
            link = r.headers.get("Link", "")
            for part in link.split(","):
                if 'rel="next"' in part:
                    api = part[part.find("<") + 1:part.find(">")]
        pages += 1
        if pages > 200:
            raise SystemExit("listing did not terminate")
    return out

files = [e for e in listing() if e.get("type") == "file"]
sizes = {e["path"]: e.get("size", 0) for e in files}
lfs = {e["path"]: e["lfs"]["oid"] for e in files if e.get("lfs")}

bad = 0
for path, want in sorted(sizes.items()):
    p = os.path.join(dest, path)
    if not os.path.isfile(p):
        print(f"  BAD  {path}: missing")
        bad += 1
    elif os.path.getsize(p) != want:
        print(f"  BAD  {path}: {os.path.getsize(p)} bytes, listing says {want}")
        bad += 1
print(f"  {'OK' if bad == 0 else 'FAIL'}: {len(sizes) - bad}/{len(sizes)} files present at listed size")
if bad:
    raise SystemExit(1)

# Shard set against the weight-map index.
wm = json.load(open(os.path.join(dest, "model.safetensors.index.json")))["weight_map"]
shards = sorted(set(wm.values()))
print(f"  index maps {len(wm)} tensors across {len(shards)} shards")
if len(shards) != 131:
    print(f"  BAD  shard count {len(shards)}, want 131")
    raise SystemExit(1)

# Content: every LFS-tracked file hashed against the Hub oid. The safetensors
# shards dominate the bytes; tokenizer/config files ride the same check when
# the Hub tracks them in LFS (they are plain blobs here, already size-checked).
todo = [p for p in shards if p in lfs]
for i, path in enumerate(todo):
    h = hashlib.sha256()
    with open(os.path.join(dest, path), "rb") as f:
        for chunk in iter(lambda: f.read(1 << 24), b""):
            h.update(chunk)
    if h.hexdigest() != lfs[path]:
        print(f"  BAD  {path}: sha256 mismatch (want {lfs[path]})")
        bad += 1
    if (i + 1) % 16 == 0:
        print(f"  ...hashed {i + 1}/{len(todo)}")
print(f"  {'OK' if bad == 0 else 'FAIL'}: {len(todo) - bad}/{len(todo)} shards sha256-match the Hub")
if bad:
    raise SystemExit(1)
PY

[[ "$fail" -eq 0 ]] || exit 1
echo "payload verified: 131 shards + index + tokenizer + configs, sizes and sha256 checked"