Frank
2e86445fc5
Every comment that cited a named law, an incident date, or an internal nickname now states the underlying mechanism and why it matters, generically: curl exit-0 artifacts and byte-count gates, argv-in-/proc secret exposure, tag mutability vs digest pinning, shape-specialized kernels refusing uncompiled slot counts, MTP heads as file tensors no flag can supply, drafter/slot coupling as an engine property, pkill -f matching its invoker, pinned-RAM pairing with LimitMEMLOCK, one-encoding-per-table, the short-run first-token noise floor, and the protected-expert rule. No behavior change: comment/string text only; suite green (371 tests).
114 lines
4.4 KiB
Bash
Executable File
114 lines
4.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# download-frank-archon-swift15.sh — payload fetch/verify for the planned
|
|
# frank-archon swap to the Swift 1.5 finetune: ukisai/Swift1.5-Qwen3.8-Flash-Next
|
|
# (BF16 source, 149 files, 360,037,355,038 bytes = 335.3 GiB: 131 safetensors
|
|
# shards + weight-map index + tokenizer + vision preprocessing configs + chat
|
|
# template + licenses). Key set verified identical to the official
|
|
# Qwen/Qwen3.8-Flash-Next base (0 added / 0 missing), so the qwen4exp recipes
|
|
# apply verbatim; the checkpoint ships BF16 with NO quantization_config.
|
|
# Repo is UNGATED — no token needed (never put one on the command line).
|
|
#
|
|
# bash scripts/download-frank-archon-swift15.sh --fetch resume/complete
|
|
# bash scripts/download-frank-archon-swift15.sh verify only
|
|
#
|
|
# Fetch uses the vendored scripts/hfget.sh (StillDeadcode's Apache-2.0
|
|
# listing/resume downloader, curl+jq only): per-file size checks against
|
|
# the Hub listing, `-C -` resume, pickle-format duplicates skipped.
|
|
# Verify (default) enforces the download contract in three parts: every listed file
|
|
# present with the listed size, the shard count matches the weight-map
|
|
# index, and every shard's sha256 equals the Hub's LFS oid. Verify needs
|
|
# only the public API.
|
|
set -euo pipefail
|
|
|
|
REPO="ukisai/Swift1.5-Qwen3.8-Flash-Next"
|
|
DEST="/home/dank/models/ukisai/Swift1.5-Qwen3.8-Flash-Next"
|
|
EXPECTED_SHARDS=131
|
|
|
|
mode="verify"
|
|
[[ "${1:-}" == "--fetch" ]] && mode="fetch"
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
if [[ "$mode" == "fetch" ]]; then
|
|
# HFGET_JOBS=4 is hfget's default (a few streams saturate the link).
|
|
bash "$SCRIPT_DIR/hfget.sh" "$REPO" "$DEST" main
|
|
fi
|
|
|
|
echo "verifying local payload: $DEST"
|
|
fail=0
|
|
|
|
INDEX="$DEST/model.safetensors.index.json"
|
|
if [[ ! -f "$INDEX" ]]; then
|
|
echo " BAD weight-map index missing: $INDEX" >&2
|
|
exit 1
|
|
fi
|
|
|
|
python3 - "$REPO" "$DEST" <<'PY' || fail=1
|
|
import hashlib, json, os, sys, urllib.request
|
|
|
|
repo, dest = sys.argv[1], sys.argv[2]
|
|
|
|
def listing():
|
|
"""The tree API, following the Link cursor exactly like hfget.sh."""
|
|
out, api, pages = [], (
|
|
f"https://huggingface.co/api/models/{repo}/tree/main"
|
|
"?recursive=true&expand=true"), 0
|
|
while api:
|
|
with urllib.request.urlopen(api) as r:
|
|
out += json.load(r)
|
|
api = None
|
|
link = r.headers.get("Link", "")
|
|
for part in link.split(","):
|
|
if 'rel="next"' in part:
|
|
api = part[part.find("<") + 1:part.find(">")]
|
|
pages += 1
|
|
if pages > 200:
|
|
raise SystemExit("listing did not terminate")
|
|
return out
|
|
|
|
files = [e for e in listing() if e.get("type") == "file"]
|
|
sizes = {e["path"]: e.get("size", 0) for e in files}
|
|
lfs = {e["path"]: e["lfs"]["oid"] for e in files if e.get("lfs")}
|
|
|
|
bad = 0
|
|
for path, want in sorted(sizes.items()):
|
|
p = os.path.join(dest, path)
|
|
if not os.path.isfile(p):
|
|
print(f" BAD {path}: missing")
|
|
bad += 1
|
|
elif os.path.getsize(p) != want:
|
|
print(f" BAD {path}: {os.path.getsize(p)} bytes, listing says {want}")
|
|
bad += 1
|
|
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(sizes) - bad}/{len(sizes)} files present at listed size")
|
|
if bad:
|
|
raise SystemExit(1)
|
|
|
|
# Shard set against the weight-map index.
|
|
wm = json.load(open(os.path.join(dest, "model.safetensors.index.json")))["weight_map"]
|
|
shards = sorted(set(wm.values()))
|
|
print(f" index maps {len(wm)} tensors across {len(shards)} shards")
|
|
if len(shards) != 131:
|
|
print(f" BAD shard count {len(shards)}, want 131")
|
|
raise SystemExit(1)
|
|
|
|
# Content: every LFS-tracked file hashed against the Hub oid. The safetensors
|
|
# shards dominate the bytes; tokenizer/config files ride the same check when
|
|
# the Hub tracks them in LFS (they are plain blobs here, already size-checked).
|
|
todo = [p for p in shards if p in lfs]
|
|
for i, path in enumerate(todo):
|
|
h = hashlib.sha256()
|
|
with open(os.path.join(dest, path), "rb") as f:
|
|
for chunk in iter(lambda: f.read(1 << 24), b""):
|
|
h.update(chunk)
|
|
if h.hexdigest() != lfs[path]:
|
|
print(f" BAD {path}: sha256 mismatch (want {lfs[path]})")
|
|
bad += 1
|
|
if (i + 1) % 16 == 0:
|
|
print(f" ...hashed {i + 1}/{len(todo)}")
|
|
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(todo) - bad}/{len(todo)} shards sha256-match the Hub")
|
|
if bad:
|
|
raise SystemExit(1)
|
|
PY
|
|
|
|
[[ "$fail" -eq 0 ]] || exit 1
|
|
echo "payload verified: 131 shards + index + tokenizer + configs, sizes and sha256 checked"
|