Frank
55f87fcdce
The 2026-10-09 wire aliases become the service names everywhere: systemd units, serve scripts, env files and FRANK_* variables, deploy-plan/deploy-remote service keys, container names, Prometheus job names + alert rules, Grafana dashboard queries, bats seams, runbook filenames, homepage/AGENTS lore, and historical prose. Rollback tags renamed in lockstep. Residue in the tree: zero. Retired frank-medium/small/npu names are history and stay. Known casualty: scripts/patches/*.patch context lines were rewritten; those retired-era vLLM-fork patches no longer apply byte-exact.
112 lines
4.2 KiB
Bash
Executable File
112 lines
4.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# download-frank-archon.sh — payload fetch/verify for frank-archon:
|
|
# the OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint (131 safetensors
|
|
# shards + weight-map index + tokenizer + vision preprocessing configs,
|
|
# ~172.8 GB). 2026-10-05: replaces the orcarouter uncensored payload of the
|
|
# FreeToken era (that script lived here; its repo was gated and needed
|
|
# HF_TOKEN — this one is UNGATED and needs no token at all).
|
|
#
|
|
# bash scripts/download-frank-archon.sh --fetch resume/complete
|
|
# bash scripts/download-frank-archon.sh verify only
|
|
#
|
|
# Fetch uses the vendored scripts/hfget.sh (StillDeadcode's Apache-2.0
|
|
# listing/resume downloader, curl+jq only): per-file size checks against
|
|
# the Hub listing, `-C -` resume, pickle-format duplicates skipped.
|
|
# Verify (default) is the AGENTS.md law in three parts: every listed file
|
|
# present with the listed size, the shard count matches the weight-map
|
|
# index, and every shard's sha256 equals the Hub's LFS oid. Verify needs
|
|
# only the public API.
|
|
set -euo pipefail
|
|
|
|
REPO="Qwen/Qwen3.8-Flash-Next-FP8"
|
|
DEST="/home/dank/models/Qwen/Qwen3.8-Flash-Next-FP8"
|
|
EXPECTED_SHARDS=131
|
|
|
|
mode="verify"
|
|
[[ "${1:-}" == "--fetch" ]] && mode="fetch"
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
if [[ "$mode" == "fetch" ]]; then
|
|
# HFGET_JOBS=4 is hfget's default (a few streams saturate the link).
|
|
bash "$SCRIPT_DIR/hfget.sh" "$REPO" "$DEST" main
|
|
fi
|
|
|
|
echo "verifying local payload: $DEST"
|
|
fail=0
|
|
|
|
INDEX="$DEST/model.safetensors.index.json"
|
|
if [[ ! -f "$INDEX" ]]; then
|
|
echo " BAD weight-map index missing: $INDEX" >&2
|
|
exit 1
|
|
fi
|
|
|
|
python3 - "$REPO" "$DEST" <<'PY' || fail=1
|
|
import hashlib, json, os, sys, urllib.request
|
|
|
|
repo, dest = sys.argv[1], sys.argv[2]
|
|
|
|
def listing():
|
|
"""The tree API, following the Link cursor exactly like hfget.sh."""
|
|
out, api, pages = [], (
|
|
f"https://huggingface.co/api/models/{repo}/tree/main"
|
|
"?recursive=true&expand=true"), 0
|
|
while api:
|
|
with urllib.request.urlopen(api) as r:
|
|
out += json.load(r)
|
|
api = None
|
|
link = r.headers.get("Link", "")
|
|
for part in link.split(","):
|
|
if 'rel="next"' in part:
|
|
api = part[part.find("<") + 1:part.find(">")]
|
|
pages += 1
|
|
if pages > 200:
|
|
raise SystemExit("listing did not terminate")
|
|
return out
|
|
|
|
files = [e for e in listing() if e.get("type") == "file"]
|
|
sizes = {e["path"]: e.get("size", 0) for e in files}
|
|
lfs = {e["path"]: e["lfs"]["oid"] for e in files if e.get("lfs")}
|
|
|
|
bad = 0
|
|
for path, want in sorted(sizes.items()):
|
|
p = os.path.join(dest, path)
|
|
if not os.path.isfile(p):
|
|
print(f" BAD {path}: missing")
|
|
bad += 1
|
|
elif os.path.getsize(p) != want:
|
|
print(f" BAD {path}: {os.path.getsize(p)} bytes, listing says {want}")
|
|
bad += 1
|
|
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(sizes) - bad}/{len(sizes)} files present at listed size")
|
|
if bad:
|
|
raise SystemExit(1)
|
|
|
|
# Shard set against the weight-map index.
|
|
wm = json.load(open(os.path.join(dest, "model.safetensors.index.json")))["weight_map"]
|
|
shards = sorted(set(wm.values()))
|
|
print(f" index maps {len(wm)} tensors across {len(shards)} shards")
|
|
if len(shards) != 131:
|
|
print(f" BAD shard count {len(shards)}, want 131")
|
|
raise SystemExit(1)
|
|
|
|
# Content: every LFS-tracked file hashed against the Hub oid. The safetensors
|
|
# shards dominate the bytes; tokenizer/config files ride the same check when
|
|
# the Hub tracks them in LFS (they are plain blobs here, already size-checked).
|
|
todo = [p for p in shards if p in lfs]
|
|
for i, path in enumerate(todo):
|
|
h = hashlib.sha256()
|
|
with open(os.path.join(dest, path), "rb") as f:
|
|
for chunk in iter(lambda: f.read(1 << 24), b""):
|
|
h.update(chunk)
|
|
if h.hexdigest() != lfs[path]:
|
|
print(f" BAD {path}: sha256 mismatch (want {lfs[path]})")
|
|
bad += 1
|
|
if (i + 1) % 16 == 0:
|
|
print(f" ...hashed {i + 1}/{len(todo)}")
|
|
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(todo) - bad}/{len(todo)} shards sha256-match the Hub")
|
|
if bad:
|
|
raise SystemExit(1)
|
|
PY
|
|
|
|
[[ "$fail" -eq 0 ]] || exit 1
|
|
echo "payload verified: 131 shards + index + tokenizer + configs, sizes and sha256 checked"
|