diff --git a/scripts/hf-manifest.py b/scripts/hf-manifest.py index ea0eea0..0dc80cb 100755 --- a/scripts/hf-manifest.py +++ b/scripts/hf-manifest.py @@ -4,9 +4,15 @@ Usage: python3 scripts/hf-manifest.py [outfile] Lists the repo tree via the HF API and, for each file, records the size and -the LFS sha256 (the X-Linked-ETag header on the resolve endpoint). Non-LFS -files (config.json, tokenizer, ...) have no LFS sha256, so they get a blank -hash field — hf-fetch.sh then checks size only for them. +the content sha256. LFS and Xet-backed files redirect the resolve endpoint to +a CDN (302), whose own headers carry the truth: X-Linked-Size is the byte +count and X-Linked-ETag is the sha256 of the file's bytes. The redirect must +NOT be followed: on Xet-backed repos the bridge answers with ETag = the Xet +CAS chunk-dag hash, which is NOT the content sha256 (seen on +Qwen/Qwen3.8-Flash-Next-FP8, 2026-09-26: every shard "failed" verification +against a hash no download could ever match). Non-LFS files (config.json, +tokenizer, ...) are answered directly (200) and carry no 64-hex etag, so they +get a blank hash field — hf-fetch.sh then checks size only for them. Output: one line per file, three space-separated fields: @@ -14,12 +20,13 @@ Output: one line per file, three space-separated fields: Token: $HF_TOKEN, else ~/.cache/huggingface/token, else the llamaswap conf (grep hf_...). Gated repos need a token that has been granted access. """ +import http.client import json import os import re import sys -import urllib.error import urllib.request +from urllib.parse import urlsplit def get_token(): @@ -40,11 +47,21 @@ def get_token(): def head(url, token): + """HEAD via http.client, which does not follow redirects. + + urllib's default opener follows the resolve endpoint's 302, and the Xet + bridge's ETag is the Xet CAS hash, not the content sha256 — so the 302's + own X-Linked-* headers must be read instead. Returns an http.client + response; the caller reads r.status and r.headers. + """ + parts = urlsplit(url) + conn = http.client.HTTPSConnection(parts.netloc, timeout=30) + path = parts.path + (f"?{parts.query}" if parts.query else "") h = {"User-Agent": "hf-manifest/1.0"} if token: h["Authorization"] = f"Bearer {token}" - req = urllib.request.Request(url, method="HEAD", headers=h) - return urllib.request.urlopen(req, timeout=30) + conn.request("HEAD", path, headers=h) + return conn.getresponse() def main(): @@ -54,8 +71,8 @@ def main(): repo = sys.argv[1] outfile = sys.argv[2] if len(sys.argv) > 2 else None token = get_token() - # 1. List the repo tree (siblings[].rfilename). Sizes and LFS sha256s - # come from the resolve endpoint's HEAD (Content-Length + X-Linked-ETag); + # 1. List the repo tree (siblings[].rfilename). Sizes and content sha256s + # come from the resolve endpoint's HEAD (not followed — see head()); # the API's siblings carry only rfilename. api = f"https://huggingface.co/api/models/{repo}" req = urllib.request.Request( @@ -71,22 +88,30 @@ def main(): name = s.get("rfilename", "") if not name or name.endswith("/"): continue - # 2. Size (Content-Length) + LFS sha256 (X-Linked-ETag) from the - # resolve endpoint's HEAD. Non-LFS files have no 64-hex etag, so the - # hash field stays blank (hf-fetch.sh checks size only for them). + # 2. Size + content sha256 from the resolve endpoint's HEAD, not + # followed (see head()). 200 = direct answer (Content-Length + + # ETag); 302 = LFS/Xet redirect (X-Linked-Size + X-Linked-ETag = + # the sha256 of the file's bytes). Non-LFS files carry no 64-hex + # etag, so the hash field stays blank (hf-fetch.sh then checks + # size only for them). size = 0 sha = "" resolve = f"https://huggingface.co/{repo}/resolve/main/{name}" try: with head(resolve, token) as r: - size = int(r.headers.get("Content-Length") or 0) - etag = (r.headers.get("X-Linked-ETag") - or r.headers.get("ETag") or "").strip('"') + if r.status == 404: + raise FileNotFoundError(name) + if r.status == 200: + size = int(r.headers.get("Content-Length") or 0) + etag = (r.headers.get("ETag") or "").strip('"') + else: + size = int(r.headers.get("X-Linked-Size") + or r.headers.get("Content-Length") or 0) + etag = (r.headers.get("X-Linked-ETag") or "").strip('"') if re.fullmatch(r"[0-9a-f]{64}", etag): sha = etag - except urllib.error.HTTPError as e: - if e.code != 404: - raise + except FileNotFoundError: + pass lines.append(f"{name} {size} {sha}") print(f" {name} {size:>12} {sha or '(size only)'}", file=sys.stderr)