diff --git a/scripts/hf-manifest.py b/scripts/hf-manifest.py index 0dc80cb..cbe3ed9 100755 --- a/scripts/hf-manifest.py +++ b/scripts/hf-manifest.py @@ -26,8 +26,7 @@ import os import re import sys import urllib.request -from urllib.parse import urlsplit - +from urllib.parse import urljoin, urlsplit def get_token(): t = os.environ.get("HF_TOKEN", "") @@ -88,12 +87,16 @@ def main(): name = s.get("rfilename", "") if not name or name.endswith("/"): continue - # 2. Size + content sha256 from the resolve endpoint's HEAD, not - # followed (see head()). 200 = direct answer (Content-Length + - # ETag); 302 = LFS/Xet redirect (X-Linked-Size + X-Linked-ETag = - # the sha256 of the file's bytes). Non-LFS files carry no 64-hex - # etag, so the hash field stays blank (hf-fetch.sh then checks - # size only for them). + # 2. Size + content sha256 from the resolve endpoint's HEAD. 200 = + # direct answer (Content-Length + ETag). 3xx WITH X-Linked-Size = + # LFS/Xet redirect: X-Linked-Size + X-Linked-ETag are the byte + # count and the sha256 of the file's bytes, and the redirect must + # NOT be followed — the Xet bridge's own ETag is the Xet CAS hash, + # not the content sha256. 3xx WITHOUT X-Linked-Size = non-LFS + # redirect (307 to HF's own /api/resolve-cache): the stub's + # Content-Length is NOT the file size, so follow ONCE; the final + # ETag there is the git blob sha (40 hex), so the 64-hex gate + # leaves the hash blank (hf-fetch.sh then checks size only). size = 0 sha = "" resolve = f"https://huggingface.co/{repo}/resolve/main/{name}" @@ -105,9 +108,18 @@ def main(): size = int(r.headers.get("Content-Length") or 0) etag = (r.headers.get("ETag") or "").strip('"') else: - size = int(r.headers.get("X-Linked-Size") - or r.headers.get("Content-Length") or 0) - etag = (r.headers.get("X-Linked-ETag") or "").strip('"') + linked = r.headers.get("X-Linked-Size") + if linked is not None: + size = int(linked) + etag = (r.headers.get("X-Linked-ETag") or "").strip('"') + else: + loc = r.headers.get("Location", "") + with head(urljoin(resolve, loc), token) as r2: + if r2.status == 200: + size = int(r2.headers.get("Content-Length") or 0) + etag = (r2.headers.get("ETag") or "").strip('"') + else: + etag = "" if re.fullmatch(r"[0-9a-f]{64}", etag): sha = etag except FileNotFoundError: