hf-manifest.py: follow the 307 resolve-cache redirect for non-LFS files
The first no-redirect HEAD fix read X-Linked-Size on every 3xx, but non-LFS files on a Xet-backed repo answer 307 to HF's own /api/resolve-cache WITHOUT any X-Linked-Size header. The fallback to the stub's Content-Length recorded the redirect page size (254-264 bytes) instead of the file size, so hf-fetch.sh failed the size check on every aux file it verified against the regenerated manifest (README.md "65400 != 254", config.json "72514 != 258", 7 files on the FP8 repo). Distinguish the two redirect shapes: 3xx WITH X-Linked-Size = LFS/Xet, whose X-Linked-* headers are the truth and whose redirect must NOT be followed (the Xet bridge ETag is the CAS hash, not the sha256); 3xx WITHOUT = non-LFS, where the Location (followed once, via urljoin) answers 200 with the real Content-Length and a git-blob-sha ETag that the 64-hex gate correctly rejects (size-only check for those files). Verified against Qwen/Qwen3.8-Flash-Next-FP8: all 7 previously-failing aux files now record their on-disk byte counts exactly (README.md 65400, config.json 72514, merges.txt 3353259, ...), shard shas unchanged (model-00002 6841fe21...), total 172.82 GiB across 144 files.
This commit is contained in:
+23
-11
@@ -26,8 +26,7 @@ import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
def get_token():
|
||||
t = os.environ.get("HF_TOKEN", "")
|
||||
@@ -88,12 +87,16 @@ def main():
|
||||
name = s.get("rfilename", "")
|
||||
if not name or name.endswith("/"):
|
||||
continue
|
||||
# 2. Size + content sha256 from the resolve endpoint's HEAD, not
|
||||
# followed (see head()). 200 = direct answer (Content-Length +
|
||||
# ETag); 302 = LFS/Xet redirect (X-Linked-Size + X-Linked-ETag =
|
||||
# the sha256 of the file's bytes). Non-LFS files carry no 64-hex
|
||||
# etag, so the hash field stays blank (hf-fetch.sh then checks
|
||||
# size only for them).
|
||||
# 2. Size + content sha256 from the resolve endpoint's HEAD. 200 =
|
||||
# direct answer (Content-Length + ETag). 3xx WITH X-Linked-Size =
|
||||
# LFS/Xet redirect: X-Linked-Size + X-Linked-ETag are the byte
|
||||
# count and the sha256 of the file's bytes, and the redirect must
|
||||
# NOT be followed — the Xet bridge's own ETag is the Xet CAS hash,
|
||||
# not the content sha256. 3xx WITHOUT X-Linked-Size = non-LFS
|
||||
# redirect (307 to HF's own /api/resolve-cache): the stub's
|
||||
# Content-Length is NOT the file size, so follow ONCE; the final
|
||||
# ETag there is the git blob sha (40 hex), so the 64-hex gate
|
||||
# leaves the hash blank (hf-fetch.sh then checks size only).
|
||||
size = 0
|
||||
sha = ""
|
||||
resolve = f"https://huggingface.co/{repo}/resolve/main/{name}"
|
||||
@@ -105,9 +108,18 @@ def main():
|
||||
size = int(r.headers.get("Content-Length") or 0)
|
||||
etag = (r.headers.get("ETag") or "").strip('"')
|
||||
else:
|
||||
size = int(r.headers.get("X-Linked-Size")
|
||||
or r.headers.get("Content-Length") or 0)
|
||||
etag = (r.headers.get("X-Linked-ETag") or "").strip('"')
|
||||
linked = r.headers.get("X-Linked-Size")
|
||||
if linked is not None:
|
||||
size = int(linked)
|
||||
etag = (r.headers.get("X-Linked-ETag") or "").strip('"')
|
||||
else:
|
||||
loc = r.headers.get("Location", "")
|
||||
with head(urljoin(resolve, loc), token) as r2:
|
||||
if r2.status == 200:
|
||||
size = int(r2.headers.get("Content-Length") or 0)
|
||||
etag = (r2.headers.get("ETag") or "").strip('"')
|
||||
else:
|
||||
etag = ""
|
||||
if re.fullmatch(r"[0-9a-f]{64}", etag):
|
||||
sha = etag
|
||||
except FileNotFoundError:
|
||||
|
||||
Reference in New Issue
Block a user