hf-manifest.py: follow the 307 resolve-cache redirect for non-LFS files
The first no-redirect HEAD fix read X-Linked-Size on every 3xx, but non-LFS files on a Xet-backed repo answer 307 to HF's own /api/resolve-cache WITHOUT any X-Linked-Size header. The fallback to the stub's Content-Length recorded the redirect page size (254-264 bytes) instead of the file size, so hf-fetch.sh failed the size check on every aux file it verified against the regenerated manifest (README.md "65400 != 254", config.json "72514 != 258", 7 files on the FP8 repo). Distinguish the two redirect shapes: 3xx WITH X-Linked-Size = LFS/Xet, whose X-Linked-* headers are the truth and whose redirect must NOT be followed (the Xet bridge ETag is the CAS hash, not the sha256); 3xx WITHOUT = non-LFS, where the Location (followed once, via urljoin) answers 200 with the real Content-Length and a git-blob-sha ETag that the 64-hex gate correctly rejects (size-only check for those files). Verified against Qwen/Qwen3.8-Flash-Next-FP8: all 7 previously-failing aux files now record their on-disk byte counts exactly (README.md 65400, config.json 72514, merges.txt 3353259, ...), shard shas unchanged (model-00002 6841fe21...), total 172.82 GiB across 144 files.
This commit is contained in:
1 parent
5bd12d3193
commit
4992e5a010
1 file changed
+23
-11
+23
-11
@@ -26,8 +26,7 @@ import os
|
|||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
import urllib.request
|
import urllib.request
|
||||||
from urllib.parse import urlsplit
|
from urllib.parse import urljoin, urlsplit
|
||||||
|
|
||||||
|
|
||||||
def get_token():
|
def get_token():
|
||||||
t = os.environ.get("HF_TOKEN", "")
|
t = os.environ.get("HF_TOKEN", "")
|
||||||
@@ -88,12 +87,16 @@ def main():
|
|||||||
name = s.get("rfilename", "")
|
name = s.get("rfilename", "")
|
||||||
if not name or name.endswith("/"):
|
if not name or name.endswith("/"):
|
||||||
continue
|
continue
|
||||||
# 2. Size + content sha256 from the resolve endpoint's HEAD, not
|
# 2. Size + content sha256 from the resolve endpoint's HEAD. 200 =
|
||||||
# followed (see head()). 200 = direct answer (Content-Length +
|
# direct answer (Content-Length + ETag). 3xx WITH X-Linked-Size =
|
||||||
# ETag); 302 = LFS/Xet redirect (X-Linked-Size + X-Linked-ETag =
|
# LFS/Xet redirect: X-Linked-Size + X-Linked-ETag are the byte
|
||||||
# the sha256 of the file's bytes). Non-LFS files carry no 64-hex
|
# count and the sha256 of the file's bytes, and the redirect must
|
||||||
# etag, so the hash field stays blank (hf-fetch.sh then checks
|
# NOT be followed — the Xet bridge's own ETag is the Xet CAS hash,
|
||||||
# size only for them).
|
# not the content sha256. 3xx WITHOUT X-Linked-Size = non-LFS
|
||||||
|
# redirect (307 to HF's own /api/resolve-cache): the stub's
|
||||||
|
# Content-Length is NOT the file size, so follow ONCE; the final
|
||||||
|
# ETag there is the git blob sha (40 hex), so the 64-hex gate
|
||||||
|
# leaves the hash blank (hf-fetch.sh then checks size only).
|
||||||
size = 0
|
size = 0
|
||||||
sha = ""
|
sha = ""
|
||||||
resolve = f"https://huggingface.co/{repo}/resolve/main/{name}"
|
resolve = f"https://huggingface.co/{repo}/resolve/main/{name}"
|
||||||
@@ -105,9 +108,18 @@ def main():
|
|||||||
size = int(r.headers.get("Content-Length") or 0)
|
size = int(r.headers.get("Content-Length") or 0)
|
||||||
etag = (r.headers.get("ETag") or "").strip('"')
|
etag = (r.headers.get("ETag") or "").strip('"')
|
||||||
else:
|
else:
|
||||||
size = int(r.headers.get("X-Linked-Size")
|
linked = r.headers.get("X-Linked-Size")
|
||||||
or r.headers.get("Content-Length") or 0)
|
if linked is not None:
|
||||||
etag = (r.headers.get("X-Linked-ETag") or "").strip('"')
|
size = int(linked)
|
||||||
|
etag = (r.headers.get("X-Linked-ETag") or "").strip('"')
|
||||||
|
else:
|
||||||
|
loc = r.headers.get("Location", "")
|
||||||
|
with head(urljoin(resolve, loc), token) as r2:
|
||||||
|
if r2.status == 200:
|
||||||
|
size = int(r2.headers.get("Content-Length") or 0)
|
||||||
|
etag = (r2.headers.get("ETag") or "").strip('"')
|
||||||
|
else:
|
||||||
|
etag = ""
|
||||||
if re.fullmatch(r"[0-9a-f]{64}", etag):
|
if re.fullmatch(r"[0-9a-f]{64}", etag):
|
||||||
sha = etag
|
sha = etag
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
|
|||||||
Reference in new issue
Block a user