From 4992e5a010952fb9c6bdbfd87694336f8ccaa47b Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Sat, 26 Sep 2026 22:50:17 -0700
Subject: [PATCH] hf-manifest.py: follow the 307 resolve-cache redirect for
 non-LFS files

The first no-redirect HEAD fix read X-Linked-Size on every 3xx, but
non-LFS files on a Xet-backed repo answer 307 to HF's own
/api/resolve-cache WITHOUT any X-Linked-Size header. The fallback to the
stub's Content-Length recorded the redirect page size (254-264 bytes)
instead of the file size, so hf-fetch.sh failed the size check on every
aux file it verified against the regenerated manifest (README.md
"65400 != 254", config.json "72514 != 258", 7 files on the FP8 repo).

Distinguish the two redirect shapes: 3xx WITH X-Linked-Size = LFS/Xet,
whose X-Linked-* headers are the truth and whose redirect must NOT be
followed (the Xet bridge ETag is the CAS hash, not the sha256); 3xx
WITHOUT = non-LFS, where the Location (followed once, via urljoin)
answers 200 with the real Content-Length and a git-blob-sha ETag that
the 64-hex gate correctly rejects (size-only check for those files).

Verified against Qwen/Qwen3.8-Flash-Next-FP8: all 7 previously-failing
aux files now record their on-disk byte counts exactly (README.md 65400,
config.json 72514, merges.txt 3353259, ...), shard shas unchanged
(model-00002 6841fe21...), total 172.82 GiB across 144 files.
---
 scripts/hf-manifest.py | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/scripts/hf-manifest.py b/scripts/hf-manifest.py
index 0dc80cb..cbe3ed9 100755
--- a/scripts/hf-manifest.py
+++ b/scripts/hf-manifest.py
@@ -26,8 +26,7 @@ import os
 import re
 import sys
 import urllib.request
-from urllib.parse import urlsplit
-
+from urllib.parse import urljoin, urlsplit
 
 def get_token():
     t = os.environ.get("HF_TOKEN", "")
@@ -88,12 +87,16 @@ def main():
         name = s.get("rfilename", "")
         if not name or name.endswith("/"):
             continue
-        # 2. Size + content sha256 from the resolve endpoint's HEAD, not
-        # followed (see head()). 200 = direct answer (Content-Length +
-        # ETag); 302 = LFS/Xet redirect (X-Linked-Size + X-Linked-ETag =
-        # the sha256 of the file's bytes). Non-LFS files carry no 64-hex
-        # etag, so the hash field stays blank (hf-fetch.sh then checks
-        # size only for them).
+        # 2. Size + content sha256 from the resolve endpoint's HEAD. 200 =
+        # direct answer (Content-Length + ETag). 3xx WITH X-Linked-Size =
+        # LFS/Xet redirect: X-Linked-Size + X-Linked-ETag are the byte
+        # count and the sha256 of the file's bytes, and the redirect must
+        # NOT be followed — the Xet bridge's own ETag is the Xet CAS hash,
+        # not the content sha256. 3xx WITHOUT X-Linked-Size = non-LFS
+        # redirect (307 to HF's own /api/resolve-cache): the stub's
+        # Content-Length is NOT the file size, so follow ONCE; the final
+        # ETag there is the git blob sha (40 hex), so the 64-hex gate
+        # leaves the hash blank (hf-fetch.sh then checks size only).
         size = 0
         sha = ""
         resolve = f"https://huggingface.co/{repo}/resolve/main/{name}"
@@ -105,9 +108,18 @@ def main():
                     size = int(r.headers.get("Content-Length") or 0)
                     etag = (r.headers.get("ETag") or "").strip('"')
                 else:
-                    size = int(r.headers.get("X-Linked-Size")
-                               or r.headers.get("Content-Length") or 0)
-                    etag = (r.headers.get("X-Linked-ETag") or "").strip('"')
+                    linked = r.headers.get("X-Linked-Size")
+                    if linked is not None:
+                        size = int(linked)
+                        etag = (r.headers.get("X-Linked-ETag") or "").strip('"')
+                    else:
+                        loc = r.headers.get("Location", "")
+                        with head(urljoin(resolve, loc), token) as r2:
+                            if r2.status == 200:
+                                size = int(r2.headers.get("Content-Length") or 0)
+                                etag = (r2.headers.get("ETag") or "").strip('"')
+                            else:
+                                etag = ""
                 if re.fullmatch(r"[0-9a-f]{64}", etag):
                     sha = etag
         except FileNotFoundError: