#!/usr/bin/env python3 """Regenerate serverAppConfigs-docs/index.md from page frontmatter. The root index is a locator (AGENTS.md -> *Documentation Policy*): one row per page, one clause, no numbers. Hand-maintaining it is how rows go missing, and the docs seam fails when a page is not linked from it, so it is generated: * preamble everything above the first `## ` in the existing file is kept verbatim (that is the curated statement of the vault's law) * section from the page's directory, overridden by its frontmatter tags (`endpoint` / `retired-endpoint` split the LLM pages) * row text the page's frontmatter `description`, verbatim A page with no description is a hard error rather than an empty cell, so the generator cannot silently produce a silent index. Sections with no pages are skipped. Deterministic: the same vault always yields the same bytes. Sub-directory indices are hand-grouped (they carry curated prose), so they are synced rather than regenerated: dead rows dropped, row text refreshed from the page's description, uncovered siblings appended. """ import pathlib import re import sys RESERVED = {"index.md", "log.md"} SECTIONS = [ ("Inference endpoints", "endpoint"), ("Retired endpoints", "retired-endpoint"), ] DIRECTORY_SECTIONS = [ ("Dankclaw", "Dankclaw services"), ("Dankserver", "Dankserver services"), ("FrankBook-Pro", "FrankBook-Pro"), ("Desktop", "Workstations"), ("Infrastructure", "Cross-host infrastructure"), ("Architecture", "Repo architecture and policy"), ("Operations", "Operations and incidents"), ("Agent", "Agent / harness"), ("Research", "Research and methodology"), ] PIPELINE_SECTION = "Pipeline Parallelism (series)" LLM_FALLBACK = "LLM technique and hardware references" CHANGE_SECTION = "Change history" ORDER = [t for t, _ in SECTIONS] + [t for _, t in DIRECTORY_SECTIONS] + \ [PIPELINE_SECTION, LLM_FALLBACK, CHANGE_SECTION] def frontmatter_fields(text): if not text.startswith("---"): return {} end = text.find("\n---", 3) if end == -1: return {} raw = text[3:end] fields = {} for key in ("description", "type"): m = re.search(rf"^{key}:\s*(.*)$", raw, re.M) if m: fields[key] = m.group(1).strip().strip("\"'") tags = [] m = re.search(r"^tags:\s*\[(.*?)\]$", raw, re.M) if m: tags = [t.strip().strip("\"'") for t in m.group(1).split(",") if t.strip()] elif re.search(r"^tags:\s*$", raw, re.M): block = re.search(r"^tags:\s*\n((?:\s+-\s+.*\n?)+)", raw, re.M) if block: tags = [l.strip().lstrip("-").strip().strip("\"'") for l in block.group(1).splitlines() if l.strip()] fields["tags"] = tags return fields def section_for(page, vault, fields): rel = page.relative_to(vault) if len(rel.parts) == 1: return CHANGE_SECTION if page.name == "log.md" else "Uncategorised" top = rel.parts[0] if top == "LLM": if PIPELINE_SECTION_DIR in rel.parts: return PIPELINE_SECTION for title, tag in SECTIONS: if tag in fields.get("tags", []): return title return LLM_FALLBACK for prefix, title in DIRECTORY_SECTIONS: if top == prefix: return title return "Uncategorised" ROW = re.compile(r"^(\| )\[\[([^\]|]+)\]\]( \|)( .*?)( \|$)") BULLET = re.compile(r"^(- )\[\[([^\]]+)\]\]( — )(.*?)$") def sync_subindex(index: pathlib.Path, vault: pathlib.Path): """Keep a hand-grouped sub-index honest without erasing its grouping. Rows are locators, so their text is the linked page's own description; a row whose page is gone is a dead promise and is dropped; a direct sibling with no row is appended under `## Pages` (that is what the link-integrity seam checks). Curated headings and prose are untouched. """ here = index.parent known = {p.stem for p in here.rglob("*.md") if ".AGENTTRASH" not in str(p) and p.name != "index.md"} lines = index.read_text().split("\n") out, dropped, refreshed = [], 0, 0 for line in lines: # A row whose first cell is empty is a description with no page: the link # was deleted when the page was destroyed, so the row is a dead promise. if re.match(r"^\|\s*\|", line.strip()): dropped += 1 continue for pat, wrap in ((ROW, "| [[{}]] |{} {} |"), (BULLET, "- [[{}]] — {}")): m = pat.match(line) if not m: continue stem = m.group(2).strip() if stem not in known: dropped += 1 line = None break cand = next((p for p in here.rglob(f"{stem}.md") if ".AGENTTRASH" not in str(p)), None) desc = frontmatter_fields(cand.read_text(errors="replace")).get("description") \ if cand else None if desc: current = m.group(4).strip() if current != desc: line = wrap.format(stem, " " if pat is ROW else "", desc) \ if pat is ROW else wrap.format(stem, desc) refreshed += 1 break if line is not None: out.append(line) text = "\n".join(out) # Presence must be the wikilink itself: a bare stem is a substring of other # page names (`frank-medium` lives inside `frank-medium-rocmfpx`), and a # sibling matched that way silently never gets a row. missing = sorted( (p.stem, frontmatter_fields(p.read_text(errors="replace")).get("description", "")) for p in here.glob("*.md") if p.name != "index.md" and f"[[{p.stem}]]" not in text ) if missing: block = ["", "## Pages", "", "| Page | Owns |", "|---|---|"] block += [f"| [[{stem}]] | {desc} |" for stem, desc in missing] text = text.rstrip("\n") + "\n" + "\n".join(block) + "\n" if text != index.read_text(): index.write_text(text) return dropped, refreshed, len(missing) PIPELINE_SECTION_DIR = "Pipeline Parallelism" def main(argv): vault = pathlib.Path(argv[1] if len(argv) > 1 else "serverAppConfigs-docs") index = vault / "index.md" if not index.is_file(): print(f"missing {index}") return 2 existing = index.read_text() preamble = existing.split("\n## ", 1)[0].rstrip("\n") pages = [p for p in vault.rglob("*.md") if ".AGENTTRASH" not in str(p) and ".obsidian" not in str(p) and p.name != "index.md" and p != index] buckets, errors = {}, [] for page in pages: fields = frontmatter_fields(page.read_text(errors="replace")) if page.name != "log.md": desc = fields.get("description") if not desc: errors.append(str(page.relative_to(vault))) continue else: desc = "Chronological incident and law log for the whole bundle." buckets.setdefault(section_for(page, vault, fields), []).append((page.stem, desc)) if errors: print("pages with no description (fix the frontmatter, not the index):") for e in sorted(errors): print(" " + e) return 1 out = [preamble, ""] for title in ORDER + sorted(set(buckets) - set(ORDER)): rows = sorted(buckets.get(title, []), key=lambda r: r[0].lower()) if not rows: continue out += [f"## {title}", "", "| Page | Owns |", "|---|---|"] out += [f"| [[{stem}]] | {desc} |" for stem, desc in rows] out.append("") out.append("") index.write_text("\n".join(out) + "\n") touched = 0 for sub in sorted(vault.rglob("index.md")): if sub == index or ".AGENTTRASH" in str(sub): continue dropped, refreshed, added = sync_subindex(sub, vault) if dropped or refreshed or added: touched += 1 print(f" {sub}: -{dropped} dead rows, {refreshed} refreshed, +{added} added") print(f"{index}: {sum(len(v) for v in buckets.values())} rows in " f"{len([t for t in ORDER if buckets.get(t)])} sections, {touched} sub-indices synced") return 0 if __name__ == "__main__": sys.exit(main(sys.argv))