serverAppConfigs/scripts/gen-vault-index.py

#!/usr/bin/env python3
"""Regenerate serverAppConfigs-docs/index.md from page frontmatter.

The root index is a locator (AGENTS.md -> *Documentation Policy*): one row per
page, one clause, no numbers. Hand-maintaining it is how rows go missing, and
the docs seam fails when a page is not linked from it, so it is generated:

  * preamble     everything above the first `## ` in the existing file is kept
                 verbatim (that is the curated statement of the vault's law)
  * section      from the page's directory, overridden by its frontmatter tags
                 (`endpoint` / `retired-endpoint` split the LLM pages)
  * row text     the page's frontmatter `description`, verbatim

A page with no description is a hard error rather than an empty cell, so the
generator cannot silently produce a silent index. Sections with no pages are
skipped. Deterministic: the same vault always yields the same bytes.

Sub-directory indices are hand-grouped (they carry curated prose), so they are
synced rather than regenerated: dead rows dropped, row text refreshed from the
page's description, uncovered siblings appended.
"""
import pathlib
import re
import sys

RESERVED = {"index.md", "log.md"}
SECTIONS = [
    ("Inference endpoints", "endpoint"),
    ("Retired endpoints", "retired-endpoint"),
]
DIRECTORY_SECTIONS = [
    ("Dankclaw", "Dankclaw services"),
    ("Dankserver", "Dankserver services"),
    ("FrankBook-Pro", "FrankBook-Pro"),
    ("Desktop", "Workstations"),
    ("Infrastructure", "Cross-host infrastructure"),
    ("Architecture", "Repo architecture and policy"),
    ("Operations", "Operations and incidents"),
    ("Agent", "Agent / harness"),
    ("Research", "Research and methodology"),
]
PIPELINE_SECTION = "Pipeline Parallelism (series)"
LLM_FALLBACK = "LLM technique and hardware references"
CHANGE_SECTION = "Change history"
ORDER = [t for t, _ in SECTIONS] + [t for _, t in DIRECTORY_SECTIONS] + \
        [PIPELINE_SECTION, LLM_FALLBACK, CHANGE_SECTION]


def frontmatter_fields(text):
    if not text.startswith("---"):
        return {}
    end = text.find("\n---", 3)
    if end == -1:
        return {}
    raw = text[3:end]
    fields = {}
    for key in ("description", "type"):
        m = re.search(rf"^{key}:\s*(.*)$", raw, re.M)
        if m:
            fields[key] = m.group(1).strip().strip("\"'")
    tags = []
    m = re.search(r"^tags:\s*\[(.*?)\]$", raw, re.M)
    if m:
        tags = [t.strip().strip("\"'") for t in m.group(1).split(",") if t.strip()]
    elif re.search(r"^tags:\s*$", raw, re.M):
        block = re.search(r"^tags:\s*\n((?:\s+-\s+.*\n?)+)", raw, re.M)
        if block:
            tags = [l.strip().lstrip("-").strip().strip("\"'")
                    for l in block.group(1).splitlines() if l.strip()]
    fields["tags"] = tags
    return fields


def section_for(page, vault, fields):
    rel = page.relative_to(vault)
    if len(rel.parts) == 1:
        return CHANGE_SECTION if page.name == "log.md" else "Uncategorised"
    top = rel.parts[0]
    if top == "LLM":
        if PIPELINE_SECTION_DIR in rel.parts:
            return PIPELINE_SECTION
        for title, tag in SECTIONS:
            if tag in fields.get("tags", []):
                return title
        return LLM_FALLBACK
    for prefix, title in DIRECTORY_SECTIONS:
        if top == prefix:
            return title
    return "Uncategorised"


ROW = re.compile(r"^(\| )\[\[([^\]|]+)\]\]( \|)( .*?)( \|$)")
BULLET = re.compile(r"^(- )\[\[([^\]]+)\]\]( — )(.*?)$")


def sync_subindex(index: pathlib.Path, vault: pathlib.Path):
    """Keep a hand-grouped sub-index honest without erasing its grouping.

    Rows are locators, so their text is the linked page's own description; a
    row whose page is gone is a dead promise and is dropped; a direct sibling
    with no row is appended under `## Pages` (that is what the link-integrity
    seam checks). Curated headings and prose are untouched.
    """
    here = index.parent
    known = {p.stem for p in here.rglob("*.md")
             if ".AGENTTRASH" not in str(p) and p.name != "index.md"}
    lines = index.read_text().split("\n")
    out, dropped, refreshed = [], 0, 0
    for line in lines:
        # A row whose first cell is empty is a description with no page: the link
        # was deleted when the page was destroyed, so the row is a dead promise.
        if re.match(r"^\|\s*\|", line.strip()):
            dropped += 1
            continue
        for pat, wrap in ((ROW, "| [[{}]] |{} {} |"), (BULLET, "- [[{}]] — {}")):
            m = pat.match(line)
            if not m:
                continue
            stem = m.group(2).strip()
            if stem not in known:
                dropped += 1
                line = None
                break
            cand = next((p for p in here.rglob(f"{stem}.md")
                         if ".AGENTTRASH" not in str(p)), None)
            desc = frontmatter_fields(cand.read_text(errors="replace")).get("description") \
                if cand else None
            if desc:
                current = m.group(4).strip()
                if current != desc:
                    line = wrap.format(stem, " " if pat is ROW else "", desc) \
                        if pat is ROW else wrap.format(stem, desc)
                    refreshed += 1
            break
        if line is not None:
            out.append(line)
    text = "\n".join(out)
    # Presence must be the wikilink itself: a bare stem is a substring of other
    # page names (`frank-medium` lives inside `frank-medium-rocmfpx`), and a
    # sibling matched that way silently never gets a row.
    missing = sorted(
        (p.stem, frontmatter_fields(p.read_text(errors="replace")).get("description", ""))
        for p in here.glob("*.md")
        if p.name != "index.md" and f"[[{p.stem}]]" not in text
    )
    if missing:
        block = ["", "## Pages", "", "| Page | Owns |", "|---|---|"]
        block += [f"| [[{stem}]] | {desc} |" for stem, desc in missing]
        text = text.rstrip("\n") + "\n" + "\n".join(block) + "\n"
    if text != index.read_text():
        index.write_text(text)
    return dropped, refreshed, len(missing)


PIPELINE_SECTION_DIR = "Pipeline Parallelism"


def main(argv):
    vault = pathlib.Path(argv[1] if len(argv) > 1 else "serverAppConfigs-docs")
    index = vault / "index.md"
    if not index.is_file():
        print(f"missing {index}")
        return 2
    existing = index.read_text()
    preamble = existing.split("\n## ", 1)[0].rstrip("\n")

    pages = [p for p in vault.rglob("*.md")
             if ".AGENTTRASH" not in str(p) and ".obsidian" not in str(p)
             and p.name != "index.md" and p != index]
    buckets, errors = {}, []
    for page in pages:
        fields = frontmatter_fields(page.read_text(errors="replace"))
        if page.name != "log.md":
            desc = fields.get("description")
            if not desc:
                errors.append(str(page.relative_to(vault)))
                continue
        else:
            desc = "Chronological incident and law log for the whole bundle."
        buckets.setdefault(section_for(page, vault, fields), []).append((page.stem, desc))
    if errors:
        print("pages with no description (fix the frontmatter, not the index):")
        for e in sorted(errors):
            print("  " + e)
        return 1

    out = [preamble, ""]
    for title in ORDER + sorted(set(buckets) - set(ORDER)):
        rows = sorted(buckets.get(title, []), key=lambda r: r[0].lower())
        if not rows:
            continue
        out += [f"## {title}", "", "| Page | Owns |", "|---|---|"]
        out += [f"| [[{stem}]] | {desc} |" for stem, desc in rows]
        out.append("")
    out.append("<!-- generated by scripts/gen-vault-index.py; edit page frontmatter instead -->")
    index.write_text("\n".join(out) + "\n")
    touched = 0
    for sub in sorted(vault.rglob("index.md")):
        if sub == index or ".AGENTTRASH" in str(sub):
            continue
        dropped, refreshed, added = sync_subindex(sub, vault)
        if dropped or refreshed or added:
            touched += 1
            print(f"  {sub}: -{dropped} dead rows, {refreshed} refreshed, +{added} added")
    print(f"{index}: {sum(len(v) for v in buckets.values())} rows in "
          f"{len([t for t in ORDER if buckets.get(t)])} sections, {touched} sub-indices synced")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))