serverAppConfigs/scripts/gen-vault-index.py

#!/usr/bin/env python3
"""Regenerate serverAppConfigs-docs/index.md from page frontmatter.

The root index is a locator (AGENTS.md -> *Documentation Policy*): one row per
page, one clause, no numbers. Hand-maintaining it is how rows go missing, and
the docs seam fails when a page is not linked from it, so it is generated:

  * preamble     everything above the first `## ` in the existing file is kept
                 verbatim (that is the curated statement of the vault's law)
  * section      from the page's directory, overridden by its frontmatter tags
                 (`endpoint` / `retired-endpoint` split the LLM pages)
  * row text     the page's frontmatter `description`, verbatim

A page with no description is a hard error rather than an empty cell, so the
generator cannot silently produce a silent index. Sections with no pages are
skipped. Deterministic: the same vault always yields the same bytes.
"""
import pathlib
import re
import sys

RESERVED = {"index.md", "log.md"}
SECTIONS = [
    ("Inference endpoints", "endpoint"),
    ("Retired endpoints", "retired-endpoint"),
]
DIRECTORY_SECTIONS = [
    ("Dankclaw", "Dankclaw services"),
    ("Dankserver", "Dankserver services"),
    ("FrankBook-Pro", "FrankBook-Pro"),
    ("Desktop", "Workstations"),
    ("Infrastructure", "Cross-host infrastructure"),
    ("Architecture", "Repo architecture and policy"),
    ("Operations", "Operations and incidents"),
    ("Agent", "Agent / harness"),
    ("Research", "Research and methodology"),
]
PIPELINE_SECTION = "Pipeline Parallelism (series)"
LLM_FALLBACK = "LLM technique and hardware references"
CHANGE_SECTION = "Change history"
ORDER = [t for t, _ in SECTIONS] + [t for _, t in DIRECTORY_SECTIONS] + \
        [PIPELINE_SECTION, LLM_FALLBACK, CHANGE_SECTION]


def frontmatter_fields(text):
    if not text.startswith("---"):
        return {}
    end = text.find("\n---", 3)
    if end == -1:
        return {}
    raw = text[3:end]
    fields = {}
    for key in ("description", "type"):
        m = re.search(rf"^{key}:\s*(.*)$", raw, re.M)
        if m:
            fields[key] = m.group(1).strip().strip("\"'")
    tags = []
    m = re.search(r"^tags:\s*\[(.*?)\]$", raw, re.M)
    if m:
        tags = [t.strip().strip("\"'") for t in m.group(1).split(",") if t.strip()]
    elif re.search(r"^tags:\s*$", raw, re.M):
        block = re.search(r"^tags:\s*\n((?:\s+-\s+.*\n?)+)", raw, re.M)
        if block:
            tags = [l.strip().lstrip("-").strip().strip("\"'")
                    for l in block.group(1).splitlines() if l.strip()]
    fields["tags"] = tags
    return fields


def section_for(page, vault, fields):
    rel = page.relative_to(vault)
    if len(rel.parts) == 1:
        return CHANGE_SECTION if page.name == "log.md" else "Uncategorised"
    top = rel.parts[0]
    if top == "LLM":
        if PIPELINE_SECTION_DIR in rel.parts:
            return PIPELINE_SECTION
        for title, tag in SECTIONS:
            if tag in fields.get("tags", []):
                return title
        return LLM_FALLBACK
    for prefix, title in DIRECTORY_SECTIONS:
        if top == prefix:
            return title
    return "Uncategorised"


PIPELINE_SECTION_DIR = "Pipeline Parallelism"


def main(argv):
    vault = pathlib.Path(argv[1] if len(argv) > 1 else "serverAppConfigs-docs")
    index = vault / "index.md"
    if not index.is_file():
        print(f"missing {index}")
        return 2
    existing = index.read_text()
    preamble = existing.split("\n## ", 1)[0].rstrip("\n")

    pages = [p for p in vault.rglob("*.md")
             if ".AGENTTRASH" not in str(p) and ".obsidian" not in str(p)
             and p.name != "index.md" and p != index]
    buckets, errors = {}, []
    for page in pages:
        fields = frontmatter_fields(page.read_text(errors="replace"))
        if page.name != "log.md":
            desc = fields.get("description")
            if not desc:
                errors.append(str(page.relative_to(vault)))
                continue
        else:
            desc = "Chronological incident and law log for the whole bundle."
        buckets.setdefault(section_for(page, vault, fields), []).append((page.stem, desc))
    if errors:
        print("pages with no description (fix the frontmatter, not the index):")
        for e in sorted(errors):
            print("  " + e)
        return 1

    out = [preamble, ""]
    for title in ORDER + sorted(set(buckets) - set(ORDER)):
        rows = sorted(buckets.get(title, []), key=lambda r: r[0].lower())
        if not rows:
            continue
        out += [f"## {title}", "", "| Page | Owns |", "|---|---|"]
        out += [f"| [[{stem}]] | {desc} |" for stem, desc in rows]
        out.append("")
    out.append("<!-- generated by scripts/gen-vault-index.py; edit page frontmatter instead -->")
    index.write_text("\n".join(out) + "\n")
    print(f"{index}: {sum(len(v) for v in buckets.values())} rows in {len([t for t in ORDER if buckets.get(t)])} sections")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))