Frank
3880503057
Add scripts/check-okf.py: enforces the OKF v0.1 hard rules the repo had been carrying by convention — parseable frontmatter with type/title/ description/timestamp, description shape (length, not-equals-title, no restated quantity), the reserved-name and index.md rules, log.md's date heading shape (bare ISO date, one entry per date, newest first), the 500-line hint, and the doubled-page signature (a second `type:` key in the body). Citations are reported as a soft tally, not a failure. The 500-line ceiling is escapable two ways, both explainable: `okf-exempt: length` in a page's own frontmatter, and pages derived as one-per- endpoint — live endpoints from scripts/, retired-endpoint era pages from their tag (AGENTS.md lets a campaign ledger live on the page that measured it). scripts/gen-vault-index.py: sibling-presence was tested as a bare stem substring, so a page whose name is a prefix of another page's name (`frank-medium` inside `frank-medium-rocmfpx`) silently never got an index row. Test the wikilink form instead. scripts/check-vault-links.py: `.AGENTTRASH` is deleted-but-held state, not vault content. It is now excluded from both the scan set and the target index, so a live link into the trash reads as unresolved instead of resolved. tests/docs_completeness.bats: the same exclusion in both page walks — trashed pages need no index row, and a trashed page must not satisfy the docs requirement for a live compose service. Adds the OKF conformance and index-freshness tests. scripts/port-census.py: the port-before-cut orphan census used to prove a merge lost no fact, kept so the next collapse can re-run it.
289 lines
11 KiB
Python
Executable File
289 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""OKF v0.1 conformance linter for a docs bundle (default serverAppConfigs-docs).
|
||
|
||
Checks the hard rules of the Open Knowledge Format spec plus the maintenance
|
||
guidance, so "conforms to OKF" is a test and not a claim:
|
||
|
||
FRONTMATTER every non-reserved .md has parseable YAML frontmatter with a
|
||
non-empty `type` from the spec's value set
|
||
TYPE the value is one of reference / howto / tutorial / explanation
|
||
DESC `description` is present, non-blank, not the title, <= 120 chars,
|
||
and does not restate a fleet quantity
|
||
TIMESTAMP `timestamp` present, ISO 8601, not in the future
|
||
LENGTH lines between 1 and 500 inclusive
|
||
INDEX-FM no index.md may carry frontmatter (index.md is reserved)
|
||
LOG-DATE log.md date headings start `## YYYY-MM-DD`, newest first
|
||
INDEX-COVER every index.md links each of its sibling pages; the root index
|
||
links every page in the bundle
|
||
INDEX-NUM an index table row restates a fleet quantity — a digit run of
|
||
three or more, a digits-plus-unit figure, a port, or a
|
||
multiplier. Names (Qwen3.6, RDNA4, RTX 3090, PICO-8, P4) are
|
||
not quantities and are allowed
|
||
CITATIONS informational: pages with factual claims and no Citations
|
||
section are counted, never failed
|
||
|
||
LENGTH lines between 1 and 500 inclusive. A page may opt out of the
|
||
ceiling with `okf-exempt: length` in its frontmatter: AGENTS.md
|
||
makes one page per endpoint permanent, and an endpoint page with
|
||
a full Eras table cannot split without creating a second owner
|
||
of the same shape (tests/docs_completeness.bats enforces that)
|
||
|
||
Exit 0 when there are no violations.
|
||
"""
|
||
import datetime
|
||
import re
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
TYPES = {"reference", "howto", "tutorial", "explanation"}
|
||
RESERVED = {"index.md", "log.md", "AGENTS.md"}
|
||
|
||
# A quantity, as opposed to a name: a comma-grouped figure, a run of 3+ digits,
|
||
# a unit-bearing figure, a port (":7770"), or a multiplier ("2x", "3 *").
|
||
QUANTITY = re.compile(
|
||
r"\d{1,3}(?:,\d{3})+" # 262,144
|
||
r"|\d{3,}" # 200000, 7770, 3090
|
||
r"|\d+(?:\.\d+)?\s*(?:GiB|MiB|GB|MB|TB|tok/s|t/s|x\b|%)"
|
||
r"|[:×]\s*\d"
|
||
r"|\d+\s*[x×]\s*\d"
|
||
)
|
||
# Product / model / standard names that legitimately contain digits.
|
||
NAMES = re.compile(
|
||
r"Qwen[\d.]*[-\w]*|Gemma\s?\d[\w.]*|MiMo[-\w.\d.]*|Ornith[-\w.\d]*"
|
||
r"|Bonsai[-\w.\d]*|Ling[-\w.\d.]*|DeepSeek[-\w.\d]*|GLM[-\w.\d]*"
|
||
r"|PICO-\d+|RDNA\s?\d+|R\d{4}|RTX\s?\d+|XDNA\d|gfx\d+|CUDA\s?[\d.]+"
|
||
r"|\d+\s?GbE|Wi-?Fi\s?\d|v?\d+\.\d+(?:\.\d+)?|b\d+|P\d\b|A\d+\b"
|
||
r"|frank-\w*2026-\d\d-\d\d|2026-\d\d-\d\d",
|
||
re.I,
|
||
)
|
||
DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}")
|
||
FM_KEY = re.compile(r"^([A-Za-z][\w-]*):\s*(.*)$")
|
||
|
||
|
||
def split_frontmatter(text):
|
||
if not text.startswith("---"):
|
||
return None, text
|
||
end = text.find("\n---", 3)
|
||
if end < 0:
|
||
return None, text
|
||
body_start = text.find("\n", end + 1)
|
||
return text[3:end], text[body_start + 1 :] if body_start >= 0 else ""
|
||
|
||
|
||
def fields(raw):
|
||
out = {}
|
||
if not raw:
|
||
return out
|
||
key = None
|
||
for line in raw.splitlines():
|
||
if not line.strip():
|
||
continue
|
||
m = FM_KEY.match(line)
|
||
if m:
|
||
key, val = m.group(1), m.group(2).strip().strip("\"'")
|
||
out[key] = val
|
||
elif line.startswith(" ") and key:
|
||
out[key] = (out[key] + " " + line.strip()).strip()
|
||
return out
|
||
|
||
|
||
def strip_names(value):
|
||
return NAMES.sub(" ", value)
|
||
|
||
|
||
def check_file(path: Path, root: Path, length_exempt=frozenset()):
|
||
v = []
|
||
rel = str(path.relative_to(root))
|
||
text = path.read_text(errors="replace")
|
||
lines = text.splitlines()
|
||
name = path.name
|
||
raw, body = split_frontmatter(text)
|
||
fm = fields(raw)
|
||
|
||
if name not in RESERVED:
|
||
if raw is None:
|
||
v.append(("FRONTMATTER", rel, "no parseable frontmatter"))
|
||
else:
|
||
t = fm.get("type", "")
|
||
if not t:
|
||
v.append(("TYPE", rel, "missing type"))
|
||
elif t not in TYPES:
|
||
v.append(("TYPE", rel, f"type '{t}' not in {sorted(TYPES)}"))
|
||
d = fm.get("description", "")
|
||
if not d:
|
||
v.append(("DESC", rel, "missing description"))
|
||
else:
|
||
if len(d) > 120:
|
||
v.append(("DESC", rel, f"description is {len(d)} chars (max 120)"))
|
||
if d.strip("# ").lower() == fm.get("title", "\0").strip("# ").lower():
|
||
v.append(("DESC", rel, "description equals title"))
|
||
if QUANTITY.search(strip_names(d)):
|
||
v.append(("DESC", rel, "description restates a quantity"))
|
||
ts = fm.get("timestamp", "")
|
||
if not ts:
|
||
v.append(("TIMESTAMP", rel, "missing timestamp"))
|
||
else:
|
||
try:
|
||
stamp = datetime.date.fromisoformat(ts[:10])
|
||
if stamp > datetime.date.today():
|
||
v.append(("TIMESTAMP", rel, f"timestamp {ts} is in the future"))
|
||
except ValueError:
|
||
v.append(("TIMESTAMP", rel, f"unparseable timestamp {ts!r}"))
|
||
|
||
if name == "index.md" and raw is not None:
|
||
v.append(("INDEX-FM", rel, "index.md must not carry frontmatter"))
|
||
|
||
if name == "log.md":
|
||
dates = []
|
||
for line in lines:
|
||
m = re.match(r"^##\s+(\S.*?)\s*$", line)
|
||
if not m:
|
||
continue
|
||
head = m.group(1)
|
||
if not DATE_RE.match(head):
|
||
continue
|
||
dates.append(head[:10])
|
||
if head != head[:10]:
|
||
v.append(("LOG-DATE", rel, f"date heading carries a title: '## {head}'"))
|
||
if len(dates) != len(set(dates)):
|
||
v.append(("LOG-DATE", rel, "duplicate date heading"))
|
||
if dates != sorted(dates, reverse=True):
|
||
v.append(("LOG-DATE", rel, "date headings not newest first"))
|
||
|
||
if raw is not None and name not in RESERVED:
|
||
n = len(lines)
|
||
body_lines = len([ln for ln in body.splitlines() if ln.strip()])
|
||
opted_out = "length" in fm.get("okf-exempt", "").lower()
|
||
if n > 500:
|
||
if rel in length_exempt:
|
||
v.append(("LENGTH-OK", rel,
|
||
f"{n} lines — over the hint, pinned by the "
|
||
"one-page-per-endpoint law"))
|
||
elif opted_out:
|
||
v.append(("LENGTH-OK", rel,
|
||
f"{n} lines — over the hint, opted out by the page"))
|
||
else:
|
||
v.append(("LENGTH", rel, f"{n} lines (max 500)"))
|
||
elif body_lines < 15:
|
||
v.append(("LENGTH", rel, "stub: fewer than 15 body lines"))
|
||
|
||
# A second `type:` key inside the body means the page carries two copies of
|
||
# itself with a frontmatter block in the middle — the shape a naive
|
||
# frontmatter rewrite produces. It doubles the length, splits the wikilinks
|
||
# and makes the page's facts ambiguous, so it is a hard failure.
|
||
if raw is not None and name not in RESERVED:
|
||
dupes = [i + len((raw or "").splitlines()) + 2
|
||
for i, ln in enumerate(body.splitlines())
|
||
if re.match(r"^(type|title|description):", ln)]
|
||
if dupes:
|
||
v.append(("DOUBLED", rel,
|
||
f"frontmatter keys inside the body at line(s) "
|
||
f"{', '.join(str(d) for d in dupes[:5])} — the page "
|
||
"contains a duplicated copy of itself"))
|
||
|
||
return v
|
||
|
||
|
||
def index_rows(text):
|
||
rows = []
|
||
for line in text.splitlines():
|
||
if line.startswith("|") and "[[" in line:
|
||
rows.append(line)
|
||
return rows
|
||
|
||
|
||
def check_links(root: Path):
|
||
v = []
|
||
pages = [
|
||
p for p in sorted(root.rglob("*.md"))
|
||
if ".AGENTTRASH" not in str(p) and ".obsidian" not in str(p)
|
||
]
|
||
for index in [p for p in pages if p.name == "index.md"]:
|
||
text = index.read_text(errors="replace")
|
||
links = set(re.findall(r"\[\[([^\]|#]+)", text))
|
||
if index.parent == root:
|
||
targets = [p for p in pages if p.name not in RESERVED]
|
||
else:
|
||
targets = [p for p in index.parent.glob("*.md") if p.name != "index.md"]
|
||
for target in targets:
|
||
if target.stem not in links and target.name not in links:
|
||
v.append(("INDEX-COVER", str(index.relative_to(root)),
|
||
f"does not link sibling {target.name}"))
|
||
for index in [p for p in pages if p.name == "index.md"]:
|
||
for row in index_rows(index.read_text(errors="replace")):
|
||
if QUANTITY.search(strip_names(row)):
|
||
v.append(("INDEX-NUM", str(index.relative_to(root)),
|
||
"row restates a quantity: " + row[:80]))
|
||
return v
|
||
|
||
|
||
def live_endpoints(root: Path):
|
||
"""Pages pinned by AGENTS.md's one-page-per-endpoint law.
|
||
|
||
Derived, never hardcoded — the same derivation as
|
||
tests/docs_completeness.bats: an endpoint is live when scripts/ holds both
|
||
its env and its unit. Those pages carry the Eras table for their whole
|
||
history, so the OKF 500-line split hint cannot apply to them without
|
||
creating a second owner of the same shape.
|
||
"""
|
||
scripts = root.parent / "scripts"
|
||
out = set()
|
||
if scripts.is_dir():
|
||
for envf in scripts.glob("frank-*.env"):
|
||
if (scripts / f"{envf.stem}.service").exists():
|
||
out.add(f"LLM/{envf.stem}.md")
|
||
return out
|
||
|
||
|
||
def retired_era_pages(root: Path):
|
||
"""Era pages of a retired endpoint: campaign ledgers, one owner each.
|
||
|
||
AGENTS.md lets a campaign ledger that exists nowhere else live on the page
|
||
that measured it, and a retired endpoint keeps exactly one page per era.
|
||
Splitting such a ledger to satisfy the 500-line hint would fragment one
|
||
campaign across two owners, so the tag opts the page out.
|
||
"""
|
||
out = set()
|
||
for path in root.rglob("*.md"):
|
||
s = str(path)
|
||
if ".AGENTTRASH" in s or ".obsidian" in s:
|
||
continue
|
||
raw, _ = split_frontmatter(path.read_text(errors="replace"))
|
||
if raw is not None and "retired-endpoint" in fields(raw).get("tags", ""):
|
||
out.add(str(path.relative_to(root)))
|
||
return out
|
||
|
||
|
||
def main(argv):
|
||
root = Path(argv[1] if len(argv) > 1 else "serverAppConfigs-docs")
|
||
exempt = frozenset(live_endpoints(root)) | retired_era_pages(root)
|
||
violations = []
|
||
notes = []
|
||
citation_gaps = 0
|
||
pages = 0
|
||
for path in sorted(root.rglob("*.md")):
|
||
s = str(path)
|
||
if ".AGENTTRASH" in s or ".obsidian" in s:
|
||
continue
|
||
pages += 1
|
||
for code, rel, detail in check_file(path, root, exempt):
|
||
(notes if code == "LENGTH-OK" else violations).append((code, rel, detail))
|
||
text = path.read_text(errors="replace")
|
||
if len(text.splitlines()) > 40 and not re.search(
|
||
r"^##\s+(Citations|Sources)\b", text, re.M):
|
||
citation_gaps += 1
|
||
violations += check_links(root)
|
||
|
||
for code, rel, detail in violations:
|
||
print(f" {code:12s} {rel}: {detail}")
|
||
for code, rel, detail in notes:
|
||
print(f" {code:12s} {rel}: {detail}")
|
||
print(f"{pages} concept pages, {len(violations)} OKF violations"
|
||
f" ({citation_gaps} pages without a Citations section — soft;"
|
||
f" {len(notes)} length exemptions honoured)")
|
||
return 1 if violations else 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main(sys.argv))
|