diff --git a/scripts/check-okf.py b/scripts/check-okf.py index f08593e..70262b6 100755 --- a/scripts/check-okf.py +++ b/scripts/check-okf.py @@ -12,7 +12,8 @@ guidance, so "conforms to OKF" is a test and not a claim: TIMESTAMP `timestamp` present, ISO 8601, not in the future LENGTH lines between 1 and 500 inclusive INDEX-FM no index.md may carry frontmatter (index.md is reserved) - LOG-DATE log.md date headings start `## YYYY-MM-DD`, newest first + LOG-DATE log.md date headings are `## YYYY-MM-DD` or `## YYYY-MM-DD — Title`, + one entry per date, newest first INDEX-COVER every index.md links each of its sibling pages; the root index links every page in the bundle INDEX-NUM an index table row restates a fleet quantity — a digit run of @@ -143,8 +144,12 @@ def check_file(path: Path, root: Path, length_exempt=frozenset()): if not DATE_RE.match(head): continue dates.append(head[:10]) - if head != head[:10]: - v.append(("LOG-DATE", rel, f"date heading carries a title: '## {head}'")) + # OKF requires the heading to USE the date format; a title after an + # em-dash is allowed and costs zero lines, so `## 2026-08-17 — Why the + # pair was rebuilt` is conformant and `## 2026-08-17Rebuilt` is not. + if head != head[:10] and not re.match(r"^\d{4}-\d{2}-\d{2}\s+—\s+\S", head): + v.append(("LOG-DATE", rel, + f"date heading must be the date alone or `date — Title`: '## {head}'")) if len(dates) != len(set(dates)): v.append(("LOG-DATE", rel, "duplicate date heading")) if dates != sorted(dates, reverse=True): diff --git a/scripts/port-census.py b/scripts/port-census.py index 3c38846..aac2b8e 100755 --- a/scripts/port-census.py +++ b/scripts/port-census.py @@ -50,23 +50,39 @@ def held(tok, corpus): def main(argv): new_dir = pathlib.Path(argv[1]) - corpus = [ - p.read_text(errors="replace") - for p in new_dir.rglob("*.md") - if ".AGENTTRASH" not in str(p) and ".obsidian" not in str(p) - ] - blob = "\n".join(corpus) - lost = 0 - checked = 0 + + def corpus_excluding(exclude): + """Every vault page except the one under test. + + The corpus is what must still hold a fact AFTER the file is gone, so a + file that lives inside the scanned directory has to be removed from it — + otherwise every token is "held" by the file itself and the census proves + nothing (it reports 0 losses for every page, which is the bug this + guards). + """ + out = [] + for p in new_dir.rglob("*.md"): + s = str(p) + if ".AGENTTRASH" in s or ".obsidian" in s: + continue + if exclude is not None and p.resolve() == exclude: + continue + out.append(p.read_text(errors="replace")) + return out + + lost = checked = 0 for spec in argv[2:]: if spec.startswith("git-"): ref, path = spec[4:].split(":", 1) old = subprocess.run(["git", "show", f"{ref}:{path}"], capture_output=True, text=True).stdout label = f"{ref}:{path}" + exclude = None else: old = pathlib.Path(spec).read_text(errors="replace") label = spec + exclude = pathlib.Path(spec).resolve() + blob = "\n".join(corpus_excluding(exclude)) for line in old.splitlines(): s = line.strip().strip("->*|# ").strip() if len(s) < 30 or s.startswith("```"): diff --git a/serverAppConfigs-docs/LLM/Pipeline Parallelism/index.md b/serverAppConfigs-docs/LLM/Pipeline Parallelism/index.md index 9e7f00f..1ffc51b 100644 --- a/serverAppConfigs-docs/LLM/Pipeline Parallelism/index.md +++ b/serverAppConfigs-docs/LLM/Pipeline Parallelism/index.md @@ -69,4 +69,3 @@ graph LR | Page | Owns | |---|---| -| [[Multi-Node Runtimes]] | How each two-machine runtime is built and launched, what it can combine with, and the mixed-vendor verdicts. | diff --git a/serverAppConfigs-docs/index.md b/serverAppConfigs-docs/index.md index 439a818..03c44b9 100644 --- a/serverAppConfigs-docs/index.md +++ b/serverAppConfigs-docs/index.md @@ -169,7 +169,6 @@ is a locator: one row, one clause, no numbers. |---|---| | [[Ethernet Bandwidth Math]] | Bytes-per-token math for pipeline versus tensor parallelism, and the link budgets that follow from it. | | [[Expected Performance]] | Estimated split-model throughput over ethernet, with the benchmark anchors the estimates rest on. | -| [[Multi-Node Runtimes]] | How each two-machine runtime is built and launched, what it can combine with, and the mixed-vendor verdicts. | | [[Qwen3.8 PP Deployment]] | The real split deployment: which node is head and which is worker, the flags, and the verified gotchas. | | [[Sample Script Set]] | An educational two-node RPC split script set, for reading only — never run it or deploy it. | diff --git a/serverAppConfigs-docs/log.md b/serverAppConfigs-docs/log.md index c415b6c..8974105 100644 --- a/serverAppConfigs-docs/log.md +++ b/serverAppConfigs-docs/log.md @@ -53,6 +53,23 @@ stale the moment main moves; re-probe `/slots` and the running cmdline before writing the number, not after.** +- **A census that includes the file under test proves nothing.** `scripts/port-census.py` + takes the new directory as its corpus, so every era page checked against + `serverAppConfigs-docs` was trivially held by *itself*: ten pages reported + `0 unheld tokens`, including `kingjones-q8agent-rocmfpx` at 481 lines whose own load + ledger cannot live anywhere else. Real numbers after the corpus excluded the page under + test: **14–63 sole-record lines per page — no era page in this vault is padding**, each + the only copy of a campaign ledger, which is what *Documentation Policy* protects. + **Law: a tool's own docstring promise ("excluding it") is not a check; assert the file + is absent from the corpus before believing a zero.** Same shape as *Key Lessons* → "A + scout verdict is a lead, never a finding": a mechanical tool is a scout too. +- So "compact the docs" reduces to deleting **restatement**, which is what landed: + `Multi-Node Runtimes.md` (−257, it restated launch material owned by + [[Qwen3.8 PP Deployment]]), 78 lines of transient deploy chatter out of this log (dropped + only where every fact token in the line is held by an owner page, preserved under + `.AGENTTRASH/20261019-log-transient/`), and 148 lines out of `AGENTS.md`. Docs flat, + operator doc down; the growth is the two enforcement tools. + ## 2026-10-18 - **frank-zealot resliced to two windows of exact KV (Frank's call, PR #21).** `--ctx-size 786432 --parallel 3` at q8_0 K+V → **`524288 / 2` at f16 K+V**, so @@ -79,7 +96,6 @@ **issue #3 research: AMD PerfOpt retires the reason for `amd_iommu=off`** - - Ask: does a recent kernel change remove the IOMMU translation penalty that justifies the 2026-09-11 `amd_iommu=off` boot line, i.e. can the XDNA2 NPU come back? Answered and documented; **no host change** at the @@ -113,11 +129,9 @@ `frank-npu.md` § *Why it is retired* corrected — the "`pt` is not a middle ground" law is dead — the AGENTS.md key-lesson bullet re-keyed, and a 4-step maintenance-window procedure that measures locally instead of - inheriting the vendor number. Nothing deployed — no service was touched. **boot line flipped: Dankclaw now boots `iommu=pt`** - - Window executed 01:24–01:35 on Frank's go-ahead, with the A/B the research demanded. Provenance first: `/etc/default/grub` had never been in the repo while the IOMMU line was the most consequential kernel decision @@ -125,7 +139,6 @@ byte-exact) and moves only through `scripts/set-dankclaw-bootline.sh` — diff, install 0644 root:root, `update-grub`, **verify the default `grub.cfg` entry**, reboot on `--reboot` only. The deployer plans nothing - under `dankclaw/host/`: kernel state is not a container restart. - **Verdict: identity DMA costs nothing on this host.** Byte-identical traffic, +2.0 % cold prefill going OFF→PT; the 13–16 % that justified `amd_iommu=off` for five weeks never existed here. Decode's apparent @@ -133,7 +146,6 @@ same boot spread 31.8–47.8 t/s. - **The XDNA2 NPU is back:** `amdxdna` bound `0000:c6:00.1`, `/dev/accel/accel0` `crw-rw---- root:render 261,0` with `dank` in - `render`. No endpoint deployed — revival needs a quadlet with `AddDevice=/dev/accel/accel0` + `GroupAdd=keep-groups`. - Deliberately NOT done: the `pp_dpm_fclk` level-7 pin (it changes iGPU timing for every workload and nothing co-runs on the NPU yet) and any @@ -142,7 +154,6 @@ - Gates after boot: 0 failed units, 32 IOMMU groups (31 identity + 1 DMA = the Genesys Logic SD reader, a kernel quirk), interrupt remapping on; frank-dragoon's ledger byte-for-byte the same shape and Gitea / Caddy - edge / Grafana answered 200. New tool `scripts/stage-frank-dragoon-eval.sh` rebuilds the A/B corpus byte-identically — run it through `bash -lc`, `uv` is in `~/.local/bin`. Docs re-keyed: the decision record `dankclaw-iommu-translation-off.md` (new § *execution*), `frank-npu.md` @@ -171,15 +182,11 @@ `SPEC_TYPE=draft-mtp` drop the flag. And `up{job="frank-zealot"}` had read 0 for the **entire retained Prometheus history**: llama.cpp answers **501** on `/metrics` unless `--metrics` is passed, so the scrape failed - beside a healthy `/health` and `LlmEndpointDown` fired for days. `METRICS=1` is now the default and the seam pins it. -- Deployed 13:16 (`deploy.sh dankclaw --service frank-zealot`) — and the first boot was **not the shipped payload**: a leftover `frank-zealot.service.d/ptq1.conf` from the bench window pointed `FRANK_ZEALOT_ENV_FILE` at `/tmp/zealot-arm-current.env`, so the unit - booted that arm. `/health` 200, `/v1/models` echoed the alias, Playwright 33/33 — **nothing in the gate reads the payload.** Drop-in moved to - `~/.AGENTTRASH/`, `daemon-reload`, restart: the shipped trio booted. (The drop-in was not a rogue — see the PTQ1_0 entry below.) - Measurement law: the first battery reported 117 and 72 t/s because it timed natural generations that stopped at 24 tokens — wall-clock aggregate @@ -218,7 +225,6 @@ **frank-dragoon: the 401,408-position pool goes live; docs re-keyed to the measured boot** - - `./scripts/deploy.sh dankclaw --service frank-dragoon` — e2e 33/33, and the drop-in check the zealot incident bought ran FIRST: `DropInPaths` empty, so the repo env governed. 2026-10-18 — frank-dragoon: the KV pool @@ -232,7 +238,6 @@ page cache loses. - Every document quoted **~10.8 GiB**, the ~28 KiB/position arithmetic; the engine reserves **11.0 GiB** and its preflight bills **11.3**. The measured - figure wins, and every "committed but not yet deployed" clause was retired with the measurement (AGENTS.md, the env lore block, both index pages, the iommu page, the runbook). - PTQ1_0 provenance corrected (see the dedicated entry): the arm that @@ -243,7 +248,6 @@ MiB freed, then scratched by Frank's 03:33 order in favour of the 9B. Nothing is owed on it. - Census of the halogen HTTP surface while doing it, because the fleet bench - law quoted the wrong engine: frank-dragoon answers `/health`, `/metrics`, `/v1/models`, `/v1/chat/completions`(POST) and 404s `/ping`, `/props`, `/completion`, `/tokenize`; the chat response carries **no `Server-Timing`** header. So the `/completion` `timings` object and the @@ -251,15 +255,12 @@ dragoon's. AGENTS.md's Benchmarking Policy re-keyed to say which engine owns which instrument. -**frank-tts phase 1: the cloned-voice endpoint, named and tested (not deployed)** - - Voice-over goal, phase 1 (Gitea issue #2). New service **frank-tts** on Dankclaw: `dankclaw/voiceservice/` (stdlib-only `app.py` + `Containerfile` + rootless quadlet), `scripts/frank-tts.env`, `tests/frank_tts_unit.bats`. One OpenAI-shaped `POST /v1/audio/speech` surface; `voice=` is honoured only when `/voices//reference.wav` exists, otherwise a 400 listing the voices that are — **no silent fallback voice**; oversized - input is a 414, not a half-narrated sentence; `/health` reports backend + voice list; no audio ever reaches the log. - **Naming law applied (Frank's call):** treat it as a first-class model endpoint → alias-named `frank-tts` (TTS endpoint, so not `frank-asr`), @@ -309,7 +310,6 @@ only and a bridge container cannot reach a host `127.0.0.1` bind — the lesson that 502'd hermes — so `host.containers.internal:8880`, which `scripts/frank-tts.env` documents, is correct only for a routable bind. - The deployed pair shares one quadlet network and the façade addresses the engine by container name. - **PR #12** (`feat/open-notebook-podcast` @ `114dd7a`) is **merged** as `1eb8b21` (Frank's OK): open-notebook + SurrealDB quadlets on the private @@ -319,9 +319,6 @@ runnable: nothing listens on 7772 until the voice stack lands, and the episode destination stays interim pending Gitea #7. -**open-notebook: the podcast generator's config lands (nothing deployed)** - - - Goal: **open-notebook** as the fleet's podcast generator on Dankclaw — notes and sources in, a multi-speaker episode out, script written by **frank-zealot**, voices rendered by **frank-tts**, no cloud provider @@ -332,7 +329,6 @@ frank-dragoon's pinning style, because `:latest` moving underneath a rootless service is how you get an unbootable pod after an unrelated `systemctl-daemon-reload`. Both health probes are argument-only - (`curl -fsS …/health`, `/surreal is-ready`), and `is-ready` refuses to pass without `SURREAL_PASS` in the environment — a credential that must never ride the command line, so `Exec=` carries flags only. - **Host 5055 was already spoken for** — seerr has held `127.0.0.1:5055` — @@ -369,20 +365,16 @@ `podman unshare`. One mount line; one deletion undoes it. - **Two UI buttons will fail, and that is cosmetic:** Test Connection does `GET /models` and the TTS probe synthesises voice `alloy`; - frank-tts serves `/health` and `/v1/audio/speech` only. Model rows are created *by name* and the voice field is free text, so `narrator1/2/3` is typed in and never discovered. Documented, not worked around. - Three deployer seams cost time and are now law-shaped: a service must be registered in **five** places — `SERVICE_ORDER`, `DANKCLAW_SERVICES`, the - `deploy-remote.sh` dispatch case, its pre-deploy requirement, and `deploy.sh`'s dispatch regex, where an omission silently plans an empty set for `--only`; a `dankclaw//*` case must precede the `dankclaw/*.container` glob because that `*` crosses `/`; and the changed-path routing lives *inside* the per-path loop, after the `scripts/*.env)` case, not at the top of the file. -**frank-tts-engine: the arm A engine becomes deployable (nothing deployed)** - - Goal: turn the engine the day's A/B picked into a **repo-owned, boot-persistent, self-healing container** — `frank-tts-engine`, `qwentts.cpp` commit `5916bf7c79dbb4e66fc059cdc507a79a23a28bb4`, resident @@ -395,7 +387,6 @@ containers therefore sit on one committed quadlet network, `frank-tts-engine.network` at **10.89.5.0/24** (the next free /24 after open-notebook's 10.89.4.0/24), the façade addresses the engine by - container name, and diagnostics — including the deployer's health gate (`wait_container_healthy`, there is no URL to dial) — run through `podman exec`. - **The boot warmer is a unit hook, not an operator habit.** The registry is @@ -403,7 +394,6 @@ reference, so `ExecStartPost=-/usr/bin/podman exec frank-tts-engine /app/warm-voices.sh` with `TimeoutStartSec=900`; the `-` prefix is what stops a failed warm from crash-looping the endpoint under - `Restart=always`, and it logs `cold=` and `warm=` per voice on every boot. - **Thread law recorded before anyone tunes it away:** no `--threads` exists; `src/backend.h` uses `hardware_concurrency()/2`, so a cpuset pin silently changes the thread count. Nothing pins this container. @@ -417,7 +407,6 @@ **frank-tts: the 7772 endpoint gets its deploy wiring, and voice naming is written down** - - Stage 2 of the voice stack: the **façade** in front of the engine is now a deployable service too. Runbook: [[Frank TTS]]. - **Two attached networks, in a meaningful order.** `dankclaw_default` @@ -446,14 +435,12 @@ - **No Prometheus job, on purpose.** The façade exports no `/metrics`, Prometheus 3 refuses HTML/JSON content types (the `comfyui` and `services` probes are documented down for exactly that), and `LlmEndpointDown` is - scoped `up{job=~"frank-.*"}` — so a `/health` scrape would page on Signal every few minutes for a healthy endpoint. That is the frank-archon never-green lesson pointed the other way, written in the page instead of quietly ignored. **voice-pack builder: the 10-second source-material gate becomes code** - - Third slice of the voice stack: `scripts/build-voice-pack.sh` turns a source recording plus a crop offset into a deployable pack — `reference.wav` (24 kHz mono `pcm_s16le`, 18–25 s), the engine's own @@ -505,18 +492,14 @@ skips `podman build` on a matching `cfhash` label, and that hash covered the `Containerfile` alone — so editing `app.py` or `tts-engine-warm.sh` (both `COPY`'d) skipped the build while the deploy reported success and - `/health` answered with old code. Each case arm now hashes the files its `Containerfile` copies. - Docs carry the lessons where an operator will hit them: a **Locally built images** section in [[deployment]], two new troubleshooting lines in [[Frank TTS Engine]] (all-voices-cold = quoted `WARM_TEXT`; façade 502 beside 200 = model loaded ≠ voices registered), and `Voice Packs.md` step 4 corrected — the warm hook re-registers **every** pack on every run, - because the engine replaces a name it already holds, so a restart is what an engine bump needs and not what a rebuilt pack needs. -**frank-tts-engine deployed: it builds, it serves, and the warm hook died on the kernel's argument cap** - - Merged as **#14** (`747c665`) and deployed `--service frank-tts-engine`; the image built clean — the first real proof of the `useradd` fix — and the unit was `healthy` after 31 s. @@ -526,7 +509,6 @@ a single argument at 131,072 bytes (`MAX_ARG_STRLEN`) and a shipped ~955 KB `reference.wav` base64's past that, so curl never executed and the hook logged that as "HTTP no response". Fourth silent fault of the voice stack, - same species as the other three: `/health` stayed green throughout. Fix: the body goes to a `mktemp` file and curl gets `@file`; the regression test is the honest shape — real listener, real `curl`, a 300 KB fixture pack — so it fails with the same kernel error the host produced. Every shipped @@ -552,7 +534,6 @@ - Merged **#15** (`0315a62`, the warmer's argument-cap fix) and **#16** (`d2ce669`, the comment sweep), then deployed both voice services in order (`--service frank-tts-engine`, then `--service frank-tts`); Playwright gate - 33/33 twice. Deployed-state proof from a LAN client: `/health` → `omni-http` with all three voices, and the same sentence in `narrator1` vs `narrator2` came back with different sha256 and different RMS — the name actually selects a voice. @@ -564,7 +545,6 @@ errors instead of degrading; and `models.yml` loads at process start while `config.yml` is read live, so a session older than the `frank-tts` row fails exactly like a dead endpoint. -- Corrected the same evening, by breaking it in a restarted session: a harness call that names no voice reaches 7772 with **no `voice` key at all** — `400 unknown voice None; available voices: …` — because no setting feeds one. The endpoint therefore grew `DEFAULT_VOICE` (its own PR): a @@ -615,7 +595,6 @@ - Also in the window: **PR #22 merged** (`0c7fe8b`, `dank/command-center`) so `agent/extensions/dankvoice.ts` landed on the auto-load path — proven in a plain `omp` session with no flags: `/say merged build check` → WAV to the - player, temp dir cleaned. Sessions open before the merge need a restart to pick the command up. **frank-zealot: 2 × 262,144 over unquantized f16 KV** @@ -638,7 +617,6 @@ ## 2026-10-17 **frank-zealot: Bonsai-2-27B PQ2_0 trial + cutover (2 × 200,704, q8_0 KV)** - - 2026-10-17 — frank-zealot: **Ternary-Bonsai-2-27B (PQ2_0) cutover**, two slots at 200,704 over q8_0 K + q8_0 V on the RTX 3090, fetched by `scripts/download-frank-zealot-bonsai2.sh` (exact bytes + sha256, @@ -672,12 +650,10 @@ `serve-frank-zealot.sh` still DEFAULTED to the retired payload's paths (a defaulted run would have served the old model — now re-keyed to the shipped trio), and the OpenWebUI `model` row's `meta.description` is live - DB state (re-keyed with a DB backup; no `open-webui` restart forced — forcing it logs every family session out for one cosmetic string). **dankserver disk purge: Swift w4dn8 rollback + orphan caches (docs re-key)** - - Ask: "dump the swift rollback and the GLM GGUF — make sure the model caches and agenttrash are empty as well." Preflight first: `lsof` empty on both payloads, the serving trio names only the live path, and a podman @@ -689,7 +665,6 @@ rollback is now a **re-download, not a local file**. Corpora (grams, kld-ref, calibration JSONL) deliberately KEPT: they are the re-convert and KLD-gate inputs. -- Post-checks: frank-archon active and `/health` ok, serving payload byte-size intact, the live kvcache tier untouched, no process references a deleted path. - Docs re-keyed: AGENTS.md fleet row + frank-archon section + both runbook @@ -701,7 +676,6 @@ **frank-dragoon cutover: native v2 abliterated payload, card's memory-saving pool** - - 2026-10-17 — frank-dragoon: cutover to the native v2 abliterated halogen payload with the card's memory-saving KV pool; census first: the n-gram table was ALREADY disk-backed and the only real lever was the KV pool. @@ -733,7 +707,6 @@ **frank-dragoon: ceiling and pool cut to 200,704; crawl reset fix** - - 2026-10-17 — frank-dragoon: context ceiling and KV pool cut to 200,704 (49 × 4096, the per-slot rung frank-zealot runs; nothing in the fleet needed the bigger window). See [[frank-dragoon]]. @@ -747,7 +720,6 @@ - Probes on the live endpoint: over the ceiling → `400 … tokens over` in 0.86 s — preflight rejects before any prefill, so probing costs the endpoint nothing. -- Deploy verdict: the endpoint dispatched healthy (`/health` 200), but the `[3/3]` Playwright gate failed on `12-crawl` and the deployer correctly said **DEPLOY INCOMPLETE**. Root cause was NOT the endpoint — it was Plex Web, signed out in the `qa-e2e` profile, running its own sign-in flow, @@ -766,7 +738,6 @@ ## 2026-10-15 **frank-archon: publisher iq4r payload swap (MODEL_FILE-only)** - - 2026-10-15 — frank-archon: publisher iq4r payload swap, `MODEL_FILE`-only — `StillDeadcode/qwen3.8-next-flash-fp8-iq4r-moe`, the engine author's published rad-convert of the OFFICIAL Qwen3.8-Flash-Next, **no other @@ -779,19 +750,15 @@ - Fetch: new `scripts/download-frank-archon-iq4r.sh` (hfget, exact bytes + Hub LFS sha256, `--verify-only` default) as a transient unit. The cutover **self-severed by design** — a detached `deploy.sh` survived the session - death when frank-archon restarted; engine healthy in ~100 s, and the prefix tier RECOVERED across the swap (arch+tokenizer unchanged). - e2e gate went red on three dankclaw specs — NOT the swap: dankclaw rebooted 01:41 and the gluetun boot-race dependency-failed `searxng`, `marreta`, and `marreta-browser` (jobs consumed while gluetun's start job - retried; `Restart=always` never armed because the services never ran). - Cascade restarted; skill codified (`gluetun-dependent-boot-race-recovery`). frank-archon's own health gate passed every deploy. **authentik: two more human accounts (Dankclaw)** - - **Live — mitch (pk 22) and eric (pk 23) provisioned via `ak shell` ORM, no repo objects**, each `@franksdank.cloud` in the human-user template shape (password via `set_password` on stdin only — never argv, @@ -805,8 +772,6 @@ power failure on dankserver reset `/sys/module/ttm/parameters/pages_limit` from the manual 200 GiB raise back to its default (half of RAM = 125,988 MiB), below the ~147 GiB pinned-RAM ledger the engine needs; the serve script's - preflight named the refusal and exited 1, and `Restart=always` re-ran it - **6,919 times in ~20 h** — `activating (auto-restart)`, exit 1 in 50 ms, no container, no engine log. Law adopted: **a sysfs raise is a countdown to the next reboot**, so the gate raises the knob itself. New `scripts/ttm-preflight.sh` is sourced by the serve script (verify, raise, @@ -879,7 +844,6 @@ (4,379 t/s), so the cliff is real and sits at ~26 GiB. The 2-slot arm that would clear it (pool 3,072 + slab 27,897) remains untested — and the paragraph above is amended: that arm's payoff is +22 % prefill at 2 slots, - not the +8.9 % the deployed pair ever showed (24,000 → 25,600 = 0 %). - **Frank's decode challenge, answered with numbers**: 512 tokens, greedy, cold — **decode 99.0 t/s single-stream at 43.5k and 93.3 at 197k** (decode is nearly window-free: −6 % for 4.5× depth), while **3 concurrent streams @@ -893,10 +857,7 @@ tokens, per-stream + aggregate) so no future shape change can be priced on prefill alone. -## 2026-10-11 -**frank-archon: unquantized (bf16) KV at 2 × 200,000** - - +## 2026-10-11 — frank-archon: unquantized (bf16) KV at 2 × 200,000 - 2026-10-11 — frank-archon: **unquantized (bf16) KV at 2 × 200,000** on exact attention state, with pool, slab and host pool rebalanced around the bigger cache. See [[frank-archon]]. @@ -941,7 +902,6 @@ `frank-archon` active/enabled (container `frank-archon`, :7770), `frank-dragoon` active/enabled (container `frank-dragoon-halogen`, :7770), `frank-zealot` active/enabled (journal shows 87–93 t/s serving - compaction on :7771). `/health` 200 ×3; `/v1/models` reports the three ids; host checkouts grep zero residue; e2e passed 32+1flaky, 32+1flaky, 33; Prometheus `frank-archon`/`frank-dragoon` up. - **Pre-existing gap found by the verification, NOT a rename regression:** @@ -950,7 +910,6 @@ it, so `up{job="frank-secondary"}` had NO series at all in the last 24 h (the zealot metrics + the queue-depth alerts on it have been dead since the Qwen3.5 swap). Fix = one flag in `serve-frank-zealot.sh` + a ~1 min - zealot restart; not applied in the rename window. ## 2026-10-09 - **Live — five authentik human accounts provisioned (Dankclaw, `ak shell` @@ -986,7 +945,6 @@ `webui.db.bak-20261009`): `model` row (`frank-zealot` → `frank-zealot`, params + `meta.description` preserved), the `access_grant` row (`model`/`frank-zealot`/`user`/`*`/`read`), and `openai.api_configs` - `model_ids` on all three backends; `systemctl --user restart open-webui.service`. Family-account proof, in a real browser as `qa-e2e`: the welcome screen selects `frank-zealot` and `POST /api/v1/chat/completions {"model":"frank-zealot"}` answers @@ -997,7 +955,6 @@ - **Live — Hermes re-keyed** (`config.yaml` `frank-dragoon` → `frank-dragoon` in the provider block, `model.default` and the three `dankclaw-large` auxiliary routes; backup `config.yaml.bak-20261009`; `hermes.service` + - `hermes-webui.service` restarted, gateway log clean, `/health` ok) and the repo baseline `dankclaw/hermes/config.yaml.example` synced to the live shape (context 262144). **Flagged, not fixed:** Hermes still routes its `dankserver-small` auxiliary tasks at the id `frank-medium` on @@ -1006,13 +963,10 @@ they need a routing decision (archon capacity), not a rename. - **Deployer blind spot found by accident:** every `deploy.sh dankserver --service frank-archon` since 2026-10-08 04:09 - restarted the **staging worktree**, not the deploy checkout — a campaign drop-in `frank-archon.service.d/bench.conf` overrode `Environment=` and - `ExecStart`. The health gate went green each time. Verified parity before removing it (serve script + unit byte-identical, env identical except the alias, staging HEAD an ancestor of `main`), moved the drop-in to `/home/dank/.AGENTTRASH/20261009-frank-archon-bench-dropin/`, and - restarted: `/v1/models` then answered `frank-archon`. Post-deploy behavioural proof is now part of the runbook. - Repo: seams re-pinned (`frank_zealot`/`frank_dragoon`/`frank_archon` unit bats), fleet harnesses retargeted (`bench-probe`, `deep-probe`, @@ -1132,13 +1086,11 @@ (refusals gone), vision back (the v2 tower loads beside the checkpoint), MTP head inside the checkpoint, 2×200,192 slots, `HALOGEN_SPEC_ADAPT=0`. Runbook: `serverAppConfigs-docs/LLM/frank-dragoon-flash-server-v2-swift15.md`. - **Incident (03:09:39–03:11:42):** the deploy started the unit while host `MemAvailable` was ~74.7 GiB, 3.34 GiB short of the engine's pin-gate requirement (62.11 GiB checkpoint + 16 GiB floor = 78.11 GiB); it crash-looped 8 boots, each dying in ~5 s at the pin gate, and self-resolved in ~2 min when the dying 27B engine freed RAM (boot #9 pinned clean at 88.4 GiB available). **Latent risk + fix:** the host was - left with only ~1.8 GiB `MemAvailable`, so a restart would crash-loop again; `serve-frank-dragoon.sh` now has a host-RAM preflight that checks `MemAvailable` against the pin gate BEFORE `podman run` and WAITS up to `FRANK_DRAGOON_RAM_WAIT` (600 s) for the RAM to clear, refusing loudly if @@ -1160,7 +1112,6 @@ `reasoning_content`; models.yml flipped accordingly). Container renamed to the unit name (Frank's call; `frank-archon-qwen` → legacy list); deployer image gate → `podman image exists` (local builds carry no - RepoDigests) and health gate reads the BODY — FreeToken 200s with `{"status":"loading"}` mid-load, which false-greened a deploy at 6 s (vLLM 503'd; the bare curl was honest there). Verified: FLEET-OK completion, reasoning split (17×24 → "408"), native `tool_calls`, vision @@ -1281,11 +1232,9 @@ 7,653 slots, budget 31.29. - Shape G v1 (0a4ff30) tried util 1.0: **both workers refused init before a single weight loaded** — `ValueError: Free memory on device cuda:N - (31.29/31.86 GiB ...)`, NRestarts=2 crash loop, zero payload progress. The CLI field is `Field(gt=0, le=1)`, but the worker demands free > budget, so budget == total is a trap, not a knife-edge. Frank: ".982 is the law, never 1.00" — now with a second proof. - - Shape G v2 (c67a2ae, deployed green incl. e2e 33/33): util back to 0.982, graph base to 0 (the shape F boot reported a NEGATIVE -0.06 GiB capture delta, so the 256 MiB base was fiction; 0 + 12 MiB/graph still prices a future 18-graph spec-decode shape at 0.21 GiB), KV_BYTES @@ -1305,7 +1254,6 @@ ## 2026-10-02 **frank-dragoon cutover: vLLM MiMo -> ROCmFPX Q8_0_ROCMFPX_AGENT + MTP** - - 2026-10-02 — frank-dragoon: vLLM MiMo → ROCmFPX `Q8_0_ROCMFPX_AGENT` payload + MTP from the repo's own bundled head, one session at the native 262,144. See [[frank-dragoon]]. @@ -1324,7 +1272,6 @@ **frank-zealot: Q5_K_M, then three contexts + vision, sized by two boots** - - 2026-10-02 — frank-zealot: Q5_K_M payload, then three contexts + vision in one day, sized by two boots (`CTX_SIZE=540672 PARALLEL=3`, f16 K + q8_0 V under flash attention, `--mmproj`). See [[frank-zealot]]. @@ -1353,7 +1300,6 @@ **frank-archon shape D: 3 x 200704, fp8, pool sized for 2 contexts** - Frank's order: shrink frank-archon to a 200704-token window with about two resident full-length contexts, cut the KV pool, and drop MNBT to 1024. The first attempt (`30904ba`, 2 x 200704 + SEQS=2) **crash-looped**; the revert @@ -1362,7 +1308,6 @@ top of the proven-good base, root cause first. - **ROOT CAUSE: `--max-num-seqs 2` cannot compile this model. It was never OOM, never MNBT, never the KV dtype.** Four single-variable boot probes - against the then-deployed shape (production stopped for each, candidate booted AS the service): 262144 x SEQS 2 fails at BOTH MNBT rungs with `torch.fx.experimental.symbolic_shapes.ConstraintViolationError: Constraints violated (L['query_start_loc'].size()[0]) ... specialized it @@ -1398,13 +1343,11 @@ top of the proven-good base, root cause first. - **Harness contract followed the window:** `models.yml` `contextWindow` 262144 -> **200704**; side effect: frank-archon's compaction request fits inside a frank-zealot slot again. A comment-only follow-up `295d578` was - deployed so the per-service revision marker sits at HEAD. Rollback for the pre-shape-D fp8+MTP4 era stays git tag `frank-archon-fp8-mtp-20261001`. Runbook [[frank-archon]]. **plex: the readiness gate probed HTTP against a TLS-only service** - Found by accident, and it had been latent since the secure-connections posture was set: `deploy_plex` gated on `curl -fsS http://127.0.0.1:32400/identity`, which **cannot** succeed. PMS drops plain HTTP off-loopback - the probe's @@ -1428,18 +1371,14 @@ and then answers with nothing, and curl reports "Empty reply from server" (52). got the same treatment. Fixed to `curl -fsSk --max-time 3 https://127.0.0.1:32400/identity` (`-k`: PMS serves its own self-signed cert on 32400). Re-deployed `39f0e0d`: plex recorded deployed, e2e - 33/33, frank-archon untouched (`health=200`, NRestarts=0). - **Operating lesson: `deploy.sh ` with no `--service` is a - fleet-wide drain.** It restarts every service whose recorded revision is behind HEAD and whose paths changed since. To touch one service, name it. **frank-archon shape E: `--expert-precise-memory` (+1,026 arena slots), a mounted compile cache, and the P2P answer** - Shape D left one question open — the plan's overhead term was reserving 2.35 GiB of compile headroom the boot never used — and the answer to Frank's free-VRAM question arrived together with a boot-cost bug that had been costing 181 s on -every restart since the cutover. - **`--expert-precise-memory` is real and it fits (option A).** Boot-probed on the shape-D base with production stopped, the candidate booted AS the service @@ -1463,11 +1402,9 @@ every restart since the cutover. 1.38x` — measured, rejected as the shipped value, kept as the conservative fallback. With 3.15e9/rank the endpoint keeps shape D's 1.99x concurrency and gains the arena, which is the shape D follow-up had actually been looking for. -- **THE COMPILE CACHE WAS BEING THROWN AWAY ON EVERY RESTART — 181 s each.** Measured: a fresh container has **no `/root/.cache` at all** (`No such file or directory`); after our boot it holds `/root/.cache/vllm` at **1,154,714,333 B in 4,665 files** plus `/root/.cache/comgr` at 229 MB, inside - the ephemeral layer, and `podman rm -f` (every deploy, restart, reboot) discards them. The engine names the loss itself: `Compiling a graph for compile range (1, 1024) takes 173.19 s` ... `torch.compile took 181.20 s in total` ... `init engine (profile, create kv cache, warmup model) took 217.26 s @@ -1520,7 +1457,6 @@ every restart since the cutover. not a crash — the container and its serve script must be polled by recorded id and their logs dumped on the exit path, or a failure looks like a bad flag. The flag was fine both times. -- Live: `health=200`, `NRestarts=0`, arena 17.22 GiB / 7,522 slots, KV 398,658 tokens / 1.99x. Docs: [[frank-archon]]. - **Deployed `1ac4d48`, e2e 33/33, and the live plan matched the probe exactly:** `'expert_precise_memory': True`, `runtime overhead … 5.25 GiB`, `expert cache @@ -1529,7 +1465,6 @@ every restart since the cutover. precise overhead and the arena growth are now production numbers, not a probe's projection. - **The compile-cache fix was then proven on the same endpoint, two consecutive - boots of the deployed shape:** cold `Compiling a graph for compile range (1, 1024) takes 179.24 s` → `torch.compile took 188.26 s`; warm `Directly load the compiled graph(s) for compile range (1, 1024) from the cache, took 2.143 s` → `torch.compile took 3.97 s in total` → `init engine (profile, @@ -1560,14 +1495,11 @@ every restart since the cutover. **frank-archon RESTORED to fp8 3 x 262144 (bf16 boot fails)** - bf16 (--kv-cache-dtype auto) at 2 x 262144 did not stay up, with OR without --expert-precise-memory: -- with precise: NRestarts 1->3, each attempt ~6.7 min then `Deactivated successfully` (API server drains after an EngineCore fatal error) — cause not captured, the serve script rm'd the container. -- without precise (default 7.60 GiB overhead, arena 14.3 GiB): NRestarts climbing, container `Exited (1)`, and the journal finally showed the real error: `RuntimeError: Worker failed with error 'Constraints violated @@ -1588,12 +1520,9 @@ Next: bisect bf16 vs SEQS in a window (bf16 + SEQS=3 isolates the dtype). **frank-archon: --expert-precise-memory + bf16 = boot crash loop** - **Reverted.** The two changes Frank asked for were applied together and the -result crash-looped: `NRestarts` climbed 1 -> 2 -> 3, each attempt running ~6.7 minutes then logging `Deactivated successfully` (vLLM's API server drains and exits 0 after an EngineCore fatal error, so `podman wait` returns 0; the -new `Restart=always` turned a silent death into a visible loop). The budget arithmetic explains it without needing the error text, because the arena is DEVICE-resident (the fp8 3-window shape's plan predicts 28.27 GiB and @@ -1621,7 +1550,6 @@ by paying for it elsewhere (smaller pool, fewer sessions, or a shorter context). **frank-archon: 2 bf16 windows + --expert-precise-memory** - Frank's final shape for the night. Two changes at once, both measured in principle before the boot: @@ -1647,7 +1575,6 @@ planned arena; it comes out of free VRAM instead. **frank-dragoon: Q4_K_XL payload + 4 x 163840 sessions** - Frank's call. Payload swapped from ajgazin's **abliterated** Swift-1.5 UD-Q8_K_XL (29.3 GiB) to **unsloth/Qwen3.8-27B-GGUF UD-Q4_K_XL** (16.35 GiB, sha256 3f227079…b01e, vanilla/Apache-2.0 — the endpoint is no longer @@ -1669,7 +1596,6 @@ weights 16.35 + KV 40.0 + drafter 1.07 + linear state 0.56 + runtime ~4 = **e2e gate flakiness (12-crawl) — do not panic at the banner** - `specs/12-crawl.spec.ts:23` ("every user open link reaches its app") failed the deploy gate twice and then passed cleanly on a manual re-run: - run 1 (deploy): `Expected "plex.franksdank.cloud"` / `Received "app.plex.tv"` @@ -1688,7 +1614,6 @@ any repo service. **frank-archon bf16 no-spec cutover** - - `/scripts/frank-archon.env` — bf16 no-spec cutover, FINAL shape after the night ladder: MAXLEN **241664** x SEQS 3, KV_BYTES **10,184,000,000**/rank, KV_DTYPE auto (bf16), MNBT 1024, GATHER CHUNKS=32/LANES=64, UTIL 0.98; @@ -1725,7 +1650,6 @@ any repo service. - **Client contract fixed (thinking + tools).** The live endpoint returned a null `reasoning` field and dumped the thinking text INLINE in `content` with a dangling `` (measured: `'We need answer…Brief.\n\n\n391'`) - — the malformed thinking block clients choke on — because the deployed argv had no `--reasoning-parser`. The GLM rollback (`d91379d`) had dropped all three contract flags and the qwen rollback inherited the gap. Restored from `ecaad36`'s values, confirmed against this fork's registries: @@ -1755,11 +1679,9 @@ any repo service. knife-edge shape change, send a large-`max_tokens` request before calling it done. FOUR slots x 262144 fp8 (KV_BYTES 8,311,013,376/rank) returns ~1.93 GiB per rank for activations/graph capture. Frank's call 2026-10-01. -- `/scripts/frank-archon.service` — **Restart=always, not on-failure** (plus `StartLimitIntervalSec=0` and `OOMScoreAdjust=-500`). A runtime worker fault makes vLLM's API server exit CLEAN, so ExecStart returned 0, systemd logged "Deactivated successfully", and on-failure NEVER fired: the endpoint stayed - down until a human noticed. An explicit `systemctl stop` still wins, so bench windows and deploys are unaffected. - **Final shape (Frank's call, 2026-10-01): 3 x 262144, pool at the FLOOR, MNBT 2048, rest to the arena.** MAXLEN 262144 x SEQS 3 = 786,432 tokens of @@ -1779,7 +1701,6 @@ any repo service. **frank-dragoon cutover: gufo -> vLLM TheRock, MiMo-V2.6 BF16 + cross-model MTP** - - **Question that started it:** can a Qwen3.5 MTP drafter attach to frank-zealot, and how does vLLM run the FULL BF16 MiMo checkpoint? Answer 1: not on frank-zealot — SGLang reads the MTP head from the TARGET checkpoint @@ -1793,7 +1714,6 @@ any repo service. prefill / 512 decode):** base 17,876 t/s prefill / 13.1 t/s decode (13.1 is ~94% of the LPDDR5x BF16 roof - decode here is bandwidth, not a bug); +DFlash-8 7,976 / 17.4 (acceptance 59%/26% per position); +MTP-2 7,380 / - 19.8 (71%/44%, token rate 57.4%). Deployed MTP-2: +51% decode. Both spec arms pay ~55-59% of prefill. `num_speculative_tokens` beyond 2 unmeasured. - **Measured numerics law:** spec arms do NOT reproduce the no-spec greedy text (bf16 batched verification reorders reductions; near-tie argmaxes @@ -1804,9 +1724,7 @@ any repo service. `reasoningContentField: "reasoning"`, and the extraBody moved to the Qwen3.5-family recipe (0.7/0.8) with `chat_template_kwargs` (a top-level `preserve_thinking` is ignored by vLLM). -- Deployer gap found pre-deploy: the spec-draft gate demanded a single `model.safetensors` (the dflash layout) and would have rejected the - deployed MTP arm (4 shards behind an index); fixed to accept both layouts, gated by bats. `z-lab/Qwen3.5-9B-DFlash`'s "vLLM PR #40898 unmerged" card claim is FALSE - `vllm/v1/spec_decode/dflash.py` ships in 0.30 (verified in-image); the drafter stays on disk as the alternative. @@ -1845,7 +1763,6 @@ any repo service. cutover), TP=2, 131,072 ctx x 2 seqs, bf16 KV, util 0.98, NBATCH 2048, `--enable-expert-offload --expert-precise-memory`, prefix caching, vision + `glm47` tool and reasoning parsers, MTP OFF, prefetch OFF. - - Measured on the deployed engine: **decode 9.9-10.0 tok/s, prefill 918 tok/s** (warm, 2.6k prompt), KV 138,561 tokens; arena 14.51 GiB = 2,383 slots per rank over 12,096 host-backed experts (73.64 GiB pinned per rank). - **The wall — and a corrected one.** The first account of the shortfall @@ -1951,7 +1868,6 @@ any repo service. per-sequence KDA state and the MTP draft's KV, and `gpu_worker` caps KV at that reserve, making a legal `--max-model-len` unbootable. Shipped `scripts/patches/gpu_worker-kv-cap.patch` (lets an explicit operator KV - request raise the cap) — NOT needed by the deployed shape. - Rollback: commit `2339fae` (qwen FP8 + MTP4); the qwen payload and the R9V state dirs are intact on disk. @@ -2131,7 +2047,6 @@ any repo service. (`20260921-frank-dragoon-assembled` 88 G, `-redownload` 30 G) and never purged. Total now staged on this host: **627 G**. - Verified after the moves: frank-dragoon and frank-zealot both `active`, - both `/health` 200, and all three live assets (Swift target, DFlash2 drafter, Gemma4 target) still present. - **frank-dragoon payload swapped to Swift-1.5 (uncensored)** (production change, Dankclaw). Frank asked whether the UD Q8 of @@ -2248,7 +2163,6 @@ any repo service. 8 s, drafter 2 s, session 3 s) against a 20-minute `LLM_WAIT` budget; RSS 30.6 GiB; a live short-prompt request returned `Server-Timing: ttft;dur=549.8, inter_token;dur=24.2` (~41 tok/s). - Verified live: `/health` 200, `/ready` 200, `/v1/models` → `frank-dragoon` at `context_length 262144`, a chat completion carrying `reasoning_content`, a `tool_calls` response with `finish_reason: tool_calls`, and SSE streaming. @@ -2315,9 +2229,6 @@ any repo service. - **LANDMINE, not changed here:** `/home/dank/models` was cleared by Frank mid-session to free space. frank-archon is still answering `:7770` from mmapped-but-unlinked inodes (`/models` is empty inside the running - container) — **the endpoint cannot be restarted until the 175 GiB payload - is re-fetched.** A reboot or a container restart takes frank-archon down. -- Deploy-state reconciliation (Frank's call: no restart) — the autoresearch trio (`f187165` PLE 1800, `41e9189` prefix caching, `ba2100e` docs) was found already MERGED to main + pushed AND already live on frank-archon: the running `frank-archon-r9v` container (booted 12:13 from the `ba2100e` checkout) carries @@ -2330,9 +2241,7 @@ any repo service. failure `11-admin` is the FrankBook-Pro Mac mini (`192.168.0.112:8989`) offline (100 % ICMP loss + TCP timeout), the documented asleep-host condition, not a regression. -- `scripts/frank-archon.env`, runbook — **prefix caching DEPLOYED on frank-archon** (Frank's go-ahead; commit `41e9189`): E8-qualified, zero - VRAM cost, 9x/63x reuse TTFT. Verified on :7770 (argv flag, health 200, completion "PREFIX-CACHE-LIVE"). Gotcha re-confirmed: the container is in dank's ROOTLESS podman store (`sudo podman` inspects an empty store and reports nothing); probe via the shim's `docker` or the unit MainPID. @@ -2464,7 +2373,6 @@ any repo service. ## 2026-08-26 - `/dankclaw/Caddyfile` — Both Plex vhosts proxy to PMS over a TLS upstream (`insecure_skip_verify`); PMS drops plain-HTTP connections from non-loopback sources (empty-body 502s). - `/dankclaw/gitea/quadlet/gitea-caddy.container` — Standalone Gitea Caddy quadlet retired; web proxy ownership moved to canonical Caddy; deploy-plan/deploy.sh/deploy-remote.sh + tests updated for the migration. -- Incident — Post-reboot: Caddy (a bare compose container — `restart: unless-stopped` does not survive shutdown's explicit stop) and Pi-hole (quadlet crash-loop, restart counter 1645) were both down. Root cause: `net.ipv4.ip_unprivileged_port_start` reset to 80 by a later-sorted sysctl drop-in AND a stale line in `/etc/sysctl.conf`, which systemd-sysctl applies AFTER all of `sysctl.d/`. - `/AGENTS.md` — New MANDATORY "Boot Persistence & Self-Healing" section. - `/dankclaw/caddy/quadlet/caddy.container`, `/dankclaw/caddy/env.example` — Caddy is now a boot-persistent quadlet (joins `dankclaw_default`, `Pull=never`, env from `/home/dank/Containers/caddy/env`); removed from compose.yaml. - `/dankclaw/pihole/sysctl/99-dnsmasq.conf` — Repo source of truth for `ip_unprivileged_port_start=53`; `deploy_pihole` stages conflicting drop-ins, comments out the stale sysctl.conf line, and applies `sysctl --system` (the boot code path).